xref: /illumos-gate/usr/src/uts/common/os/dumpsubr.c (revision 1c802681)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5ae115bc7Smrj  * Common Development and Distribution License (the "License").
6ae115bc7Smrj  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
21342440ecSPrasad Singamsetty 
227c478bd9Sstevel@tonic-gate /*
2386f21945SDave Plauger  * Copyright (c) 1998, 2010, Oracle and/or its affiliates. All rights reserved.
24dfec2ecfSJohn Levon  * Copyright 2018 Joyent, Inc.
256ccea422SJoyce McIntosh  * Copyright 2018 Nexenta Systems, Inc. All rights reserved.
267c478bd9Sstevel@tonic-gate  */
277c478bd9Sstevel@tonic-gate 
287c478bd9Sstevel@tonic-gate #include <sys/types.h>
297c478bd9Sstevel@tonic-gate #include <sys/param.h>
307c478bd9Sstevel@tonic-gate #include <sys/systm.h>
317c478bd9Sstevel@tonic-gate #include <sys/vm.h>
327c478bd9Sstevel@tonic-gate #include <sys/proc.h>
337c478bd9Sstevel@tonic-gate #include <sys/file.h>
347c478bd9Sstevel@tonic-gate #include <sys/conf.h>
357c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
367c478bd9Sstevel@tonic-gate #include <sys/mem.h>
377c478bd9Sstevel@tonic-gate #include <sys/mman.h>
387c478bd9Sstevel@tonic-gate #include <sys/vnode.h>
397c478bd9Sstevel@tonic-gate #include <sys/errno.h>
407c478bd9Sstevel@tonic-gate #include <sys/memlist.h>
417c478bd9Sstevel@tonic-gate #include <sys/dumphdr.h>
427c478bd9Sstevel@tonic-gate #include <sys/dumpadm.h>
437c478bd9Sstevel@tonic-gate #include <sys/ksyms.h>
447c478bd9Sstevel@tonic-gate #include <sys/compress.h>
457c478bd9Sstevel@tonic-gate #include <sys/stream.h>
467c478bd9Sstevel@tonic-gate #include <sys/strsun.h>
477c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
487c478bd9Sstevel@tonic-gate #include <sys/bitmap.h>
497c478bd9Sstevel@tonic-gate #include <sys/modctl.h>
507c478bd9Sstevel@tonic-gate #include <sys/utsname.h>
517c478bd9Sstevel@tonic-gate #include <sys/systeminfo.h>
527c478bd9Sstevel@tonic-gate #include <sys/vmem.h>
537c478bd9Sstevel@tonic-gate #include <sys/log.h>
547c478bd9Sstevel@tonic-gate #include <sys/var.h>
557c478bd9Sstevel@tonic-gate #include <sys/debug.h>
567c478bd9Sstevel@tonic-gate #include <sys/sunddi.h>
577c478bd9Sstevel@tonic-gate #include <fs/fs_subr.h>
587c478bd9Sstevel@tonic-gate #include <sys/fs/snode.h>
597c478bd9Sstevel@tonic-gate #include <sys/ontrap.h>
607c478bd9Sstevel@tonic-gate #include <sys/panic.h>
617c478bd9Sstevel@tonic-gate #include <sys/dkio.h>
627c478bd9Sstevel@tonic-gate #include <sys/vtoc.h>
637c478bd9Sstevel@tonic-gate #include <sys/errorq.h>
647c478bd9Sstevel@tonic-gate #include <sys/fm/util.h>
65e7cbe64fSgw #include <sys/fs/zfs.h>
667c478bd9Sstevel@tonic-gate 
677c478bd9Sstevel@tonic-gate #include <vm/hat.h>
687c478bd9Sstevel@tonic-gate #include <vm/as.h>
697c478bd9Sstevel@tonic-gate #include <vm/page.h>
70ca3e8d88SDave Plauger #include <vm/pvn.h>
717c478bd9Sstevel@tonic-gate #include <vm/seg.h>
727c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h>
73d3d50737SRafael Vanoni #include <sys/clock_impl.h>
74b280b2a3SStuart Maybee #include <sys/hold_page.h>
756ccea422SJoyce McIntosh #include <sys/cpu.h>
767c478bd9Sstevel@tonic-gate 
77ca3e8d88SDave Plauger #include <bzip2/bzlib.h>
78ca3e8d88SDave Plauger 
79dfec2ecfSJohn Levon #define	ONE_GIG	(1024 * 1024 * 1024UL)
80dfec2ecfSJohn Levon 
81ca3e8d88SDave Plauger /*
82ca3e8d88SDave Plauger  * Crash dump time is dominated by disk write time.  To reduce this,
83ca3e8d88SDave Plauger  * the stronger compression method bzip2 is applied to reduce the dump
84ca3e8d88SDave Plauger  * size and hence reduce I/O time.  However, bzip2 is much more
85ca3e8d88SDave Plauger  * computationally expensive than the existing lzjb algorithm, so to
86ca3e8d88SDave Plauger  * avoid increasing compression time, CPUs that are otherwise idle
87ca3e8d88SDave Plauger  * during panic are employed to parallelize the compression task.
88ca3e8d88SDave Plauger  * Many helper CPUs are needed to prevent bzip2 from being a
89ca3e8d88SDave Plauger  * bottleneck, and on systems with too few CPUs, the lzjb algorithm is
90ca3e8d88SDave Plauger  * parallelized instead. Lastly, I/O and compression are performed by
91ca3e8d88SDave Plauger  * different CPUs, and are hence overlapped in time, unlike the older
92ca3e8d88SDave Plauger  * serial code.
93ca3e8d88SDave Plauger  *
94ca3e8d88SDave Plauger  * Another important consideration is the speed of the dump
95ca3e8d88SDave Plauger  * device. Faster disks need less CPUs in order to benefit from
96ca3e8d88SDave Plauger  * parallel lzjb versus parallel bzip2. Therefore, the CPU count
97ca3e8d88SDave Plauger  * threshold for switching from parallel lzjb to paralled bzip2 is
98ca3e8d88SDave Plauger  * elevated for faster disks. The dump device speed is adduced from
99ca3e8d88SDave Plauger  * the setting for dumpbuf.iosize, see dump_update_clevel.
100ca3e8d88SDave Plauger  */
101ca3e8d88SDave Plauger 
102ca3e8d88SDave Plauger /*
103ca3e8d88SDave Plauger  * exported vars
104ca3e8d88SDave Plauger  */
105ca3e8d88SDave Plauger kmutex_t	dump_lock;		/* lock for dump configuration */
106ca3e8d88SDave Plauger dumphdr_t	*dumphdr;		/* dump header */
1077c478bd9Sstevel@tonic-gate int		dump_conflags = DUMP_KERNEL; /* dump configuration flags */
108ca3e8d88SDave Plauger vnode_t		*dumpvp;		/* dump device vnode pointer */
109ca3e8d88SDave Plauger u_offset_t	dumpvp_size;		/* size of dump device, in bytes */
110ca3e8d88SDave Plauger char		*dumppath;		/* pathname of dump device */
111ca3e8d88SDave Plauger int		dump_timeout = 120;	/* timeout for dumping pages */
112ca3e8d88SDave Plauger int		dump_timeleft;		/* portion of dump_timeout remaining */
113ca3e8d88SDave Plauger int		dump_ioerr;		/* dump i/o error */
114ca3e8d88SDave Plauger int		dump_check_used;	/* enable check for used pages */
115f6e214c7SGavin Maltby char	    *dump_stack_scratch; /* scratch area for saving stack summary */
116ca3e8d88SDave Plauger 
117ca3e8d88SDave Plauger /*
118ca3e8d88SDave Plauger  * Tunables for dump compression and parallelism. These can be set via
119ca3e8d88SDave Plauger  * /etc/system.
120ca3e8d88SDave Plauger  *
121ca3e8d88SDave Plauger  * dump_ncpu_low	number of helpers for parallel lzjb
122ca3e8d88SDave Plauger  *	This is also the minimum configuration.
123ca3e8d88SDave Plauger  *
124ca3e8d88SDave Plauger  * dump_bzip2_level	bzip2 compression level: 1-9
125ca3e8d88SDave Plauger  *	Higher numbers give greater compression, but take more memory
126ca3e8d88SDave Plauger  *	and time. Memory used per helper is ~(dump_bzip2_level * 1MB).
127ca3e8d88SDave Plauger  *
128ca3e8d88SDave Plauger  * dump_plat_mincpu	the cross-over limit for using bzip2 (per platform):
129ca3e8d88SDave Plauger  *	if dump_plat_mincpu == 0, then always do single threaded dump
130ca3e8d88SDave Plauger  *	if ncpu >= dump_plat_mincpu then try to use bzip2
131ca3e8d88SDave Plauger  *
132ca3e8d88SDave Plauger  * dump_metrics_on	if set, metrics are collected in the kernel, passed
133ca3e8d88SDave Plauger  *	to savecore via the dump file, and recorded by savecore in
134ca3e8d88SDave Plauger  *	METRICS.txt.
135ca3e8d88SDave Plauger  */
136ca3e8d88SDave Plauger uint_t dump_ncpu_low = 4;	/* minimum config for parallel lzjb */
137ca3e8d88SDave Plauger uint_t dump_bzip2_level = 1;	/* bzip2 level (1-9) */
138ca3e8d88SDave Plauger 
1394cca9c84SDave Plauger /* Use dump_plat_mincpu_default unless this variable is set by /etc/system */
1404cca9c84SDave Plauger #define	MINCPU_NOT_SET	((uint_t)-1)
1414cca9c84SDave Plauger uint_t dump_plat_mincpu = MINCPU_NOT_SET;
1424cca9c84SDave Plauger 
1439dd77bc8SDave Plauger /* tunables for pre-reserved heap */
1449dd77bc8SDave Plauger uint_t dump_kmem_permap = 1024;
145dfec2ecfSJohn Levon uint_t dump_kmem_pages = 0;
1469dd77bc8SDave Plauger 
147ca3e8d88SDave Plauger /* Define multiple buffers per helper to avoid stalling */
148ca3e8d88SDave Plauger #define	NCBUF_PER_HELPER	2
149ca3e8d88SDave Plauger #define	NCMAP_PER_HELPER	4
150ca3e8d88SDave Plauger 
151ca3e8d88SDave Plauger /* minimum number of helpers configured */
152ca3e8d88SDave Plauger #define	MINHELPERS	(dump_ncpu_low)
153ca3e8d88SDave Plauger #define	MINCBUFS	(MINHELPERS * NCBUF_PER_HELPER)
154ca3e8d88SDave Plauger 
155ca3e8d88SDave Plauger /*
156ca3e8d88SDave Plauger  * Define constant parameters.
157ca3e8d88SDave Plauger  *
158ca3e8d88SDave Plauger  * CBUF_SIZE		size of an output buffer
159ca3e8d88SDave Plauger  *
160ca3e8d88SDave Plauger  * CBUF_MAPSIZE		size of virtual range for mapping pages
161ca3e8d88SDave Plauger  *
162ca3e8d88SDave Plauger  * CBUF_MAPNP		size of virtual range in pages
163ca3e8d88SDave Plauger  *
164ca3e8d88SDave Plauger  */
165ca3e8d88SDave Plauger #define	DUMP_1KB	((size_t)1 << 10)
166ca3e8d88SDave Plauger #define	DUMP_1MB	((size_t)1 << 20)
167ca3e8d88SDave Plauger #define	CBUF_SIZE	((size_t)1 << 17)
168ca3e8d88SDave Plauger #define	CBUF_MAPSHIFT	(22)
169ca3e8d88SDave Plauger #define	CBUF_MAPSIZE	((size_t)1 << CBUF_MAPSHIFT)
170ca3e8d88SDave Plauger #define	CBUF_MAPNP	((size_t)1 << (CBUF_MAPSHIFT - PAGESHIFT))
171ca3e8d88SDave Plauger 
172ca3e8d88SDave Plauger /*
173ca3e8d88SDave Plauger  * Compression metrics are accumulated nano-second subtotals. The
174ca3e8d88SDave Plauger  * results are normalized by the number of pages dumped. A report is
175ca3e8d88SDave Plauger  * generated when dumpsys() completes and is saved in the dump image
176ca3e8d88SDave Plauger  * after the trailing dump header.
177ca3e8d88SDave Plauger  *
178ca3e8d88SDave Plauger  * Metrics are always collected. Set the variable dump_metrics_on to
179ca3e8d88SDave Plauger  * cause metrics to be saved in the crash file, where savecore will
180ca3e8d88SDave Plauger  * save it in the file METRICS.txt.
181ca3e8d88SDave Plauger  */
182ca3e8d88SDave Plauger #define	PERPAGES \
183ca3e8d88SDave Plauger 	PERPAGE(bitmap) PERPAGE(map) PERPAGE(unmap) \
184ca3e8d88SDave Plauger 	PERPAGE(copy) PERPAGE(compress) \
185ca3e8d88SDave Plauger 	PERPAGE(write) \
186ca3e8d88SDave Plauger 	PERPAGE(inwait) PERPAGE(outwait)
187ca3e8d88SDave Plauger 
188ca3e8d88SDave Plauger typedef struct perpage {
189ca3e8d88SDave Plauger #define	PERPAGE(x) hrtime_t x;
190ca3e8d88SDave Plauger 	PERPAGES
191ca3e8d88SDave Plauger #undef PERPAGE
192ca3e8d88SDave Plauger } perpage_t;
193ca3e8d88SDave Plauger 
194ca3e8d88SDave Plauger /*
195ca3e8d88SDave Plauger  * This macro controls the code generation for collecting dump
196ca3e8d88SDave Plauger  * performance information. By default, the code is generated, but
197ca3e8d88SDave Plauger  * automatic saving of the information is disabled. If dump_metrics_on
198ca3e8d88SDave Plauger  * is set to 1, the timing information is passed to savecore via the
199ca3e8d88SDave Plauger  * crash file, where it is appended to the file dump-dir/METRICS.txt.
200ca3e8d88SDave Plauger  */
201ca3e8d88SDave Plauger #define	COLLECT_METRICS
202ca3e8d88SDave Plauger 
203ca3e8d88SDave Plauger #ifdef COLLECT_METRICS
204ca3e8d88SDave Plauger uint_t dump_metrics_on = 0;	/* set to 1 to enable recording metrics */
205ca3e8d88SDave Plauger 
206ca3e8d88SDave Plauger #define	HRSTART(v, m)		v##ts.m = gethrtime()
207ca3e8d88SDave Plauger #define	HRSTOP(v, m)		v.m += gethrtime() - v##ts.m
208ca3e8d88SDave Plauger #define	HRBEGIN(v, m, s)	v##ts.m = gethrtime(); v.size += s
209ca3e8d88SDave Plauger #define	HREND(v, m)		v.m += gethrtime() - v##ts.m
210ca3e8d88SDave Plauger #define	HRNORM(v, m, n)		v.m /= (n)
2117c478bd9Sstevel@tonic-gate 
2127c478bd9Sstevel@tonic-gate #else
213ca3e8d88SDave Plauger #define	HRSTART(v, m)
214ca3e8d88SDave Plauger #define	HRSTOP(v, m)
215ca3e8d88SDave Plauger #define	HRBEGIN(v, m, s)
216ca3e8d88SDave Plauger #define	HREND(v, m)
217ca3e8d88SDave Plauger #define	HRNORM(v, m, n)
218ca3e8d88SDave Plauger #endif	/* COLLECT_METRICS */
219ca3e8d88SDave Plauger 
220ca3e8d88SDave Plauger /*
221ca3e8d88SDave Plauger  * Buffers for copying and compressing memory pages.
222ca3e8d88SDave Plauger  *
223ca3e8d88SDave Plauger  * cbuf_t buffer controllers: used for both input and output.
224ca3e8d88SDave Plauger  *
225ca3e8d88SDave Plauger  * The buffer state indicates how it is being used:
226ca3e8d88SDave Plauger  *
227ca3e8d88SDave Plauger  * CBUF_FREEMAP: CBUF_MAPSIZE virtual address range is available for
228ca3e8d88SDave Plauger  * mapping input pages.
229ca3e8d88SDave Plauger  *
230ca3e8d88SDave Plauger  * CBUF_INREADY: input pages are mapped and ready for compression by a
231ca3e8d88SDave Plauger  * helper.
232ca3e8d88SDave Plauger  *
233ca3e8d88SDave Plauger  * CBUF_USEDMAP: mapping has been consumed by a helper. Needs unmap.
234ca3e8d88SDave Plauger  *
235ca3e8d88SDave Plauger  * CBUF_FREEBUF: CBUF_SIZE output buffer, which is available.
236ca3e8d88SDave Plauger  *
237ca3e8d88SDave Plauger  * CBUF_WRITE: CBUF_SIZE block of compressed pages from a helper,
238ca3e8d88SDave Plauger  * ready to write out.
239ca3e8d88SDave Plauger  *
240ca3e8d88SDave Plauger  * CBUF_ERRMSG: CBUF_SIZE block of error messages from a helper
241ca3e8d88SDave Plauger  * (reports UE errors.)
242ca3e8d88SDave Plauger  */
243ca3e8d88SDave Plauger 
244ca3e8d88SDave Plauger typedef enum cbufstate {
245ca3e8d88SDave Plauger 	CBUF_FREEMAP,
246ca3e8d88SDave Plauger 	CBUF_INREADY,
247ca3e8d88SDave Plauger 	CBUF_USEDMAP,
248ca3e8d88SDave Plauger 	CBUF_FREEBUF,
249ca3e8d88SDave Plauger 	CBUF_WRITE,
250ca3e8d88SDave Plauger 	CBUF_ERRMSG
251ca3e8d88SDave Plauger } cbufstate_t;
252ca3e8d88SDave Plauger 
253ca3e8d88SDave Plauger typedef struct cbuf cbuf_t;
254ca3e8d88SDave Plauger 
255ca3e8d88SDave Plauger struct cbuf {
256ca3e8d88SDave Plauger 	cbuf_t *next;			/* next in list */
257ca3e8d88SDave Plauger 	cbufstate_t state;		/* processing state */
258ca3e8d88SDave Plauger 	size_t used;			/* amount used */
259ca3e8d88SDave Plauger 	size_t size;			/* mem size */
260ca3e8d88SDave Plauger 	char *buf;			/* kmem or vmem */
261ca3e8d88SDave Plauger 	pgcnt_t pagenum;		/* index to pfn map */
262ca3e8d88SDave Plauger 	pgcnt_t bitnum;			/* first set bitnum */
263ca3e8d88SDave Plauger 	pfn_t pfn;			/* first pfn in mapped range */
264ca3e8d88SDave Plauger 	int off;			/* byte offset to first pfn */
265ca3e8d88SDave Plauger };
266ca3e8d88SDave Plauger 
267f6e214c7SGavin Maltby static char dump_osimage_uuid[36 + 1];
268f6e214c7SGavin Maltby 
269f6e214c7SGavin Maltby #define	isdigit(ch)	((ch) >= '0' && (ch) <= '9')
270f6e214c7SGavin Maltby #define	isxdigit(ch)	(isdigit(ch) || ((ch) >= 'a' && (ch) <= 'f') || \
271f6e214c7SGavin Maltby 			((ch) >= 'A' && (ch) <= 'F'))
272f6e214c7SGavin Maltby 
273ca3e8d88SDave Plauger /*
274ca3e8d88SDave Plauger  * cqueue_t queues: a uni-directional channel for communication
275ca3e8d88SDave Plauger  * from the master to helper tasks or vice-versa using put and
276ca3e8d88SDave Plauger  * get primitives. Both mappings and data buffers are passed via
277ca3e8d88SDave Plauger  * queues. Producers close a queue when done. The number of
278ca3e8d88SDave Plauger  * active producers is reference counted so the consumer can
279ca3e8d88SDave Plauger  * detect end of data. Concurrent access is mediated by atomic
280ca3e8d88SDave Plauger  * operations for panic dump, or mutex/cv for live dump.
281ca3e8d88SDave Plauger  *
282ca3e8d88SDave Plauger  * There a four queues, used as follows:
283ca3e8d88SDave Plauger  *
284ca3e8d88SDave Plauger  * Queue		Dataflow		NewState
285ca3e8d88SDave Plauger  * --------------------------------------------------
286ca3e8d88SDave Plauger  * mainq		master -> master	FREEMAP
287ca3e8d88SDave Plauger  * master has initialized or unmapped an input buffer
288ca3e8d88SDave Plauger  * --------------------------------------------------
289ca3e8d88SDave Plauger  * helperq		master -> helper	INREADY
290ca3e8d88SDave Plauger  * master has mapped input for use by helper
291ca3e8d88SDave Plauger  * --------------------------------------------------
292ca3e8d88SDave Plauger  * mainq		master <- helper	USEDMAP
293ca3e8d88SDave Plauger  * helper is done with input
294ca3e8d88SDave Plauger  * --------------------------------------------------
295ca3e8d88SDave Plauger  * freebufq		master -> helper	FREEBUF
296ca3e8d88SDave Plauger  * master has initialized or written an output buffer
297ca3e8d88SDave Plauger  * --------------------------------------------------
298ca3e8d88SDave Plauger  * mainq		master <- helper	WRITE
299ca3e8d88SDave Plauger  * block of compressed pages from a helper
300ca3e8d88SDave Plauger  * --------------------------------------------------
301ca3e8d88SDave Plauger  * mainq		master <- helper	ERRMSG
302ca3e8d88SDave Plauger  * error messages from a helper (memory error case)
303ca3e8d88SDave Plauger  * --------------------------------------------------
304ca3e8d88SDave Plauger  * writerq		master <- master	WRITE
305ca3e8d88SDave Plauger  * non-blocking queue of blocks to write
306ca3e8d88SDave Plauger  * --------------------------------------------------
307ca3e8d88SDave Plauger  */
308ca3e8d88SDave Plauger typedef struct cqueue {
309ca3e8d88SDave Plauger 	cbuf_t *volatile first;		/* first in list */
310ca3e8d88SDave Plauger 	cbuf_t *last;			/* last in list */
311ca3e8d88SDave Plauger 	hrtime_t ts;			/* timestamp */
312ca3e8d88SDave Plauger 	hrtime_t empty;			/* total time empty */
313ca3e8d88SDave Plauger 	kmutex_t mutex;			/* live state lock */
314ca3e8d88SDave Plauger 	kcondvar_t cv;			/* live wait var */
315ca3e8d88SDave Plauger 	lock_t spinlock;		/* panic mode spin lock */
316ca3e8d88SDave Plauger 	volatile uint_t open;		/* producer ref count */
317ca3e8d88SDave Plauger } cqueue_t;
318ca3e8d88SDave Plauger 
319ca3e8d88SDave Plauger /*
320ca3e8d88SDave Plauger  * Convenience macros for using the cqueue functions
321ca3e8d88SDave Plauger  * Note that the caller must have defined "dumpsync_t *ds"
322ca3e8d88SDave Plauger  */
323ca3e8d88SDave Plauger #define	CQ_IS_EMPTY(q)					\
324ca3e8d88SDave Plauger 	(ds->q.first == NULL)
325ca3e8d88SDave Plauger 
326ca3e8d88SDave Plauger #define	CQ_OPEN(q)					\
327ca3e8d88SDave Plauger 	atomic_inc_uint(&ds->q.open)
328ca3e8d88SDave Plauger 
329ca3e8d88SDave Plauger #define	CQ_CLOSE(q)					\
330ca3e8d88SDave Plauger 	dumpsys_close_cq(&ds->q, ds->live)
331ca3e8d88SDave Plauger 
332ca3e8d88SDave Plauger #define	CQ_PUT(q, cp, st)				\
333ca3e8d88SDave Plauger 	dumpsys_put_cq(&ds->q, cp, st, ds->live)
334ca3e8d88SDave Plauger 
335ca3e8d88SDave Plauger #define	CQ_GET(q)					\
336ca3e8d88SDave Plauger 	dumpsys_get_cq(&ds->q, ds->live)
337ca3e8d88SDave Plauger 
338ca3e8d88SDave Plauger /*
339ca3e8d88SDave Plauger  * Dynamic state when dumpsys() is running.
340ca3e8d88SDave Plauger  */
341ca3e8d88SDave Plauger typedef struct dumpsync {
342ca3e8d88SDave Plauger 	pgcnt_t npages;			/* subtotal of pages dumped */
343ca3e8d88SDave Plauger 	pgcnt_t pages_mapped;		/* subtotal of pages mapped */
344ca3e8d88SDave Plauger 	pgcnt_t pages_used;		/* subtotal of pages used per map */
345ca3e8d88SDave Plauger 	size_t nwrite;			/* subtotal of bytes written */
346ca3e8d88SDave Plauger 	uint_t live;			/* running live dump */
347ca3e8d88SDave Plauger 	uint_t neednl;			/* will need to print a newline */
348ca3e8d88SDave Plauger 	uint_t percent;			/* dump progress */
349ca3e8d88SDave Plauger 	uint_t percent_done;		/* dump progress reported */
3502fb0949cSJoshua M. Clulow 	int sec_done;			/* dump progress last report time */
351ca3e8d88SDave Plauger 	cqueue_t freebufq;		/* free kmem bufs for writing */
352ca3e8d88SDave Plauger 	cqueue_t mainq;			/* input for main task */
353ca3e8d88SDave Plauger 	cqueue_t helperq;		/* input for helpers */
354ca3e8d88SDave Plauger 	cqueue_t writerq;		/* input for writer */
355ca3e8d88SDave Plauger 	hrtime_t start;			/* start time */
356ca3e8d88SDave Plauger 	hrtime_t elapsed;		/* elapsed time when completed */
357ca3e8d88SDave Plauger 	hrtime_t iotime;		/* time spent writing nwrite bytes */
358ca3e8d88SDave Plauger 	hrtime_t iowait;		/* time spent waiting for output */
359ca3e8d88SDave Plauger 	hrtime_t iowaitts;		/* iowait timestamp */
360ca3e8d88SDave Plauger 	perpage_t perpage;		/* metrics */
361ca3e8d88SDave Plauger 	perpage_t perpagets;
362ca3e8d88SDave Plauger 	int dumpcpu;			/* master cpu */
363ca3e8d88SDave Plauger } dumpsync_t;
364ca3e8d88SDave Plauger 
365ca3e8d88SDave Plauger static dumpsync_t dumpsync;		/* synchronization vars */
366ca3e8d88SDave Plauger 
367ca3e8d88SDave Plauger /*
368ca3e8d88SDave Plauger  * helper_t helpers: contains the context for a stream. CPUs run in
369ca3e8d88SDave Plauger  * parallel at dump time; each CPU creates a single stream of
370ca3e8d88SDave Plauger  * compression data.  Stream data is divided into CBUF_SIZE blocks.
371ca3e8d88SDave Plauger  * The blocks are written in order within a stream. But, blocks from
372ca3e8d88SDave Plauger  * multiple streams can be interleaved. Each stream is identified by a
373ca3e8d88SDave Plauger  * unique tag.
374ca3e8d88SDave Plauger  */
375ca3e8d88SDave Plauger typedef struct helper {
376ca3e8d88SDave Plauger 	int helper;			/* bound helper id */
377ca3e8d88SDave Plauger 	int tag;			/* compression stream tag */
378ca3e8d88SDave Plauger 	perpage_t perpage;		/* per page metrics */
379ca3e8d88SDave Plauger 	perpage_t perpagets;		/* per page metrics (timestamps) */
380ca3e8d88SDave Plauger 	taskqid_t taskqid;		/* live dump task ptr */
381ca3e8d88SDave Plauger 	int in, out;			/* buffer offsets */
382ca3e8d88SDave Plauger 	cbuf_t *cpin, *cpout, *cperr;	/* cbuf objects in process */
383ca3e8d88SDave Plauger 	dumpsync_t *ds;			/* pointer to sync vars */
384ca3e8d88SDave Plauger 	size_t used;			/* counts input consumed */
385ca3e8d88SDave Plauger 	char *page;			/* buffer for page copy */
386ca3e8d88SDave Plauger 	char *lzbuf;			/* lzjb output */
387ca3e8d88SDave Plauger 	bz_stream bzstream;		/* bzip2 state */
388ca3e8d88SDave Plauger } helper_t;
389ca3e8d88SDave Plauger 
390ca3e8d88SDave Plauger #define	MAINHELPER	(-1)		/* helper is also the main task */
391ca3e8d88SDave Plauger #define	FREEHELPER	(-2)		/* unbound helper */
392ca3e8d88SDave Plauger #define	DONEHELPER	(-3)		/* helper finished */
3937c478bd9Sstevel@tonic-gate 
394ca3e8d88SDave Plauger /*
395ca3e8d88SDave Plauger  * configuration vars for dumpsys
396ca3e8d88SDave Plauger  */
397ca3e8d88SDave Plauger typedef struct dumpcfg {
398ca3e8d88SDave Plauger 	int	threshold;	/* ncpu threshold for bzip2 */
399ca3e8d88SDave Plauger 	int	nhelper;	/* number of helpers */
400ca3e8d88SDave Plauger 	int	nhelper_used;	/* actual number of helpers used */
401ca3e8d88SDave Plauger 	int	ncmap;		/* number VA pages for compression */
402ca3e8d88SDave Plauger 	int	ncbuf;		/* number of bufs for compression */
403ca3e8d88SDave Plauger 	int	ncbuf_used;	/* number of bufs in use */
404ca3e8d88SDave Plauger 	uint_t	clevel;		/* dump compression level */
405ca3e8d88SDave Plauger 	helper_t *helper;	/* array of helpers */
406ca3e8d88SDave Plauger 	cbuf_t	*cmap;		/* array of input (map) buffers */
407ca3e8d88SDave Plauger 	cbuf_t	*cbuf;		/* array of output  buffers */
408ca3e8d88SDave Plauger 	ulong_t	*helpermap;	/* set of dumpsys helper CPU ids */
409ca3e8d88SDave Plauger 	ulong_t	*bitmap;	/* bitmap for marking pages to dump */
410ca3e8d88SDave Plauger 	ulong_t	*rbitmap;	/* bitmap for used CBUF_MAPSIZE ranges */
411ca3e8d88SDave Plauger 	pgcnt_t	bitmapsize;	/* size of bitmap */
412ca3e8d88SDave Plauger 	pgcnt_t	rbitmapsize;	/* size of bitmap for ranges */
413ca3e8d88SDave Plauger 	pgcnt_t found4m;	/* number ranges allocated by dump */
414ca3e8d88SDave Plauger 	pgcnt_t foundsm;	/* number small pages allocated by dump */
415ca3e8d88SDave Plauger 	pid_t	*pids;		/* list of process IDs at dump time */
416ca3e8d88SDave Plauger 	size_t	maxsize;	/* memory size needed at dump time */
417ca3e8d88SDave Plauger 	size_t	maxvmsize;	/* size of reserved VM */
418ca3e8d88SDave Plauger 	char	*maxvm;		/* reserved VM for spare pages */
419ca3e8d88SDave Plauger 	lock_t	helper_lock;	/* protect helper state */
420ca3e8d88SDave Plauger 	char	helpers_wanted;	/* flag to enable parallelism */
421ca3e8d88SDave Plauger } dumpcfg_t;
422ca3e8d88SDave Plauger 
423ca3e8d88SDave Plauger static dumpcfg_t dumpcfg;	/* config vars */
424ca3e8d88SDave Plauger 
425ca3e8d88SDave Plauger /*
426ca3e8d88SDave Plauger  * The dump I/O buffer.
427ca3e8d88SDave Plauger  *
428ca3e8d88SDave Plauger  * There is one I/O buffer used by dumpvp_write and dumvp_flush. It is
429ca3e8d88SDave Plauger  * sized according to the optimum device transfer speed.
430ca3e8d88SDave Plauger  */
431ca3e8d88SDave Plauger typedef struct dumpbuf {
432ca3e8d88SDave Plauger 	vnode_t	*cdev_vp;	/* VCHR open of the dump device */
433ca3e8d88SDave Plauger 	len_t	vp_limit;	/* maximum write offset */
434ca3e8d88SDave Plauger 	offset_t vp_off;	/* current dump device offset */
435ca3e8d88SDave Plauger 	char	*cur;		/* dump write pointer */
436ca3e8d88SDave Plauger 	char	*start;		/* dump buffer address */
437ca3e8d88SDave Plauger 	char	*end;		/* dump buffer end */
438ca3e8d88SDave Plauger 	size_t	size;		/* size of dumpbuf in bytes */
439ca3e8d88SDave Plauger 	size_t	iosize;		/* best transfer size for device */
440ca3e8d88SDave Plauger } dumpbuf_t;
441ca3e8d88SDave Plauger 
442ca3e8d88SDave Plauger dumpbuf_t dumpbuf;		/* I/O buffer */
443ca3e8d88SDave Plauger 
4446ccea422SJoyce McIntosh /*
4456ccea422SJoyce McIntosh  * For parallel dump, defines maximum time main task thread will wait
4466ccea422SJoyce McIntosh  * for at least one helper to register in dumpcfg.helpermap, before
4476ccea422SJoyce McIntosh  * assuming there are no helpers and falling back to serial mode.
4486ccea422SJoyce McIntosh  * Value is chosen arbitrary and provides *really* long wait for any
4496ccea422SJoyce McIntosh  * available helper to register.
4506ccea422SJoyce McIntosh  */
4516ccea422SJoyce McIntosh #define	DUMP_HELPER_MAX_WAIT	1000	/* millisec */
4526ccea422SJoyce McIntosh 
453ca3e8d88SDave Plauger /*
454ca3e8d88SDave Plauger  * The dump I/O buffer must be at least one page, at most xfer_size
455ca3e8d88SDave Plauger  * bytes, and should scale with physmem in between.  The transfer size
456ca3e8d88SDave Plauger  * passed in will either represent a global default (maxphys) or the
457ca3e8d88SDave Plauger  * best size for the device.  The size of the dumpbuf I/O buffer is
458ca3e8d88SDave Plauger  * limited by dumpbuf_limit (8MB by default) because the dump
459ca3e8d88SDave Plauger  * performance saturates beyond a certain size.  The default is to
460ca3e8d88SDave Plauger  * select 1/4096 of the memory.
4617c478bd9Sstevel@tonic-gate  */
462ca3e8d88SDave Plauger static int	dumpbuf_fraction = 12;	/* memory size scale factor */
463ca3e8d88SDave Plauger static size_t	dumpbuf_limit = 8 * DUMP_1MB;	/* max I/O buf size */
464ca3e8d88SDave Plauger 
4657c478bd9Sstevel@tonic-gate static size_t
dumpbuf_iosize(size_t xfer_size)4667c478bd9Sstevel@tonic-gate dumpbuf_iosize(size_t xfer_size)
4677c478bd9Sstevel@tonic-gate {
468ca3e8d88SDave Plauger 	size_t iosize = ptob(physmem >> dumpbuf_fraction);
469ca3e8d88SDave Plauger 
470ca3e8d88SDave Plauger 	if (iosize < PAGESIZE)
471ca3e8d88SDave Plauger 		iosize = PAGESIZE;
472ca3e8d88SDave Plauger 	else if (iosize > xfer_size)
473ca3e8d88SDave Plauger 		iosize = xfer_size;
474ca3e8d88SDave Plauger 	if (iosize > dumpbuf_limit)
475ca3e8d88SDave Plauger 		iosize = dumpbuf_limit;
476ca3e8d88SDave Plauger 	return (iosize & PAGEMASK);
4777c478bd9Sstevel@tonic-gate }
4787c478bd9Sstevel@tonic-gate 
479ca3e8d88SDave Plauger /*
480ca3e8d88SDave Plauger  * resize the I/O buffer
481ca3e8d88SDave Plauger  */
4827c478bd9Sstevel@tonic-gate static void
dumpbuf_resize(void)4837c478bd9Sstevel@tonic-gate dumpbuf_resize(void)
4847c478bd9Sstevel@tonic-gate {
485ca3e8d88SDave Plauger 	char *old_buf = dumpbuf.start;
486ca3e8d88SDave Plauger 	size_t old_size = dumpbuf.size;
4877c478bd9Sstevel@tonic-gate 	char *new_buf;
4887c478bd9Sstevel@tonic-gate 	size_t new_size;
4897c478bd9Sstevel@tonic-gate 
4907c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&dump_lock));
4917c478bd9Sstevel@tonic-gate 
492ca3e8d88SDave Plauger 	new_size = dumpbuf_iosize(MAX(dumpbuf.iosize, maxphys));
493ca3e8d88SDave Plauger 	if (new_size <= old_size)
4947c478bd9Sstevel@tonic-gate 		return; /* no need to reallocate buffer */
4957c478bd9Sstevel@tonic-gate 
4967c478bd9Sstevel@tonic-gate 	new_buf = kmem_alloc(new_size, KM_SLEEP);
497ca3e8d88SDave Plauger 	dumpbuf.size = new_size;
498ca3e8d88SDave Plauger 	dumpbuf.start = new_buf;
499ca3e8d88SDave Plauger 	dumpbuf.end = new_buf + new_size;
5007c478bd9Sstevel@tonic-gate 	kmem_free(old_buf, old_size);
5017c478bd9Sstevel@tonic-gate }
5027c478bd9Sstevel@tonic-gate 
503ca3e8d88SDave Plauger /*
504ca3e8d88SDave Plauger  * dump_update_clevel is called when dumpadm configures the dump device.
5056ccea422SJoyce McIntosh  *	Calculate number of helpers and buffers.
5066ccea422SJoyce McIntosh  *	Allocate the minimum configuration for now.
507ca3e8d88SDave Plauger  *
508ca3e8d88SDave Plauger  * When the dump file is configured we reserve a minimum amount of
509ca3e8d88SDave Plauger  * memory for use at crash time. But we reserve VA for all the memory
510ca3e8d88SDave Plauger  * we really want in order to do the fastest dump possible. The VA is
511ca3e8d88SDave Plauger  * backed by pages not being dumped, according to the bitmap. If
512ca3e8d88SDave Plauger  * there is insufficient spare memory, however, we fall back to the
513ca3e8d88SDave Plauger  * minimum.
514ca3e8d88SDave Plauger  *
515ca3e8d88SDave Plauger  * Live dump (savecore -L) always uses the minimum config.
516ca3e8d88SDave Plauger  *
517ca3e8d88SDave Plauger  * clevel 0 is single threaded lzjb
518ca3e8d88SDave Plauger  * clevel 1 is parallel lzjb
519ca3e8d88SDave Plauger  * clevel 2 is parallel bzip2
520ca3e8d88SDave Plauger  *
521ca3e8d88SDave Plauger  * The ncpu threshold is selected with dump_plat_mincpu.
522ca3e8d88SDave Plauger  * On OPL, set_platform_defaults() overrides the sun4u setting.
523ca3e8d88SDave Plauger  * The actual values are defined via DUMP_PLAT_*_MINCPU macros.
524ca3e8d88SDave Plauger  *
525ca3e8d88SDave Plauger  * Architecture		Threshold	Algorithm
5266ccea422SJoyce McIntosh  * sun4u		<  51		parallel lzjb
5276ccea422SJoyce McIntosh  * sun4u		>= 51		parallel bzip2(*)
5286ccea422SJoyce McIntosh  * sun4u OPL		<  8		parallel lzjb
5296ccea422SJoyce McIntosh  * sun4u OPL		>= 8		parallel bzip2(*)
5306ccea422SJoyce McIntosh  * sun4v		<  128		parallel lzjb
5316ccea422SJoyce McIntosh  * sun4v		>= 128		parallel bzip2(*)
532ca3e8d88SDave Plauger  * x86			< 11		parallel lzjb
533ca3e8d88SDave Plauger  * x86			>= 11		parallel bzip2(*)
5346ccea422SJoyce McIntosh  * 32-bit		N/A		single-threaded lzjb
535ca3e8d88SDave Plauger  *
536ca3e8d88SDave Plauger  * (*) bzip2 is only chosen if there is sufficient available
537ca3e8d88SDave Plauger  * memory for buffers at dump time. See dumpsys_get_maxmem().
538ca3e8d88SDave Plauger  *
539ca3e8d88SDave Plauger  * Faster dump devices have larger I/O buffers. The threshold value is
540ca3e8d88SDave Plauger  * increased according to the size of the dump I/O buffer, because
541ca3e8d88SDave Plauger  * parallel lzjb performs better with faster disks. For buffers >= 1MB
542ca3e8d88SDave Plauger  * the threshold is 3X; for buffers >= 256K threshold is 2X.
543ca3e8d88SDave Plauger  *
544ca3e8d88SDave Plauger  * For parallel dumps, the number of helpers is ncpu-1. The CPU
545ca3e8d88SDave Plauger  * running panic runs the main task. For single-threaded dumps, the
546ca3e8d88SDave Plauger  * panic CPU does lzjb compression (it is tagged as MAINHELPER.)
547ca3e8d88SDave Plauger  *
548ca3e8d88SDave Plauger  * Need multiple buffers per helper so that they do not block waiting
549ca3e8d88SDave Plauger  * for the main task.
550ca3e8d88SDave Plauger  *				parallel	single-threaded
551ca3e8d88SDave Plauger  * Number of output buffers:	nhelper*2		1
552ca3e8d88SDave Plauger  * Number of mapping buffers:	nhelper*4		1
553ca3e8d88SDave Plauger  *
554ca3e8d88SDave Plauger  */
555ca3e8d88SDave Plauger static void
dump_update_clevel()556ca3e8d88SDave Plauger dump_update_clevel()
557ca3e8d88SDave Plauger {
558ca3e8d88SDave Plauger 	int tag;
559ca3e8d88SDave Plauger 	size_t bz2size;
560ca3e8d88SDave Plauger 	helper_t *hp, *hpend;
561ca3e8d88SDave Plauger 	cbuf_t *cp, *cpend;
562ca3e8d88SDave Plauger 	dumpcfg_t *old = &dumpcfg;
563ca3e8d88SDave Plauger 	dumpcfg_t newcfg = *old;
564ca3e8d88SDave Plauger 	dumpcfg_t *new = &newcfg;
565ca3e8d88SDave Plauger 
566ca3e8d88SDave Plauger 	ASSERT(MUTEX_HELD(&dump_lock));
567ca3e8d88SDave Plauger 
568ca3e8d88SDave Plauger 	/*
569ca3e8d88SDave Plauger 	 * Free the previously allocated bufs and VM.
570ca3e8d88SDave Plauger 	 */
571ca3e8d88SDave Plauger 	if (old->helper != NULL) {
572ca3e8d88SDave Plauger 
573ca3e8d88SDave Plauger 		/* helpers */
574ca3e8d88SDave Plauger 		hpend = &old->helper[old->nhelper];
575ca3e8d88SDave Plauger 		for (hp = old->helper; hp != hpend; hp++) {
576ca3e8d88SDave Plauger 			if (hp->lzbuf != NULL)
577ca3e8d88SDave Plauger 				kmem_free(hp->lzbuf, PAGESIZE);
578ca3e8d88SDave Plauger 			if (hp->page != NULL)
579ca3e8d88SDave Plauger 				kmem_free(hp->page, PAGESIZE);
580ca3e8d88SDave Plauger 		}
581ca3e8d88SDave Plauger 		kmem_free(old->helper, old->nhelper * sizeof (helper_t));
582ca3e8d88SDave Plauger 
583ca3e8d88SDave Plauger 		/* VM space for mapping pages */
584ca3e8d88SDave Plauger 		cpend = &old->cmap[old->ncmap];
585ca3e8d88SDave Plauger 		for (cp = old->cmap; cp != cpend; cp++)
586ca3e8d88SDave Plauger 			vmem_xfree(heap_arena, cp->buf, CBUF_MAPSIZE);
587ca3e8d88SDave Plauger 		kmem_free(old->cmap, old->ncmap * sizeof (cbuf_t));
588ca3e8d88SDave Plauger 
589ca3e8d88SDave Plauger 		/* output bufs */
590ca3e8d88SDave Plauger 		cpend = &old->cbuf[old->ncbuf];
591ca3e8d88SDave Plauger 		for (cp = old->cbuf; cp != cpend; cp++)
592ca3e8d88SDave Plauger 			if (cp->buf != NULL)
593ca3e8d88SDave Plauger 				kmem_free(cp->buf, cp->size);
594ca3e8d88SDave Plauger 		kmem_free(old->cbuf, old->ncbuf * sizeof (cbuf_t));
595ca3e8d88SDave Plauger 
596ca3e8d88SDave Plauger 		/* reserved VM for dumpsys_get_maxmem */
597ca3e8d88SDave Plauger 		if (old->maxvmsize > 0)
598ca3e8d88SDave Plauger 			vmem_xfree(heap_arena, old->maxvm, old->maxvmsize);
599ca3e8d88SDave Plauger 	}
600ca3e8d88SDave Plauger 
601ca3e8d88SDave Plauger 	/*
602ca3e8d88SDave Plauger 	 * Allocate memory and VM.
603ca3e8d88SDave Plauger 	 * One CPU runs dumpsys, the rest are helpers.
604ca3e8d88SDave Plauger 	 */
605ca3e8d88SDave Plauger 	new->nhelper = ncpus - 1;
606ca3e8d88SDave Plauger 	if (new->nhelper < 1)
607ca3e8d88SDave Plauger 		new->nhelper = 1;
608ca3e8d88SDave Plauger 
609ca3e8d88SDave Plauger 	if (new->nhelper > DUMP_MAX_NHELPER)
610ca3e8d88SDave Plauger 		new->nhelper = DUMP_MAX_NHELPER;
611ca3e8d88SDave Plauger 
6124cca9c84SDave Plauger 	/* use platform default, unless /etc/system overrides */
6134cca9c84SDave Plauger 	if (dump_plat_mincpu == MINCPU_NOT_SET)
6144cca9c84SDave Plauger 		dump_plat_mincpu = dump_plat_mincpu_default;
6154cca9c84SDave Plauger 
616ca3e8d88SDave Plauger 	/* increase threshold for faster disks */
617ca3e8d88SDave Plauger 	new->threshold = dump_plat_mincpu;
618ca3e8d88SDave Plauger 	if (dumpbuf.iosize >= DUMP_1MB)
619ca3e8d88SDave Plauger 		new->threshold *= 3;
620ca3e8d88SDave Plauger 	else if (dumpbuf.iosize >= (256 * DUMP_1KB))
621ca3e8d88SDave Plauger 		new->threshold *= 2;
622ca3e8d88SDave Plauger 
623ca3e8d88SDave Plauger 	/* figure compression level based upon the computed threshold. */
624ca3e8d88SDave Plauger 	if (dump_plat_mincpu == 0 || new->nhelper < 2) {
625ca3e8d88SDave Plauger 		new->clevel = 0;
626ca3e8d88SDave Plauger 		new->nhelper = 1;
627ca3e8d88SDave Plauger 	} else if ((new->nhelper + 1) >= new->threshold) {
628ca3e8d88SDave Plauger 		new->clevel = DUMP_CLEVEL_BZIP2;
629ca3e8d88SDave Plauger 	} else {
630ca3e8d88SDave Plauger 		new->clevel = DUMP_CLEVEL_LZJB;
631ca3e8d88SDave Plauger 	}
632ca3e8d88SDave Plauger 
633ca3e8d88SDave Plauger 	if (new->clevel == 0) {
634ca3e8d88SDave Plauger 		new->ncbuf = 1;
635ca3e8d88SDave Plauger 		new->ncmap = 1;
636ca3e8d88SDave Plauger 	} else {
637ca3e8d88SDave Plauger 		new->ncbuf = NCBUF_PER_HELPER * new->nhelper;
638ca3e8d88SDave Plauger 		new->ncmap = NCMAP_PER_HELPER * new->nhelper;
639ca3e8d88SDave Plauger 	}
640ca3e8d88SDave Plauger 
641ca3e8d88SDave Plauger 	/*
642ca3e8d88SDave Plauger 	 * Allocate new data structures and buffers for MINHELPERS,
643ca3e8d88SDave Plauger 	 * and also figure the max desired size.
644ca3e8d88SDave Plauger 	 */
645ca3e8d88SDave Plauger 	bz2size = BZ2_bzCompressInitSize(dump_bzip2_level);
646ca3e8d88SDave Plauger 	new->maxsize = 0;
647ca3e8d88SDave Plauger 	new->maxvmsize = 0;
648ca3e8d88SDave Plauger 	new->maxvm = NULL;
649ca3e8d88SDave Plauger 	tag = 1;
650ca3e8d88SDave Plauger 	new->helper = kmem_zalloc(new->nhelper * sizeof (helper_t), KM_SLEEP);
651ca3e8d88SDave Plauger 	hpend = &new->helper[new->nhelper];
652ca3e8d88SDave Plauger 	for (hp = new->helper; hp != hpend; hp++) {
653ca3e8d88SDave Plauger 		hp->tag = tag++;
654ca3e8d88SDave Plauger 		if (hp < &new->helper[MINHELPERS]) {
655ca3e8d88SDave Plauger 			hp->lzbuf = kmem_alloc(PAGESIZE, KM_SLEEP);
656ca3e8d88SDave Plauger 			hp->page = kmem_alloc(PAGESIZE, KM_SLEEP);
657ca3e8d88SDave Plauger 		} else if (new->clevel < DUMP_CLEVEL_BZIP2) {
658ca3e8d88SDave Plauger 			new->maxsize += 2 * PAGESIZE;
659ca3e8d88SDave Plauger 		} else {
660ca3e8d88SDave Plauger 			new->maxsize += PAGESIZE;
661ca3e8d88SDave Plauger 		}
662ca3e8d88SDave Plauger 		if (new->clevel >= DUMP_CLEVEL_BZIP2)
663ca3e8d88SDave Plauger 			new->maxsize += bz2size;
664ca3e8d88SDave Plauger 	}
665ca3e8d88SDave Plauger 
666ca3e8d88SDave Plauger 	new->cbuf = kmem_zalloc(new->ncbuf * sizeof (cbuf_t), KM_SLEEP);
667ca3e8d88SDave Plauger 	cpend = &new->cbuf[new->ncbuf];
668ca3e8d88SDave Plauger 	for (cp = new->cbuf; cp != cpend; cp++) {
669ca3e8d88SDave Plauger 		cp->state = CBUF_FREEBUF;
670ca3e8d88SDave Plauger 		cp->size = CBUF_SIZE;
671ca3e8d88SDave Plauger 		if (cp < &new->cbuf[MINCBUFS])
672ca3e8d88SDave Plauger 			cp->buf = kmem_alloc(cp->size, KM_SLEEP);
673ca3e8d88SDave Plauger 		else
674ca3e8d88SDave Plauger 			new->maxsize += cp->size;
675ca3e8d88SDave Plauger 	}
676ca3e8d88SDave Plauger 
677ca3e8d88SDave Plauger 	new->cmap = kmem_zalloc(new->ncmap * sizeof (cbuf_t), KM_SLEEP);
678ca3e8d88SDave Plauger 	cpend = &new->cmap[new->ncmap];
679ca3e8d88SDave Plauger 	for (cp = new->cmap; cp != cpend; cp++) {
680ca3e8d88SDave Plauger 		cp->state = CBUF_FREEMAP;
681ca3e8d88SDave Plauger 		cp->size = CBUF_MAPSIZE;
682ca3e8d88SDave Plauger 		cp->buf = vmem_xalloc(heap_arena, CBUF_MAPSIZE, CBUF_MAPSIZE,
683ca3e8d88SDave Plauger 		    0, 0, NULL, NULL, VM_SLEEP);
684ca3e8d88SDave Plauger 	}
685ca3e8d88SDave Plauger 
686ca3e8d88SDave Plauger 	/* reserve VA to be backed with spare pages at crash time */
687ca3e8d88SDave Plauger 	if (new->maxsize > 0) {
688ca3e8d88SDave Plauger 		new->maxsize = P2ROUNDUP(new->maxsize, PAGESIZE);
689ca3e8d88SDave Plauger 		new->maxvmsize = P2ROUNDUP(new->maxsize, CBUF_MAPSIZE);
690ca3e8d88SDave Plauger 		new->maxvm = vmem_xalloc(heap_arena, new->maxvmsize,
691ca3e8d88SDave Plauger 		    CBUF_MAPSIZE, 0, 0, NULL, NULL, VM_SLEEP);
692ca3e8d88SDave Plauger 	}
693ca3e8d88SDave Plauger 
6949dd77bc8SDave Plauger 	/*
695dfec2ecfSJohn Levon 	 * Reserve memory for kmem allocation calls made during crash dump.  The
696dfec2ecfSJohn Levon 	 * hat layer allocates memory for each mapping created, and the I/O path
697dfec2ecfSJohn Levon 	 * allocates buffers and data structs.
698dfec2ecfSJohn Levon 	 *
699dfec2ecfSJohn Levon 	 * On larger systems, we easily exceed the lower amount, so we need some
700dfec2ecfSJohn Levon 	 * more space; the cut-over point is relatively arbitrary.  If we run
701dfec2ecfSJohn Levon 	 * out, the only impact is that kmem state in the dump becomes
702dfec2ecfSJohn Levon 	 * inconsistent.
7039dd77bc8SDave Plauger 	 */
704dfec2ecfSJohn Levon 
705dfec2ecfSJohn Levon 	if (dump_kmem_pages == 0) {
706dfec2ecfSJohn Levon 		if (physmem > (16 * ONE_GIG) / PAGESIZE)
707dfec2ecfSJohn Levon 			dump_kmem_pages = 20;
708dfec2ecfSJohn Levon 		else
709dfec2ecfSJohn Levon 			dump_kmem_pages = 8;
710dfec2ecfSJohn Levon 	}
711dfec2ecfSJohn Levon 
7129dd77bc8SDave Plauger 	kmem_dump_init((new->ncmap * dump_kmem_permap) +
7139dd77bc8SDave Plauger 	    (dump_kmem_pages * PAGESIZE));
7149dd77bc8SDave Plauger 
715ca3e8d88SDave Plauger 	/* set new config pointers */
716ca3e8d88SDave Plauger 	*old = *new;
717ca3e8d88SDave Plauger }
718ca3e8d88SDave Plauger 
719ca3e8d88SDave Plauger /*
720ca3e8d88SDave Plauger  * Define a struct memlist walker to optimize bitnum to pfn
721ca3e8d88SDave Plauger  * lookup. The walker maintains the state of the list traversal.
722ca3e8d88SDave Plauger  */
723ca3e8d88SDave Plauger typedef struct dumpmlw {
724ca3e8d88SDave Plauger 	struct memlist	*mp;		/* current memlist */
725ca3e8d88SDave Plauger 	pgcnt_t		basenum;	/* bitnum base offset */
726ca3e8d88SDave Plauger 	pgcnt_t		mppages;	/* current memlist size */
727ca3e8d88SDave Plauger 	pgcnt_t		mpleft;		/* size to end of current memlist */
728ca3e8d88SDave Plauger 	pfn_t		mpaddr;		/* first pfn in memlist */
729ca3e8d88SDave Plauger } dumpmlw_t;
730ca3e8d88SDave Plauger 
731ca3e8d88SDave Plauger /* initialize the walker */
732ca3e8d88SDave Plauger static inline void
dump_init_memlist_walker(dumpmlw_t * pw)733ca3e8d88SDave Plauger dump_init_memlist_walker(dumpmlw_t *pw)
734ca3e8d88SDave Plauger {
735ca3e8d88SDave Plauger 	pw->mp = phys_install;
736ca3e8d88SDave Plauger 	pw->basenum = 0;
73756f33205SJonathan Adams 	pw->mppages = pw->mp->ml_size >> PAGESHIFT;
738ca3e8d88SDave Plauger 	pw->mpleft = pw->mppages;
73956f33205SJonathan Adams 	pw->mpaddr = pw->mp->ml_address >> PAGESHIFT;
740ca3e8d88SDave Plauger }
741ca3e8d88SDave Plauger 
742ca3e8d88SDave Plauger /*
743ca3e8d88SDave Plauger  * Lookup pfn given bitnum. The memlist can be quite long on some
744ca3e8d88SDave Plauger  * systems (e.g.: one per board). To optimize sequential lookups, the
745ca3e8d88SDave Plauger  * caller initializes and presents a memlist walker.
746ca3e8d88SDave Plauger  */
747ca3e8d88SDave Plauger static pfn_t
dump_bitnum_to_pfn(pgcnt_t bitnum,dumpmlw_t * pw)748ca3e8d88SDave Plauger dump_bitnum_to_pfn(pgcnt_t bitnum, dumpmlw_t *pw)
749ca3e8d88SDave Plauger {
750ca3e8d88SDave Plauger 	bitnum -= pw->basenum;
751ca3e8d88SDave Plauger 	while (pw->mp != NULL) {
752ca3e8d88SDave Plauger 		if (bitnum < pw->mppages) {
753ca3e8d88SDave Plauger 			pw->mpleft = pw->mppages - bitnum;
754ca3e8d88SDave Plauger 			return (pw->mpaddr + bitnum);
755ca3e8d88SDave Plauger 		}
756ca3e8d88SDave Plauger 		bitnum -= pw->mppages;
757ca3e8d88SDave Plauger 		pw->basenum += pw->mppages;
75856f33205SJonathan Adams 		pw->mp = pw->mp->ml_next;
759ca3e8d88SDave Plauger 		if (pw->mp != NULL) {
76056f33205SJonathan Adams 			pw->mppages = pw->mp->ml_size >> PAGESHIFT;
761ca3e8d88SDave Plauger 			pw->mpleft = pw->mppages;
76256f33205SJonathan Adams 			pw->mpaddr = pw->mp->ml_address >> PAGESHIFT;
763ca3e8d88SDave Plauger 		}
764ca3e8d88SDave Plauger 	}
765ca3e8d88SDave Plauger 	return (PFN_INVALID);
766ca3e8d88SDave Plauger }
767ca3e8d88SDave Plauger 
768ca3e8d88SDave Plauger static pgcnt_t
dump_pfn_to_bitnum(pfn_t pfn)769ca3e8d88SDave Plauger dump_pfn_to_bitnum(pfn_t pfn)
770ca3e8d88SDave Plauger {
771ca3e8d88SDave Plauger 	struct memlist *mp;
772ca3e8d88SDave Plauger 	pgcnt_t bitnum = 0;
773ca3e8d88SDave Plauger 
77456f33205SJonathan Adams 	for (mp = phys_install; mp != NULL; mp = mp->ml_next) {
77556f33205SJonathan Adams 		if (pfn >= (mp->ml_address >> PAGESHIFT) &&
77656f33205SJonathan Adams 		    pfn < ((mp->ml_address + mp->ml_size) >> PAGESHIFT))
77756f33205SJonathan Adams 			return (bitnum + pfn - (mp->ml_address >> PAGESHIFT));
77856f33205SJonathan Adams 		bitnum += mp->ml_size >> PAGESHIFT;
779ca3e8d88SDave Plauger 	}
780ca3e8d88SDave Plauger 	return ((pgcnt_t)-1);
781ca3e8d88SDave Plauger }
782ca3e8d88SDave Plauger 
783ca3e8d88SDave Plauger /*
784ca3e8d88SDave Plauger  * Set/test bitmap for a CBUF_MAPSIZE range which includes pfn. The
785ca3e8d88SDave Plauger  * mapping of pfn to range index is imperfect because pfn and bitnum
786ca3e8d88SDave Plauger  * do not have the same phase. To make sure a CBUF_MAPSIZE range is
787ca3e8d88SDave Plauger  * covered, call this for both ends:
788ca3e8d88SDave Plauger  *	dump_set_used(base)
789ca3e8d88SDave Plauger  *	dump_set_used(base+CBUF_MAPNP-1)
790ca3e8d88SDave Plauger  *
791ca3e8d88SDave Plauger  * This is used during a panic dump to mark pages allocated by
792ca3e8d88SDave Plauger  * dumpsys_get_maxmem(). The macro IS_DUMP_PAGE(pp) is used by
793ca3e8d88SDave Plauger  * page_get_mnode_freelist() to make sure pages used by dump are never
794ca3e8d88SDave Plauger  * allocated.
795ca3e8d88SDave Plauger  */
796ca3e8d88SDave Plauger #define	CBUF_MAPP2R(pfn)	((pfn) >> (CBUF_MAPSHIFT - PAGESHIFT))
797ca3e8d88SDave Plauger 
798ca3e8d88SDave Plauger static void
dump_set_used(pfn_t pfn)799ca3e8d88SDave Plauger dump_set_used(pfn_t pfn)
800ca3e8d88SDave Plauger {
801ca3e8d88SDave Plauger 
802ca3e8d88SDave Plauger 	pgcnt_t bitnum, rbitnum;
803ca3e8d88SDave Plauger 
804ca3e8d88SDave Plauger 	bitnum = dump_pfn_to_bitnum(pfn);
805ca3e8d88SDave Plauger 	ASSERT(bitnum != (pgcnt_t)-1);
806ca3e8d88SDave Plauger 
807ca3e8d88SDave Plauger 	rbitnum = CBUF_MAPP2R(bitnum);
808ca3e8d88SDave Plauger 	ASSERT(rbitnum < dumpcfg.rbitmapsize);
809ca3e8d88SDave Plauger 
810ca3e8d88SDave Plauger 	BT_SET(dumpcfg.rbitmap, rbitnum);
811ca3e8d88SDave Plauger }
812ca3e8d88SDave Plauger 
813ca3e8d88SDave Plauger int
dump_test_used(pfn_t pfn)814ca3e8d88SDave Plauger dump_test_used(pfn_t pfn)
815ca3e8d88SDave Plauger {
816ca3e8d88SDave Plauger 	pgcnt_t bitnum, rbitnum;
817ca3e8d88SDave Plauger 
818ca3e8d88SDave Plauger 	bitnum = dump_pfn_to_bitnum(pfn);
819ca3e8d88SDave Plauger 	ASSERT(bitnum != (pgcnt_t)-1);
820ca3e8d88SDave Plauger 
821ca3e8d88SDave Plauger 	rbitnum = CBUF_MAPP2R(bitnum);
822ca3e8d88SDave Plauger 	ASSERT(rbitnum < dumpcfg.rbitmapsize);
823ca3e8d88SDave Plauger 
824ca3e8d88SDave Plauger 	return (BT_TEST(dumpcfg.rbitmap, rbitnum));
825ca3e8d88SDave Plauger }
826ca3e8d88SDave Plauger 
827ca3e8d88SDave Plauger /*
828ca3e8d88SDave Plauger  * dumpbzalloc and dumpbzfree are callbacks from the bzip2 library.
829ca3e8d88SDave Plauger  * dumpsys_get_maxmem() uses them for BZ2_bzCompressInit().
830ca3e8d88SDave Plauger  */
831ca3e8d88SDave Plauger static void *
dumpbzalloc(void * opaque,int items,int size)832ca3e8d88SDave Plauger dumpbzalloc(void *opaque, int items, int size)
833ca3e8d88SDave Plauger {
834ca3e8d88SDave Plauger 	size_t *sz;
835ca3e8d88SDave Plauger 	char *ret;
836ca3e8d88SDave Plauger 
837ca3e8d88SDave Plauger 	ASSERT(opaque != NULL);
838ca3e8d88SDave Plauger 	sz = opaque;
839ca3e8d88SDave Plauger 	ret = dumpcfg.maxvm + *sz;
840ca3e8d88SDave Plauger 	*sz += items * size;
841ca3e8d88SDave Plauger 	*sz = P2ROUNDUP(*sz, BZ2_BZALLOC_ALIGN);
842ca3e8d88SDave Plauger 	ASSERT(*sz <= dumpcfg.maxvmsize);
843ca3e8d88SDave Plauger 	return (ret);
844ca3e8d88SDave Plauger }
845ca3e8d88SDave Plauger 
846ca3e8d88SDave Plauger /*ARGSUSED*/
847ca3e8d88SDave Plauger static void
dumpbzfree(void * opaque,void * addr)848ca3e8d88SDave Plauger dumpbzfree(void *opaque, void *addr)
849ca3e8d88SDave Plauger {
850ca3e8d88SDave Plauger }
851ca3e8d88SDave Plauger 
852ca3e8d88SDave Plauger /*
853ca3e8d88SDave Plauger  * Perform additional checks on the page to see if we can really use
854ca3e8d88SDave Plauger  * it. The kernel (kas) pages are always set in the bitmap. However,
855ca3e8d88SDave Plauger  * boot memory pages (prom_ppages or P_BOOTPAGES) are not in the
856ca3e8d88SDave Plauger  * bitmap. So we check for them.
857ca3e8d88SDave Plauger  */
858ca3e8d88SDave Plauger static inline int
dump_pfn_check(pfn_t pfn)859ca3e8d88SDave Plauger dump_pfn_check(pfn_t pfn)
860ca3e8d88SDave Plauger {
861ca3e8d88SDave Plauger 	page_t *pp = page_numtopp_nolock(pfn);
862ca3e8d88SDave Plauger 	if (pp == NULL || pp->p_pagenum != pfn ||
863ca3e8d88SDave Plauger #if defined(__sparc)
864af4c679fSSean McEnroe 	    pp->p_vnode == &promvp ||
865ca3e8d88SDave Plauger #else
866ca3e8d88SDave Plauger 	    PP_ISBOOTPAGES(pp) ||
867ca3e8d88SDave Plauger #endif
868ca3e8d88SDave Plauger 	    pp->p_toxic != 0)
869ca3e8d88SDave Plauger 		return (0);
870ca3e8d88SDave Plauger 	return (1);
871ca3e8d88SDave Plauger }
872ca3e8d88SDave Plauger 
873ca3e8d88SDave Plauger /*
874ca3e8d88SDave Plauger  * Check a range to see if all contained pages are available and
875ca3e8d88SDave Plauger  * return non-zero if the range can be used.
876ca3e8d88SDave Plauger  */
877ca3e8d88SDave Plauger static inline int
dump_range_check(pgcnt_t start,pgcnt_t end,pfn_t pfn)878ca3e8d88SDave Plauger dump_range_check(pgcnt_t start, pgcnt_t end, pfn_t pfn)
879ca3e8d88SDave Plauger {
880ca3e8d88SDave Plauger 	for (; start < end; start++, pfn++) {
881ca3e8d88SDave Plauger 		if (BT_TEST(dumpcfg.bitmap, start))
882ca3e8d88SDave Plauger 			return (0);
883ca3e8d88SDave Plauger 		if (!dump_pfn_check(pfn))
884ca3e8d88SDave Plauger 			return (0);
885ca3e8d88SDave Plauger 	}
886ca3e8d88SDave Plauger 	return (1);
887ca3e8d88SDave Plauger }
888ca3e8d88SDave Plauger 
889ca3e8d88SDave Plauger /*
890ca3e8d88SDave Plauger  * dumpsys_get_maxmem() is called during panic. Find unused ranges
891ca3e8d88SDave Plauger  * and use them for buffers. If we find enough memory switch to
892ca3e8d88SDave Plauger  * parallel bzip2, otherwise use parallel lzjb.
893ca3e8d88SDave Plauger  *
894ca3e8d88SDave Plauger  * It searches the dump bitmap in 2 passes. The first time it looks
895ca3e8d88SDave Plauger  * for CBUF_MAPSIZE ranges. On the second pass it uses small pages.
896ca3e8d88SDave Plauger  */
897ca3e8d88SDave Plauger static void
dumpsys_get_maxmem()898ca3e8d88SDave Plauger dumpsys_get_maxmem()
899ca3e8d88SDave Plauger {
900ca3e8d88SDave Plauger 	dumpcfg_t *cfg = &dumpcfg;
901ca3e8d88SDave Plauger 	cbuf_t *endcp = &cfg->cbuf[cfg->ncbuf];
902ca3e8d88SDave Plauger 	helper_t *endhp = &cfg->helper[cfg->nhelper];
903ca3e8d88SDave Plauger 	pgcnt_t bitnum, end;
904ca3e8d88SDave Plauger 	size_t sz, endsz, bz2size;
905ca3e8d88SDave Plauger 	pfn_t pfn, off;
906ca3e8d88SDave Plauger 	cbuf_t *cp;
907ca3e8d88SDave Plauger 	helper_t *hp, *ohp;
908ca3e8d88SDave Plauger 	dumpmlw_t mlw;
909ca3e8d88SDave Plauger 	int k;
910ca3e8d88SDave Plauger 
9114cca9c84SDave Plauger 	/*
9129c84d1a2SDave Plauger 	 * Setting dump_plat_mincpu to 0 at any time forces a serial
9139c84d1a2SDave Plauger 	 * dump.
9144cca9c84SDave Plauger 	 */
9159c84d1a2SDave Plauger 	if (dump_plat_mincpu == 0) {
9164cca9c84SDave Plauger 		cfg->clevel = 0;
9174cca9c84SDave Plauger 		return;
9184cca9c84SDave Plauger 	}
9194cca9c84SDave Plauger 
9204cca9c84SDave Plauger 	/*
9214cca9c84SDave Plauger 	 * There may be no point in looking for spare memory. If
9224cca9c84SDave Plauger 	 * dumping all memory, then none is spare. If doing a serial
9234cca9c84SDave Plauger 	 * dump, then already have buffers.
9244cca9c84SDave Plauger 	 */
925ca3e8d88SDave Plauger 	if (cfg->maxsize == 0 || cfg->clevel < DUMP_CLEVEL_LZJB ||
92686f21945SDave Plauger 	    (dump_conflags & DUMP_ALL) != 0) {
92786f21945SDave Plauger 		if (cfg->clevel > DUMP_CLEVEL_LZJB)
92886f21945SDave Plauger 			cfg->clevel = DUMP_CLEVEL_LZJB;
929ca3e8d88SDave Plauger 		return;
93086f21945SDave Plauger 	}
931ca3e8d88SDave Plauger 
932ca3e8d88SDave Plauger 	sz = 0;
933ca3e8d88SDave Plauger 	cfg->found4m = 0;
934ca3e8d88SDave Plauger 	cfg->foundsm = 0;
935ca3e8d88SDave Plauger 
936ca3e8d88SDave Plauger 	/* bitmap of ranges used to estimate which pfns are being used */
937ca3e8d88SDave Plauger 	bzero(dumpcfg.rbitmap, BT_SIZEOFMAP(dumpcfg.rbitmapsize));
938ca3e8d88SDave Plauger 
939ca3e8d88SDave Plauger 	/* find ranges that are not being dumped to use for buffers */
940ca3e8d88SDave Plauger 	dump_init_memlist_walker(&mlw);
941ca3e8d88SDave Plauger 	for (bitnum = 0; bitnum < dumpcfg.bitmapsize; bitnum = end) {
942ca3e8d88SDave Plauger 		dump_timeleft = dump_timeout;
943ca3e8d88SDave Plauger 		end = bitnum + CBUF_MAPNP;
944ca3e8d88SDave Plauger 		pfn = dump_bitnum_to_pfn(bitnum, &mlw);
945ca3e8d88SDave Plauger 		ASSERT(pfn != PFN_INVALID);
946ca3e8d88SDave Plauger 
947ca3e8d88SDave Plauger 		/* skip partial range at end of mem segment */
948ca3e8d88SDave Plauger 		if (mlw.mpleft < CBUF_MAPNP) {
949ca3e8d88SDave Plauger 			end = bitnum + mlw.mpleft;
950ca3e8d88SDave Plauger 			continue;
951ca3e8d88SDave Plauger 		}
952ca3e8d88SDave Plauger 
953ca3e8d88SDave Plauger 		/* skip non aligned pages */
954ca3e8d88SDave Plauger 		off = P2PHASE(pfn, CBUF_MAPNP);
955ca3e8d88SDave Plauger 		if (off != 0) {
956ca3e8d88SDave Plauger 			end -= off;
957ca3e8d88SDave Plauger 			continue;
958ca3e8d88SDave Plauger 		}
959ca3e8d88SDave Plauger 
960ca3e8d88SDave Plauger 		if (!dump_range_check(bitnum, end, pfn))
961ca3e8d88SDave Plauger 			continue;
962ca3e8d88SDave Plauger 
963ca3e8d88SDave Plauger 		ASSERT((sz + CBUF_MAPSIZE) <= cfg->maxvmsize);
964ca3e8d88SDave Plauger 		hat_devload(kas.a_hat, cfg->maxvm + sz, CBUF_MAPSIZE, pfn,
965ca3e8d88SDave Plauger 		    PROT_READ | PROT_WRITE, HAT_LOAD_NOCONSIST);
966ca3e8d88SDave Plauger 		sz += CBUF_MAPSIZE;
967ca3e8d88SDave Plauger 		cfg->found4m++;
968ca3e8d88SDave Plauger 
969ca3e8d88SDave Plauger 		/* set the bitmap for both ends to be sure to cover the range */
970ca3e8d88SDave Plauger 		dump_set_used(pfn);
971ca3e8d88SDave Plauger 		dump_set_used(pfn + CBUF_MAPNP - 1);
972ca3e8d88SDave Plauger 
973ca3e8d88SDave Plauger 		if (sz >= cfg->maxsize)
974ca3e8d88SDave Plauger 			goto foundmax;
975ca3e8d88SDave Plauger 	}
976ca3e8d88SDave Plauger 
977ca3e8d88SDave Plauger 	/* Add small pages if we can't find enough large pages. */
978ca3e8d88SDave Plauger 	dump_init_memlist_walker(&mlw);
979ca3e8d88SDave Plauger 	for (bitnum = 0; bitnum < dumpcfg.bitmapsize; bitnum = end) {
980ca3e8d88SDave Plauger 		dump_timeleft = dump_timeout;
981ca3e8d88SDave Plauger 		end = bitnum + CBUF_MAPNP;
982ca3e8d88SDave Plauger 		pfn = dump_bitnum_to_pfn(bitnum, &mlw);
983ca3e8d88SDave Plauger 		ASSERT(pfn != PFN_INVALID);
984ca3e8d88SDave Plauger 
985ca3e8d88SDave Plauger 		/* Find any non-aligned pages at start and end of segment. */
986ca3e8d88SDave Plauger 		off = P2PHASE(pfn, CBUF_MAPNP);
987ca3e8d88SDave Plauger 		if (mlw.mpleft < CBUF_MAPNP) {
988ca3e8d88SDave Plauger 			end = bitnum + mlw.mpleft;
989ca3e8d88SDave Plauger 		} else if (off != 0) {
990ca3e8d88SDave Plauger 			end -= off;
991ca3e8d88SDave Plauger 		} else if (cfg->found4m && dump_test_used(pfn)) {
992ca3e8d88SDave Plauger 			continue;
993ca3e8d88SDave Plauger 		}
994ca3e8d88SDave Plauger 
995ca3e8d88SDave Plauger 		for (; bitnum < end; bitnum++, pfn++) {
996ca3e8d88SDave Plauger 			dump_timeleft = dump_timeout;
997ca3e8d88SDave Plauger 			if (BT_TEST(dumpcfg.bitmap, bitnum))
998ca3e8d88SDave Plauger 				continue;
999ca3e8d88SDave Plauger 			if (!dump_pfn_check(pfn))
1000ca3e8d88SDave Plauger 				continue;
1001ca3e8d88SDave Plauger 			ASSERT((sz + PAGESIZE) <= cfg->maxvmsize);
1002ca3e8d88SDave Plauger 			hat_devload(kas.a_hat, cfg->maxvm + sz, PAGESIZE, pfn,
1003ca3e8d88SDave Plauger 			    PROT_READ | PROT_WRITE, HAT_LOAD_NOCONSIST);
1004ca3e8d88SDave Plauger 			sz += PAGESIZE;
1005ca3e8d88SDave Plauger 			cfg->foundsm++;
1006ca3e8d88SDave Plauger 			dump_set_used(pfn);
1007ca3e8d88SDave Plauger 			if (sz >= cfg->maxsize)
1008ca3e8d88SDave Plauger 				goto foundmax;
1009ca3e8d88SDave Plauger 		}
1010ca3e8d88SDave Plauger 	}
1011ca3e8d88SDave Plauger 
1012ca3e8d88SDave Plauger 	/* Fall back to lzjb if we did not get enough memory for bzip2. */
1013ca3e8d88SDave Plauger 	endsz = (cfg->maxsize * cfg->threshold) / cfg->nhelper;
1014ca3e8d88SDave Plauger 	if (sz < endsz) {
1015ca3e8d88SDave Plauger 		cfg->clevel = DUMP_CLEVEL_LZJB;
1016ca3e8d88SDave Plauger 	}
1017ca3e8d88SDave Plauger 
1018ca3e8d88SDave Plauger 	/* Allocate memory for as many helpers as we can. */
1019ca3e8d88SDave Plauger foundmax:
1020ca3e8d88SDave Plauger 
1021ca3e8d88SDave Plauger 	/* Byte offsets into memory found and mapped above */
1022ca3e8d88SDave Plauger 	endsz = sz;
1023ca3e8d88SDave Plauger 	sz = 0;
1024ca3e8d88SDave Plauger 
1025ca3e8d88SDave Plauger 	/* Set the size for bzip2 state. Only bzip2 needs it. */
1026ca3e8d88SDave Plauger 	bz2size = BZ2_bzCompressInitSize(dump_bzip2_level);
1027ca3e8d88SDave Plauger 
1028ca3e8d88SDave Plauger 	/* Skip the preallocate output buffers. */
1029ca3e8d88SDave Plauger 	cp = &cfg->cbuf[MINCBUFS];
1030ca3e8d88SDave Plauger 
1031ca3e8d88SDave Plauger 	/* Use this to move memory up from the preallocated helpers. */
1032ca3e8d88SDave Plauger 	ohp = cfg->helper;
1033ca3e8d88SDave Plauger 
1034ca3e8d88SDave Plauger 	/* Loop over all helpers and allocate memory. */
1035ca3e8d88SDave Plauger 	for (hp = cfg->helper; hp < endhp; hp++) {
1036ca3e8d88SDave Plauger 
1037ca3e8d88SDave Plauger 		/* Skip preallocated helpers by checking hp->page. */
1038ca3e8d88SDave Plauger 		if (hp->page == NULL) {
1039ca3e8d88SDave Plauger 			if (cfg->clevel <= DUMP_CLEVEL_LZJB) {
1040ca3e8d88SDave Plauger 				/* lzjb needs 2 1-page buffers */
1041ca3e8d88SDave Plauger 				if ((sz + (2 * PAGESIZE)) > endsz)
1042ca3e8d88SDave Plauger 					break;
1043ca3e8d88SDave Plauger 				hp->page = cfg->maxvm + sz;
1044ca3e8d88SDave Plauger 				sz += PAGESIZE;
1045ca3e8d88SDave Plauger 				hp->lzbuf = cfg->maxvm + sz;
1046ca3e8d88SDave Plauger 				sz += PAGESIZE;
1047ca3e8d88SDave Plauger 
1048ca3e8d88SDave Plauger 			} else if (ohp->lzbuf != NULL) {
1049ca3e8d88SDave Plauger 				/* re-use the preallocted lzjb page for bzip2 */
1050ca3e8d88SDave Plauger 				hp->page = ohp->lzbuf;
1051ca3e8d88SDave Plauger 				ohp->lzbuf = NULL;
1052ca3e8d88SDave Plauger 				++ohp;
1053ca3e8d88SDave Plauger 
1054ca3e8d88SDave Plauger 			} else {
1055ca3e8d88SDave Plauger 				/* bzip2 needs a 1-page buffer */
1056ca3e8d88SDave Plauger 				if ((sz + PAGESIZE) > endsz)
1057ca3e8d88SDave Plauger 					break;
1058ca3e8d88SDave Plauger 				hp->page = cfg->maxvm + sz;
1059ca3e8d88SDave Plauger 				sz += PAGESIZE;
1060ca3e8d88SDave Plauger 			}
1061ca3e8d88SDave Plauger 		}
1062ca3e8d88SDave Plauger 
1063ca3e8d88SDave Plauger 		/*
1064ca3e8d88SDave Plauger 		 * Add output buffers per helper. The number of
1065ca3e8d88SDave Plauger 		 * buffers per helper is determined by the ratio of
1066ca3e8d88SDave Plauger 		 * ncbuf to nhelper.
1067ca3e8d88SDave Plauger 		 */
1068ca3e8d88SDave Plauger 		for (k = 0; cp < endcp && (sz + CBUF_SIZE) <= endsz &&
1069ca3e8d88SDave Plauger 		    k < NCBUF_PER_HELPER; k++) {
1070ca3e8d88SDave Plauger 			cp->state = CBUF_FREEBUF;
1071ca3e8d88SDave Plauger 			cp->size = CBUF_SIZE;
1072ca3e8d88SDave Plauger 			cp->buf = cfg->maxvm + sz;
1073ca3e8d88SDave Plauger 			sz += CBUF_SIZE;
1074ca3e8d88SDave Plauger 			++cp;
1075ca3e8d88SDave Plauger 		}
1076ca3e8d88SDave Plauger 
1077ca3e8d88SDave Plauger 		/*
1078ca3e8d88SDave Plauger 		 * bzip2 needs compression state. Use the dumpbzalloc
1079ca3e8d88SDave Plauger 		 * and dumpbzfree callbacks to allocate the memory.
1080ca3e8d88SDave Plauger 		 * bzip2 does allocation only at init time.
1081ca3e8d88SDave Plauger 		 */
1082ca3e8d88SDave Plauger 		if (cfg->clevel >= DUMP_CLEVEL_BZIP2) {
1083ca3e8d88SDave Plauger 			if ((sz + bz2size) > endsz) {
1084ca3e8d88SDave Plauger 				hp->page = NULL;
1085ca3e8d88SDave Plauger 				break;
1086ca3e8d88SDave Plauger 			} else {
1087ca3e8d88SDave Plauger 				hp->bzstream.opaque = &sz;
1088ca3e8d88SDave Plauger 				hp->bzstream.bzalloc = dumpbzalloc;
1089ca3e8d88SDave Plauger 				hp->bzstream.bzfree = dumpbzfree;
1090ca3e8d88SDave Plauger 				(void) BZ2_bzCompressInit(&hp->bzstream,
1091ca3e8d88SDave Plauger 				    dump_bzip2_level, 0, 0);
1092ca3e8d88SDave Plauger 				hp->bzstream.opaque = NULL;
1093ca3e8d88SDave Plauger 			}
1094ca3e8d88SDave Plauger 		}
1095ca3e8d88SDave Plauger 	}
1096ca3e8d88SDave Plauger 
1097ca3e8d88SDave Plauger 	/* Finish allocating output buffers */
1098ca3e8d88SDave Plauger 	for (; cp < endcp && (sz + CBUF_SIZE) <= endsz; cp++) {
1099ca3e8d88SDave Plauger 		cp->state = CBUF_FREEBUF;
1100ca3e8d88SDave Plauger 		cp->size = CBUF_SIZE;
1101ca3e8d88SDave Plauger 		cp->buf = cfg->maxvm + sz;
1102ca3e8d88SDave Plauger 		sz += CBUF_SIZE;
1103ca3e8d88SDave Plauger 	}
1104ca3e8d88SDave Plauger 
1105ca3e8d88SDave Plauger 	/* Enable IS_DUMP_PAGE macro, which checks for pages we took. */
1106ca3e8d88SDave Plauger 	if (cfg->found4m || cfg->foundsm)
1107ca3e8d88SDave Plauger 		dump_check_used = 1;
1108ca3e8d88SDave Plauger 
1109ca3e8d88SDave Plauger 	ASSERT(sz <= endsz);
1110ca3e8d88SDave Plauger }
1111ca3e8d88SDave Plauger 
11127c478bd9Sstevel@tonic-gate static void
dumphdr_init(void)11137c478bd9Sstevel@tonic-gate dumphdr_init(void)
11147c478bd9Sstevel@tonic-gate {
11157c478bd9Sstevel@tonic-gate 	pgcnt_t npages = 0;
11167c478bd9Sstevel@tonic-gate 
11177c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&dump_lock));
11187c478bd9Sstevel@tonic-gate 
11197c478bd9Sstevel@tonic-gate 	if (dumphdr == NULL) {
11207c478bd9Sstevel@tonic-gate 		dumphdr = kmem_zalloc(sizeof (dumphdr_t), KM_SLEEP);
11217c478bd9Sstevel@tonic-gate 		dumphdr->dump_magic = DUMP_MAGIC;
11227c478bd9Sstevel@tonic-gate 		dumphdr->dump_version = DUMP_VERSION;
11237c478bd9Sstevel@tonic-gate 		dumphdr->dump_wordsize = DUMP_WORDSIZE;
11247c478bd9Sstevel@tonic-gate 		dumphdr->dump_pageshift = PAGESHIFT;
11257c478bd9Sstevel@tonic-gate 		dumphdr->dump_pagesize = PAGESIZE;
11267c478bd9Sstevel@tonic-gate 		dumphdr->dump_utsname = utsname;
11277c478bd9Sstevel@tonic-gate 		(void) strcpy(dumphdr->dump_platform, platform);
1128ca3e8d88SDave Plauger 		dumpbuf.size = dumpbuf_iosize(maxphys);
1129ca3e8d88SDave Plauger 		dumpbuf.start = kmem_alloc(dumpbuf.size, KM_SLEEP);
1130ca3e8d88SDave Plauger 		dumpbuf.end = dumpbuf.start + dumpbuf.size;
1131ca3e8d88SDave Plauger 		dumpcfg.pids = kmem_alloc(v.v_proc * sizeof (pid_t), KM_SLEEP);
1132ca3e8d88SDave Plauger 		dumpcfg.helpermap = kmem_zalloc(BT_SIZEOFMAP(NCPU), KM_SLEEP);
1133ca3e8d88SDave Plauger 		LOCK_INIT_HELD(&dumpcfg.helper_lock);
1134f6e214c7SGavin Maltby 		dump_stack_scratch = kmem_alloc(STACK_BUF_SIZE, KM_SLEEP);
1135f6e214c7SGavin Maltby 		(void) strncpy(dumphdr->dump_uuid, dump_get_uuid(),
1136f6e214c7SGavin Maltby 		    sizeof (dumphdr->dump_uuid));
11377c478bd9Sstevel@tonic-gate 	}
11387c478bd9Sstevel@tonic-gate 
1139843e1988Sjohnlev 	npages = num_phys_pages();
11407c478bd9Sstevel@tonic-gate 
1141ca3e8d88SDave Plauger 	if (dumpcfg.bitmapsize != npages) {
1142ca3e8d88SDave Plauger 		size_t rlen = CBUF_MAPP2R(P2ROUNDUP(npages, CBUF_MAPNP));
11437c478bd9Sstevel@tonic-gate 		void *map = kmem_alloc(BT_SIZEOFMAP(npages), KM_SLEEP);
1144ca3e8d88SDave Plauger 		void *rmap = kmem_alloc(BT_SIZEOFMAP(rlen), KM_SLEEP);
1145ca3e8d88SDave Plauger 
1146ca3e8d88SDave Plauger 		if (dumpcfg.bitmap != NULL)
1147ca3e8d88SDave Plauger 			kmem_free(dumpcfg.bitmap, BT_SIZEOFMAP(dumpcfg.
1148ca3e8d88SDave Plauger 			    bitmapsize));
1149ca3e8d88SDave Plauger 		if (dumpcfg.rbitmap != NULL)
1150ca3e8d88SDave Plauger 			kmem_free(dumpcfg.rbitmap, BT_SIZEOFMAP(dumpcfg.
1151ca3e8d88SDave Plauger 			    rbitmapsize));
1152ca3e8d88SDave Plauger 		dumpcfg.bitmap = map;
1153ca3e8d88SDave Plauger 		dumpcfg.bitmapsize = npages;
1154ca3e8d88SDave Plauger 		dumpcfg.rbitmap = rmap;
1155ca3e8d88SDave Plauger 		dumpcfg.rbitmapsize = rlen;
11567c478bd9Sstevel@tonic-gate 	}
11577c478bd9Sstevel@tonic-gate }
11587c478bd9Sstevel@tonic-gate 
11597c478bd9Sstevel@tonic-gate /*
11607c478bd9Sstevel@tonic-gate  * Establish a new dump device.
11617c478bd9Sstevel@tonic-gate  */
11627c478bd9Sstevel@tonic-gate int
dumpinit(vnode_t * vp,char * name,int justchecking)11637c478bd9Sstevel@tonic-gate dumpinit(vnode_t *vp, char *name, int justchecking)
11647c478bd9Sstevel@tonic-gate {
11657c478bd9Sstevel@tonic-gate 	vnode_t *cvp;
11667c478bd9Sstevel@tonic-gate 	vattr_t vattr;
11677c478bd9Sstevel@tonic-gate 	vnode_t *cdev_vp;
11687c478bd9Sstevel@tonic-gate 	int error = 0;
11697c478bd9Sstevel@tonic-gate 
11707c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&dump_lock));
11717c478bd9Sstevel@tonic-gate 
11727c478bd9Sstevel@tonic-gate 	dumphdr_init();
11737c478bd9Sstevel@tonic-gate 
11747c478bd9Sstevel@tonic-gate 	cvp = common_specvp(vp);
11757c478bd9Sstevel@tonic-gate 	if (cvp == dumpvp)
11767c478bd9Sstevel@tonic-gate 		return (0);
11777c478bd9Sstevel@tonic-gate 
11787c478bd9Sstevel@tonic-gate 	/*
11797c478bd9Sstevel@tonic-gate 	 * Determine whether this is a plausible dump device.  We want either:
11807c478bd9Sstevel@tonic-gate 	 * (1) a real device that's not mounted and has a cb_dump routine, or
11817c478bd9Sstevel@tonic-gate 	 * (2) a swapfile on some filesystem that has a vop_dump routine.
11827c478bd9Sstevel@tonic-gate 	 */
1183da6c28aaSamw 	if ((error = VOP_OPEN(&cvp, FREAD | FWRITE, kcred, NULL)) != 0)
11847c478bd9Sstevel@tonic-gate 		return (error);
11857c478bd9Sstevel@tonic-gate 
11867c478bd9Sstevel@tonic-gate 	vattr.va_mask = AT_SIZE | AT_TYPE | AT_RDEV;
1187da6c28aaSamw 	if ((error = VOP_GETATTR(cvp, &vattr, 0, kcred, NULL)) == 0) {
11887c478bd9Sstevel@tonic-gate 		if (vattr.va_type == VBLK || vattr.va_type == VCHR) {
11897c478bd9Sstevel@tonic-gate 			if (devopsp[getmajor(vattr.va_rdev)]->
11907c478bd9Sstevel@tonic-gate 			    devo_cb_ops->cb_dump == nodev)
11917c478bd9Sstevel@tonic-gate 				error = ENOTSUP;
11927c478bd9Sstevel@tonic-gate 			else if (vfs_devismounted(vattr.va_rdev))
11937c478bd9Sstevel@tonic-gate 				error = EBUSY;
1194681d9761SEric Taylor 			if (strcmp(ddi_driver_name(VTOS(cvp)->s_dip),
1195681d9761SEric Taylor 			    ZFS_DRIVER) == 0 &&
1196681d9761SEric Taylor 			    IS_SWAPVP(common_specvp(cvp)))
1197681d9761SEric Taylor 					error = EBUSY;
11987c478bd9Sstevel@tonic-gate 		} else {
11997c478bd9Sstevel@tonic-gate 			if (vn_matchopval(cvp, VOPNAME_DUMP, fs_nosys) ||
12007c478bd9Sstevel@tonic-gate 			    !IS_SWAPVP(cvp))
12017c478bd9Sstevel@tonic-gate 				error = ENOTSUP;
12027c478bd9Sstevel@tonic-gate 		}
12037c478bd9Sstevel@tonic-gate 	}
12047c478bd9Sstevel@tonic-gate 
12057c478bd9Sstevel@tonic-gate 	if (error == 0 && vattr.va_size < 2 * DUMP_LOGSIZE + DUMP_ERPTSIZE)
12067c478bd9Sstevel@tonic-gate 		error = ENOSPC;
12077c478bd9Sstevel@tonic-gate 
12087c478bd9Sstevel@tonic-gate 	if (error || justchecking) {
1209da6c28aaSamw 		(void) VOP_CLOSE(cvp, FREAD | FWRITE, 1, (offset_t)0,
1210da6c28aaSamw 		    kcred, NULL);
12117c478bd9Sstevel@tonic-gate 		return (error);
12127c478bd9Sstevel@tonic-gate 	}
12137c478bd9Sstevel@tonic-gate 
12147c478bd9Sstevel@tonic-gate 	VN_HOLD(cvp);
12157c478bd9Sstevel@tonic-gate 
12167c478bd9Sstevel@tonic-gate 	if (dumpvp != NULL)
12177c478bd9Sstevel@tonic-gate 		dumpfini();	/* unconfigure the old dump device */
12187c478bd9Sstevel@tonic-gate 
12197c478bd9Sstevel@tonic-gate 	dumpvp = cvp;
12207c478bd9Sstevel@tonic-gate 	dumpvp_size = vattr.va_size & -DUMP_OFFSET;
12217c478bd9Sstevel@tonic-gate 	dumppath = kmem_alloc(strlen(name) + 1, KM_SLEEP);
12227c478bd9Sstevel@tonic-gate 	(void) strcpy(dumppath, name);
1223ca3e8d88SDave Plauger 	dumpbuf.iosize = 0;
12247c478bd9Sstevel@tonic-gate 
12257c478bd9Sstevel@tonic-gate 	/*
12267c478bd9Sstevel@tonic-gate 	 * If the dump device is a block device, attempt to open up the
12277c478bd9Sstevel@tonic-gate 	 * corresponding character device and determine its maximum transfer
12287c478bd9Sstevel@tonic-gate 	 * size.  We use this information to potentially resize dumpbuf to a
12297c478bd9Sstevel@tonic-gate 	 * larger and more optimal size for performing i/o to the dump device.
12307c478bd9Sstevel@tonic-gate 	 */
12317c478bd9Sstevel@tonic-gate 	if (cvp->v_type == VBLK &&
12327c478bd9Sstevel@tonic-gate 	    (cdev_vp = makespecvp(VTOS(cvp)->s_dev, VCHR)) != NULL) {
1233da6c28aaSamw 		if (VOP_OPEN(&cdev_vp, FREAD | FWRITE, kcred, NULL) == 0) {
12347c478bd9Sstevel@tonic-gate 			size_t blk_size;
12357c478bd9Sstevel@tonic-gate 			struct dk_cinfo dki;
123665908c77Syu, larry liu - Sun Microsystems - Beijing China 			struct dk_minfo minf;
12377c478bd9Sstevel@tonic-gate 
123865908c77Syu, larry liu - Sun Microsystems - Beijing China 			if (VOP_IOCTL(cdev_vp, DKIOCGMEDIAINFO,
123965908c77Syu, larry liu - Sun Microsystems - Beijing China 			    (intptr_t)&minf, FKIOCTL, kcred, NULL, NULL)
124065908c77Syu, larry liu - Sun Microsystems - Beijing China 			    == 0 && minf.dki_lbsize != 0)
124165908c77Syu, larry liu - Sun Microsystems - Beijing China 				blk_size = minf.dki_lbsize;
12427c478bd9Sstevel@tonic-gate 			else
12437c478bd9Sstevel@tonic-gate 				blk_size = DEV_BSIZE;
12447c478bd9Sstevel@tonic-gate 
12457c478bd9Sstevel@tonic-gate 			if (VOP_IOCTL(cdev_vp, DKIOCINFO, (intptr_t)&dki,
1246da6c28aaSamw 			    FKIOCTL, kcred, NULL, NULL) == 0) {
1247ca3e8d88SDave Plauger 				dumpbuf.iosize = dki.dki_maxtransfer * blk_size;
12487c478bd9Sstevel@tonic-gate 				dumpbuf_resize();
12497c478bd9Sstevel@tonic-gate 			}
1250e7cbe64fSgw 			/*
1251681d9761SEric Taylor 			 * If we are working with a zvol then dumpify it
1252681d9761SEric Taylor 			 * if it's not being used as swap.
1253e7cbe64fSgw 			 */
1254e7cbe64fSgw 			if (strcmp(dki.dki_dname, ZVOL_DRIVER) == 0) {
1255681d9761SEric Taylor 				if (IS_SWAPVP(common_specvp(cvp)))
1256681d9761SEric Taylor 					error = EBUSY;
1257681d9761SEric Taylor 				else if ((error = VOP_IOCTL(cdev_vp,
12587e12ceb3SToomas Soome 				    DKIOCDUMPINIT, 0, FKIOCTL, kcred,
1259681d9761SEric Taylor 				    NULL, NULL)) != 0)
1260e7cbe64fSgw 					dumpfini();
1261e7cbe64fSgw 			}
12627c478bd9Sstevel@tonic-gate 
1263da6c28aaSamw 			(void) VOP_CLOSE(cdev_vp, FREAD | FWRITE, 1, 0,
1264da6c28aaSamw 			    kcred, NULL);
12657c478bd9Sstevel@tonic-gate 		}
12667c478bd9Sstevel@tonic-gate 
12677c478bd9Sstevel@tonic-gate 		VN_RELE(cdev_vp);
12687c478bd9Sstevel@tonic-gate 	}
12697c478bd9Sstevel@tonic-gate 
12707c478bd9Sstevel@tonic-gate 	cmn_err(CE_CONT, "?dump on %s size %llu MB\n", name, dumpvp_size >> 20);
12717c478bd9Sstevel@tonic-gate 
1272ca3e8d88SDave Plauger 	dump_update_clevel();
1273ca3e8d88SDave Plauger 
1274e7cbe64fSgw 	return (error);
12757c478bd9Sstevel@tonic-gate }
12767c478bd9Sstevel@tonic-gate 
12777c478bd9Sstevel@tonic-gate void
dumpfini(void)12787c478bd9Sstevel@tonic-gate dumpfini(void)
12797c478bd9Sstevel@tonic-gate {
1280e7cbe64fSgw 	vattr_t vattr;
1281e7cbe64fSgw 	boolean_t is_zfs = B_FALSE;
1282e7cbe64fSgw 	vnode_t *cdev_vp;
12837c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&dump_lock));
12847c478bd9Sstevel@tonic-gate 
12857c478bd9Sstevel@tonic-gate 	kmem_free(dumppath, strlen(dumppath) + 1);
12867c478bd9Sstevel@tonic-gate 
1287e7cbe64fSgw 	/*
1288e7cbe64fSgw 	 * Determine if we are using zvols for our dump device
1289e7cbe64fSgw 	 */
1290e7cbe64fSgw 	vattr.va_mask = AT_RDEV;
1291e7cbe64fSgw 	if (VOP_GETATTR(dumpvp, &vattr, 0, kcred, NULL) == 0) {
1292e7cbe64fSgw 		is_zfs = (getmajor(vattr.va_rdev) ==
1293e7cbe64fSgw 		    ddi_name_to_major(ZFS_DRIVER)) ? B_TRUE : B_FALSE;
1294e7cbe64fSgw 	}
1295e7cbe64fSgw 
1296e7cbe64fSgw 	/*
1297e7cbe64fSgw 	 * If we have a zvol dump device then we call into zfs so
1298e7cbe64fSgw 	 * that it may have a chance to cleanup.
1299e7cbe64fSgw 	 */
1300e7cbe64fSgw 	if (is_zfs &&
1301e7cbe64fSgw 	    (cdev_vp = makespecvp(VTOS(dumpvp)->s_dev, VCHR)) != NULL) {
1302e7cbe64fSgw 		if (VOP_OPEN(&cdev_vp, FREAD | FWRITE, kcred, NULL) == 0) {
13037e12ceb3SToomas Soome 			(void) VOP_IOCTL(cdev_vp, DKIOCDUMPFINI, 0, FKIOCTL,
1304e7cbe64fSgw 			    kcred, NULL, NULL);
1305e7cbe64fSgw 			(void) VOP_CLOSE(cdev_vp, FREAD | FWRITE, 1, 0,
1306e7cbe64fSgw 			    kcred, NULL);
1307e7cbe64fSgw 		}
1308e7cbe64fSgw 		VN_RELE(cdev_vp);
1309e7cbe64fSgw 	}
1310e7cbe64fSgw 
1311da6c28aaSamw 	(void) VOP_CLOSE(dumpvp, FREAD | FWRITE, 1, (offset_t)0, kcred, NULL);
13127c478bd9Sstevel@tonic-gate 
13137c478bd9Sstevel@tonic-gate 	VN_RELE(dumpvp);
13147c478bd9Sstevel@tonic-gate 
13157c478bd9Sstevel@tonic-gate 	dumpvp = NULL;
13167c478bd9Sstevel@tonic-gate 	dumpvp_size = 0;
13177c478bd9Sstevel@tonic-gate 	dumppath = NULL;
13187c478bd9Sstevel@tonic-gate }
13197c478bd9Sstevel@tonic-gate 
1320ca3e8d88SDave Plauger static offset_t
dumpvp_flush(void)1321ca3e8d88SDave Plauger dumpvp_flush(void)
13227c478bd9Sstevel@tonic-gate {
1323ca3e8d88SDave Plauger 	size_t size = P2ROUNDUP(dumpbuf.cur - dumpbuf.start, PAGESIZE);
1324ca3e8d88SDave Plauger 	hrtime_t iotime;
1325ca3e8d88SDave Plauger 	int err;
13267c478bd9Sstevel@tonic-gate 
1327ca3e8d88SDave Plauger 	if (dumpbuf.vp_off + size > dumpbuf.vp_limit) {
1328ca3e8d88SDave Plauger 		dump_ioerr = ENOSPC;
1329ca3e8d88SDave Plauger 		dumpbuf.vp_off = dumpbuf.vp_limit;
1330ca3e8d88SDave Plauger 	} else if (size != 0) {
1331ca3e8d88SDave Plauger 		iotime = gethrtime();
1332ca3e8d88SDave Plauger 		dumpsync.iowait += iotime - dumpsync.iowaitts;
1333ca3e8d88SDave Plauger 		if (panicstr)
1334ca3e8d88SDave Plauger 			err = VOP_DUMP(dumpvp, dumpbuf.start,
1335ca3e8d88SDave Plauger 			    lbtodb(dumpbuf.vp_off), btod(size), NULL);
1336ca3e8d88SDave Plauger 		else
1337ca3e8d88SDave Plauger 			err = vn_rdwr(UIO_WRITE, dumpbuf.cdev_vp != NULL ?
1338ca3e8d88SDave Plauger 			    dumpbuf.cdev_vp : dumpvp, dumpbuf.start, size,
1339ca3e8d88SDave Plauger 			    dumpbuf.vp_off, UIO_SYSSPACE, 0, dumpbuf.vp_limit,
1340ca3e8d88SDave Plauger 			    kcred, 0);
1341ca3e8d88SDave Plauger 		if (err && dump_ioerr == 0)
1342ca3e8d88SDave Plauger 			dump_ioerr = err;
1343ca3e8d88SDave Plauger 		dumpsync.iowaitts = gethrtime();
1344ca3e8d88SDave Plauger 		dumpsync.iotime += dumpsync.iowaitts - iotime;
1345ca3e8d88SDave Plauger 		dumpsync.nwrite += size;
1346ca3e8d88SDave Plauger 		dumpbuf.vp_off += size;
13477c478bd9Sstevel@tonic-gate 	}
1348ca3e8d88SDave Plauger 	dumpbuf.cur = dumpbuf.start;
1349ca3e8d88SDave Plauger 	dump_timeleft = dump_timeout;
1350ca3e8d88SDave Plauger 	return (dumpbuf.vp_off);
13517c478bd9Sstevel@tonic-gate }
13527c478bd9Sstevel@tonic-gate 
1353ca3e8d88SDave Plauger /* maximize write speed by keeping seek offset aligned with size */
1354ca3e8d88SDave Plauger void
dumpvp_write(const void * va,size_t size)1355ca3e8d88SDave Plauger dumpvp_write(const void *va, size_t size)
13567c478bd9Sstevel@tonic-gate {
1357ca3e8d88SDave Plauger 	size_t len, off, sz;
13587c478bd9Sstevel@tonic-gate 
1359ca3e8d88SDave Plauger 	while (size != 0) {
1360ca3e8d88SDave Plauger 		len = MIN(size, dumpbuf.end - dumpbuf.cur);
1361ca3e8d88SDave Plauger 		if (len == 0) {
1362ca3e8d88SDave Plauger 			off = P2PHASE(dumpbuf.vp_off, dumpbuf.size);
1363ca3e8d88SDave Plauger 			if (off == 0 || !ISP2(dumpbuf.size)) {
1364ca3e8d88SDave Plauger 				(void) dumpvp_flush();
1365ca3e8d88SDave Plauger 			} else {
1366ca3e8d88SDave Plauger 				sz = dumpbuf.size - off;
1367ca3e8d88SDave Plauger 				dumpbuf.cur = dumpbuf.start + sz;
1368ca3e8d88SDave Plauger 				(void) dumpvp_flush();
1369ca3e8d88SDave Plauger 				ovbcopy(dumpbuf.start + sz, dumpbuf.start, off);
1370ca3e8d88SDave Plauger 				dumpbuf.cur += off;
1371ca3e8d88SDave Plauger 			}
1372ca3e8d88SDave Plauger 		} else {
1373ca3e8d88SDave Plauger 			bcopy(va, dumpbuf.cur, len);
1374ca3e8d88SDave Plauger 			va = (char *)va + len;
1375ca3e8d88SDave Plauger 			dumpbuf.cur += len;
1376ca3e8d88SDave Plauger 			size -= len;
1377ca3e8d88SDave Plauger 		}
13787c478bd9Sstevel@tonic-gate 	}
13797c478bd9Sstevel@tonic-gate }
13807c478bd9Sstevel@tonic-gate 
1381ca3e8d88SDave Plauger /*ARGSUSED*/
1382ca3e8d88SDave Plauger static void
dumpvp_ksyms_write(const void * src,void * dst,size_t size)1383ca3e8d88SDave Plauger dumpvp_ksyms_write(const void *src, void *dst, size_t size)
13847c478bd9Sstevel@tonic-gate {
1385ca3e8d88SDave Plauger 	dumpvp_write(src, size);
1386ca3e8d88SDave Plauger }
13877c478bd9Sstevel@tonic-gate 
1388ca3e8d88SDave Plauger /*
1389ca3e8d88SDave Plauger  * Mark 'pfn' in the bitmap and dump its translation table entry.
1390ca3e8d88SDave Plauger  */
1391ca3e8d88SDave Plauger void
dump_addpage(struct as * as,void * va,pfn_t pfn)1392ca3e8d88SDave Plauger dump_addpage(struct as *as, void *va, pfn_t pfn)
1393ca3e8d88SDave Plauger {
1394ca3e8d88SDave Plauger 	mem_vtop_t mem_vtop;
1395ca3e8d88SDave Plauger 	pgcnt_t bitnum;
1396ca3e8d88SDave Plauger 
1397ca3e8d88SDave Plauger 	if ((bitnum = dump_pfn_to_bitnum(pfn)) != (pgcnt_t)-1) {
1398ca3e8d88SDave Plauger 		if (!BT_TEST(dumpcfg.bitmap, bitnum)) {
1399ca3e8d88SDave Plauger 			dumphdr->dump_npages++;
1400ca3e8d88SDave Plauger 			BT_SET(dumpcfg.bitmap, bitnum);
1401ca3e8d88SDave Plauger 		}
1402ca3e8d88SDave Plauger 		dumphdr->dump_nvtop++;
1403ca3e8d88SDave Plauger 		mem_vtop.m_as = as;
1404ca3e8d88SDave Plauger 		mem_vtop.m_va = va;
1405ca3e8d88SDave Plauger 		mem_vtop.m_pfn = pfn;
1406ca3e8d88SDave Plauger 		dumpvp_write(&mem_vtop, sizeof (mem_vtop_t));
1407ca3e8d88SDave Plauger 	}
1408ca3e8d88SDave Plauger 	dump_timeleft = dump_timeout;
1409ca3e8d88SDave Plauger }
1410ca3e8d88SDave Plauger 
1411ca3e8d88SDave Plauger /*
1412ca3e8d88SDave Plauger  * Mark 'pfn' in the bitmap
1413ca3e8d88SDave Plauger  */
1414ca3e8d88SDave Plauger void
dump_page(pfn_t pfn)1415ca3e8d88SDave Plauger dump_page(pfn_t pfn)
1416ca3e8d88SDave Plauger {
1417ca3e8d88SDave Plauger 	pgcnt_t bitnum;
1418ca3e8d88SDave Plauger 
1419ca3e8d88SDave Plauger 	if ((bitnum = dump_pfn_to_bitnum(pfn)) != (pgcnt_t)-1) {
1420ca3e8d88SDave Plauger 		if (!BT_TEST(dumpcfg.bitmap, bitnum)) {
1421ca3e8d88SDave Plauger 			dumphdr->dump_npages++;
1422ca3e8d88SDave Plauger 			BT_SET(dumpcfg.bitmap, bitnum);
1423ca3e8d88SDave Plauger 		}
1424ca3e8d88SDave Plauger 	}
1425ca3e8d88SDave Plauger 	dump_timeleft = dump_timeout;
1426ca3e8d88SDave Plauger }
1427ca3e8d88SDave Plauger 
1428ca3e8d88SDave Plauger /*
1429ca3e8d88SDave Plauger  * Dump the <as, va, pfn> information for a given address space.
1430ca3e8d88SDave Plauger  * SEGOP_DUMP() will call dump_addpage() for each page in the segment.
1431ca3e8d88SDave Plauger  */
1432ca3e8d88SDave Plauger static void
dump_as(struct as * as)1433ca3e8d88SDave Plauger dump_as(struct as *as)
1434ca3e8d88SDave Plauger {
1435ca3e8d88SDave Plauger 	struct seg *seg;
1436ca3e8d88SDave Plauger 
1437dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_ENTER(as, RW_READER);
1438ca3e8d88SDave Plauger 	for (seg = AS_SEGFIRST(as); seg; seg = AS_SEGNEXT(as, seg)) {
1439ca3e8d88SDave Plauger 		if (seg->s_as != as)
1440ca3e8d88SDave Plauger 			break;
1441ca3e8d88SDave Plauger 		if (seg->s_ops == NULL)
1442ca3e8d88SDave Plauger 			continue;
1443ca3e8d88SDave Plauger 		SEGOP_DUMP(seg);
1444ca3e8d88SDave Plauger 	}
1445dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_EXIT(as);
1446ca3e8d88SDave Plauger 
1447ca3e8d88SDave Plauger 	if (seg != NULL)
1448ca3e8d88SDave Plauger 		cmn_err(CE_WARN, "invalid segment %p in address space %p",
1449ca3e8d88SDave Plauger 		    (void *)seg, (void *)as);
1450ca3e8d88SDave Plauger }
1451ca3e8d88SDave Plauger 
1452ca3e8d88SDave Plauger static int
dump_process(pid_t pid)1453ca3e8d88SDave Plauger dump_process(pid_t pid)
1454ca3e8d88SDave Plauger {
1455ca3e8d88SDave Plauger 	proc_t *p = sprlock(pid);
1456ca3e8d88SDave Plauger 
1457ca3e8d88SDave Plauger 	if (p == NULL)
1458ca3e8d88SDave Plauger 		return (-1);
1459ca3e8d88SDave Plauger 	if (p->p_as != &kas) {
1460ca3e8d88SDave Plauger 		mutex_exit(&p->p_lock);
1461ca3e8d88SDave Plauger 		dump_as(p->p_as);
1462ca3e8d88SDave Plauger 		mutex_enter(&p->p_lock);
1463ca3e8d88SDave Plauger 	}
1464ca3e8d88SDave Plauger 
1465ca3e8d88SDave Plauger 	sprunlock(p);
1466ca3e8d88SDave Plauger 
1467ca3e8d88SDave Plauger 	return (0);
1468ca3e8d88SDave Plauger }
1469ca3e8d88SDave Plauger 
1470f6e214c7SGavin Maltby /*
1471f6e214c7SGavin Maltby  * The following functions (dump_summary(), dump_ereports(), and
1472f6e214c7SGavin Maltby  * dump_messages()), write data to an uncompressed area within the
1473f6e214c7SGavin Maltby  * crashdump. The layout of these is
1474f6e214c7SGavin Maltby  *
1475f6e214c7SGavin Maltby  * +------------------------------------------------------------+
1476f6e214c7SGavin Maltby  * |     compressed pages       | summary | ereports | messages |
1477f6e214c7SGavin Maltby  * +------------------------------------------------------------+
1478f6e214c7SGavin Maltby  *
1479f6e214c7SGavin Maltby  * With the advent of saving a compressed crash dump by default, we
1480f6e214c7SGavin Maltby  * need to save a little more data to describe the failure mode in
1481f6e214c7SGavin Maltby  * an uncompressed buffer available before savecore uncompresses
1482f6e214c7SGavin Maltby  * the dump. Initially this is a copy of the stack trace. Additional
1483f6e214c7SGavin Maltby  * summary information should be added here.
1484f6e214c7SGavin Maltby  */
1485f6e214c7SGavin Maltby 
1486f6e214c7SGavin Maltby void
dump_summary(void)1487f6e214c7SGavin Maltby dump_summary(void)
1488f6e214c7SGavin Maltby {
1489f6e214c7SGavin Maltby 	u_offset_t dumpvp_start;
1490f6e214c7SGavin Maltby 	summary_dump_t sd;
1491f6e214c7SGavin Maltby 
1492f6e214c7SGavin Maltby 	if (dumpvp == NULL || dumphdr == NULL)
1493f6e214c7SGavin Maltby 		return;
1494f6e214c7SGavin Maltby 
1495f6e214c7SGavin Maltby 	dumpbuf.cur = dumpbuf.start;
1496f6e214c7SGavin Maltby 
1497f6e214c7SGavin Maltby 	dumpbuf.vp_limit = dumpvp_size - (DUMP_OFFSET + DUMP_LOGSIZE +
1498f6e214c7SGavin Maltby 	    DUMP_ERPTSIZE);
1499f6e214c7SGavin Maltby 	dumpvp_start = dumpbuf.vp_limit - DUMP_SUMMARYSIZE;
1500f6e214c7SGavin Maltby 	dumpbuf.vp_off = dumpvp_start;
1501f6e214c7SGavin Maltby 
1502f6e214c7SGavin Maltby 	sd.sd_magic = SUMMARY_MAGIC;
1503f6e214c7SGavin Maltby 	sd.sd_ssum = checksum32(dump_stack_scratch, STACK_BUF_SIZE);
1504f6e214c7SGavin Maltby 	dumpvp_write(&sd, sizeof (sd));
1505f6e214c7SGavin Maltby 	dumpvp_write(dump_stack_scratch, STACK_BUF_SIZE);
1506f6e214c7SGavin Maltby 
1507f6e214c7SGavin Maltby 	sd.sd_magic = 0; /* indicate end of summary */
1508f6e214c7SGavin Maltby 	dumpvp_write(&sd, sizeof (sd));
1509f6e214c7SGavin Maltby 	(void) dumpvp_flush();
1510f6e214c7SGavin Maltby }
1511f6e214c7SGavin Maltby 
1512ca3e8d88SDave Plauger void
dump_ereports(void)1513ca3e8d88SDave Plauger dump_ereports(void)
1514ca3e8d88SDave Plauger {
1515ca3e8d88SDave Plauger 	u_offset_t dumpvp_start;
1516ca3e8d88SDave Plauger 	erpt_dump_t ed;
1517ca3e8d88SDave Plauger 
1518ca3e8d88SDave Plauger 	if (dumpvp == NULL || dumphdr == NULL)
1519ca3e8d88SDave Plauger 		return;
1520ca3e8d88SDave Plauger 
1521ca3e8d88SDave Plauger 	dumpbuf.cur = dumpbuf.start;
1522ca3e8d88SDave Plauger 	dumpbuf.vp_limit = dumpvp_size - (DUMP_OFFSET + DUMP_LOGSIZE);
1523ca3e8d88SDave Plauger 	dumpvp_start = dumpbuf.vp_limit - DUMP_ERPTSIZE;
1524ca3e8d88SDave Plauger 	dumpbuf.vp_off = dumpvp_start;
1525ca3e8d88SDave Plauger 
1526ca3e8d88SDave Plauger 	fm_ereport_dump();
1527ca3e8d88SDave Plauger 	if (panicstr)
1528ca3e8d88SDave Plauger 		errorq_dump();
1529ca3e8d88SDave Plauger 
1530ca3e8d88SDave Plauger 	bzero(&ed, sizeof (ed)); /* indicate end of ereports */
1531ca3e8d88SDave Plauger 	dumpvp_write(&ed, sizeof (ed));
1532ca3e8d88SDave Plauger 	(void) dumpvp_flush();
1533ca3e8d88SDave Plauger 
1534ca3e8d88SDave Plauger 	if (!panicstr) {
1535ca3e8d88SDave Plauger 		(void) VOP_PUTPAGE(dumpvp, dumpvp_start,
1536ca3e8d88SDave Plauger 		    (size_t)(dumpbuf.vp_off - dumpvp_start),
1537ca3e8d88SDave Plauger 		    B_INVAL | B_FORCE, kcred, NULL);
1538ca3e8d88SDave Plauger 	}
1539ca3e8d88SDave Plauger }
1540ca3e8d88SDave Plauger 
1541ca3e8d88SDave Plauger void
dump_messages(void)1542ca3e8d88SDave Plauger dump_messages(void)
1543ca3e8d88SDave Plauger {
1544ca3e8d88SDave Plauger 	log_dump_t ld;
1545ca3e8d88SDave Plauger 	mblk_t *mctl, *mdata;
1546ca3e8d88SDave Plauger 	queue_t *q, *qlast;
1547ca3e8d88SDave Plauger 	u_offset_t dumpvp_start;
1548ca3e8d88SDave Plauger 
1549ca3e8d88SDave Plauger 	if (dumpvp == NULL || dumphdr == NULL || log_consq == NULL)
1550ca3e8d88SDave Plauger 		return;
1551ca3e8d88SDave Plauger 
1552ca3e8d88SDave Plauger 	dumpbuf.cur = dumpbuf.start;
1553ca3e8d88SDave Plauger 	dumpbuf.vp_limit = dumpvp_size - DUMP_OFFSET;
1554ca3e8d88SDave Plauger 	dumpvp_start = dumpbuf.vp_limit - DUMP_LOGSIZE;
1555ca3e8d88SDave Plauger 	dumpbuf.vp_off = dumpvp_start;
1556ca3e8d88SDave Plauger 
1557ca3e8d88SDave Plauger 	qlast = NULL;
1558ca3e8d88SDave Plauger 	do {
1559ca3e8d88SDave Plauger 		for (q = log_consq; q->q_next != qlast; q = q->q_next)
1560ca3e8d88SDave Plauger 			continue;
1561ca3e8d88SDave Plauger 		for (mctl = q->q_first; mctl != NULL; mctl = mctl->b_next) {
1562ca3e8d88SDave Plauger 			dump_timeleft = dump_timeout;
1563ca3e8d88SDave Plauger 			mdata = mctl->b_cont;
1564ca3e8d88SDave Plauger 			ld.ld_magic = LOG_MAGIC;
1565ca3e8d88SDave Plauger 			ld.ld_msgsize = MBLKL(mctl->b_cont);
1566ca3e8d88SDave Plauger 			ld.ld_csum = checksum32(mctl->b_rptr, MBLKL(mctl));
1567ca3e8d88SDave Plauger 			ld.ld_msum = checksum32(mdata->b_rptr, MBLKL(mdata));
1568ca3e8d88SDave Plauger 			dumpvp_write(&ld, sizeof (ld));
1569ca3e8d88SDave Plauger 			dumpvp_write(mctl->b_rptr, MBLKL(mctl));
1570ca3e8d88SDave Plauger 			dumpvp_write(mdata->b_rptr, MBLKL(mdata));
1571ca3e8d88SDave Plauger 		}
1572ca3e8d88SDave Plauger 	} while ((qlast = q) != log_consq);
1573ca3e8d88SDave Plauger 
1574ca3e8d88SDave Plauger 	ld.ld_magic = 0;		/* indicate end of messages */
1575ca3e8d88SDave Plauger 	dumpvp_write(&ld, sizeof (ld));
1576ca3e8d88SDave Plauger 	(void) dumpvp_flush();
1577ca3e8d88SDave Plauger 	if (!panicstr) {
1578ca3e8d88SDave Plauger 		(void) VOP_PUTPAGE(dumpvp, dumpvp_start,
1579ca3e8d88SDave Plauger 		    (size_t)(dumpbuf.vp_off - dumpvp_start),
1580ca3e8d88SDave Plauger 		    B_INVAL | B_FORCE, kcred, NULL);
1581ca3e8d88SDave Plauger 	}
1582ca3e8d88SDave Plauger }
1583ca3e8d88SDave Plauger 
1584ca3e8d88SDave Plauger /*
1585ca3e8d88SDave Plauger  * The following functions are called on multiple CPUs during dump.
1586ca3e8d88SDave Plauger  * They must not use most kernel services, because all cross-calls are
1587ca3e8d88SDave Plauger  * disabled during panic. Therefore, blocking locks and cache flushes
1588ca3e8d88SDave Plauger  * will not work.
1589ca3e8d88SDave Plauger  */
1590ca3e8d88SDave Plauger 
15919dd77bc8SDave Plauger /*
15929dd77bc8SDave Plauger  * Copy pages, trapping ECC errors. Also, for robustness, trap data
15939dd77bc8SDave Plauger  * access in case something goes wrong in the hat layer and the
15949dd77bc8SDave Plauger  * mapping is broken.
15959dd77bc8SDave Plauger  */
1596ca3e8d88SDave Plauger static int
dump_pagecopy(void * src,void * dst)1597ca3e8d88SDave Plauger dump_pagecopy(void *src, void *dst)
1598ca3e8d88SDave Plauger {
1599ca3e8d88SDave Plauger 	long *wsrc = (long *)src;
1600ca3e8d88SDave Plauger 	long *wdst = (long *)dst;
1601ca3e8d88SDave Plauger 	const ulong_t ncopies = PAGESIZE / sizeof (long);
1602ca3e8d88SDave Plauger 	volatile int w = 0;
1603ca3e8d88SDave Plauger 	volatile int ueoff = -1;
1604ca3e8d88SDave Plauger 	on_trap_data_t otd;
1605ca3e8d88SDave Plauger 
16069dd77bc8SDave Plauger 	if (on_trap(&otd, OT_DATA_EC | OT_DATA_ACCESS)) {
1607ca3e8d88SDave Plauger 		if (ueoff == -1)
1608ca3e8d88SDave Plauger 			ueoff = w * sizeof (long);
16099dd77bc8SDave Plauger 		/* report "bad ECC" or "bad address" */
1610ca3e8d88SDave Plauger #ifdef _LP64
16119dd77bc8SDave Plauger 		if (otd.ot_trap & OT_DATA_EC)
16129dd77bc8SDave Plauger 			wdst[w++] = 0x00badecc00badecc;
16139dd77bc8SDave Plauger 		else
16149dd77bc8SDave Plauger 			wdst[w++] = 0x00badadd00badadd;
1615ca3e8d88SDave Plauger #else
16169dd77bc8SDave Plauger 		if (otd.ot_trap & OT_DATA_EC)
16179dd77bc8SDave Plauger 			wdst[w++] = 0x00badecc;
16189dd77bc8SDave Plauger 		else
16199dd77bc8SDave Plauger 			wdst[w++] = 0x00badadd;
1620ca3e8d88SDave Plauger #endif
1621ca3e8d88SDave Plauger 	}
1622ca3e8d88SDave Plauger 	while (w < ncopies) {
1623ca3e8d88SDave Plauger 		wdst[w] = wsrc[w];
1624ca3e8d88SDave Plauger 		w++;
1625ca3e8d88SDave Plauger 	}
1626ca3e8d88SDave Plauger 	no_trap();
1627ca3e8d88SDave Plauger 	return (ueoff);
1628ca3e8d88SDave Plauger }
1629ca3e8d88SDave Plauger 
1630ca3e8d88SDave Plauger static void
dumpsys_close_cq(cqueue_t * cq,int live)1631ca3e8d88SDave Plauger dumpsys_close_cq(cqueue_t *cq, int live)
1632ca3e8d88SDave Plauger {
1633ca3e8d88SDave Plauger 	if (live) {
1634ca3e8d88SDave Plauger 		mutex_enter(&cq->mutex);
1635ca3e8d88SDave Plauger 		atomic_dec_uint(&cq->open);
1636ca3e8d88SDave Plauger 		cv_signal(&cq->cv);
1637ca3e8d88SDave Plauger 		mutex_exit(&cq->mutex);
1638ca3e8d88SDave Plauger 	} else {
1639ca3e8d88SDave Plauger 		atomic_dec_uint(&cq->open);
1640ca3e8d88SDave Plauger 	}
1641ca3e8d88SDave Plauger }
1642ca3e8d88SDave Plauger 
1643ca3e8d88SDave Plauger static inline void
dumpsys_spinlock(lock_t * lp)1644ca3e8d88SDave Plauger dumpsys_spinlock(lock_t *lp)
1645ca3e8d88SDave Plauger {
1646ca3e8d88SDave Plauger 	uint_t backoff = 0;
1647ca3e8d88SDave Plauger 	int loop_count = 0;
1648ca3e8d88SDave Plauger 
1649ca3e8d88SDave Plauger 	while (LOCK_HELD(lp) || !lock_spin_try(lp)) {
1650ca3e8d88SDave Plauger 		if (++loop_count >= ncpus) {
1651ca3e8d88SDave Plauger 			backoff = mutex_lock_backoff(0);
1652ca3e8d88SDave Plauger 			loop_count = 0;
1653ca3e8d88SDave Plauger 		} else {
1654ca3e8d88SDave Plauger 			backoff = mutex_lock_backoff(backoff);
1655ca3e8d88SDave Plauger 		}
1656ca3e8d88SDave Plauger 		mutex_lock_delay(backoff);
1657ca3e8d88SDave Plauger 	}
1658ca3e8d88SDave Plauger }
1659ca3e8d88SDave Plauger 
1660ca3e8d88SDave Plauger static inline void
dumpsys_spinunlock(lock_t * lp)1661ca3e8d88SDave Plauger dumpsys_spinunlock(lock_t *lp)
1662ca3e8d88SDave Plauger {
1663ca3e8d88SDave Plauger 	lock_clear(lp);
1664ca3e8d88SDave Plauger }
1665ca3e8d88SDave Plauger 
1666ca3e8d88SDave Plauger static inline void
dumpsys_lock(cqueue_t * cq,int live)1667ca3e8d88SDave Plauger dumpsys_lock(cqueue_t *cq, int live)
1668ca3e8d88SDave Plauger {
1669ca3e8d88SDave Plauger 	if (live)
1670ca3e8d88SDave Plauger 		mutex_enter(&cq->mutex);
1671ca3e8d88SDave Plauger 	else
1672ca3e8d88SDave Plauger 		dumpsys_spinlock(&cq->spinlock);
1673ca3e8d88SDave Plauger }
1674ca3e8d88SDave Plauger 
1675ca3e8d88SDave Plauger static inline void
dumpsys_unlock(cqueue_t * cq,int live,int signal)1676ca3e8d88SDave Plauger dumpsys_unlock(cqueue_t *cq, int live, int signal)
1677ca3e8d88SDave Plauger {
1678ca3e8d88SDave Plauger 	if (live) {
1679ca3e8d88SDave Plauger 		if (signal)
1680ca3e8d88SDave Plauger 			cv_signal(&cq->cv);
1681ca3e8d88SDave Plauger 		mutex_exit(&cq->mutex);
1682ca3e8d88SDave Plauger 	} else {
1683ca3e8d88SDave Plauger 		dumpsys_spinunlock(&cq->spinlock);
1684ca3e8d88SDave Plauger 	}
1685ca3e8d88SDave Plauger }
1686ca3e8d88SDave Plauger 
1687ca3e8d88SDave Plauger static void
dumpsys_wait_cq(cqueue_t * cq,int live)1688ca3e8d88SDave Plauger dumpsys_wait_cq(cqueue_t *cq, int live)
1689ca3e8d88SDave Plauger {
1690ca3e8d88SDave Plauger 	if (live) {
1691ca3e8d88SDave Plauger 		cv_wait(&cq->cv, &cq->mutex);
1692ca3e8d88SDave Plauger 	} else {
1693ca3e8d88SDave Plauger 		dumpsys_spinunlock(&cq->spinlock);
1694ca3e8d88SDave Plauger 		while (cq->open)
1695ca3e8d88SDave Plauger 			if (cq->first)
1696ca3e8d88SDave Plauger 				break;
1697ca3e8d88SDave Plauger 		dumpsys_spinlock(&cq->spinlock);
1698ca3e8d88SDave Plauger 	}
1699ca3e8d88SDave Plauger }
1700ca3e8d88SDave Plauger 
1701ca3e8d88SDave Plauger static void
dumpsys_put_cq(cqueue_t * cq,cbuf_t * cp,int newstate,int live)1702ca3e8d88SDave Plauger dumpsys_put_cq(cqueue_t *cq, cbuf_t *cp, int newstate, int live)
1703ca3e8d88SDave Plauger {
1704ca3e8d88SDave Plauger 	if (cp == NULL)
1705ca3e8d88SDave Plauger 		return;
1706ca3e8d88SDave Plauger 
1707ca3e8d88SDave Plauger 	dumpsys_lock(cq, live);
1708ca3e8d88SDave Plauger 
1709ca3e8d88SDave Plauger 	if (cq->ts != 0) {
1710ca3e8d88SDave Plauger 		cq->empty += gethrtime() - cq->ts;
1711ca3e8d88SDave Plauger 		cq->ts = 0;
1712ca3e8d88SDave Plauger 	}
1713ca3e8d88SDave Plauger 
1714ca3e8d88SDave Plauger 	cp->state = newstate;
1715ca3e8d88SDave Plauger 	cp->next = NULL;
1716ca3e8d88SDave Plauger 	if (cq->last == NULL)
1717ca3e8d88SDave Plauger 		cq->first = cp;
1718ca3e8d88SDave Plauger 	else
1719ca3e8d88SDave Plauger 		cq->last->next = cp;
1720ca3e8d88SDave Plauger 	cq->last = cp;
1721ca3e8d88SDave Plauger 
1722ca3e8d88SDave Plauger 	dumpsys_unlock(cq, live, 1);
1723ca3e8d88SDave Plauger }
1724ca3e8d88SDave Plauger 
1725ca3e8d88SDave Plauger static cbuf_t *
dumpsys_get_cq(cqueue_t * cq,int live)1726ca3e8d88SDave Plauger dumpsys_get_cq(cqueue_t *cq, int live)
1727ca3e8d88SDave Plauger {
1728ca3e8d88SDave Plauger 	cbuf_t *cp;
1729ca3e8d88SDave Plauger 	hrtime_t now = gethrtime();
1730ca3e8d88SDave Plauger 
1731ca3e8d88SDave Plauger 	dumpsys_lock(cq, live);
1732ca3e8d88SDave Plauger 
1733ca3e8d88SDave Plauger 	/* CONSTCOND */
1734ca3e8d88SDave Plauger 	while (1) {
1735ca3e8d88SDave Plauger 		cp = (cbuf_t *)cq->first;
1736ca3e8d88SDave Plauger 		if (cp == NULL) {
1737ca3e8d88SDave Plauger 			if (cq->open == 0)
1738ca3e8d88SDave Plauger 				break;
1739ca3e8d88SDave Plauger 			dumpsys_wait_cq(cq, live);
1740ca3e8d88SDave Plauger 			continue;
1741ca3e8d88SDave Plauger 		}
1742ca3e8d88SDave Plauger 		cq->first = cp->next;
1743ca3e8d88SDave Plauger 		if (cq->first == NULL) {
1744ca3e8d88SDave Plauger 			cq->last = NULL;
1745ca3e8d88SDave Plauger 			cq->ts = now;
1746ca3e8d88SDave Plauger 		}
1747ca3e8d88SDave Plauger 		break;
1748ca3e8d88SDave Plauger 	}
1749ca3e8d88SDave Plauger 
1750ca3e8d88SDave Plauger 	dumpsys_unlock(cq, live, cq->first != NULL || cq->open == 0);
1751ca3e8d88SDave Plauger 	return (cp);
1752ca3e8d88SDave Plauger }
1753ca3e8d88SDave Plauger 
1754ca3e8d88SDave Plauger /*
1755ca3e8d88SDave Plauger  * Send an error message to the console. If the main task is running
1756ca3e8d88SDave Plauger  * just write the message via uprintf. If a helper is running the
1757ca3e8d88SDave Plauger  * message has to be put on a queue for the main task. Setting fmt to
1758ca3e8d88SDave Plauger  * NULL means flush the error message buffer. If fmt is not NULL, just
1759ca3e8d88SDave Plauger  * add the text to the existing buffer.
1760ca3e8d88SDave Plauger  */
1761ca3e8d88SDave Plauger static void
dumpsys_errmsg(helper_t * hp,const char * fmt,...)1762ca3e8d88SDave Plauger dumpsys_errmsg(helper_t *hp, const char *fmt, ...)
1763ca3e8d88SDave Plauger {
1764ca3e8d88SDave Plauger 	dumpsync_t *ds = hp->ds;
1765ca3e8d88SDave Plauger 	cbuf_t *cp = hp->cperr;
1766ca3e8d88SDave Plauger 	va_list adx;
1767ca3e8d88SDave Plauger 
1768ca3e8d88SDave Plauger 	if (hp->helper == MAINHELPER) {
1769ca3e8d88SDave Plauger 		if (fmt != NULL) {
1770ca3e8d88SDave Plauger 			if (ds->neednl) {
1771ca3e8d88SDave Plauger 				uprintf("\n");
1772ca3e8d88SDave Plauger 				ds->neednl = 0;
1773ca3e8d88SDave Plauger 			}
1774ca3e8d88SDave Plauger 			va_start(adx, fmt);
1775ca3e8d88SDave Plauger 			vuprintf(fmt, adx);
1776ca3e8d88SDave Plauger 			va_end(adx);
1777ca3e8d88SDave Plauger 		}
1778ca3e8d88SDave Plauger 	} else if (fmt == NULL) {
1779ca3e8d88SDave Plauger 		if (cp != NULL) {
1780ca3e8d88SDave Plauger 			CQ_PUT(mainq, cp, CBUF_ERRMSG);
1781ca3e8d88SDave Plauger 			hp->cperr = NULL;
1782ca3e8d88SDave Plauger 		}
1783ca3e8d88SDave Plauger 	} else {
1784ca3e8d88SDave Plauger 		if (hp->cperr == NULL) {
1785ca3e8d88SDave Plauger 			cp = CQ_GET(freebufq);
1786ca3e8d88SDave Plauger 			hp->cperr = cp;
1787ca3e8d88SDave Plauger 			cp->used = 0;
1788ca3e8d88SDave Plauger 		}
1789ca3e8d88SDave Plauger 		va_start(adx, fmt);
1790ca3e8d88SDave Plauger 		cp->used += vsnprintf(cp->buf + cp->used, cp->size - cp->used,
1791ca3e8d88SDave Plauger 		    fmt, adx);
1792ca3e8d88SDave Plauger 		va_end(adx);
1793ca3e8d88SDave Plauger 		if ((cp->used + LOG_MSGSIZE) > cp->size) {
1794ca3e8d88SDave Plauger 			CQ_PUT(mainq, cp, CBUF_ERRMSG);
1795ca3e8d88SDave Plauger 			hp->cperr = NULL;
1796ca3e8d88SDave Plauger 		}
1797ca3e8d88SDave Plauger 	}
1798ca3e8d88SDave Plauger }
1799ca3e8d88SDave Plauger 
1800ca3e8d88SDave Plauger /*
1801ca3e8d88SDave Plauger  * Write an output buffer to the dump file. If the main task is
1802ca3e8d88SDave Plauger  * running just write the data. If a helper is running the output is
1803ca3e8d88SDave Plauger  * placed on a queue for the main task.
1804ca3e8d88SDave Plauger  */
1805ca3e8d88SDave Plauger static void
dumpsys_swrite(helper_t * hp,cbuf_t * cp,size_t used)1806ca3e8d88SDave Plauger dumpsys_swrite(helper_t *hp, cbuf_t *cp, size_t used)
1807ca3e8d88SDave Plauger {
1808ca3e8d88SDave Plauger 	dumpsync_t *ds = hp->ds;
1809ca3e8d88SDave Plauger 
1810ca3e8d88SDave Plauger 	if (hp->helper == MAINHELPER) {
1811ca3e8d88SDave Plauger 		HRSTART(ds->perpage, write);
1812ca3e8d88SDave Plauger 		dumpvp_write(cp->buf, used);
1813ca3e8d88SDave Plauger 		HRSTOP(ds->perpage, write);
1814ca3e8d88SDave Plauger 		CQ_PUT(freebufq, cp, CBUF_FREEBUF);
1815ca3e8d88SDave Plauger 	} else {
1816ca3e8d88SDave Plauger 		cp->used = used;
1817ca3e8d88SDave Plauger 		CQ_PUT(mainq, cp, CBUF_WRITE);
1818ca3e8d88SDave Plauger 	}
1819ca3e8d88SDave Plauger }
1820ca3e8d88SDave Plauger 
1821ca3e8d88SDave Plauger /*
1822ca3e8d88SDave Plauger  * Copy one page within the mapped range. The offset starts at 0 and
1823ca3e8d88SDave Plauger  * is relative to the first pfn. cp->buf + cp->off is the address of
1824ca3e8d88SDave Plauger  * the first pfn. If dump_pagecopy returns a UE offset, create an
1825ca3e8d88SDave Plauger  * error message.  Returns the offset to the next pfn in the range
1826ca3e8d88SDave Plauger  * selected by the bitmap.
1827ca3e8d88SDave Plauger  */
1828ca3e8d88SDave Plauger static int
dumpsys_copy_page(helper_t * hp,int offset)1829ca3e8d88SDave Plauger dumpsys_copy_page(helper_t *hp, int offset)
1830ca3e8d88SDave Plauger {
1831ca3e8d88SDave Plauger 	cbuf_t *cp = hp->cpin;
1832ca3e8d88SDave Plauger 	int ueoff;
1833ca3e8d88SDave Plauger 
1834ca3e8d88SDave Plauger 	ASSERT(cp->off + offset + PAGESIZE <= cp->size);
1835ca3e8d88SDave Plauger 	ASSERT(BT_TEST(dumpcfg.bitmap, cp->bitnum));
1836ca3e8d88SDave Plauger 
1837ca3e8d88SDave Plauger 	ueoff = dump_pagecopy(cp->buf + cp->off + offset, hp->page);
1838ca3e8d88SDave Plauger 
1839ca3e8d88SDave Plauger 	/* ueoff is the offset in the page to a UE error */
1840ca3e8d88SDave Plauger 	if (ueoff != -1) {
1841ca3e8d88SDave Plauger 		uint64_t pa = ptob(cp->pfn) + offset + ueoff;
1842ca3e8d88SDave Plauger 
18439dd77bc8SDave Plauger 		dumpsys_errmsg(hp, "cpu %d: memory error at PA 0x%08x.%08x\n",
18449dd77bc8SDave Plauger 		    CPU->cpu_id, (uint32_t)(pa >> 32), (uint32_t)pa);
1845ca3e8d88SDave Plauger 	}
1846ca3e8d88SDave Plauger 
1847ca3e8d88SDave Plauger 	/*
1848ca3e8d88SDave Plauger 	 * Advance bitnum and offset to the next input page for the
1849ca3e8d88SDave Plauger 	 * next call to this function.
1850ca3e8d88SDave Plauger 	 */
1851ca3e8d88SDave Plauger 	offset += PAGESIZE;
1852ca3e8d88SDave Plauger 	cp->bitnum++;
1853ca3e8d88SDave Plauger 	while (cp->off + offset < cp->size) {
1854ca3e8d88SDave Plauger 		if (BT_TEST(dumpcfg.bitmap, cp->bitnum))
1855ca3e8d88SDave Plauger 			break;
1856ca3e8d88SDave Plauger 		offset += PAGESIZE;
1857ca3e8d88SDave Plauger 		cp->bitnum++;
1858ca3e8d88SDave Plauger 	}
1859ca3e8d88SDave Plauger 
1860ca3e8d88SDave Plauger 	return (offset);
1861ca3e8d88SDave Plauger }
1862ca3e8d88SDave Plauger 
1863ca3e8d88SDave Plauger /*
1864ca3e8d88SDave Plauger  * Read the helper queue, and copy one mapped page. Return 0 when
1865ca3e8d88SDave Plauger  * done. Return 1 when a page has been copied into hp->page.
1866ca3e8d88SDave Plauger  */
1867ca3e8d88SDave Plauger static int
dumpsys_sread(helper_t * hp)1868ca3e8d88SDave Plauger dumpsys_sread(helper_t *hp)
1869ca3e8d88SDave Plauger {
1870ca3e8d88SDave Plauger 	dumpsync_t *ds = hp->ds;
1871ca3e8d88SDave Plauger 
1872ca3e8d88SDave Plauger 	/* CONSTCOND */
1873ca3e8d88SDave Plauger 	while (1) {
1874ca3e8d88SDave Plauger 
1875ca3e8d88SDave Plauger 		/* Find the next input buffer. */
1876ca3e8d88SDave Plauger 		if (hp->cpin == NULL) {
1877ca3e8d88SDave Plauger 			HRSTART(hp->perpage, inwait);
1878ca3e8d88SDave Plauger 
1879ca3e8d88SDave Plauger 			/* CONSTCOND */
1880ca3e8d88SDave Plauger 			while (1) {
1881ca3e8d88SDave Plauger 				hp->cpin = CQ_GET(helperq);
1882ca3e8d88SDave Plauger 				dump_timeleft = dump_timeout;
1883ca3e8d88SDave Plauger 
1884ca3e8d88SDave Plauger 				/*
1885ca3e8d88SDave Plauger 				 * NULL return means the helper queue
1886ca3e8d88SDave Plauger 				 * is closed and empty.
1887ca3e8d88SDave Plauger 				 */
1888ca3e8d88SDave Plauger 				if (hp->cpin == NULL)
1889ca3e8d88SDave Plauger 					break;
1890ca3e8d88SDave Plauger 
1891ca3e8d88SDave Plauger 				/* Have input, check for dump I/O error. */
1892ca3e8d88SDave Plauger 				if (!dump_ioerr)
1893ca3e8d88SDave Plauger 					break;
1894ca3e8d88SDave Plauger 
1895ca3e8d88SDave Plauger 				/*
1896ca3e8d88SDave Plauger 				 * If an I/O error occurs, stay in the
1897ca3e8d88SDave Plauger 				 * loop in order to empty the helper
1898ca3e8d88SDave Plauger 				 * queue. Return the buffers to the
1899ca3e8d88SDave Plauger 				 * main task to unmap and free it.
1900ca3e8d88SDave Plauger 				 */
1901ca3e8d88SDave Plauger 				hp->cpin->used = 0;
1902ca3e8d88SDave Plauger 				CQ_PUT(mainq, hp->cpin, CBUF_USEDMAP);
1903ca3e8d88SDave Plauger 			}
1904ca3e8d88SDave Plauger 			HRSTOP(hp->perpage, inwait);
1905ca3e8d88SDave Plauger 
1906ca3e8d88SDave Plauger 			/* Stop here when the helper queue is closed. */
1907ca3e8d88SDave Plauger 			if (hp->cpin == NULL)
1908ca3e8d88SDave Plauger 				break;
1909ca3e8d88SDave Plauger 
1910ca3e8d88SDave Plauger 			/* Set the offset=0 to get the first pfn. */
1911ca3e8d88SDave Plauger 			hp->in = 0;
1912ca3e8d88SDave Plauger 
1913ca3e8d88SDave Plauger 			/* Set the total processed to 0 */
1914ca3e8d88SDave Plauger 			hp->used = 0;
1915ca3e8d88SDave Plauger 		}
1916ca3e8d88SDave Plauger 
1917ca3e8d88SDave Plauger 		/* Process the next page. */
1918ca3e8d88SDave Plauger 		if (hp->used < hp->cpin->used) {
1919ca3e8d88SDave Plauger 
1920ca3e8d88SDave Plauger 			/*
1921ca3e8d88SDave Plauger 			 * Get the next page from the input buffer and
1922ca3e8d88SDave Plauger 			 * return a copy.
1923ca3e8d88SDave Plauger 			 */
1924ca3e8d88SDave Plauger 			ASSERT(hp->in != -1);
1925ca3e8d88SDave Plauger 			HRSTART(hp->perpage, copy);
1926ca3e8d88SDave Plauger 			hp->in = dumpsys_copy_page(hp, hp->in);
1927ca3e8d88SDave Plauger 			hp->used += PAGESIZE;
1928ca3e8d88SDave Plauger 			HRSTOP(hp->perpage, copy);
1929ca3e8d88SDave Plauger 			break;
1930ca3e8d88SDave Plauger 
1931ca3e8d88SDave Plauger 		} else {
1932ca3e8d88SDave Plauger 
1933ca3e8d88SDave Plauger 			/*
1934ca3e8d88SDave Plauger 			 * Done with the input. Flush the VM and
1935ca3e8d88SDave Plauger 			 * return the buffer to the main task.
1936ca3e8d88SDave Plauger 			 */
1937ca3e8d88SDave Plauger 			if (panicstr && hp->helper != MAINHELPER)
1938ca3e8d88SDave Plauger 				hat_flush_range(kas.a_hat,
1939ca3e8d88SDave Plauger 				    hp->cpin->buf, hp->cpin->size);
1940ca3e8d88SDave Plauger 			dumpsys_errmsg(hp, NULL);
1941ca3e8d88SDave Plauger 			CQ_PUT(mainq, hp->cpin, CBUF_USEDMAP);
1942ca3e8d88SDave Plauger 			hp->cpin = NULL;
1943ca3e8d88SDave Plauger 		}
1944ca3e8d88SDave Plauger 	}
1945ca3e8d88SDave Plauger 
1946ca3e8d88SDave Plauger 	return (hp->cpin != NULL);
1947ca3e8d88SDave Plauger }
1948ca3e8d88SDave Plauger 
1949ca3e8d88SDave Plauger /*
1950ca3e8d88SDave Plauger  * Compress size bytes starting at buf with bzip2
1951ca3e8d88SDave Plauger  * mode:
1952ca3e8d88SDave Plauger  *	BZ_RUN		add one more compressed page
1953ca3e8d88SDave Plauger  *	BZ_FINISH	no more input, flush the state
1954ca3e8d88SDave Plauger  */
1955ca3e8d88SDave Plauger static void
dumpsys_bzrun(helper_t * hp,void * buf,size_t size,int mode)1956ca3e8d88SDave Plauger dumpsys_bzrun(helper_t *hp, void *buf, size_t size, int mode)
1957ca3e8d88SDave Plauger {
1958ca3e8d88SDave Plauger 	dumpsync_t *ds = hp->ds;
1959ca3e8d88SDave Plauger 	const int CSIZE = sizeof (dumpcsize_t);
1960ca3e8d88SDave Plauger 	bz_stream *ps = &hp->bzstream;
1961ca3e8d88SDave Plauger 	int rc = 0;
1962ca3e8d88SDave Plauger 	uint32_t csize;
1963ca3e8d88SDave Plauger 	dumpcsize_t cs;
1964ca3e8d88SDave Plauger 
1965ca3e8d88SDave Plauger 	/* Set input pointers to new input page */
1966ca3e8d88SDave Plauger 	if (size > 0) {
1967ca3e8d88SDave Plauger 		ps->avail_in = size;
1968ca3e8d88SDave Plauger 		ps->next_in = buf;
1969ca3e8d88SDave Plauger 	}
1970ca3e8d88SDave Plauger 
1971ca3e8d88SDave Plauger 	/* CONSTCOND */
1972ca3e8d88SDave Plauger 	while (1) {
1973ca3e8d88SDave Plauger 
1974ca3e8d88SDave Plauger 		/* Quit when all input has been consumed */
1975ca3e8d88SDave Plauger 		if (ps->avail_in == 0 && mode == BZ_RUN)
1976ca3e8d88SDave Plauger 			break;
1977ca3e8d88SDave Plauger 
1978ca3e8d88SDave Plauger 		/* Get a new output buffer */
1979ca3e8d88SDave Plauger 		if (hp->cpout == NULL) {
1980ca3e8d88SDave Plauger 			HRSTART(hp->perpage, outwait);
1981ca3e8d88SDave Plauger 			hp->cpout = CQ_GET(freebufq);
1982ca3e8d88SDave Plauger 			HRSTOP(hp->perpage, outwait);
1983ca3e8d88SDave Plauger 			ps->avail_out = hp->cpout->size - CSIZE;
1984ca3e8d88SDave Plauger 			ps->next_out = hp->cpout->buf + CSIZE;
1985ca3e8d88SDave Plauger 		}
1986ca3e8d88SDave Plauger 
1987ca3e8d88SDave Plauger 		/* Compress input, or finalize */
1988ca3e8d88SDave Plauger 		HRSTART(hp->perpage, compress);
1989ca3e8d88SDave Plauger 		rc = BZ2_bzCompress(ps, mode);
1990ca3e8d88SDave Plauger 		HRSTOP(hp->perpage, compress);
1991ca3e8d88SDave Plauger 
1992ca3e8d88SDave Plauger 		/* Check for error */
1993ca3e8d88SDave Plauger 		if (mode == BZ_RUN && rc != BZ_RUN_OK) {
1994ca3e8d88SDave Plauger 			dumpsys_errmsg(hp, "%d: BZ_RUN error %s at page %lx\n",
1995ca3e8d88SDave Plauger 			    hp->helper, BZ2_bzErrorString(rc),
1996ca3e8d88SDave Plauger 			    hp->cpin->pagenum);
1997ca3e8d88SDave Plauger 			break;
1998ca3e8d88SDave Plauger 		}
1999ca3e8d88SDave Plauger 
2000ca3e8d88SDave Plauger 		/* Write the buffer if it is full, or we are flushing */
2001ca3e8d88SDave Plauger 		if (ps->avail_out == 0 || mode == BZ_FINISH) {
2002ca3e8d88SDave Plauger 			csize = hp->cpout->size - CSIZE - ps->avail_out;
2003ca3e8d88SDave Plauger 			cs = DUMP_SET_TAG(csize, hp->tag);
2004ca3e8d88SDave Plauger 			if (csize > 0) {
2005ca3e8d88SDave Plauger 				(void) memcpy(hp->cpout->buf, &cs, CSIZE);
2006ca3e8d88SDave Plauger 				dumpsys_swrite(hp, hp->cpout, csize + CSIZE);
2007ca3e8d88SDave Plauger 				hp->cpout = NULL;
2008ca3e8d88SDave Plauger 			}
2009ca3e8d88SDave Plauger 		}
2010ca3e8d88SDave Plauger 
2011ca3e8d88SDave Plauger 		/* Check for final complete */
2012ca3e8d88SDave Plauger 		if (mode == BZ_FINISH) {
2013ca3e8d88SDave Plauger 			if (rc == BZ_STREAM_END)
2014ca3e8d88SDave Plauger 				break;
2015ca3e8d88SDave Plauger 			if (rc != BZ_FINISH_OK) {
2016ca3e8d88SDave Plauger 				dumpsys_errmsg(hp, "%d: BZ_FINISH error %s\n",
2017ca3e8d88SDave Plauger 				    hp->helper, BZ2_bzErrorString(rc));
2018ca3e8d88SDave Plauger 				break;
2019ca3e8d88SDave Plauger 			}
2020ca3e8d88SDave Plauger 		}
2021ca3e8d88SDave Plauger 	}
2022ca3e8d88SDave Plauger 
2023ca3e8d88SDave Plauger 	/* Cleanup state and buffers */
2024ca3e8d88SDave Plauger 	if (mode == BZ_FINISH) {
2025ca3e8d88SDave Plauger 
2026ca3e8d88SDave Plauger 		/* Reset state so that it is re-usable. */
2027ca3e8d88SDave Plauger 		(void) BZ2_bzCompressReset(&hp->bzstream);
2028ca3e8d88SDave Plauger 
2029ca3e8d88SDave Plauger 		/* Give any unused outout buffer to the main task */
2030ca3e8d88SDave Plauger 		if (hp->cpout != NULL) {
2031ca3e8d88SDave Plauger 			hp->cpout->used = 0;
2032ca3e8d88SDave Plauger 			CQ_PUT(mainq, hp->cpout, CBUF_ERRMSG);
2033ca3e8d88SDave Plauger 			hp->cpout = NULL;
2034ca3e8d88SDave Plauger 		}
2035ca3e8d88SDave Plauger 	}
2036ca3e8d88SDave Plauger }
2037ca3e8d88SDave Plauger 
2038ca3e8d88SDave Plauger static void
dumpsys_bz2compress(helper_t * hp)2039ca3e8d88SDave Plauger dumpsys_bz2compress(helper_t *hp)
2040ca3e8d88SDave Plauger {
2041ca3e8d88SDave Plauger 	dumpsync_t *ds = hp->ds;
2042ca3e8d88SDave Plauger 	dumpstreamhdr_t sh;
2043ca3e8d88SDave Plauger 
2044ca3e8d88SDave Plauger 	(void) strcpy(sh.stream_magic, DUMP_STREAM_MAGIC);
2045ca3e8d88SDave Plauger 	sh.stream_pagenum = (pgcnt_t)-1;
2046ca3e8d88SDave Plauger 	sh.stream_npages = 0;
2047ca3e8d88SDave Plauger 	hp->cpin = NULL;
2048ca3e8d88SDave Plauger 	hp->cpout = NULL;
2049ca3e8d88SDave Plauger 	hp->cperr = NULL;
2050ca3e8d88SDave Plauger 	hp->in = 0;
2051ca3e8d88SDave Plauger 	hp->out = 0;
2052ca3e8d88SDave Plauger 	hp->bzstream.avail_in = 0;
2053ca3e8d88SDave Plauger 
2054ca3e8d88SDave Plauger 	/* Bump reference to mainq while we are running */
2055ca3e8d88SDave Plauger 	CQ_OPEN(mainq);
2056ca3e8d88SDave Plauger 
2057ca3e8d88SDave Plauger 	/* Get one page at a time */
2058ca3e8d88SDave Plauger 	while (dumpsys_sread(hp)) {
2059ca3e8d88SDave Plauger 		if (sh.stream_pagenum != hp->cpin->pagenum) {
2060ca3e8d88SDave Plauger 			sh.stream_pagenum = hp->cpin->pagenum;
2061ca3e8d88SDave Plauger 			sh.stream_npages = btop(hp->cpin->used);
2062ca3e8d88SDave Plauger 			dumpsys_bzrun(hp, &sh, sizeof (sh), BZ_RUN);
2063ca3e8d88SDave Plauger 		}
2064ca3e8d88SDave Plauger 		dumpsys_bzrun(hp, hp->page, PAGESIZE, 0);
2065ca3e8d88SDave Plauger 	}
2066ca3e8d88SDave Plauger 
2067ca3e8d88SDave Plauger 	/* Done with input, flush any partial buffer */
2068ca3e8d88SDave Plauger 	if (sh.stream_pagenum != (pgcnt_t)-1) {
2069ca3e8d88SDave Plauger 		dumpsys_bzrun(hp, NULL, 0, BZ_FINISH);
2070ca3e8d88SDave Plauger 		dumpsys_errmsg(hp, NULL);
2071ca3e8d88SDave Plauger 	}
2072ca3e8d88SDave Plauger 
2073ca3e8d88SDave Plauger 	ASSERT(hp->cpin == NULL && hp->cpout == NULL && hp->cperr == NULL);
2074ca3e8d88SDave Plauger 
2075ca3e8d88SDave Plauger 	/* Decrement main queue count, we are done */
2076ca3e8d88SDave Plauger 	CQ_CLOSE(mainq);
2077ca3e8d88SDave Plauger }
2078ca3e8d88SDave Plauger 
2079ca3e8d88SDave Plauger /*
2080ca3e8d88SDave Plauger  * Compress with lzjb
2081ca3e8d88SDave Plauger  * write stream block if full or size==0
2082ca3e8d88SDave Plauger  * if csize==0 write stream header, else write <csize, data>
2083ca3e8d88SDave Plauger  * size==0 is a call to flush a buffer
2084ca3e8d88SDave Plauger  * hp->cpout is the buffer we are flushing or filling
2085ca3e8d88SDave Plauger  * hp->out is the next index to fill data
2086ca3e8d88SDave Plauger  * osize is either csize+data, or the size of a stream header
2087ca3e8d88SDave Plauger  */
2088ca3e8d88SDave Plauger static void
dumpsys_lzjbrun(helper_t * hp,size_t csize,void * buf,size_t size)2089ca3e8d88SDave Plauger dumpsys_lzjbrun(helper_t *hp, size_t csize, void *buf, size_t size)
2090ca3e8d88SDave Plauger {
2091ca3e8d88SDave Plauger 	dumpsync_t *ds = hp->ds;
2092ca3e8d88SDave Plauger 	const int CSIZE = sizeof (dumpcsize_t);
2093ca3e8d88SDave Plauger 	dumpcsize_t cs;
2094ca3e8d88SDave Plauger 	size_t osize = csize > 0 ? CSIZE + size : size;
2095ca3e8d88SDave Plauger 
2096ca3e8d88SDave Plauger 	/* If flush, and there is no buffer, just return */
2097ca3e8d88SDave Plauger 	if (size == 0 && hp->cpout == NULL)
2098ca3e8d88SDave Plauger 		return;
2099ca3e8d88SDave Plauger 
2100ca3e8d88SDave Plauger 	/* If flush, or cpout is full, write it out */
2101ca3e8d88SDave Plauger 	if (size == 0 ||
2102ca3e8d88SDave Plauger 	    hp->cpout != NULL && hp->out + osize > hp->cpout->size) {
2103ca3e8d88SDave Plauger 
2104ca3e8d88SDave Plauger 		/* Set tag+size word at the front of the stream block. */
2105ca3e8d88SDave Plauger 		cs = DUMP_SET_TAG(hp->out - CSIZE, hp->tag);
2106ca3e8d88SDave Plauger 		(void) memcpy(hp->cpout->buf, &cs, CSIZE);
2107ca3e8d88SDave Plauger 
2108ca3e8d88SDave Plauger 		/* Write block to dump file. */
2109ca3e8d88SDave Plauger 		dumpsys_swrite(hp, hp->cpout, hp->out);
2110ca3e8d88SDave Plauger 
2111ca3e8d88SDave Plauger 		/* Clear pointer to indicate we need a new buffer */
2112ca3e8d88SDave Plauger 		hp->cpout = NULL;
2113ca3e8d88SDave Plauger 
2114ca3e8d88SDave Plauger 		/* flushing, we are done */
2115ca3e8d88SDave Plauger 		if (size == 0)
2116ca3e8d88SDave Plauger 			return;
21177c478bd9Sstevel@tonic-gate 	}
21187c478bd9Sstevel@tonic-gate 
2119ca3e8d88SDave Plauger 	/* Get an output buffer if we dont have one. */
2120ca3e8d88SDave Plauger 	if (hp->cpout == NULL) {
2121ca3e8d88SDave Plauger 		HRSTART(hp->perpage, outwait);
2122ca3e8d88SDave Plauger 		hp->cpout = CQ_GET(freebufq);
2123ca3e8d88SDave Plauger 		HRSTOP(hp->perpage, outwait);
2124ca3e8d88SDave Plauger 		hp->out = CSIZE;
21257c478bd9Sstevel@tonic-gate 	}
2126ca3e8d88SDave Plauger 
2127ca3e8d88SDave Plauger 	/* Store csize word. This is the size of compressed data. */
2128ca3e8d88SDave Plauger 	if (csize > 0) {
2129ca3e8d88SDave Plauger 		cs = DUMP_SET_TAG(csize, 0);
2130ca3e8d88SDave Plauger 		(void) memcpy(hp->cpout->buf + hp->out, &cs, CSIZE);
2131ca3e8d88SDave Plauger 		hp->out += CSIZE;
2132ca3e8d88SDave Plauger 	}
2133ca3e8d88SDave Plauger 
2134ca3e8d88SDave Plauger 	/* Store the data. */
2135ca3e8d88SDave Plauger 	(void) memcpy(hp->cpout->buf + hp->out, buf, size);
2136ca3e8d88SDave Plauger 	hp->out += size;
21377c478bd9Sstevel@tonic-gate }
21387c478bd9Sstevel@tonic-gate 
21397c478bd9Sstevel@tonic-gate static void
dumpsys_lzjbcompress(helper_t * hp)2140ca3e8d88SDave Plauger dumpsys_lzjbcompress(helper_t *hp)
21417c478bd9Sstevel@tonic-gate {
2142ca3e8d88SDave Plauger 	dumpsync_t *ds = hp->ds;
2143ca3e8d88SDave Plauger 	size_t csize;
2144ca3e8d88SDave Plauger 	dumpstreamhdr_t sh;
2145ca3e8d88SDave Plauger 
2146ca3e8d88SDave Plauger 	(void) strcpy(sh.stream_magic, DUMP_STREAM_MAGIC);
2147ca3e8d88SDave Plauger 	sh.stream_pagenum = (pfn_t)-1;
2148ca3e8d88SDave Plauger 	sh.stream_npages = 0;
2149ca3e8d88SDave Plauger 	hp->cpin = NULL;
2150ca3e8d88SDave Plauger 	hp->cpout = NULL;
2151ca3e8d88SDave Plauger 	hp->cperr = NULL;
2152ca3e8d88SDave Plauger 	hp->in = 0;
2153ca3e8d88SDave Plauger 	hp->out = 0;
2154ca3e8d88SDave Plauger 
2155ca3e8d88SDave Plauger 	/* Bump reference to mainq while we are running */
2156ca3e8d88SDave Plauger 	CQ_OPEN(mainq);
2157ca3e8d88SDave Plauger 
2158ca3e8d88SDave Plauger 	/* Get one page at a time */
2159ca3e8d88SDave Plauger 	while (dumpsys_sread(hp)) {
2160ca3e8d88SDave Plauger 
2161ca3e8d88SDave Plauger 		/* Create a stream header for each new input map */
2162ca3e8d88SDave Plauger 		if (sh.stream_pagenum != hp->cpin->pagenum) {
2163ca3e8d88SDave Plauger 			sh.stream_pagenum = hp->cpin->pagenum;
2164ca3e8d88SDave Plauger 			sh.stream_npages = btop(hp->cpin->used);
2165ca3e8d88SDave Plauger 			dumpsys_lzjbrun(hp, 0, &sh, sizeof (sh));
2166ca3e8d88SDave Plauger 		}
2167ca3e8d88SDave Plauger 
2168ca3e8d88SDave Plauger 		/* Compress one page */
2169ca3e8d88SDave Plauger 		HRSTART(hp->perpage, compress);
2170ca3e8d88SDave Plauger 		csize = compress(hp->page, hp->lzbuf, PAGESIZE);
2171ca3e8d88SDave Plauger 		HRSTOP(hp->perpage, compress);
2172ca3e8d88SDave Plauger 
2173ca3e8d88SDave Plauger 		/* Add csize+data to output block */
2174ca3e8d88SDave Plauger 		ASSERT(csize > 0 && csize <= PAGESIZE);
2175ca3e8d88SDave Plauger 		dumpsys_lzjbrun(hp, csize, hp->lzbuf, csize);
2176ca3e8d88SDave Plauger 	}
2177ca3e8d88SDave Plauger 
2178ca3e8d88SDave Plauger 	/* Done with input, flush any partial buffer */
2179ca3e8d88SDave Plauger 	if (sh.stream_pagenum != (pfn_t)-1) {
2180ca3e8d88SDave Plauger 		dumpsys_lzjbrun(hp, 0, NULL, 0);
2181ca3e8d88SDave Plauger 		dumpsys_errmsg(hp, NULL);
2182ca3e8d88SDave Plauger 	}
2183ca3e8d88SDave Plauger 
2184ca3e8d88SDave Plauger 	ASSERT(hp->cpin == NULL && hp->cpout == NULL && hp->cperr == NULL);
2185ca3e8d88SDave Plauger 
2186ca3e8d88SDave Plauger 	/* Decrement main queue count, we are done */
2187ca3e8d88SDave Plauger 	CQ_CLOSE(mainq);
21887c478bd9Sstevel@tonic-gate }
21897c478bd9Sstevel@tonic-gate 
21907c478bd9Sstevel@tonic-gate /*
2191ca3e8d88SDave Plauger  * Dump helper called from panic_idle() to compress pages.  CPUs in
2192ca3e8d88SDave Plauger  * this path must not call most kernel services.
2193ca3e8d88SDave Plauger  *
2194ca3e8d88SDave Plauger  * During panic, all but one of the CPUs is idle. These CPUs are used
2195ca3e8d88SDave Plauger  * as helpers working in parallel to copy and compress memory
2196ca3e8d88SDave Plauger  * pages. During a panic, however, these processors cannot call any
2197ca3e8d88SDave Plauger  * kernel services. This is because mutexes become no-ops during
2198ca3e8d88SDave Plauger  * panic, and, cross-call interrupts are inhibited.  Therefore, during
2199ca3e8d88SDave Plauger  * panic dump the helper CPUs communicate with the panic CPU using
2200ca3e8d88SDave Plauger  * memory variables. All memory mapping and I/O is performed by the
2201ca3e8d88SDave Plauger  * panic CPU.
220280f5ed81SDave Plauger  *
220380f5ed81SDave Plauger  * At dump configuration time, helper_lock is set and helpers_wanted
220480f5ed81SDave Plauger  * is 0. dumpsys() decides whether to set helpers_wanted before
220580f5ed81SDave Plauger  * clearing helper_lock.
220680f5ed81SDave Plauger  *
220780f5ed81SDave Plauger  * At panic time, idle CPUs spin-wait on helper_lock, then alternately
220880f5ed81SDave Plauger  * take the lock and become a helper, or return.
22097c478bd9Sstevel@tonic-gate  */
22107c478bd9Sstevel@tonic-gate void
dumpsys_helper()2211ca3e8d88SDave Plauger dumpsys_helper()
22127c478bd9Sstevel@tonic-gate {
2213ca3e8d88SDave Plauger 	dumpsys_spinlock(&dumpcfg.helper_lock);
2214ca3e8d88SDave Plauger 	if (dumpcfg.helpers_wanted) {
2215ca3e8d88SDave Plauger 		helper_t *hp, *hpend = &dumpcfg.helper[dumpcfg.nhelper];
22167c478bd9Sstevel@tonic-gate 
2217ca3e8d88SDave Plauger 		for (hp = dumpcfg.helper; hp != hpend; hp++) {
2218ca3e8d88SDave Plauger 			if (hp->helper == FREEHELPER) {
2219ca3e8d88SDave Plauger 				hp->helper = CPU->cpu_id;
2220ca3e8d88SDave Plauger 				BT_SET(dumpcfg.helpermap, CPU->cpu_seqid);
2221ca3e8d88SDave Plauger 
2222ca3e8d88SDave Plauger 				dumpsys_spinunlock(&dumpcfg.helper_lock);
2223ca3e8d88SDave Plauger 
2224ca3e8d88SDave Plauger 				if (dumpcfg.clevel < DUMP_CLEVEL_BZIP2)
2225ca3e8d88SDave Plauger 					dumpsys_lzjbcompress(hp);
2226ca3e8d88SDave Plauger 				else
2227ca3e8d88SDave Plauger 					dumpsys_bz2compress(hp);
2228ca3e8d88SDave Plauger 
2229ca3e8d88SDave Plauger 				hp->helper = DONEHELPER;
2230ca3e8d88SDave Plauger 				return;
2231ca3e8d88SDave Plauger 			}
22327c478bd9Sstevel@tonic-gate 		}
223380f5ed81SDave Plauger 
223480f5ed81SDave Plauger 		/* No more helpers are needed. */
223580f5ed81SDave Plauger 		dumpcfg.helpers_wanted = 0;
223680f5ed81SDave Plauger 
22377c478bd9Sstevel@tonic-gate 	}
2238ca3e8d88SDave Plauger 	dumpsys_spinunlock(&dumpcfg.helper_lock);
22397c478bd9Sstevel@tonic-gate }
22407c478bd9Sstevel@tonic-gate 
224180f5ed81SDave Plauger /*
224280f5ed81SDave Plauger  * No-wait helper callable in spin loops.
224380f5ed81SDave Plauger  *
224480f5ed81SDave Plauger  * Do not wait for helper_lock. Just check helpers_wanted. The caller
224580f5ed81SDave Plauger  * may decide to continue. This is the "c)ontinue, s)ync, r)eset? s"
224680f5ed81SDave Plauger  * case.
224780f5ed81SDave Plauger  */
224880f5ed81SDave Plauger void
dumpsys_helper_nw()224980f5ed81SDave Plauger dumpsys_helper_nw()
225080f5ed81SDave Plauger {
225180f5ed81SDave Plauger 	if (dumpcfg.helpers_wanted)
225280f5ed81SDave Plauger 		dumpsys_helper();
225380f5ed81SDave Plauger }
225480f5ed81SDave Plauger 
22557c478bd9Sstevel@tonic-gate /*
2256ca3e8d88SDave Plauger  * Dump helper for live dumps.
2257ca3e8d88SDave Plauger  * These run as a system task.
22587c478bd9Sstevel@tonic-gate  */
2259ca3e8d88SDave Plauger static void
dumpsys_live_helper(void * arg)2260ca3e8d88SDave Plauger dumpsys_live_helper(void *arg)
22617c478bd9Sstevel@tonic-gate {
2262ca3e8d88SDave Plauger 	helper_t *hp = arg;
22637c478bd9Sstevel@tonic-gate 
2264ca3e8d88SDave Plauger 	BT_ATOMIC_SET(dumpcfg.helpermap, CPU->cpu_seqid);
2265ca3e8d88SDave Plauger 	if (dumpcfg.clevel < DUMP_CLEVEL_BZIP2)
2266ca3e8d88SDave Plauger 		dumpsys_lzjbcompress(hp);
2267ca3e8d88SDave Plauger 	else
2268ca3e8d88SDave Plauger 		dumpsys_bz2compress(hp);
22697c478bd9Sstevel@tonic-gate }
22707c478bd9Sstevel@tonic-gate 
22717c478bd9Sstevel@tonic-gate /*
2272ca3e8d88SDave Plauger  * Compress one page with lzjb (single threaded case)
22737c478bd9Sstevel@tonic-gate  */
22747c478bd9Sstevel@tonic-gate static void
dumpsys_lzjb_page(helper_t * hp,cbuf_t * cp)2275ca3e8d88SDave Plauger dumpsys_lzjb_page(helper_t *hp, cbuf_t *cp)
22767c478bd9Sstevel@tonic-gate {
2277ca3e8d88SDave Plauger 	dumpsync_t *ds = hp->ds;
2278ca3e8d88SDave Plauger 	uint32_t csize;
22797c478bd9Sstevel@tonic-gate 
2280ca3e8d88SDave Plauger 	hp->helper = MAINHELPER;
2281ca3e8d88SDave Plauger 	hp->in = 0;
2282ca3e8d88SDave Plauger 	hp->used = 0;
2283ca3e8d88SDave Plauger 	hp->cpin = cp;
2284ca3e8d88SDave Plauger 	while (hp->used < cp->used) {
2285ca3e8d88SDave Plauger 		HRSTART(hp->perpage, copy);
2286ca3e8d88SDave Plauger 		hp->in = dumpsys_copy_page(hp, hp->in);
2287ca3e8d88SDave Plauger 		hp->used += PAGESIZE;
2288ca3e8d88SDave Plauger 		HRSTOP(hp->perpage, copy);
2289ca3e8d88SDave Plauger 
2290ca3e8d88SDave Plauger 		HRSTART(hp->perpage, compress);
2291ca3e8d88SDave Plauger 		csize = compress(hp->page, hp->lzbuf, PAGESIZE);
2292ca3e8d88SDave Plauger 		HRSTOP(hp->perpage, compress);
2293ca3e8d88SDave Plauger 
2294ca3e8d88SDave Plauger 		HRSTART(hp->perpage, write);
2295ca3e8d88SDave Plauger 		dumpvp_write(&csize, sizeof (csize));
2296ca3e8d88SDave Plauger 		dumpvp_write(hp->lzbuf, csize);
2297ca3e8d88SDave Plauger 		HRSTOP(hp->perpage, write);
22987c478bd9Sstevel@tonic-gate 	}
2299ca3e8d88SDave Plauger 	CQ_PUT(mainq, hp->cpin, CBUF_USEDMAP);
2300ca3e8d88SDave Plauger 	hp->cpin = NULL;
23017c478bd9Sstevel@tonic-gate }
23027c478bd9Sstevel@tonic-gate 
2303ca3e8d88SDave Plauger /*
2304ca3e8d88SDave Plauger  * Main task to dump pages. This is called on the dump CPU.
2305ca3e8d88SDave Plauger  */
2306ca3e8d88SDave Plauger static void
dumpsys_main_task(void * arg)2307ca3e8d88SDave Plauger dumpsys_main_task(void *arg)
23087c478bd9Sstevel@tonic-gate {
2309ca3e8d88SDave Plauger 	dumpsync_t *ds = arg;
2310ca3e8d88SDave Plauger 	pgcnt_t pagenum = 0, bitnum = 0, hibitnum;
2311ca3e8d88SDave Plauger 	dumpmlw_t mlw;
2312ca3e8d88SDave Plauger 	cbuf_t *cp;
2313ca3e8d88SDave Plauger 	pgcnt_t baseoff, pfnoff;
2314ca3e8d88SDave Plauger 	pfn_t base, pfn;
23156ccea422SJoyce McIntosh 	boolean_t dumpserial;
23166ccea422SJoyce McIntosh 	int i;
23179c84d1a2SDave Plauger 
23189c84d1a2SDave Plauger 	/*
23199c84d1a2SDave Plauger 	 * Fall back to serial mode if there are no helpers.
23209c84d1a2SDave Plauger 	 * dump_plat_mincpu can be set to 0 at any time.
23219c84d1a2SDave Plauger 	 * dumpcfg.helpermap must contain at least one member.
23226ccea422SJoyce McIntosh 	 *
23236ccea422SJoyce McIntosh 	 * It is possible that the helpers haven't registered
23246ccea422SJoyce McIntosh 	 * in helpermap yet; wait up to DUMP_HELPER_MAX_WAIT for
23256ccea422SJoyce McIntosh 	 * at least one helper to register.
23269c84d1a2SDave Plauger 	 */
23276ccea422SJoyce McIntosh 	dumpserial = B_TRUE;
23289c84d1a2SDave Plauger 	if (dump_plat_mincpu != 0 && dumpcfg.clevel != 0) {
23296ccea422SJoyce McIntosh 		hrtime_t hrtmax = MSEC2NSEC(DUMP_HELPER_MAX_WAIT);
23306ccea422SJoyce McIntosh 		hrtime_t hrtstart = gethrtime();
23316ccea422SJoyce McIntosh 
23326ccea422SJoyce McIntosh 		for (;;) {
23336ccea422SJoyce McIntosh 			for (i = 0; i < BT_BITOUL(NCPU); ++i) {
23346ccea422SJoyce McIntosh 				if (dumpcfg.helpermap[i] != 0) {
23356ccea422SJoyce McIntosh 					dumpserial = B_FALSE;
23366ccea422SJoyce McIntosh 					break;
23376ccea422SJoyce McIntosh 				}
23386ccea422SJoyce McIntosh 			}
23396ccea422SJoyce McIntosh 
23406ccea422SJoyce McIntosh 			if ((!dumpserial) ||
23416ccea422SJoyce McIntosh 			    ((gethrtime() - hrtstart) >= hrtmax)) {
23429c84d1a2SDave Plauger 				break;
23439c84d1a2SDave Plauger 			}
23446ccea422SJoyce McIntosh 
23456ccea422SJoyce McIntosh 			SMT_PAUSE();
23469c84d1a2SDave Plauger 		}
23479c84d1a2SDave Plauger 
23486ccea422SJoyce McIntosh 		if (dumpserial) {
23496ccea422SJoyce McIntosh 			dumpcfg.clevel = 0;
23506ccea422SJoyce McIntosh 			if (dumpcfg.helper[0].lzbuf == NULL) {
23516ccea422SJoyce McIntosh 				dumpcfg.helper[0].lzbuf =
23526ccea422SJoyce McIntosh 				    dumpcfg.helper[1].page;
23536ccea422SJoyce McIntosh 			}
23546ccea422SJoyce McIntosh 		}
23559c84d1a2SDave Plauger 	}
2356ca3e8d88SDave Plauger 
2357ca3e8d88SDave Plauger 	dump_init_memlist_walker(&mlw);
2358ca3e8d88SDave Plauger 
23592fb0949cSJoshua M. Clulow 	for (;;) {
23602fb0949cSJoshua M. Clulow 		int sec = (gethrtime() - ds->start) / NANOSEC;
2361ca3e8d88SDave Plauger 
23622fb0949cSJoshua M. Clulow 		/*
23632fb0949cSJoshua M. Clulow 		 * Render a simple progress display on the system console to
23642fb0949cSJoshua M. Clulow 		 * make clear to the operator that the system has not hung.
23652fb0949cSJoshua M. Clulow 		 * Emit an update when dump progress has advanced by one
23662fb0949cSJoshua M. Clulow 		 * percent, or when no update has been drawn in the last
23672fb0949cSJoshua M. Clulow 		 * second.
23682fb0949cSJoshua M. Clulow 		 */
23692fb0949cSJoshua M. Clulow 		if (ds->percent > ds->percent_done || sec > ds->sec_done) {
23702fb0949cSJoshua M. Clulow 			ds->sec_done = sec;
2371ca3e8d88SDave Plauger 			ds->percent_done = ds->percent;
23722fb0949cSJoshua M. Clulow 			uprintf("^\rdumping: %2d:%02d %3d%% done",
2373ca3e8d88SDave Plauger 			    sec / 60, sec % 60, ds->percent);
2374ca3e8d88SDave Plauger 			ds->neednl = 1;
2375ca3e8d88SDave Plauger 		}
23767c478bd9Sstevel@tonic-gate 
2377ca3e8d88SDave Plauger 		while (CQ_IS_EMPTY(mainq) && !CQ_IS_EMPTY(writerq)) {
23787c478bd9Sstevel@tonic-gate 
2379ca3e8d88SDave Plauger 			/* the writerq never blocks */
2380ca3e8d88SDave Plauger 			cp = CQ_GET(writerq);
2381ca3e8d88SDave Plauger 			if (cp == NULL)
2382ca3e8d88SDave Plauger 				break;
23837c478bd9Sstevel@tonic-gate 
2384ca3e8d88SDave Plauger 			dump_timeleft = dump_timeout;
23857c478bd9Sstevel@tonic-gate 
2386ca3e8d88SDave Plauger 			HRSTART(ds->perpage, write);
2387ca3e8d88SDave Plauger 			dumpvp_write(cp->buf, cp->used);
2388ca3e8d88SDave Plauger 			HRSTOP(ds->perpage, write);
23897c478bd9Sstevel@tonic-gate 
2390ca3e8d88SDave Plauger 			CQ_PUT(freebufq, cp, CBUF_FREEBUF);
2391ca3e8d88SDave Plauger 		}
23927c478bd9Sstevel@tonic-gate 
2393ca3e8d88SDave Plauger 		/*
2394ca3e8d88SDave Plauger 		 * Wait here for some buffers to process. Returns NULL
2395ca3e8d88SDave Plauger 		 * when all helpers have terminated and all buffers
2396ca3e8d88SDave Plauger 		 * have been processed.
2397ca3e8d88SDave Plauger 		 */
2398ca3e8d88SDave Plauger 		cp = CQ_GET(mainq);
23997c478bd9Sstevel@tonic-gate 
2400ca3e8d88SDave Plauger 		if (cp == NULL) {
24017c478bd9Sstevel@tonic-gate 
2402ca3e8d88SDave Plauger 			/* Drain the write queue. */
2403ca3e8d88SDave Plauger 			if (!CQ_IS_EMPTY(writerq))
2404ca3e8d88SDave Plauger 				continue;
24057c478bd9Sstevel@tonic-gate 
2406ca3e8d88SDave Plauger 			/* Main task exits here. */
2407ca3e8d88SDave Plauger 			break;
2408ca3e8d88SDave Plauger 		}
24097c478bd9Sstevel@tonic-gate 
2410ca3e8d88SDave Plauger 		dump_timeleft = dump_timeout;
24117c478bd9Sstevel@tonic-gate 
2412ca3e8d88SDave Plauger 		switch (cp->state) {
24137c478bd9Sstevel@tonic-gate 
2414ca3e8d88SDave Plauger 		case CBUF_FREEMAP:
24157c478bd9Sstevel@tonic-gate 
2416ca3e8d88SDave Plauger 			/*
2417ca3e8d88SDave Plauger 			 * Note that we drop CBUF_FREEMAP buffers on
2418ca3e8d88SDave Plauger 			 * the floor (they will not be on any cqueue)
2419ca3e8d88SDave Plauger 			 * when we no longer need them.
2420ca3e8d88SDave Plauger 			 */
2421ca3e8d88SDave Plauger 			if (bitnum >= dumpcfg.bitmapsize)
2422ca3e8d88SDave Plauger 				break;
2423ca3e8d88SDave Plauger 
2424ca3e8d88SDave Plauger 			if (dump_ioerr) {
2425ca3e8d88SDave Plauger 				bitnum = dumpcfg.bitmapsize;
2426ca3e8d88SDave Plauger 				CQ_CLOSE(helperq);
2427ca3e8d88SDave Plauger 				break;
2428ca3e8d88SDave Plauger 			}
2429ca3e8d88SDave Plauger 
2430ca3e8d88SDave Plauger 			HRSTART(ds->perpage, bitmap);
2431ca3e8d88SDave Plauger 			for (; bitnum < dumpcfg.bitmapsize; bitnum++)
2432ca3e8d88SDave Plauger 				if (BT_TEST(dumpcfg.bitmap, bitnum))
2433ca3e8d88SDave Plauger 					break;
2434ca3e8d88SDave Plauger 			HRSTOP(ds->perpage, bitmap);
24357c478bd9Sstevel@tonic-gate 			dump_timeleft = dump_timeout;
24367c478bd9Sstevel@tonic-gate 
2437ca3e8d88SDave Plauger 			if (bitnum >= dumpcfg.bitmapsize) {
2438ca3e8d88SDave Plauger 				CQ_CLOSE(helperq);
2439ca3e8d88SDave Plauger 				break;
2440ca3e8d88SDave Plauger 			}
2441ca3e8d88SDave Plauger 
2442ca3e8d88SDave Plauger 			/*
2443ca3e8d88SDave Plauger 			 * Try to map CBUF_MAPSIZE ranges. Can't
2444ca3e8d88SDave Plauger 			 * assume that memory segment size is a
2445ca3e8d88SDave Plauger 			 * multiple of CBUF_MAPSIZE. Can't assume that
2446ca3e8d88SDave Plauger 			 * the segment starts on a CBUF_MAPSIZE
2447ca3e8d88SDave Plauger 			 * boundary.
2448ca3e8d88SDave Plauger 			 */
2449ca3e8d88SDave Plauger 			pfn = dump_bitnum_to_pfn(bitnum, &mlw);
2450ca3e8d88SDave Plauger 			ASSERT(pfn != PFN_INVALID);
2451ca3e8d88SDave Plauger 			ASSERT(bitnum + mlw.mpleft <= dumpcfg.bitmapsize);
2452ca3e8d88SDave Plauger 
2453ca3e8d88SDave Plauger 			base = P2ALIGN(pfn, CBUF_MAPNP);
2454ca3e8d88SDave Plauger 			if (base < mlw.mpaddr) {
2455ca3e8d88SDave Plauger 				base = mlw.mpaddr;
2456ca3e8d88SDave Plauger 				baseoff = P2PHASE(base, CBUF_MAPNP);
2457ca3e8d88SDave Plauger 			} else {
2458ca3e8d88SDave Plauger 				baseoff = 0;
2459ca3e8d88SDave Plauger 			}
2460ca3e8d88SDave Plauger 
2461ca3e8d88SDave Plauger 			pfnoff = pfn - base;
2462ca3e8d88SDave Plauger 			if (pfnoff + mlw.mpleft < CBUF_MAPNP) {
2463ca3e8d88SDave Plauger 				hibitnum = bitnum + mlw.mpleft;
2464ca3e8d88SDave Plauger 				cp->size = ptob(pfnoff + mlw.mpleft);
2465ca3e8d88SDave Plauger 			} else {
2466ca3e8d88SDave Plauger 				hibitnum = bitnum - pfnoff + CBUF_MAPNP -
2467ca3e8d88SDave Plauger 				    baseoff;
2468ca3e8d88SDave Plauger 				cp->size = CBUF_MAPSIZE - ptob(baseoff);
2469ca3e8d88SDave Plauger 			}
2470ca3e8d88SDave Plauger 
2471ca3e8d88SDave Plauger 			cp->pfn = pfn;
2472ca3e8d88SDave Plauger 			cp->bitnum = bitnum++;
2473ca3e8d88SDave Plauger 			cp->pagenum = pagenum++;
2474ca3e8d88SDave Plauger 			cp->off = ptob(pfnoff);
2475ca3e8d88SDave Plauger 
2476ca3e8d88SDave Plauger 			for (; bitnum < hibitnum; bitnum++)
2477ca3e8d88SDave Plauger 				if (BT_TEST(dumpcfg.bitmap, bitnum))
2478ca3e8d88SDave Plauger 					pagenum++;
2479ca3e8d88SDave Plauger 
2480ca3e8d88SDave Plauger 			dump_timeleft = dump_timeout;
2481ca3e8d88SDave Plauger 			cp->used = ptob(pagenum - cp->pagenum);
2482ca3e8d88SDave Plauger 
2483ca3e8d88SDave Plauger 			HRSTART(ds->perpage, map);
2484ca3e8d88SDave Plauger 			hat_devload(kas.a_hat, cp->buf, cp->size, base,
2485ca3e8d88SDave Plauger 			    PROT_READ, HAT_LOAD_NOCONSIST);
2486ca3e8d88SDave Plauger 			HRSTOP(ds->perpage, map);
2487ca3e8d88SDave Plauger 
2488ca3e8d88SDave Plauger 			ds->pages_mapped += btop(cp->size);
2489ca3e8d88SDave Plauger 			ds->pages_used += pagenum - cp->pagenum;
2490ca3e8d88SDave Plauger 
2491ca3e8d88SDave Plauger 			CQ_OPEN(mainq);
2492ca3e8d88SDave Plauger 
2493ca3e8d88SDave Plauger 			/*
2494ca3e8d88SDave Plauger 			 * If there are no helpers the main task does
2495ca3e8d88SDave Plauger 			 * non-streams lzjb compress.
2496ca3e8d88SDave Plauger 			 */
24979c84d1a2SDave Plauger 			if (dumpserial) {
2498ca3e8d88SDave Plauger 				dumpsys_lzjb_page(dumpcfg.helper, cp);
24996ccea422SJoyce McIntosh 			} else {
25006ccea422SJoyce McIntosh 				/* pass mapped pages to a helper */
25016ccea422SJoyce McIntosh 				CQ_PUT(helperq, cp, CBUF_INREADY);
2502ca3e8d88SDave Plauger 			}
2503ca3e8d88SDave Plauger 
2504ca3e8d88SDave Plauger 			/* the last page was done */
2505ca3e8d88SDave Plauger 			if (bitnum >= dumpcfg.bitmapsize)
2506ca3e8d88SDave Plauger 				CQ_CLOSE(helperq);
2507ca3e8d88SDave Plauger 
2508ca3e8d88SDave Plauger 			break;
2509ca3e8d88SDave Plauger 
2510ca3e8d88SDave Plauger 		case CBUF_USEDMAP:
2511ca3e8d88SDave Plauger 
2512ca3e8d88SDave Plauger 			ds->npages += btop(cp->used);
2513ca3e8d88SDave Plauger 
2514ca3e8d88SDave Plauger 			HRSTART(ds->perpage, unmap);
2515ca3e8d88SDave Plauger 			hat_unload(kas.a_hat, cp->buf, cp->size, HAT_UNLOAD);
2516ca3e8d88SDave Plauger 			HRSTOP(ds->perpage, unmap);
2517ca3e8d88SDave Plauger 
2518ca3e8d88SDave Plauger 			if (bitnum < dumpcfg.bitmapsize)
2519ca3e8d88SDave Plauger 				CQ_PUT(mainq, cp, CBUF_FREEMAP);
2520ca3e8d88SDave Plauger 			CQ_CLOSE(mainq);
2521ca3e8d88SDave Plauger 
2522ca3e8d88SDave Plauger 			ASSERT(ds->npages <= dumphdr->dump_npages);
2523ca3e8d88SDave Plauger 			ds->percent = ds->npages * 100LL / dumphdr->dump_npages;
2524ca3e8d88SDave Plauger 			break;
2525ca3e8d88SDave Plauger 
2526ca3e8d88SDave Plauger 		case CBUF_WRITE:
2527ca3e8d88SDave Plauger 
2528ca3e8d88SDave Plauger 			CQ_PUT(writerq, cp, CBUF_WRITE);
2529ca3e8d88SDave Plauger 			break;
2530ca3e8d88SDave Plauger 
2531ca3e8d88SDave Plauger 		case CBUF_ERRMSG:
2532ca3e8d88SDave Plauger 
2533ca3e8d88SDave Plauger 			if (cp->used > 0) {
2534ca3e8d88SDave Plauger 				cp->buf[cp->size - 2] = '\n';
2535ca3e8d88SDave Plauger 				cp->buf[cp->size - 1] = '\0';
2536ca3e8d88SDave Plauger 				if (ds->neednl) {
2537ca3e8d88SDave Plauger 					uprintf("\n%s", cp->buf);
2538ca3e8d88SDave Plauger 					ds->neednl = 0;
2539ca3e8d88SDave Plauger 				} else {
2540ca3e8d88SDave Plauger 					uprintf("%s", cp->buf);
2541ca3e8d88SDave Plauger 				}
25429dd77bc8SDave Plauger 				/* wait for console output */
25439dd77bc8SDave Plauger 				drv_usecwait(200000);
25449dd77bc8SDave Plauger 				dump_timeleft = dump_timeout;
2545ca3e8d88SDave Plauger 			}
2546ca3e8d88SDave Plauger 			CQ_PUT(freebufq, cp, CBUF_FREEBUF);
2547ca3e8d88SDave Plauger 			break;
2548ca3e8d88SDave Plauger 
2549ca3e8d88SDave Plauger 		default:
2550ca3e8d88SDave Plauger 			uprintf("dump: unexpected buffer state %d, "
2551ca3e8d88SDave Plauger 			    "buffer will be lost\n", cp->state);
2552ca3e8d88SDave Plauger 			break;
2553ca3e8d88SDave Plauger 
2554ca3e8d88SDave Plauger 		} /* end switch */
25552fb0949cSJoshua M. Clulow 	}
25567c478bd9Sstevel@tonic-gate }
25577c478bd9Sstevel@tonic-gate 
2558ca3e8d88SDave Plauger #ifdef	COLLECT_METRICS
2559ca3e8d88SDave Plauger size_t
dumpsys_metrics(dumpsync_t * ds,char * buf,size_t size)2560ca3e8d88SDave Plauger dumpsys_metrics(dumpsync_t *ds, char *buf, size_t size)
25617c478bd9Sstevel@tonic-gate {
2562ca3e8d88SDave Plauger 	dumpcfg_t *cfg = &dumpcfg;
2563ca3e8d88SDave Plauger 	int myid = CPU->cpu_seqid;
2564ca3e8d88SDave Plauger 	int i, compress_ratio;
2565ca3e8d88SDave Plauger 	int sec, iorate;
2566ca3e8d88SDave Plauger 	helper_t *hp, *hpend = &cfg->helper[cfg->nhelper];
2567ca3e8d88SDave Plauger 	char *e = buf + size;
2568ca3e8d88SDave Plauger 	char *p = buf;
2569ca3e8d88SDave Plauger 
2570ca3e8d88SDave Plauger 	sec = ds->elapsed / (1000 * 1000 * 1000ULL);
2571ca3e8d88SDave Plauger 	if (sec < 1)
2572ca3e8d88SDave Plauger 		sec = 1;
2573ca3e8d88SDave Plauger 
2574ca3e8d88SDave Plauger 	if (ds->iotime < 1)
2575ca3e8d88SDave Plauger 		ds->iotime = 1;
2576ca3e8d88SDave Plauger 	iorate = (ds->nwrite * 100000ULL) / ds->iotime;
2577ca3e8d88SDave Plauger 
2578ca3e8d88SDave Plauger 	compress_ratio = 100LL * ds->npages / btopr(ds->nwrite + 1);
2579ca3e8d88SDave Plauger 
2580ca3e8d88SDave Plauger #define	P(...) (p += p < e ? snprintf(p, e - p, __VA_ARGS__) : 0)
2581ca3e8d88SDave Plauger 
2582ca3e8d88SDave Plauger 	P("Master cpu_seqid,%d\n", CPU->cpu_seqid);
2583ca3e8d88SDave Plauger 	P("Master cpu_id,%d\n", CPU->cpu_id);
2584ca3e8d88SDave Plauger 	P("dump_flags,0x%x\n", dumphdr->dump_flags);
2585ca3e8d88SDave Plauger 	P("dump_ioerr,%d\n", dump_ioerr);
2586ca3e8d88SDave Plauger 
2587ca3e8d88SDave Plauger 	P("Helpers:\n");
2588ca3e8d88SDave Plauger 	for (i = 0; i < ncpus; i++) {
2589ca3e8d88SDave Plauger 		if ((i & 15) == 0)
2590ca3e8d88SDave Plauger 			P(",,%03d,", i);
2591ca3e8d88SDave Plauger 		if (i == myid)
2592ca3e8d88SDave Plauger 			P("   M");
2593ca3e8d88SDave Plauger 		else if (BT_TEST(cfg->helpermap, i))
2594ca3e8d88SDave Plauger 			P("%4d", cpu_seq[i]->cpu_id);
2595ca3e8d88SDave Plauger 		else
2596ca3e8d88SDave Plauger 			P("   *");
2597ca3e8d88SDave Plauger 		if ((i & 15) == 15)
2598ca3e8d88SDave Plauger 			P("\n");
2599ca3e8d88SDave Plauger 	}
26007c478bd9Sstevel@tonic-gate 
2601ca3e8d88SDave Plauger 	P("ncbuf_used,%d\n", cfg->ncbuf_used);
2602ca3e8d88SDave Plauger 	P("ncmap,%d\n", cfg->ncmap);
2603ca3e8d88SDave Plauger 
2604ca3e8d88SDave Plauger 	P("Found %ldM ranges,%ld\n", (CBUF_MAPSIZE / DUMP_1MB), cfg->found4m);
2605ca3e8d88SDave Plauger 	P("Found small pages,%ld\n", cfg->foundsm);
2606ca3e8d88SDave Plauger 
2607ca3e8d88SDave Plauger 	P("Compression level,%d\n", cfg->clevel);
26086ccea422SJoyce McIntosh 	P("Compression type,%s %s", cfg->clevel == 0 ? "serial" : "parallel",
2609ca3e8d88SDave Plauger 	    cfg->clevel >= DUMP_CLEVEL_BZIP2 ? "bzip2" : "lzjb");
26106ccea422SJoyce McIntosh 	if (cfg->clevel >= DUMP_CLEVEL_BZIP2)
26116ccea422SJoyce McIntosh 		P(" (level %d)\n", dump_bzip2_level);
26126ccea422SJoyce McIntosh 	else
26136ccea422SJoyce McIntosh 		P("\n");
2614ca3e8d88SDave Plauger 	P("Compression ratio,%d.%02d\n", compress_ratio / 100, compress_ratio %
2615ca3e8d88SDave Plauger 	    100);
2616ca3e8d88SDave Plauger 	P("nhelper_used,%d\n", cfg->nhelper_used);
2617ca3e8d88SDave Plauger 
2618ca3e8d88SDave Plauger 	P("Dump I/O rate MBS,%d.%02d\n", iorate / 100, iorate % 100);
2619ca3e8d88SDave Plauger 	P("..total bytes,%lld\n", (u_longlong_t)ds->nwrite);
2620ca3e8d88SDave Plauger 	P("..total nsec,%lld\n", (u_longlong_t)ds->iotime);
2621ca3e8d88SDave Plauger 	P("dumpbuf.iosize,%ld\n", dumpbuf.iosize);
2622ca3e8d88SDave Plauger 	P("dumpbuf.size,%ld\n", dumpbuf.size);
2623ca3e8d88SDave Plauger 
2624ca3e8d88SDave Plauger 	P("Dump pages/sec,%llu\n", (u_longlong_t)ds->npages / sec);
2625ca3e8d88SDave Plauger 	P("Dump pages,%llu\n", (u_longlong_t)ds->npages);
2626ca3e8d88SDave Plauger 	P("Dump time,%d\n", sec);
2627ca3e8d88SDave Plauger 
2628ca3e8d88SDave Plauger 	if (ds->pages_mapped > 0)
2629ca3e8d88SDave Plauger 		P("per-cent map utilization,%d\n", (int)((100 * ds->pages_used)
2630ca3e8d88SDave Plauger 		    / ds->pages_mapped));
2631ca3e8d88SDave Plauger 
2632ca3e8d88SDave Plauger 	P("\nPer-page metrics:\n");
2633ca3e8d88SDave Plauger 	if (ds->npages > 0) {
2634ca3e8d88SDave Plauger 		for (hp = cfg->helper; hp != hpend; hp++) {
2635ca3e8d88SDave Plauger #define	PERPAGE(x)	ds->perpage.x += hp->perpage.x;
2636ca3e8d88SDave Plauger 			PERPAGES;
2637ca3e8d88SDave Plauger #undef PERPAGE
26387c478bd9Sstevel@tonic-gate 		}
2639ca3e8d88SDave Plauger #define	PERPAGE(x) \
2640ca3e8d88SDave Plauger 		P("%s nsec/page,%d\n", #x, (int)(ds->perpage.x / ds->npages));
2641ca3e8d88SDave Plauger 		PERPAGES;
2642ca3e8d88SDave Plauger #undef PERPAGE
2643ca3e8d88SDave Plauger 		P("freebufq.empty,%d\n", (int)(ds->freebufq.empty /
2644ca3e8d88SDave Plauger 		    ds->npages));
2645ca3e8d88SDave Plauger 		P("helperq.empty,%d\n", (int)(ds->helperq.empty /
2646ca3e8d88SDave Plauger 		    ds->npages));
2647ca3e8d88SDave Plauger 		P("writerq.empty,%d\n", (int)(ds->writerq.empty /
2648ca3e8d88SDave Plauger 		    ds->npages));
2649ca3e8d88SDave Plauger 		P("mainq.empty,%d\n", (int)(ds->mainq.empty / ds->npages));
2650ca3e8d88SDave Plauger 
2651ca3e8d88SDave Plauger 		P("I/O wait nsec/page,%llu\n", (u_longlong_t)(ds->iowait /
2652ca3e8d88SDave Plauger 		    ds->npages));
26537c478bd9Sstevel@tonic-gate 	}
2654ca3e8d88SDave Plauger #undef P
2655ca3e8d88SDave Plauger 	if (p < e)
2656ca3e8d88SDave Plauger 		bzero(p, e - p);
2657ca3e8d88SDave Plauger 	return (p - buf);
26587c478bd9Sstevel@tonic-gate }
2659ca3e8d88SDave Plauger #endif	/* COLLECT_METRICS */
26607c478bd9Sstevel@tonic-gate 
26617c478bd9Sstevel@tonic-gate /*
26627c478bd9Sstevel@tonic-gate  * Dump the system.
26637c478bd9Sstevel@tonic-gate  */
26647c478bd9Sstevel@tonic-gate void
dumpsys(void)26657c478bd9Sstevel@tonic-gate dumpsys(void)
26667c478bd9Sstevel@tonic-gate {
2667ca3e8d88SDave Plauger 	dumpsync_t *ds = &dumpsync;
2668ca3e8d88SDave Plauger 	taskq_t *livetaskq = NULL;
26697c478bd9Sstevel@tonic-gate 	pfn_t pfn;
26707c478bd9Sstevel@tonic-gate 	pgcnt_t bitnum;
26717c478bd9Sstevel@tonic-gate 	proc_t *p;
2672ca3e8d88SDave Plauger 	helper_t *hp, *hpend = &dumpcfg.helper[dumpcfg.nhelper];
2673ca3e8d88SDave Plauger 	cbuf_t *cp;
26747c478bd9Sstevel@tonic-gate 	pid_t npids, pidx;
26757c478bd9Sstevel@tonic-gate 	char *content;
26769dd77bc8SDave Plauger 	char *buf;
26779dd77bc8SDave Plauger 	size_t size;
2678ca3e8d88SDave Plauger 	int save_dump_clevel;
2679ca3e8d88SDave Plauger 	dumpmlw_t mlw;
2680ca3e8d88SDave Plauger 	dumpcsize_t datatag;
2681ca3e8d88SDave Plauger 	dumpdatahdr_t datahdr;
26827c478bd9Sstevel@tonic-gate 
26837c478bd9Sstevel@tonic-gate 	if (dumpvp == NULL || dumphdr == NULL) {
26847c478bd9Sstevel@tonic-gate 		uprintf("skipping system dump - no dump device configured\n");
2685ca3e8d88SDave Plauger 		if (panicstr) {
2686ca3e8d88SDave Plauger 			dumpcfg.helpers_wanted = 0;
2687ca3e8d88SDave Plauger 			dumpsys_spinunlock(&dumpcfg.helper_lock);
2688ca3e8d88SDave Plauger 		}
26897c478bd9Sstevel@tonic-gate 		return;
26907c478bd9Sstevel@tonic-gate 	}
2691ca3e8d88SDave Plauger 	dumpbuf.cur = dumpbuf.start;
2692ca3e8d88SDave Plauger 
2693ca3e8d88SDave Plauger 	/* clear the sync variables */
2694ca3e8d88SDave Plauger 	ASSERT(dumpcfg.nhelper > 0);
2695ca3e8d88SDave Plauger 	bzero(ds, sizeof (*ds));
2696ca3e8d88SDave Plauger 	ds->dumpcpu = CPU->cpu_id;
26977c478bd9Sstevel@tonic-gate 
26987c478bd9Sstevel@tonic-gate 	/*
26997c478bd9Sstevel@tonic-gate 	 * Calculate the starting block for dump.  If we're dumping on a
27007c478bd9Sstevel@tonic-gate 	 * swap device, start 1/5 of the way in; otherwise, start at the
27017c478bd9Sstevel@tonic-gate 	 * beginning.  And never use the first page -- it may be a disk label.
27027c478bd9Sstevel@tonic-gate 	 */
27037c478bd9Sstevel@tonic-gate 	if (dumpvp->v_flag & VISSWAP)
27047c478bd9Sstevel@tonic-gate 		dumphdr->dump_start = P2ROUNDUP(dumpvp_size / 5, DUMP_OFFSET);
27057c478bd9Sstevel@tonic-gate 	else
27067c478bd9Sstevel@tonic-gate 		dumphdr->dump_start = DUMP_OFFSET;
27077c478bd9Sstevel@tonic-gate 
2708ca3e8d88SDave Plauger 	dumphdr->dump_flags = DF_VALID | DF_COMPLETE | DF_LIVE | DF_COMPRESSED;
27097c478bd9Sstevel@tonic-gate 	dumphdr->dump_crashtime = gethrestime_sec();
27107c478bd9Sstevel@tonic-gate 	dumphdr->dump_npages = 0;
27117c478bd9Sstevel@tonic-gate 	dumphdr->dump_nvtop = 0;
2712ca3e8d88SDave Plauger 	bzero(dumpcfg.bitmap, BT_SIZEOFMAP(dumpcfg.bitmapsize));
27137c478bd9Sstevel@tonic-gate 	dump_timeleft = dump_timeout;
27147c478bd9Sstevel@tonic-gate 
27157c478bd9Sstevel@tonic-gate 	if (panicstr) {
27167c478bd9Sstevel@tonic-gate 		dumphdr->dump_flags &= ~DF_LIVE;
2717da6c28aaSamw 		(void) VOP_DUMPCTL(dumpvp, DUMP_FREE, NULL, NULL);
2718da6c28aaSamw 		(void) VOP_DUMPCTL(dumpvp, DUMP_ALLOC, NULL, NULL);
27197c478bd9Sstevel@tonic-gate 		(void) vsnprintf(dumphdr->dump_panicstring, DUMP_PANICSIZE,
27207c478bd9Sstevel@tonic-gate 		    panicstr, panicargs);
2721ca3e8d88SDave Plauger 
27227c478bd9Sstevel@tonic-gate 	}
27237c478bd9Sstevel@tonic-gate 
27247c478bd9Sstevel@tonic-gate 	if (dump_conflags & DUMP_ALL)
27257c478bd9Sstevel@tonic-gate 		content = "all";
27267c478bd9Sstevel@tonic-gate 	else if (dump_conflags & DUMP_CURPROC)
27277c478bd9Sstevel@tonic-gate 		content = "kernel + curproc";
27287c478bd9Sstevel@tonic-gate 	else
27297c478bd9Sstevel@tonic-gate 		content = "kernel";
27307c478bd9Sstevel@tonic-gate 	uprintf("dumping to %s, offset %lld, content: %s\n", dumppath,
27317c478bd9Sstevel@tonic-gate 	    dumphdr->dump_start, content);
27327c478bd9Sstevel@tonic-gate 
2733ca3e8d88SDave Plauger 	/* Make sure nodename is current */
2734ca3e8d88SDave Plauger 	bcopy(utsname.nodename, dumphdr->dump_utsname.nodename, SYS_NMLN);
2735ca3e8d88SDave Plauger 
2736ca3e8d88SDave Plauger 	/*
2737ca3e8d88SDave Plauger 	 * If this is a live dump, try to open a VCHR vnode for better
2738ca3e8d88SDave Plauger 	 * performance. We must take care to flush the buffer cache
2739ca3e8d88SDave Plauger 	 * first.
2740ca3e8d88SDave Plauger 	 */
2741ca3e8d88SDave Plauger 	if (!panicstr) {
2742ca3e8d88SDave Plauger 		vnode_t *cdev_vp, *cmn_cdev_vp;
2743ca3e8d88SDave Plauger 
2744ca3e8d88SDave Plauger 		ASSERT(dumpbuf.cdev_vp == NULL);
2745ca3e8d88SDave Plauger 		cdev_vp = makespecvp(VTOS(dumpvp)->s_dev, VCHR);
2746ca3e8d88SDave Plauger 		if (cdev_vp != NULL) {
2747ca3e8d88SDave Plauger 			cmn_cdev_vp = common_specvp(cdev_vp);
2748ca3e8d88SDave Plauger 			if (VOP_OPEN(&cmn_cdev_vp, FREAD | FWRITE, kcred, NULL)
2749ca3e8d88SDave Plauger 			    == 0) {
2750ca3e8d88SDave Plauger 				if (vn_has_cached_data(dumpvp))
2751ca3e8d88SDave Plauger 					(void) pvn_vplist_dirty(dumpvp, 0, NULL,
2752ca3e8d88SDave Plauger 					    B_INVAL | B_TRUNC, kcred);
2753ca3e8d88SDave Plauger 				dumpbuf.cdev_vp = cmn_cdev_vp;
2754ca3e8d88SDave Plauger 			} else {
2755ca3e8d88SDave Plauger 				VN_RELE(cdev_vp);
2756ca3e8d88SDave Plauger 			}
2757ca3e8d88SDave Plauger 		}
2758ca3e8d88SDave Plauger 	}
2759ca3e8d88SDave Plauger 
2760d3d50737SRafael Vanoni 	/*
2761d3d50737SRafael Vanoni 	 * Store a hires timestamp so we can look it up during debugging.
2762d3d50737SRafael Vanoni 	 */
2763d3d50737SRafael Vanoni 	lbolt_debug_entry();
2764d3d50737SRafael Vanoni 
27657c478bd9Sstevel@tonic-gate 	/*
27667c478bd9Sstevel@tonic-gate 	 * Leave room for the message and ereport save areas and terminal dump
27677c478bd9Sstevel@tonic-gate 	 * header.
27687c478bd9Sstevel@tonic-gate 	 */
2769ca3e8d88SDave Plauger 	dumpbuf.vp_limit = dumpvp_size - DUMP_LOGSIZE - DUMP_OFFSET -
2770ca3e8d88SDave Plauger 	    DUMP_ERPTSIZE;
27717c478bd9Sstevel@tonic-gate 
27727c478bd9Sstevel@tonic-gate 	/*
27737c478bd9Sstevel@tonic-gate 	 * Write out the symbol table.  It's no longer compressed,
27747c478bd9Sstevel@tonic-gate 	 * so its 'size' and 'csize' are equal.
27757c478bd9Sstevel@tonic-gate 	 */
2776ca3e8d88SDave Plauger 	dumpbuf.vp_off = dumphdr->dump_ksyms = dumphdr->dump_start + PAGESIZE;
27777c478bd9Sstevel@tonic-gate 	dumphdr->dump_ksyms_size = dumphdr->dump_ksyms_csize =
27787c478bd9Sstevel@tonic-gate 	    ksyms_snapshot(dumpvp_ksyms_write, NULL, LONG_MAX);
27797c478bd9Sstevel@tonic-gate 
27807c478bd9Sstevel@tonic-gate 	/*
27817c478bd9Sstevel@tonic-gate 	 * Write out the translation map.
27827c478bd9Sstevel@tonic-gate 	 */
27837c478bd9Sstevel@tonic-gate 	dumphdr->dump_map = dumpvp_flush();
27847c478bd9Sstevel@tonic-gate 	dump_as(&kas);
2785ae115bc7Smrj 	dumphdr->dump_nvtop += dump_plat_addr();
27867c478bd9Sstevel@tonic-gate 
27877c478bd9Sstevel@tonic-gate 	/*
27887c478bd9Sstevel@tonic-gate 	 * call into hat, which may have unmapped pages that also need to
27897c478bd9Sstevel@tonic-gate 	 * be in the dump
27907c478bd9Sstevel@tonic-gate 	 */
27917c478bd9Sstevel@tonic-gate 	hat_dump();
27927c478bd9Sstevel@tonic-gate 
27937c478bd9Sstevel@tonic-gate 	if (dump_conflags & DUMP_ALL) {
27947c478bd9Sstevel@tonic-gate 		mutex_enter(&pidlock);
27957c478bd9Sstevel@tonic-gate 
27967c478bd9Sstevel@tonic-gate 		for (npids = 0, p = practive; p != NULL; p = p->p_next)
2797ca3e8d88SDave Plauger 			dumpcfg.pids[npids++] = p->p_pid;
27987c478bd9Sstevel@tonic-gate 
27997c478bd9Sstevel@tonic-gate 		mutex_exit(&pidlock);
28007c478bd9Sstevel@tonic-gate 
28017c478bd9Sstevel@tonic-gate 		for (pidx = 0; pidx < npids; pidx++)
2802ca3e8d88SDave Plauger 			(void) dump_process(dumpcfg.pids[pidx]);
28037c478bd9Sstevel@tonic-gate 
280486f21945SDave Plauger 		dump_init_memlist_walker(&mlw);
2805ca3e8d88SDave Plauger 		for (bitnum = 0; bitnum < dumpcfg.bitmapsize; bitnum++) {
28067c478bd9Sstevel@tonic-gate 			dump_timeleft = dump_timeout;
2807b280b2a3SStuart Maybee 			pfn = dump_bitnum_to_pfn(bitnum, &mlw);
2808b280b2a3SStuart Maybee 			/*
2809b280b2a3SStuart Maybee 			 * Some hypervisors do not have all pages available to
2810b280b2a3SStuart Maybee 			 * be accessed by the guest OS.  Check for page
2811b280b2a3SStuart Maybee 			 * accessibility.
2812b280b2a3SStuart Maybee 			 */
2813b280b2a3SStuart Maybee 			if (plat_hold_page(pfn, PLAT_HOLD_NO_LOCK, NULL) !=
2814b280b2a3SStuart Maybee 			    PLAT_HOLD_OK)
2815b280b2a3SStuart Maybee 				continue;
2816ca3e8d88SDave Plauger 			BT_SET(dumpcfg.bitmap, bitnum);
28177c478bd9Sstevel@tonic-gate 		}
2818ca3e8d88SDave Plauger 		dumphdr->dump_npages = dumpcfg.bitmapsize;
28197c478bd9Sstevel@tonic-gate 		dumphdr->dump_flags |= DF_ALL;
28207c478bd9Sstevel@tonic-gate 
28217c478bd9Sstevel@tonic-gate 	} else if (dump_conflags & DUMP_CURPROC) {
28227c478bd9Sstevel@tonic-gate 		/*
28237c478bd9Sstevel@tonic-gate 		 * Determine which pid is to be dumped.  If we're panicking, we
28247c478bd9Sstevel@tonic-gate 		 * dump the process associated with panic_thread (if any).  If
28257c478bd9Sstevel@tonic-gate 		 * this is a live dump, we dump the process associated with
28267c478bd9Sstevel@tonic-gate 		 * curthread.
28277c478bd9Sstevel@tonic-gate 		 */
28287c478bd9Sstevel@tonic-gate 		npids = 0;
28297c478bd9Sstevel@tonic-gate 		if (panicstr) {
28307c478bd9Sstevel@tonic-gate 			if (panic_thread != NULL &&
28317c478bd9Sstevel@tonic-gate 			    panic_thread->t_procp != NULL &&
28327c478bd9Sstevel@tonic-gate 			    panic_thread->t_procp != &p0) {
2833ca3e8d88SDave Plauger 				dumpcfg.pids[npids++] =
28347c478bd9Sstevel@tonic-gate 				    panic_thread->t_procp->p_pid;
28357c478bd9Sstevel@tonic-gate 			}
28367c478bd9Sstevel@tonic-gate 		} else {
2837ca3e8d88SDave Plauger 			dumpcfg.pids[npids++] = curthread->t_procp->p_pid;
28387c478bd9Sstevel@tonic-gate 		}
28397c478bd9Sstevel@tonic-gate 
2840ca3e8d88SDave Plauger 		if (npids && dump_process(dumpcfg.pids[0]) == 0)
28417c478bd9Sstevel@tonic-gate 			dumphdr->dump_flags |= DF_CURPROC;
28427c478bd9Sstevel@tonic-gate 		else
28437c478bd9Sstevel@tonic-gate 			dumphdr->dump_flags |= DF_KERNEL;
28447c478bd9Sstevel@tonic-gate 
28457c478bd9Sstevel@tonic-gate 	} else {
28467c478bd9Sstevel@tonic-gate 		dumphdr->dump_flags |= DF_KERNEL;
28477c478bd9Sstevel@tonic-gate 	}
28487c478bd9Sstevel@tonic-gate 
28497c478bd9Sstevel@tonic-gate 	dumphdr->dump_hashmask = (1 << highbit(dumphdr->dump_nvtop - 1)) - 1;
28507c478bd9Sstevel@tonic-gate 
28517c478bd9Sstevel@tonic-gate 	/*
28527c478bd9Sstevel@tonic-gate 	 * Write out the pfn table.
28537c478bd9Sstevel@tonic-gate 	 */
28547c478bd9Sstevel@tonic-gate 	dumphdr->dump_pfn = dumpvp_flush();
2855ca3e8d88SDave Plauger 	dump_init_memlist_walker(&mlw);
2856ca3e8d88SDave Plauger 	for (bitnum = 0; bitnum < dumpcfg.bitmapsize; bitnum++) {
28577c478bd9Sstevel@tonic-gate 		dump_timeleft = dump_timeout;
2858ca3e8d88SDave Plauger 		if (!BT_TEST(dumpcfg.bitmap, bitnum))
28597c478bd9Sstevel@tonic-gate 			continue;
2860ca3e8d88SDave Plauger 		pfn = dump_bitnum_to_pfn(bitnum, &mlw);
28617c478bd9Sstevel@tonic-gate 		ASSERT(pfn != PFN_INVALID);
28627c478bd9Sstevel@tonic-gate 		dumpvp_write(&pfn, sizeof (pfn_t));
28637c478bd9Sstevel@tonic-gate 	}
2864ae115bc7Smrj 	dump_plat_pfn();
28657c478bd9Sstevel@tonic-gate 
28667c478bd9Sstevel@tonic-gate 	/*
28677c478bd9Sstevel@tonic-gate 	 * Write out all the pages.
2868ca3e8d88SDave Plauger 	 * Map pages, copy them handling UEs, compress, and write them out.
2869ca3e8d88SDave Plauger 	 * Cooperate with any helpers running on CPUs in panic_idle().
28707c478bd9Sstevel@tonic-gate 	 */
28717c478bd9Sstevel@tonic-gate 	dumphdr->dump_data = dumpvp_flush();
2872ca3e8d88SDave Plauger 
2873ca3e8d88SDave Plauger 	bzero(dumpcfg.helpermap, BT_SIZEOFMAP(NCPU));
2874ca3e8d88SDave Plauger 	ds->live = dumpcfg.clevel > 0 &&
2875ca3e8d88SDave Plauger 	    (dumphdr->dump_flags & DF_LIVE) != 0;
2876ca3e8d88SDave Plauger 
2877ca3e8d88SDave Plauger 	save_dump_clevel = dumpcfg.clevel;
2878ca3e8d88SDave Plauger 	if (panicstr)
2879ca3e8d88SDave Plauger 		dumpsys_get_maxmem();
2880ca3e8d88SDave Plauger 	else if (dumpcfg.clevel >= DUMP_CLEVEL_BZIP2)
2881ca3e8d88SDave Plauger 		dumpcfg.clevel = DUMP_CLEVEL_LZJB;
2882ca3e8d88SDave Plauger 
2883ca3e8d88SDave Plauger 	dumpcfg.nhelper_used = 0;
2884ca3e8d88SDave Plauger 	for (hp = dumpcfg.helper; hp != hpend; hp++) {
2885ca3e8d88SDave Plauger 		if (hp->page == NULL) {
2886ca3e8d88SDave Plauger 			hp->helper = DONEHELPER;
28877c478bd9Sstevel@tonic-gate 			continue;
2888ca3e8d88SDave Plauger 		}
2889ca3e8d88SDave Plauger 		++dumpcfg.nhelper_used;
2890ca3e8d88SDave Plauger 		hp->helper = FREEHELPER;
28917e12ceb3SToomas Soome 		hp->taskqid = 0;
2892ca3e8d88SDave Plauger 		hp->ds = ds;
2893ca3e8d88SDave Plauger 		bzero(&hp->perpage, sizeof (hp->perpage));
2894ca3e8d88SDave Plauger 		if (dumpcfg.clevel >= DUMP_CLEVEL_BZIP2)
2895ca3e8d88SDave Plauger 			(void) BZ2_bzCompressReset(&hp->bzstream);
2896ca3e8d88SDave Plauger 	}
28977c478bd9Sstevel@tonic-gate 
2898ca3e8d88SDave Plauger 	CQ_OPEN(freebufq);
2899ca3e8d88SDave Plauger 	CQ_OPEN(helperq);
2900ca3e8d88SDave Plauger 
2901ca3e8d88SDave Plauger 	dumpcfg.ncbuf_used = 0;
2902ca3e8d88SDave Plauger 	for (cp = dumpcfg.cbuf; cp != &dumpcfg.cbuf[dumpcfg.ncbuf]; cp++) {
2903ca3e8d88SDave Plauger 		if (cp->buf != NULL) {
2904ca3e8d88SDave Plauger 			CQ_PUT(freebufq, cp, CBUF_FREEBUF);
2905ca3e8d88SDave Plauger 			++dumpcfg.ncbuf_used;
29067c478bd9Sstevel@tonic-gate 		}
2907ca3e8d88SDave Plauger 	}
2908ca3e8d88SDave Plauger 
2909ca3e8d88SDave Plauger 	for (cp = dumpcfg.cmap; cp != &dumpcfg.cmap[dumpcfg.ncmap]; cp++)
2910ca3e8d88SDave Plauger 		CQ_PUT(mainq, cp, CBUF_FREEMAP);
2911ca3e8d88SDave Plauger 
2912ca3e8d88SDave Plauger 	ds->start = gethrtime();
2913ca3e8d88SDave Plauger 	ds->iowaitts = ds->start;
2914ca3e8d88SDave Plauger 
2915ca3e8d88SDave Plauger 	/* start helpers */
2916ca3e8d88SDave Plauger 	if (ds->live) {
2917ca3e8d88SDave Plauger 		int n = dumpcfg.nhelper_used;
2918ca3e8d88SDave Plauger 		int pri = MINCLSYSPRI - 25;
2919ca3e8d88SDave Plauger 
2920ca3e8d88SDave Plauger 		livetaskq = taskq_create("LiveDump", n, pri, n, n,
2921ca3e8d88SDave Plauger 		    TASKQ_PREPOPULATE);
2922ca3e8d88SDave Plauger 		for (hp = dumpcfg.helper; hp != hpend; hp++) {
2923ca3e8d88SDave Plauger 			if (hp->page == NULL)
2924ca3e8d88SDave Plauger 				continue;
2925ca3e8d88SDave Plauger 			hp->helper = hp - dumpcfg.helper;
2926ca3e8d88SDave Plauger 			hp->taskqid = taskq_dispatch(livetaskq,
2927ca3e8d88SDave Plauger 			    dumpsys_live_helper, (void *)hp, TQ_NOSLEEP);
29287c478bd9Sstevel@tonic-gate 		}
2929ca3e8d88SDave Plauger 
2930ca3e8d88SDave Plauger 	} else {
29319dd77bc8SDave Plauger 		if (panicstr)
29329dd77bc8SDave Plauger 			kmem_dump_begin();
2933ca3e8d88SDave Plauger 		dumpcfg.helpers_wanted = dumpcfg.clevel > 0;
2934ca3e8d88SDave Plauger 		dumpsys_spinunlock(&dumpcfg.helper_lock);
29357c478bd9Sstevel@tonic-gate 	}
29367c478bd9Sstevel@tonic-gate 
2937ca3e8d88SDave Plauger 	/* run main task */
2938ca3e8d88SDave Plauger 	dumpsys_main_task(ds);
2939ca3e8d88SDave Plauger 
2940ca3e8d88SDave Plauger 	ds->elapsed = gethrtime() - ds->start;
2941ca3e8d88SDave Plauger 	if (ds->elapsed < 1)
2942ca3e8d88SDave Plauger 		ds->elapsed = 1;
2943ca3e8d88SDave Plauger 
2944ca3e8d88SDave Plauger 	if (livetaskq != NULL)
2945ca3e8d88SDave Plauger 		taskq_destroy(livetaskq);
2946ca3e8d88SDave Plauger 
2947ca3e8d88SDave Plauger 	if (ds->neednl) {
2948ca3e8d88SDave Plauger 		uprintf("\n");
2949ca3e8d88SDave Plauger 		ds->neednl = 0;
2950ca3e8d88SDave Plauger 	}
2951ca3e8d88SDave Plauger 
2952ca3e8d88SDave Plauger 	/* record actual pages dumped */
2953ca3e8d88SDave Plauger 	dumphdr->dump_npages = ds->npages;
2954ca3e8d88SDave Plauger 
2955ca3e8d88SDave Plauger 	/* platform-specific data */
2956ca3e8d88SDave Plauger 	dumphdr->dump_npages += dump_plat_data(dumpcfg.cbuf[0].buf);
2957ca3e8d88SDave Plauger 
2958ca3e8d88SDave Plauger 	/* note any errors by clearing DF_COMPLETE */
2959ca3e8d88SDave Plauger 	if (dump_ioerr || ds->npages < dumphdr->dump_npages)
2960ca3e8d88SDave Plauger 		dumphdr->dump_flags &= ~DF_COMPLETE;
2961ca3e8d88SDave Plauger 
2962ca3e8d88SDave Plauger 	/* end of stream blocks */
2963ca3e8d88SDave Plauger 	datatag = 0;
2964ca3e8d88SDave Plauger 	dumpvp_write(&datatag, sizeof (datatag));
2965ca3e8d88SDave Plauger 
2966ca3e8d88SDave Plauger 	bzero(&datahdr, sizeof (datahdr));
29679dd77bc8SDave Plauger 
29689dd77bc8SDave Plauger 	/* buffer for metrics */
29699dd77bc8SDave Plauger 	buf = dumpcfg.cbuf[0].buf;
29709dd77bc8SDave Plauger 	size = MIN(dumpcfg.cbuf[0].size, DUMP_OFFSET - sizeof (dumphdr_t) -
29719dd77bc8SDave Plauger 	    sizeof (dumpdatahdr_t));
29729dd77bc8SDave Plauger 
29739dd77bc8SDave Plauger 	/* finish the kmem intercepts, collect kmem verbose info */
29749dd77bc8SDave Plauger 	if (panicstr) {
29759dd77bc8SDave Plauger 		datahdr.dump_metrics = kmem_dump_finish(buf, size);
29769dd77bc8SDave Plauger 		buf += datahdr.dump_metrics;
29779dd77bc8SDave Plauger 		size -= datahdr.dump_metrics;
29789dd77bc8SDave Plauger 	}
29799dd77bc8SDave Plauger 
2980f6e214c7SGavin Maltby 	/* record in the header whether this is a fault-management panic */
2981f6e214c7SGavin Maltby 	if (panicstr)
2982f6e214c7SGavin Maltby 		dumphdr->dump_fm_panic = is_fm_panic();
2983f6e214c7SGavin Maltby 
29849dd77bc8SDave Plauger 	/* compression info in data header */
2985ca3e8d88SDave Plauger 	datahdr.dump_datahdr_magic = DUMP_DATAHDR_MAGIC;
2986ca3e8d88SDave Plauger 	datahdr.dump_datahdr_version = DUMP_DATAHDR_VERSION;
2987ca3e8d88SDave Plauger 	datahdr.dump_maxcsize = CBUF_SIZE;
2988ca3e8d88SDave Plauger 	datahdr.dump_maxrange = CBUF_MAPSIZE / PAGESIZE;
2989ca3e8d88SDave Plauger 	datahdr.dump_nstreams = dumpcfg.nhelper_used;
2990ca3e8d88SDave Plauger 	datahdr.dump_clevel = dumpcfg.clevel;
2991ca3e8d88SDave Plauger #ifdef COLLECT_METRICS
2992ca3e8d88SDave Plauger 	if (dump_metrics_on)
29939dd77bc8SDave Plauger 		datahdr.dump_metrics += dumpsys_metrics(ds, buf, size);
2994ca3e8d88SDave Plauger #endif
2995ca3e8d88SDave Plauger 	datahdr.dump_data_csize = dumpvp_flush() - dumphdr->dump_data;
29967c478bd9Sstevel@tonic-gate 
29977c478bd9Sstevel@tonic-gate 	/*
29987c478bd9Sstevel@tonic-gate 	 * Write out the initial and terminal dump headers.
29997c478bd9Sstevel@tonic-gate 	 */
3000ca3e8d88SDave Plauger 	dumpbuf.vp_off = dumphdr->dump_start;
30017c478bd9Sstevel@tonic-gate 	dumpvp_write(dumphdr, sizeof (dumphdr_t));
30027c478bd9Sstevel@tonic-gate 	(void) dumpvp_flush();
30037c478bd9Sstevel@tonic-gate 
3004ca3e8d88SDave Plauger 	dumpbuf.vp_limit = dumpvp_size;
3005ca3e8d88SDave Plauger 	dumpbuf.vp_off = dumpbuf.vp_limit - DUMP_OFFSET;
30067c478bd9Sstevel@tonic-gate 	dumpvp_write(dumphdr, sizeof (dumphdr_t));
3007ca3e8d88SDave Plauger 	dumpvp_write(&datahdr, sizeof (dumpdatahdr_t));
3008ca3e8d88SDave Plauger 	dumpvp_write(dumpcfg.cbuf[0].buf, datahdr.dump_metrics);
30097c478bd9Sstevel@tonic-gate 
3010ca3e8d88SDave Plauger 	(void) dumpvp_flush();
30117c478bd9Sstevel@tonic-gate 
3012ca3e8d88SDave Plauger 	uprintf("\r%3d%% done: %llu pages dumped, ",
3013ca3e8d88SDave Plauger 	    ds->percent_done, (u_longlong_t)ds->npages);
30147c478bd9Sstevel@tonic-gate 
30157c478bd9Sstevel@tonic-gate 	if (dump_ioerr == 0) {
30167c478bd9Sstevel@tonic-gate 		uprintf("dump succeeded\n");
30177c478bd9Sstevel@tonic-gate 	} else {
30187c478bd9Sstevel@tonic-gate 		uprintf("dump failed: error %d\n", dump_ioerr);
3019ca3e8d88SDave Plauger #ifdef DEBUG
3020ca3e8d88SDave Plauger 		if (panicstr)
30217c478bd9Sstevel@tonic-gate 			debug_enter("dump failed");
3022ca3e8d88SDave Plauger #endif
30237c478bd9Sstevel@tonic-gate 	}
30247c478bd9Sstevel@tonic-gate 
30257c478bd9Sstevel@tonic-gate 	/*
30267c478bd9Sstevel@tonic-gate 	 * Write out all undelivered messages.  This has to be the *last*
30277c478bd9Sstevel@tonic-gate 	 * thing we do because the dump process itself emits messages.
30287c478bd9Sstevel@tonic-gate 	 */
30297c478bd9Sstevel@tonic-gate 	if (panicstr) {
3030f6e214c7SGavin Maltby 		dump_summary();
30317c478bd9Sstevel@tonic-gate 		dump_ereports();
30327c478bd9Sstevel@tonic-gate 		dump_messages();
30337c478bd9Sstevel@tonic-gate 	}
30347c478bd9Sstevel@tonic-gate 
30357c478bd9Sstevel@tonic-gate 	delay(2 * hz);	/* let people see the 'done' message */
30367c478bd9Sstevel@tonic-gate 	dump_timeleft = 0;
30377c478bd9Sstevel@tonic-gate 	dump_ioerr = 0;
3038ca3e8d88SDave Plauger 
3039ca3e8d88SDave Plauger 	/* restore settings after live dump completes */
3040ca3e8d88SDave Plauger 	if (!panicstr) {
3041ca3e8d88SDave Plauger 		dumpcfg.clevel = save_dump_clevel;
3042ca3e8d88SDave Plauger 
3043ca3e8d88SDave Plauger 		/* release any VCHR open of the dump device */
3044ca3e8d88SDave Plauger 		if (dumpbuf.cdev_vp != NULL) {
3045ca3e8d88SDave Plauger 			(void) VOP_CLOSE(dumpbuf.cdev_vp, FREAD | FWRITE, 1, 0,
3046ca3e8d88SDave Plauger 			    kcred, NULL);
3047ca3e8d88SDave Plauger 			VN_RELE(dumpbuf.cdev_vp);
3048ca3e8d88SDave Plauger 			dumpbuf.cdev_vp = NULL;
3049ca3e8d88SDave Plauger 		}
3050ca3e8d88SDave Plauger 	}
30517c478bd9Sstevel@tonic-gate }
30527c478bd9Sstevel@tonic-gate 
30537c478bd9Sstevel@tonic-gate /*
30547c478bd9Sstevel@tonic-gate  * This function is called whenever the memory size, as represented
30557c478bd9Sstevel@tonic-gate  * by the phys_install list, changes.
30567c478bd9Sstevel@tonic-gate  */
30577c478bd9Sstevel@tonic-gate void
dump_resize()30587c478bd9Sstevel@tonic-gate dump_resize()
30597c478bd9Sstevel@tonic-gate {
30607c478bd9Sstevel@tonic-gate 	mutex_enter(&dump_lock);
30617c478bd9Sstevel@tonic-gate 	dumphdr_init();
30627c478bd9Sstevel@tonic-gate 	dumpbuf_resize();
3063ca3e8d88SDave Plauger 	dump_update_clevel();
30647c478bd9Sstevel@tonic-gate 	mutex_exit(&dump_lock);
30657c478bd9Sstevel@tonic-gate }
3066e7cbe64fSgw 
3067e7cbe64fSgw /*
3068e7cbe64fSgw  * This function allows for dynamic resizing of a dump area. It assumes that
3069e7cbe64fSgw  * the underlying device has update its appropriate size(9P).
3070e7cbe64fSgw  */
3071e7cbe64fSgw int
dumpvp_resize()3072e7cbe64fSgw dumpvp_resize()
3073e7cbe64fSgw {
3074e7cbe64fSgw 	int error;
3075e7cbe64fSgw 	vattr_t vattr;
3076e7cbe64fSgw 
3077e7cbe64fSgw 	mutex_enter(&dump_lock);
3078e7cbe64fSgw 	vattr.va_mask = AT_SIZE;
3079e7cbe64fSgw 	if ((error = VOP_GETATTR(dumpvp, &vattr, 0, kcred, NULL)) != 0) {
3080e7cbe64fSgw 		mutex_exit(&dump_lock);
3081e7cbe64fSgw 		return (error);
3082e7cbe64fSgw 	}
3083e7cbe64fSgw 
3084e7cbe64fSgw 	if (error == 0 && vattr.va_size < 2 * DUMP_LOGSIZE + DUMP_ERPTSIZE) {
3085e7cbe64fSgw 		mutex_exit(&dump_lock);
3086e7cbe64fSgw 		return (ENOSPC);
3087e7cbe64fSgw 	}
3088e7cbe64fSgw 
3089e7cbe64fSgw 	dumpvp_size = vattr.va_size & -DUMP_OFFSET;
3090e7cbe64fSgw 	mutex_exit(&dump_lock);
3091e7cbe64fSgw 	return (0);
3092e7cbe64fSgw }
3093f6e214c7SGavin Maltby 
3094f6e214c7SGavin Maltby int
dump_set_uuid(const char * uuidstr)3095f6e214c7SGavin Maltby dump_set_uuid(const char *uuidstr)
3096f6e214c7SGavin Maltby {
3097f6e214c7SGavin Maltby 	const char *ptr;
3098f6e214c7SGavin Maltby 	int i;
3099f6e214c7SGavin Maltby 
3100f6e214c7SGavin Maltby 	if (uuidstr == NULL || strnlen(uuidstr, 36 + 1) != 36)
3101f6e214c7SGavin Maltby 		return (EINVAL);
3102f6e214c7SGavin Maltby 
3103f6e214c7SGavin Maltby 	/* uuid_parse is not common code so check manually */
3104f6e214c7SGavin Maltby 	for (i = 0, ptr = uuidstr; i < 36; i++, ptr++) {
3105f6e214c7SGavin Maltby 		switch (i) {
3106f6e214c7SGavin Maltby 		case 8:
3107f6e214c7SGavin Maltby 		case 13:
3108f6e214c7SGavin Maltby 		case 18:
3109f6e214c7SGavin Maltby 		case 23:
3110f6e214c7SGavin Maltby 			if (*ptr != '-')
3111f6e214c7SGavin Maltby 				return (EINVAL);
3112f6e214c7SGavin Maltby 			break;
3113f6e214c7SGavin Maltby 
3114f6e214c7SGavin Maltby 		default:
3115f6e214c7SGavin Maltby 			if (!isxdigit(*ptr))
3116f6e214c7SGavin Maltby 				return (EINVAL);
3117f6e214c7SGavin Maltby 			break;
3118f6e214c7SGavin Maltby 		}
3119f6e214c7SGavin Maltby 	}
3120f6e214c7SGavin Maltby 
3121f6e214c7SGavin Maltby 	if (dump_osimage_uuid[0] != '\0')
3122f6e214c7SGavin Maltby 		return (EALREADY);
3123f6e214c7SGavin Maltby 
3124f6e214c7SGavin Maltby 	(void) strncpy(dump_osimage_uuid, uuidstr, 36 + 1);
3125f6e214c7SGavin Maltby 
3126*1c802681SLuca Minoja 	cmn_err(CE_CONT, "?This illumos instance has UUID %s\n",
3127f6e214c7SGavin Maltby 	    dump_osimage_uuid);
3128f6e214c7SGavin Maltby 
3129f6e214c7SGavin Maltby 	return (0);
3130f6e214c7SGavin Maltby }
3131f6e214c7SGavin Maltby 
3132f6e214c7SGavin Maltby const char *
dump_get_uuid(void)3133f6e214c7SGavin Maltby dump_get_uuid(void)
3134f6e214c7SGavin Maltby {
3135f6e214c7SGavin Maltby 	return (dump_osimage_uuid[0] != '\0' ? dump_osimage_uuid : "");
3136f6e214c7SGavin Maltby }
3137