1#
2# CDDL HEADER START
3#
4# The contents of this file are subject to the terms of the
5# Common Development and Distribution License (the "License").
6# You may not use this file except in compliance with the License.
7#
8# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9# or http://www.opensolaris.org/os/licensing.
10# See the License for the specific language governing permissions
11# and limitations under the License.
12#
13# When distributing Covered Code, include this CDDL HEADER in each
14# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15# If applicable, add the following below this CDDL HEADER, with the
16# fields enclosed by brackets "[]" replaced with your own identifying
17# information: Portions Copyright [yyyy] [name of copyright owner]
18#
19# CDDL HEADER END
20#
21
22#
23# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24# Use is subject to license terms.
25#
26
27#
28# Copyright (c) 2013, 2016 by Delphix. All rights reserved.
29#
30
31. $STF_SUITE/include/libtest.shlib
32. $STF_SUITE/tests/functional/rsend/rsend.cfg
33
34#
35# Set up test model which includes various datasets
36#
37#               @final
38#               @snapB
39#               @init
40#               |
41#   ______ pclone
42#  |      /
43#  |@psnap
44#  ||                         @final
45#  ||@final       @final      @snapC
46#  ||@snapC       @snapC      @snapB
47#  ||@snapA       @snapB      @snapA
48#  ||@init        @init       @init
49#  |||            |           |
50# $pool -------- $FS ------- fs1 ------- fs2
51#    \             \\_____     \          |
52#     vol           vol   \____ \         @fsnap
53#      |              |        \ \              \
54#      @init          @vsnap   |  ------------ fclone
55#      @snapA         @init \  |                    |
56#      @final         @snapB \ |                    @init
57#                     @snapC  vclone                @snapA
58#                     @final       |                @final
59#                                 @init
60#                                 @snapC
61#                                 @final
62#
63# $1 pool name
64#
65function setup_test_model
66{
67	typeset pool=$1
68
69	log_must zfs create -p $pool/$FS/fs1/fs2
70
71	log_must zfs snapshot $pool@psnap
72	log_must zfs clone $pool@psnap $pool/pclone
73
74	if is_global_zone ; then
75		log_must zfs create -V 16M $pool/vol
76		log_must zfs create -V 16M $pool/$FS/vol
77
78		log_must zfs snapshot $pool/$FS/vol@vsnap
79		log_must zfs clone $pool/$FS/vol@vsnap $pool/$FS/vclone
80	fi
81
82	log_must snapshot_tree $pool/$FS/fs1/fs2@fsnap
83	log_must zfs clone $pool/$FS/fs1/fs2@fsnap $pool/$FS/fs1/fclone
84	log_must zfs snapshot -r $pool@init
85
86	log_must snapshot_tree $pool@snapA
87	log_must snapshot_tree $pool@snapC
88	log_must snapshot_tree $pool/pclone@snapB
89	log_must snapshot_tree $pool/$FS@snapB
90	log_must snapshot_tree $pool/$FS@snapC
91	log_must snapshot_tree $pool/$FS/fs1@snapA
92	log_must snapshot_tree $pool/$FS/fs1@snapB
93	log_must snapshot_tree $pool/$FS/fs1@snapC
94	log_must snapshot_tree $pool/$FS/fs1/fclone@snapA
95
96	if is_global_zone ; then
97		log_must zfs snapshot $pool/vol@snapA
98		log_must zfs snapshot $pool/$FS/vol@snapB
99		log_must zfs snapshot $pool/$FS/vol@snapC
100		log_must zfs snapshot $pool/$FS/vclone@snapC
101	fi
102
103	log_must zfs snapshot -r $pool@final
104
105	return 0
106}
107
108#
109# Cleanup the BACKDIR and given pool content and all the sub datasets
110#
111# $1 pool name
112#
113function cleanup_pool
114{
115	typeset pool=$1
116	log_must rm -rf $BACKDIR/*
117
118	if is_global_zone ; then
119		log_must zfs destroy -Rf $pool
120	else
121		typeset list=$(zfs list -H -r -t filesystem,snapshot,volume -o name $pool)
122		for ds in $list ; do
123			if [[ $ds != $pool ]] ; then
124				if datasetexists $ds ; then
125					log_must zfs destroy -Rf $ds
126				fi
127			fi
128		done
129	fi
130
131	typeset mntpnt=$(get_prop mountpoint $pool)
132	if ! ismounted $pool ; then
133		# Make sure mountpoint directory is empty
134		if [[ -d $mntpnt ]]; then
135			log_must rm -rf $mntpnt/*
136		fi
137
138		log_must zfs mount $pool
139	fi
140	if [[ -d $mntpnt ]]; then
141		log_must rm -rf $mntpnt/*
142	fi
143
144	return 0
145}
146
147#
148# Detect if the given two filesystems have same sub-datasets
149#
150# $1 source filesystem
151# $2 destination filesystem
152#
153function cmp_ds_subs
154{
155	typeset src_fs=$1
156	typeset dst_fs=$2
157
158	zfs list -r -H -t filesystem,snapshot,volume -o name $src_fs > $BACKDIR/src1
159	zfs list -r -H -t filesystem,snapshot,volume -o name $dst_fs > $BACKDIR/dst1
160
161	eval sed -e 's:^$src_fs:PREFIX:g' < $BACKDIR/src1 > $BACKDIR/src
162	eval sed -e 's:^$dst_fs:PREFIX:g' < $BACKDIR/dst1 > $BACKDIR/dst
163
164	diff $BACKDIR/src $BACKDIR/dst
165	typeset -i ret=$?
166
167	rm -f $BACKDIR/src $BACKDIR/dst $BACKDIR/src1 $BACKDIR/dst1
168
169	return $ret
170}
171
172#
173# Compare all the directores and files in two filesystems
174#
175# $1 source filesystem
176# $2 destination filesystem
177#
178function cmp_ds_cont
179{
180	typeset src_fs=$1
181	typeset dst_fs=$2
182
183	typeset srcdir dstdir
184	srcdir=$(get_prop mountpoint $src_fs)
185	dstdir=$(get_prop mountpoint $dst_fs)
186
187	diff -r $srcdir $dstdir > /dev/null 2>&1
188	return $?
189}
190
191#
192# Compare the given two dataset properties
193#
194# $1 dataset 1
195# $2 dataset 2
196#
197function cmp_ds_prop
198{
199	typeset dtst1=$1
200	typeset dtst2=$2
201
202	for item in "type" "origin" "volblocksize" "aclinherit" "aclmode" \
203	    "atime" "canmount" "checksum" "compression" "copies" "devices" \
204	    "exec" "quota" "readonly" "recordsize" "reservation" "setuid" \
205	    "sharenfs" "snapdir" "version" "volsize" "xattr" "zoned" \
206	    "mountpoint";
207	do
208		zfs get -H -o property,value,source $item $dtst1 >> \
209		    $BACKDIR/dtst1
210		zfs get -H -o property,value,source $item $dtst2 >> \
211		    $BACKDIR/dtst2
212	done
213
214	eval sed -e 's:$dtst1:PREFIX:g' < $BACKDIR/dtst1 > $BACKDIR/dtst1
215	eval sed -e 's:$dtst2:PREFIX:g' < $BACKDIR/dtst2 > $BACKDIR/dtst2
216
217	diff $BACKDIR/dtst1 $BACKDIR/dtst2
218	typeset -i ret=$?
219
220	rm -f $BACKDIR/dtst1 $BACKDIR/dtst2
221
222	return $ret
223
224}
225
226#
227# Random create directories and files
228#
229# $1 directory
230#
231function random_tree
232{
233	typeset dir=$1
234
235	if [[ -d $dir ]]; then
236		rm -rf $dir
237	fi
238	mkdir -p $dir
239	typeset -i ret=$?
240
241	typeset -i nl nd nf
242	((nl = RANDOM % 6 + 1))
243	((nd = RANDOM % 3 ))
244	((nf = RANDOM % 5 ))
245	mktree -b $dir -l $nl -d $nd -f $nf
246	((ret |= $?))
247
248	return $ret
249}
250
251#
252# Put data in filesystem and take snapshot
253#
254# $1 snapshot name
255#
256function snapshot_tree
257{
258	typeset snap=$1
259	typeset ds=${snap%%@*}
260	typeset type=$(get_prop "type" $ds)
261
262	typeset -i ret=0
263	if [[ $type == "filesystem" ]]; then
264		typeset mntpnt=$(get_prop mountpoint $ds)
265		((ret |= $?))
266
267		if ((ret == 0)) ; then
268			eval random_tree $mntpnt/${snap##$ds}
269			((ret |= $?))
270		fi
271	fi
272
273	if ((ret == 0)) ; then
274		zfs snapshot $snap
275		((ret |= $?))
276	fi
277
278	return $ret
279}
280
281#
282# Destroy the given snapshot and stuff
283#
284# $1 snapshot
285#
286function destroy_tree
287{
288	typeset -i ret=0
289	typeset snap
290	for snap in "$@" ; do
291		zfs destroy $snap
292		ret=$?
293
294		typeset ds=${snap%%@*}
295		typeset type=$(get_prop "type" $ds)
296		if [[ $type == "filesystem" ]]; then
297			typeset mntpnt=$(get_prop mountpoint $ds)
298			((ret |= $?))
299
300			if ((ret != 0)); then
301				rm -r $mntpnt/$snap
302				((ret |= $?))
303			fi
304		fi
305
306		if ((ret != 0)); then
307			return $ret
308		fi
309	done
310
311	return 0
312}
313
314#
315# Get all the sub-datasets of give dataset with specific suffix
316#
317# $1 Given dataset
318# $2 Suffix
319#
320function getds_with_suffix
321{
322	typeset ds=$1
323	typeset suffix=$2
324
325	typeset list=$(zfs list -r -H -t filesystem,snapshot,volume -o name $ds \
326	    | grep "$suffix$")
327
328	echo $list
329}
330
331#
332# Output inherited properties whitch is edited for file system
333#
334function fs_inherit_prop
335{
336	typeset fs_prop
337	if is_global_zone ; then
338		fs_prop=$(zfs inherit 2>&1 | \
339		    awk '$2=="YES" && $3=="YES" {print $1}')
340		if ! is_te_enabled ; then
341		        fs_prop=$(echo $fs_prop | grep -v "mlslabel")
342		fi
343	else
344		fs_prop=$(zfs inherit 2>&1 | \
345		    awk '$2=="YES" && $3=="YES" {print $1}'|
346		    egrep -v "devices|mlslabel|sharenfs|sharesmb|zoned")
347	fi
348
349	echo $fs_prop
350}
351
352#
353# Output inherited properties for volume
354#
355function vol_inherit_prop
356{
357	echo "checksum readonly"
358}
359
360#
361# Get the destination dataset to compare
362#
363function get_dst_ds
364{
365	typeset srcfs=$1
366	typeset dstfs=$2
367
368	#
369	# If the srcfs is not pool
370	#
371	if ! zpool list $srcfs > /dev/null 2>&1 ; then
372		eval dstfs="$dstfs/${srcfs#*/}"
373	fi
374
375	echo $dstfs
376}
377
378#
379# Make test files
380#
381# $1 Number of files to create
382# $2 Maximum file size
383# $3 File ID offset
384# $4 File system to create the files on
385#
386function mk_files
387{
388	nfiles=$1
389	maxsize=$2
390	file_id_offset=$3
391	fs=$4
392
393	for ((i=0; i<$nfiles; i=i+1)); do
394		dd if=/dev/urandom \
395		    of=/$fs/file-$maxsize-$((i+$file_id_offset)) \
396		    bs=$(($RANDOM * $RANDOM % $maxsize)) \
397		    count=1 >/dev/null 2>&1 || log_fail \
398		    "Failed to create /$fs/file-$maxsize-$((i+$file_id_offset))"
399	done
400	echo Created $nfiles files of random sizes up to $maxsize bytes
401}
402
403#
404# Remove test files
405#
406# $1 Number of files to remove
407# $2 Maximum file size
408# $3 File ID offset
409# $4 File system to remove the files from
410#
411function rm_files
412{
413	nfiles=$1
414	maxsize=$2
415	file_id_offset=$3
416	fs=$4
417
418	for ((i=0; i<$nfiles; i=i+1)); do
419		rm -f /$fs/file-$maxsize-$((i+$file_id_offset))
420	done
421	echo Removed $nfiles files of random sizes up to $maxsize bytes
422}
423
424#
425# Mess up file contents
426#
427# $1 The file path
428#
429function mess_file
430{
431	file=$1
432
433	filesize=$(stat -c '%s' $file)
434	offset=$(($RANDOM * $RANDOM % $filesize))
435	if (($RANDOM % 7 <= 1)); then
436		#
437		# We corrupt 2 bytes to minimize the chance that we
438		# write the same value that's already there.
439		#
440		log_must eval "dd if=/dev/random of=$file conv=notrunc " \
441		    "bs=1 count=2 oseek=$offset >/dev/null 2>&1"
442	else
443		log_must truncate -s $offset $file
444	fi
445}
446
447#
448# Diff the send/receive filesystems
449#
450# $1 The sent filesystem
451# $2 The received filesystem
452#
453function file_check
454{
455	sendfs=$1
456	recvfs=$2
457
458	if [[ -d /$recvfs/.zfs/snapshot/a && -d \
459	    /$sendfs/.zfs/snapshot/a ]]; then
460		diff -r /$recvfs/.zfs/snapshot/a /$sendfs/.zfs/snapshot/a
461		[[ $? -eq 0 ]] || log_fail "Differences found in snap a"
462	fi
463	if [[ -d /$recvfs/.zfs/snapshot/b && -d \
464	    /$sendfs/.zfs/snapshot/b ]]; then
465		diff -r /$recvfs/.zfs/snapshot/b /$sendfs/.zfs/snapshot/b
466		[[ $? -eq 0 ]] || log_fail "Differences found in snap b"
467	fi
468}
469
470#
471# Resume test helper
472#
473# $1 The ZFS send command
474# $2 The filesystem where the streams are sent
475# $3 The receive filesystem
476#
477function resume_test
478{
479	sendcmd=$1
480	streamfs=$2
481	recvfs=$3
482
483	stream_num=1
484	log_must eval "$sendcmd >/$streamfs/$stream_num"
485
486	for ((i=0; i<2; i=i+1)); do
487		mess_file /$streamfs/$stream_num
488		log_mustnot zfs recv -sv $recvfs </$streamfs/$stream_num
489		stream_num=$((stream_num+1))
490
491		token=$(zfs get -Hp -o value receive_resume_token $recvfs)
492		log_must eval "zfs send -v -t $token >/$streamfs/$stream_num"
493		[[ -f /$streamfs/$stream_num ]] || \
494		    log_fail "NO FILE /$streamfs/$stream_num"
495	done
496	log_must zfs recv -sv $recvfs </$streamfs/$stream_num
497}
498
499#
500# Setup filesystems for the resumable send/receive tests
501#
502# $1 The pool to set up with the "send" filesystems
503# $2 The pool for receive
504#
505function test_fs_setup
506{
507	sendpool=$1
508	recvpool=$2
509
510	sendfs=$sendpool/sendfs
511	recvfs=$recvpool/recvfs
512	streamfs=$sendpool/stream
513
514	if datasetexists $recvfs; then
515		log_must zfs destroy -r $recvfs
516	fi
517	if datasetexists $sendfs; then
518		log_must zfs destroy -r $sendfs
519	fi
520	if $(zfs create -o compress=lz4 $sendfs); then
521		mk_files 1000 256 0 $sendfs &
522		mk_files 1000 131072 0 $sendfs &
523		mk_files 100 1048576 0 $sendfs &
524		mk_files 10 10485760 0 $sendfs &
525		mk_files 1 104857600 0 $sendfs &
526		log_must wait
527		log_must zfs snapshot $sendfs@a
528
529		rm_files 200 256 0 $sendfs &
530		rm_files 200 131072 0 $sendfs &
531		rm_files 20 1048576 0 $sendfs &
532		rm_files 2 10485760 0 $sendfs &
533		log_must wait
534
535		mk_files 400 256 0 $sendfs &
536		mk_files 400 131072 0 $sendfs &
537		mk_files 40 1048576 0 $sendfs &
538		mk_files 4 10485760 0 $sendfs &
539		log_must wait
540
541		log_must zfs snapshot $sendfs@b
542		log_must eval "zfs send -v $sendfs@a >/$sendpool/initial.zsend"
543		log_must eval "zfs send -v -i @a $sendfs@b " \
544		    ">/$sendpool/incremental.zsend"
545	fi
546
547	if datasetexists $streamfs; then
548		log_must zfs destroy -r $streamfs
549	fi
550	log_must zfs create -o compress=lz4 $sendpool/stream
551}
552