1#
2# CDDL HEADER START
3#
4# The contents of this file are subject to the terms of the
5# Common Development and Distribution License (the "License").
6# You may not use this file except in compliance with the License.
7#
8# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9# or http://www.opensolaris.org/os/licensing.
10# See the License for the specific language governing permissions
11# and limitations under the License.
12#
13# When distributing Covered Code, include this CDDL HEADER in each
14# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15# If applicable, add the following below this CDDL HEADER, with the
16# fields enclosed by brackets "[]" replaced with your own identifying
17# information: Portions Copyright [yyyy] [name of copyright owner]
18#
19# CDDL HEADER END
20#
21
22#
23# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24# Use is subject to license terms.
25#
26
27#
28# Copyright (c) 2013, 2016 by Delphix. All rights reserved.
29#
30
31. $STF_SUITE/include/libtest.shlib
32. $STF_SUITE/tests/functional/rsend/rsend.cfg
33
34#
35# Set up test model which includes various datasets
36#
37#               @final
38#               @snapB
39#               @init
40#               |
41#   ______ pclone
42#  |      /
43#  |@psnap
44#  ||                         @final
45#  ||@final       @final      @snapC
46#  ||@snapC       @snapC      @snapB
47#  ||@snapA       @snapB      @snapA
48#  ||@init        @init       @init
49#  |||            |           |
50# $pool -------- $FS ------- fs1 ------- fs2
51#    \             \\_____     \          |
52#     vol           vol   \____ \         @fsnap
53#      |              |        \ \              \
54#      @init          @vsnap   |  ------------ fclone
55#      @snapA         @init \  |                    |
56#      @final         @snapB \ |                    @init
57#                     @snapC  vclone                @snapA
58#                     @final       |                @final
59#                                 @init
60#                                 @snapC
61#                                 @final
62#
63# $1 pool name
64#
65function setup_test_model
66{
67	typeset pool=$1
68
69	log_must zfs create -p $pool/$FS/fs1/fs2
70
71	log_must zfs snapshot $pool@psnap
72	log_must zfs clone $pool@psnap $pool/pclone
73
74	if is_global_zone ; then
75		log_must zfs create -V 16M $pool/vol
76		log_must zfs create -V 16M $pool/$FS/vol
77
78		log_must zfs snapshot $pool/$FS/vol@vsnap
79		log_must zfs clone $pool/$FS/vol@vsnap $pool/$FS/vclone
80	fi
81
82	log_must snapshot_tree $pool/$FS/fs1/fs2@fsnap
83	log_must zfs clone $pool/$FS/fs1/fs2@fsnap $pool/$FS/fs1/fclone
84	log_must zfs snapshot -r $pool@init
85
86	log_must snapshot_tree $pool@snapA
87	log_must snapshot_tree $pool@snapC
88	log_must snapshot_tree $pool/pclone@snapB
89	log_must snapshot_tree $pool/$FS@snapB
90	log_must snapshot_tree $pool/$FS@snapC
91	log_must snapshot_tree $pool/$FS/fs1@snapA
92	log_must snapshot_tree $pool/$FS/fs1@snapB
93	log_must snapshot_tree $pool/$FS/fs1@snapC
94	log_must snapshot_tree $pool/$FS/fs1/fclone@snapA
95
96	if is_global_zone ; then
97		log_must zfs snapshot $pool/vol@snapA
98		log_must zfs snapshot $pool/$FS/vol@snapB
99		log_must zfs snapshot $pool/$FS/vol@snapC
100		log_must zfs snapshot $pool/$FS/vclone@snapC
101	fi
102
103	log_must zfs snapshot -r $pool@final
104
105	return 0
106}
107
108#
109# Cleanup the BACKDIR and given pool content and all the sub datasets
110#
111# $1 pool name
112#
113function cleanup_pool
114{
115	typeset pool=$1
116	log_must rm -rf $BACKDIR/*
117
118	if is_global_zone ; then
119		log_must zfs destroy -Rf $pool
120	else
121		typeset list=$(zfs list -H -r -t all -o name $pool)
122		for ds in $list ; do
123			if [[ $ds != $pool ]] ; then
124				if datasetexists $ds ; then
125					log_must zfs destroy -Rf $ds
126				fi
127			fi
128		done
129	fi
130
131	typeset mntpnt=$(get_prop mountpoint $pool)
132	if ! ismounted $pool ; then
133		# Make sure mountpoint directory is empty
134		if [[ -d $mntpnt ]]; then
135			log_must rm -rf $mntpnt/*
136		fi
137
138		log_must zfs mount $pool
139	fi
140	if [[ -d $mntpnt ]]; then
141		rm -rf $mntpnt/*
142	fi
143
144	return 0
145}
146
147function cleanup_pools
148{
149	cleanup_pool $POOL2
150	destroy_pool $POOL3
151}
152
153#
154# Detect if the given two filesystems have same sub-datasets
155#
156# $1 source filesystem
157# $2 destination filesystem
158#
159function cmp_ds_subs
160{
161	typeset src_fs=$1
162	typeset dst_fs=$2
163
164	zfs list -r -H -t all -o name $src_fs > $BACKDIR/src1
165	zfs list -r -H -t all -o name $dst_fs > $BACKDIR/dst1
166
167	eval sed -e 's:^$src_fs:PREFIX:g' < $BACKDIR/src1 > $BACKDIR/src
168	eval sed -e 's:^$dst_fs:PREFIX:g' < $BACKDIR/dst1 > $BACKDIR/dst
169
170	diff $BACKDIR/src $BACKDIR/dst
171	typeset -i ret=$?
172
173	rm -f $BACKDIR/src $BACKDIR/dst $BACKDIR/src1 $BACKDIR/dst1
174
175	return $ret
176}
177
178#
179# Compare all the directores and files in two filesystems
180#
181# $1 source filesystem
182# $2 destination filesystem
183#
184function cmp_ds_cont
185{
186	typeset src_fs=$1
187	typeset dst_fs=$2
188
189	typeset srcdir dstdir
190	srcdir=$(get_prop mountpoint $src_fs)
191	dstdir=$(get_prop mountpoint $dst_fs)
192
193	diff -r $srcdir $dstdir > /dev/null 2>&1
194	return $?
195}
196
197#
198# Compare the given two dataset properties
199#
200# $1 dataset 1
201# $2 dataset 2
202#
203function cmp_ds_prop
204{
205	typeset dtst1=$1
206	typeset dtst2=$2
207
208	for item in "type" "origin" "volblocksize" "aclinherit" "aclmode" \
209	    "atime" "canmount" "checksum" "compression" "copies" "devices" \
210	    "exec" "quota" "readonly" "recordsize" "reservation" "setuid" \
211	    "sharenfs" "snapdir" "version" "volsize" "xattr" "zoned" \
212	    "mountpoint";
213	do
214		zfs get -H -o property,value,source $item $dtst1 >> \
215		    $BACKDIR/dtst1
216		zfs get -H -o property,value,source $item $dtst2 >> \
217		    $BACKDIR/dtst2
218	done
219
220	eval sed -e 's:$dtst1:PREFIX:g' < $BACKDIR/dtst1 > $BACKDIR/dtst1
221	eval sed -e 's:$dtst2:PREFIX:g' < $BACKDIR/dtst2 > $BACKDIR/dtst2
222
223	diff $BACKDIR/dtst1 $BACKDIR/dtst2
224	typeset -i ret=$?
225
226	rm -f $BACKDIR/dtst1 $BACKDIR/dtst2
227
228	return $ret
229
230}
231
232#
233# Random create directories and files
234#
235# $1 directory
236#
237function random_tree
238{
239	typeset dir=$1
240
241	if [[ -d $dir ]]; then
242		rm -rf $dir
243	fi
244	mkdir -p $dir
245	typeset -i ret=$?
246
247	typeset -i nl nd nf
248	((nl = RANDOM % 6 + 1))
249	((nd = RANDOM % 3 ))
250	((nf = RANDOM % 5 ))
251	mktree -b $dir -l $nl -d $nd -f $nf
252	((ret |= $?))
253
254	return $ret
255}
256
257#
258# Put data in filesystem and take snapshot
259#
260# $1 snapshot name
261#
262function snapshot_tree
263{
264	typeset snap=$1
265	typeset ds=${snap%%@*}
266	typeset type=$(get_prop "type" $ds)
267
268	typeset -i ret=0
269	if [[ $type == "filesystem" ]]; then
270		typeset mntpnt=$(get_prop mountpoint $ds)
271		((ret |= $?))
272
273		if ((ret == 0)) ; then
274			eval random_tree $mntpnt/${snap##$ds}
275			((ret |= $?))
276		fi
277	fi
278
279	if ((ret == 0)) ; then
280		zfs snapshot $snap
281		((ret |= $?))
282	fi
283
284	return $ret
285}
286
287#
288# Destroy the given snapshot and stuff
289#
290# $1 snapshot
291#
292function destroy_tree
293{
294	typeset -i ret=0
295	typeset snap
296	for snap in "$@" ; do
297		zfs destroy $snap
298		ret=$?
299
300		typeset ds=${snap%%@*}
301		typeset type=$(get_prop "type" $ds)
302		if [[ $type == "filesystem" ]]; then
303			typeset mntpnt=$(get_prop mountpoint $ds)
304			((ret |= $?))
305
306			if ((ret != 0)); then
307				rm -r $mntpnt/$snap
308				((ret |= $?))
309			fi
310		fi
311
312		if ((ret != 0)); then
313			return $ret
314		fi
315	done
316
317	return 0
318}
319
320#
321# Get all the sub-datasets of give dataset with specific suffix
322#
323# $1 Given dataset
324# $2 Suffix
325#
326function getds_with_suffix
327{
328	typeset ds=$1
329	typeset suffix=$2
330
331	typeset list=$(zfs list -r -H -t all -o name $ds | grep "$suffix$")
332
333	echo $list
334}
335
336#
337# Output inherited properties whitch is edited for file system
338#
339function fs_inherit_prop
340{
341	typeset fs_prop
342	if is_global_zone ; then
343		fs_prop=$(zfs inherit 2>&1 | \
344		    awk '$2=="YES" && $3=="YES" {print $1}')
345		if ! is_te_enabled ; then
346		        fs_prop=$(echo $fs_prop | grep -v "mlslabel")
347		fi
348	else
349		fs_prop=$(zfs inherit 2>&1 | \
350		    awk '$2=="YES" && $3=="YES" {print $1}'|
351		    egrep -v "devices|mlslabel|sharenfs|sharesmb|zoned")
352	fi
353
354	echo $fs_prop
355}
356
357#
358# Output inherited properties for volume
359#
360function vol_inherit_prop
361{
362	echo "checksum readonly"
363}
364
365#
366# Get the destination dataset to compare
367#
368function get_dst_ds
369{
370	typeset srcfs=$1
371	typeset dstfs=$2
372
373	#
374	# If the srcfs is not pool
375	#
376	if ! zpool list $srcfs > /dev/null 2>&1 ; then
377		eval dstfs="$dstfs/${srcfs#*/}"
378	fi
379
380	echo $dstfs
381}
382
383#
384# Make test files
385#
386# $1 Number of files to create
387# $2 Maximum file size
388# $3 File ID offset
389# $4 File system to create the files on
390#
391function mk_files
392{
393	nfiles=$1
394	maxsize=$2
395	file_id_offset=$3
396	fs=$4
397
398	for ((i=0; i<$nfiles; i=i+1)); do
399		dd if=/dev/urandom \
400		    of=/$fs/file-$maxsize-$((i+$file_id_offset)) \
401		    bs=$(($RANDOM * $RANDOM % $maxsize)) \
402		    count=1 >/dev/null 2>&1 || log_fail \
403		    "Failed to create /$fs/file-$maxsize-$((i+$file_id_offset))"
404	done
405	echo Created $nfiles files of random sizes up to $maxsize bytes
406}
407
408#
409# Remove test files
410#
411# $1 Number of files to remove
412# $2 Maximum file size
413# $3 File ID offset
414# $4 File system to remove the files from
415#
416function rm_files
417{
418	nfiles=$1
419	maxsize=$2
420	file_id_offset=$3
421	fs=$4
422
423	for ((i=0; i<$nfiles; i=i+1)); do
424		rm -f /$fs/file-$maxsize-$((i+$file_id_offset))
425	done
426	echo Removed $nfiles files of random sizes up to $maxsize bytes
427}
428
429#
430# Mess up file contents
431#
432# $1 The file path
433#
434function mess_file
435{
436	file=$1
437
438	filesize=$(stat -c '%s' $file)
439	offset=$(($RANDOM * $RANDOM % $filesize))
440	if (($RANDOM % 7 <= 1)); then
441		#
442		# We corrupt 2 bytes to minimize the chance that we
443		# write the same value that's already there.
444		#
445		log_must eval "dd if=/dev/random of=$file conv=notrunc " \
446		    "bs=1 count=2 oseek=$offset >/dev/null 2>&1"
447	else
448		log_must truncate -s $offset $file
449	fi
450}
451
452#
453# Diff the send/receive filesystems
454#
455# $1 The sent filesystem
456# $2 The received filesystem
457#
458function file_check
459{
460	sendfs=$1
461	recvfs=$2
462
463	if [[ -d /$recvfs/.zfs/snapshot/a && -d \
464	    /$sendfs/.zfs/snapshot/a ]]; then
465		diff -r /$recvfs/.zfs/snapshot/a /$sendfs/.zfs/snapshot/a
466		[[ $? -eq 0 ]] || log_fail "Differences found in snap a"
467	fi
468	if [[ -d /$recvfs/.zfs/snapshot/b && -d \
469	    /$sendfs/.zfs/snapshot/b ]]; then
470		diff -r /$recvfs/.zfs/snapshot/b /$sendfs/.zfs/snapshot/b
471		[[ $? -eq 0 ]] || log_fail "Differences found in snap b"
472	fi
473}
474
475#
476# Resume test helper
477#
478# $1 The ZFS send command
479# $2 The filesystem where the streams are sent
480# $3 The receive filesystem
481#
482function resume_test
483{
484	sendcmd=$1
485	streamfs=$2
486	recvfs=$3
487
488	stream_num=1
489	log_must eval "$sendcmd >/$streamfs/$stream_num"
490
491	for ((i=0; i<2; i=i+1)); do
492		mess_file /$streamfs/$stream_num
493		log_mustnot zfs recv -suv $recvfs </$streamfs/$stream_num
494		stream_num=$((stream_num+1))
495
496		token=$(zfs get -Hp -o value receive_resume_token $recvfs)
497		log_must eval "zfs send -v -t $token >/$streamfs/$stream_num"
498		[[ -f /$streamfs/$stream_num ]] || \
499		    log_fail "NO FILE /$streamfs/$stream_num"
500	done
501	log_must zfs recv -suv $recvfs </$streamfs/$stream_num
502}
503
504#
505# Setup filesystems for the resumable send/receive tests
506#
507# $1 The "send" filesystem
508# $2 The "recv" filesystem
509#
510function test_fs_setup
511{
512	typeset sendfs=$1
513	typeset recvfs=$2
514	typeset sendpool=${sendfs%%/*}
515	typeset recvpool=${recvfs%%/*}
516
517	datasetexists $sendfs && log_must $ZFS destroy -r $sendpool
518	datasetexists $recvfs && log_must $ZFS destroy -r $recvpool
519
520	if $(datasetexists $sendfs || zfs create -o compress=lz4 $sendfs); then
521		mk_files 1000 256 0 $sendfs &
522		mk_files 1000 131072 0 $sendfs &
523		mk_files 100 1048576 0 $sendfs &
524		mk_files 10 10485760 0 $sendfs &
525		mk_files 1 104857600 0 $sendfs &
526		log_must wait
527		log_must zfs snapshot $sendfs@a
528
529		rm_files 200 256 0 $sendfs &
530		rm_files 200 131072 0 $sendfs &
531		rm_files 20 1048576 0 $sendfs &
532		rm_files 2 10485760 0 $sendfs &
533		log_must wait
534
535		mk_files 400 256 0 $sendfs &
536		mk_files 400 131072 0 $sendfs &
537		mk_files 40 1048576 0 $sendfs &
538		mk_files 4 10485760 0 $sendfs &
539		log_must wait
540
541		log_must zfs snapshot $sendfs@b
542		log_must eval "zfs send -v $sendfs@a >/$sendpool/initial.zsend"
543		log_must eval "zfs send -v -i @a $sendfs@b " \
544		    ">/$sendpool/incremental.zsend"
545	fi
546
547	if datasetexists $streamfs; then
548		log_must zfs destroy -r $streamfs
549	fi
550	log_must zfs create -o compress=lz4 $sendpool/stream
551}
552