1# 2# CDDL HEADER START 3# 4# The contents of this file are subject to the terms of the 5# Common Development and Distribution License (the "License"). 6# You may not use this file except in compliance with the License. 7# 8# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9# or http://www.opensolaris.org/os/licensing. 10# See the License for the specific language governing permissions 11# and limitations under the License. 12# 13# When distributing Covered Code, include this CDDL HEADER in each 14# file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15# If applicable, add the following below this CDDL HEADER, with the 16# fields enclosed by brackets "[]" replaced with your own identifying 17# information: Portions Copyright [yyyy] [name of copyright owner] 18# 19# CDDL HEADER END 20# 21 22# 23# Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24# Use is subject to license terms. 25# 26 27# 28# Copyright (c) 2013, 2016 by Delphix. All rights reserved. 29# 30 31. $STF_SUITE/include/libtest.shlib 32. $STF_SUITE/tests/functional/rsend/rsend.cfg 33 34# 35# Set up test model which includes various datasets 36# 37# @final 38# @snapB 39# @init 40# | 41# ______ pclone 42# | / 43# |@psnap 44# || @final 45# ||@final @final @snapC 46# ||@snapC @snapC @snapB 47# ||@snapA @snapB @snapA 48# ||@init @init @init 49# ||| | | 50# $pool -------- $FS ------- fs1 ------- fs2 51# \ \\_____ \ | 52# vol vol \____ \ @fsnap 53# | | \ \ \ 54# @init @vsnap | ------------ fclone 55# @snapA @init \ | | 56# @final @snapB \ | @init 57# @snapC vclone @snapA 58# @final | @final 59# @init 60# @snapC 61# @final 62# 63# $1 pool name 64# 65function setup_test_model 66{ 67 typeset pool=$1 68 69 log_must zfs create -p $pool/$FS/fs1/fs2 70 71 log_must zfs snapshot $pool@psnap 72 log_must zfs clone $pool@psnap $pool/pclone 73 74 if is_global_zone ; then 75 log_must zfs create -V 16M $pool/vol 76 log_must zfs create -V 16M $pool/$FS/vol 77 78 log_must zfs snapshot $pool/$FS/vol@vsnap 79 log_must zfs clone $pool/$FS/vol@vsnap $pool/$FS/vclone 80 fi 81 82 log_must snapshot_tree $pool/$FS/fs1/fs2@fsnap 83 log_must zfs clone $pool/$FS/fs1/fs2@fsnap $pool/$FS/fs1/fclone 84 log_must zfs snapshot -r $pool@init 85 86 log_must snapshot_tree $pool@snapA 87 log_must snapshot_tree $pool@snapC 88 log_must snapshot_tree $pool/pclone@snapB 89 log_must snapshot_tree $pool/$FS@snapB 90 log_must snapshot_tree $pool/$FS@snapC 91 log_must snapshot_tree $pool/$FS/fs1@snapA 92 log_must snapshot_tree $pool/$FS/fs1@snapB 93 log_must snapshot_tree $pool/$FS/fs1@snapC 94 log_must snapshot_tree $pool/$FS/fs1/fclone@snapA 95 96 if is_global_zone ; then 97 log_must zfs snapshot $pool/vol@snapA 98 log_must zfs snapshot $pool/$FS/vol@snapB 99 log_must zfs snapshot $pool/$FS/vol@snapC 100 log_must zfs snapshot $pool/$FS/vclone@snapC 101 fi 102 103 log_must zfs snapshot -r $pool@final 104 105 return 0 106} 107 108# 109# Cleanup the BACKDIR and given pool content and all the sub datasets 110# 111# $1 pool name 112# 113function cleanup_pool 114{ 115 typeset pool=$1 116 log_must rm -rf $BACKDIR/* 117 118 if is_global_zone ; then 119 log_must zfs destroy -Rf $pool 120 else 121 typeset list=$(zfs list -H -r -t all -o name $pool) 122 for ds in $list ; do 123 if [[ $ds != $pool ]] ; then 124 if datasetexists $ds ; then 125 log_must zfs destroy -Rf $ds 126 fi 127 fi 128 done 129 fi 130 131 typeset mntpnt=$(get_prop mountpoint $pool) 132 if ! ismounted $pool ; then 133 # Make sure mountpoint directory is empty 134 if [[ -d $mntpnt ]]; then 135 log_must rm -rf $mntpnt/* 136 fi 137 138 log_must zfs mount $pool 139 fi 140 if [[ -d $mntpnt ]]; then 141 rm -rf $mntpnt/* 142 fi 143 144 return 0 145} 146 147function cleanup_pools 148{ 149 cleanup_pool $POOL2 150 destroy_pool $POOL3 151} 152 153# 154# Detect if the given two filesystems have same sub-datasets 155# 156# $1 source filesystem 157# $2 destination filesystem 158# 159function cmp_ds_subs 160{ 161 typeset src_fs=$1 162 typeset dst_fs=$2 163 164 zfs list -r -H -t all -o name $src_fs > $BACKDIR/src1 165 zfs list -r -H -t all -o name $dst_fs > $BACKDIR/dst1 166 167 eval sed -e 's:^$src_fs:PREFIX:g' < $BACKDIR/src1 > $BACKDIR/src 168 eval sed -e 's:^$dst_fs:PREFIX:g' < $BACKDIR/dst1 > $BACKDIR/dst 169 170 diff $BACKDIR/src $BACKDIR/dst 171 typeset -i ret=$? 172 173 rm -f $BACKDIR/src $BACKDIR/dst $BACKDIR/src1 $BACKDIR/dst1 174 175 return $ret 176} 177 178# 179# Compare all the directores and files in two filesystems 180# 181# $1 source filesystem 182# $2 destination filesystem 183# 184function cmp_ds_cont 185{ 186 typeset src_fs=$1 187 typeset dst_fs=$2 188 189 typeset srcdir dstdir 190 srcdir=$(get_prop mountpoint $src_fs) 191 dstdir=$(get_prop mountpoint $dst_fs) 192 193 diff -r $srcdir $dstdir > /dev/null 2>&1 194 return $? 195} 196 197# 198# Compare the given two dataset properties 199# 200# $1 dataset 1 201# $2 dataset 2 202# 203function cmp_ds_prop 204{ 205 typeset dtst1=$1 206 typeset dtst2=$2 207 208 for item in "type" "origin" "volblocksize" "aclinherit" "aclmode" \ 209 "atime" "canmount" "checksum" "compression" "copies" "devices" \ 210 "exec" "quota" "readonly" "recordsize" "reservation" "setuid" \ 211 "sharenfs" "snapdir" "version" "volsize" "xattr" "zoned" \ 212 "mountpoint"; 213 do 214 zfs get -H -o property,value,source $item $dtst1 >> \ 215 $BACKDIR/dtst1 216 zfs get -H -o property,value,source $item $dtst2 >> \ 217 $BACKDIR/dtst2 218 done 219 220 eval sed -e 's:$dtst1:PREFIX:g' < $BACKDIR/dtst1 > $BACKDIR/dtst1 221 eval sed -e 's:$dtst2:PREFIX:g' < $BACKDIR/dtst2 > $BACKDIR/dtst2 222 223 diff $BACKDIR/dtst1 $BACKDIR/dtst2 224 typeset -i ret=$? 225 226 rm -f $BACKDIR/dtst1 $BACKDIR/dtst2 227 228 return $ret 229 230} 231 232# 233# Random create directories and files 234# 235# $1 directory 236# 237function random_tree 238{ 239 typeset dir=$1 240 241 if [[ -d $dir ]]; then 242 rm -rf $dir 243 fi 244 mkdir -p $dir 245 typeset -i ret=$? 246 247 typeset -i nl nd nf 248 ((nl = RANDOM % 6 + 1)) 249 ((nd = RANDOM % 3 )) 250 ((nf = RANDOM % 5 )) 251 mktree -b $dir -l $nl -d $nd -f $nf 252 ((ret |= $?)) 253 254 return $ret 255} 256 257# 258# Put data in filesystem and take snapshot 259# 260# $1 snapshot name 261# 262function snapshot_tree 263{ 264 typeset snap=$1 265 typeset ds=${snap%%@*} 266 typeset type=$(get_prop "type" $ds) 267 268 typeset -i ret=0 269 if [[ $type == "filesystem" ]]; then 270 typeset mntpnt=$(get_prop mountpoint $ds) 271 ((ret |= $?)) 272 273 if ((ret == 0)) ; then 274 eval random_tree $mntpnt/${snap##$ds} 275 ((ret |= $?)) 276 fi 277 fi 278 279 if ((ret == 0)) ; then 280 zfs snapshot $snap 281 ((ret |= $?)) 282 fi 283 284 return $ret 285} 286 287# 288# Destroy the given snapshot and stuff 289# 290# $1 snapshot 291# 292function destroy_tree 293{ 294 typeset -i ret=0 295 typeset snap 296 for snap in "$@" ; do 297 zfs destroy $snap 298 ret=$? 299 300 typeset ds=${snap%%@*} 301 typeset type=$(get_prop "type" $ds) 302 if [[ $type == "filesystem" ]]; then 303 typeset mntpnt=$(get_prop mountpoint $ds) 304 ((ret |= $?)) 305 306 if ((ret != 0)); then 307 rm -r $mntpnt/$snap 308 ((ret |= $?)) 309 fi 310 fi 311 312 if ((ret != 0)); then 313 return $ret 314 fi 315 done 316 317 return 0 318} 319 320# 321# Get all the sub-datasets of give dataset with specific suffix 322# 323# $1 Given dataset 324# $2 Suffix 325# 326function getds_with_suffix 327{ 328 typeset ds=$1 329 typeset suffix=$2 330 331 typeset list=$(zfs list -r -H -t all -o name $ds | grep "$suffix$") 332 333 echo $list 334} 335 336# 337# Output inherited properties whitch is edited for file system 338# 339function fs_inherit_prop 340{ 341 typeset fs_prop 342 if is_global_zone ; then 343 fs_prop=$(zfs inherit 2>&1 | \ 344 awk '$2=="YES" && $3=="YES" {print $1}') 345 if ! is_te_enabled ; then 346 fs_prop=$(echo $fs_prop | grep -v "mlslabel") 347 fi 348 else 349 fs_prop=$(zfs inherit 2>&1 | \ 350 awk '$2=="YES" && $3=="YES" {print $1}'| 351 egrep -v "devices|mlslabel|sharenfs|sharesmb|zoned") 352 fi 353 354 echo $fs_prop 355} 356 357# 358# Output inherited properties for volume 359# 360function vol_inherit_prop 361{ 362 echo "checksum readonly" 363} 364 365# 366# Get the destination dataset to compare 367# 368function get_dst_ds 369{ 370 typeset srcfs=$1 371 typeset dstfs=$2 372 373 # 374 # If the srcfs is not pool 375 # 376 if ! zpool list $srcfs > /dev/null 2>&1 ; then 377 eval dstfs="$dstfs/${srcfs#*/}" 378 fi 379 380 echo $dstfs 381} 382 383# 384# Make test files 385# 386# $1 Number of files to create 387# $2 Maximum file size 388# $3 File ID offset 389# $4 File system to create the files on 390# 391function mk_files 392{ 393 nfiles=$1 394 maxsize=$2 395 file_id_offset=$3 396 fs=$4 397 398 for ((i=0; i<$nfiles; i=i+1)); do 399 dd if=/dev/urandom \ 400 of=/$fs/file-$maxsize-$((i+$file_id_offset)) \ 401 bs=$(($RANDOM * $RANDOM % $maxsize)) \ 402 count=1 >/dev/null 2>&1 || log_fail \ 403 "Failed to create /$fs/file-$maxsize-$((i+$file_id_offset))" 404 done 405 echo Created $nfiles files of random sizes up to $maxsize bytes 406} 407 408# 409# Remove test files 410# 411# $1 Number of files to remove 412# $2 Maximum file size 413# $3 File ID offset 414# $4 File system to remove the files from 415# 416function rm_files 417{ 418 nfiles=$1 419 maxsize=$2 420 file_id_offset=$3 421 fs=$4 422 423 for ((i=0; i<$nfiles; i=i+1)); do 424 rm -f /$fs/file-$maxsize-$((i+$file_id_offset)) 425 done 426 echo Removed $nfiles files of random sizes up to $maxsize bytes 427} 428 429# 430# Mess up file contents 431# 432# $1 The file path 433# 434function mess_file 435{ 436 file=$1 437 438 filesize=$(stat -c '%s' $file) 439 offset=$(($RANDOM * $RANDOM % $filesize)) 440 if (($RANDOM % 7 <= 1)); then 441 # 442 # We corrupt 2 bytes to minimize the chance that we 443 # write the same value that's already there. 444 # 445 log_must eval "dd if=/dev/random of=$file conv=notrunc " \ 446 "bs=1 count=2 oseek=$offset >/dev/null 2>&1" 447 else 448 log_must truncate -s $offset $file 449 fi 450} 451 452# 453# Diff the send/receive filesystems 454# 455# $1 The sent filesystem 456# $2 The received filesystem 457# 458function file_check 459{ 460 sendfs=$1 461 recvfs=$2 462 463 if [[ -d /$recvfs/.zfs/snapshot/a && -d \ 464 /$sendfs/.zfs/snapshot/a ]]; then 465 diff -r /$recvfs/.zfs/snapshot/a /$sendfs/.zfs/snapshot/a 466 [[ $? -eq 0 ]] || log_fail "Differences found in snap a" 467 fi 468 if [[ -d /$recvfs/.zfs/snapshot/b && -d \ 469 /$sendfs/.zfs/snapshot/b ]]; then 470 diff -r /$recvfs/.zfs/snapshot/b /$sendfs/.zfs/snapshot/b 471 [[ $? -eq 0 ]] || log_fail "Differences found in snap b" 472 fi 473} 474 475# 476# Resume test helper 477# 478# $1 The ZFS send command 479# $2 The filesystem where the streams are sent 480# $3 The receive filesystem 481# 482function resume_test 483{ 484 sendcmd=$1 485 streamfs=$2 486 recvfs=$3 487 488 stream_num=1 489 log_must eval "$sendcmd >/$streamfs/$stream_num" 490 491 for ((i=0; i<2; i=i+1)); do 492 mess_file /$streamfs/$stream_num 493 log_mustnot zfs recv -suv $recvfs </$streamfs/$stream_num 494 stream_num=$((stream_num+1)) 495 496 token=$(zfs get -Hp -o value receive_resume_token $recvfs) 497 log_must eval "zfs send -v -t $token >/$streamfs/$stream_num" 498 [[ -f /$streamfs/$stream_num ]] || \ 499 log_fail "NO FILE /$streamfs/$stream_num" 500 done 501 log_must zfs recv -suv $recvfs </$streamfs/$stream_num 502} 503 504# 505# Setup filesystems for the resumable send/receive tests 506# 507# $1 The "send" filesystem 508# $2 The "recv" filesystem 509# 510function test_fs_setup 511{ 512 typeset sendfs=$1 513 typeset recvfs=$2 514 typeset sendpool=${sendfs%%/*} 515 typeset recvpool=${recvfs%%/*} 516 517 datasetexists $sendfs && log_must $ZFS destroy -r $sendpool 518 datasetexists $recvfs && log_must $ZFS destroy -r $recvpool 519 520 if $(datasetexists $sendfs || zfs create -o compress=lz4 $sendfs); then 521 mk_files 1000 256 0 $sendfs & 522 mk_files 1000 131072 0 $sendfs & 523 mk_files 100 1048576 0 $sendfs & 524 mk_files 10 10485760 0 $sendfs & 525 mk_files 1 104857600 0 $sendfs & 526 log_must wait 527 log_must zfs snapshot $sendfs@a 528 529 rm_files 200 256 0 $sendfs & 530 rm_files 200 131072 0 $sendfs & 531 rm_files 20 1048576 0 $sendfs & 532 rm_files 2 10485760 0 $sendfs & 533 log_must wait 534 535 mk_files 400 256 0 $sendfs & 536 mk_files 400 131072 0 $sendfs & 537 mk_files 40 1048576 0 $sendfs & 538 mk_files 4 10485760 0 $sendfs & 539 log_must wait 540 541 log_must zfs snapshot $sendfs@b 542 log_must eval "zfs send -v $sendfs@a >/$sendpool/initial.zsend" 543 log_must eval "zfs send -v -i @a $sendfs@b " \ 544 ">/$sendpool/incremental.zsend" 545 fi 546 547 if datasetexists $streamfs; then 548 log_must zfs destroy -r $streamfs 549 fi 550 log_must zfs create -o compress=lz4 $sendpool/stream 551} 552