1f38cb554SJohn Wren Kennedy# 2f38cb554SJohn Wren Kennedy# CDDL HEADER START 3f38cb554SJohn Wren Kennedy# 4f38cb554SJohn Wren Kennedy# The contents of this file are subject to the terms of the 5f38cb554SJohn Wren Kennedy# Common Development and Distribution License (the "License"). 6f38cb554SJohn Wren Kennedy# You may not use this file except in compliance with the License. 7f38cb554SJohn Wren Kennedy# 8f38cb554SJohn Wren Kennedy# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9f38cb554SJohn Wren Kennedy# or http://www.opensolaris.org/os/licensing. 10f38cb554SJohn Wren Kennedy# See the License for the specific language governing permissions 11f38cb554SJohn Wren Kennedy# and limitations under the License. 12f38cb554SJohn Wren Kennedy# 13f38cb554SJohn Wren Kennedy# When distributing Covered Code, include this CDDL HEADER in each 14f38cb554SJohn Wren Kennedy# file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15f38cb554SJohn Wren Kennedy# If applicable, add the following below this CDDL HEADER, with the 16f38cb554SJohn Wren Kennedy# fields enclosed by brackets "[]" replaced with your own identifying 17f38cb554SJohn Wren Kennedy# information: Portions Copyright [yyyy] [name of copyright owner] 18f38cb554SJohn Wren Kennedy# 19f38cb554SJohn Wren Kennedy# CDDL HEADER END 20f38cb554SJohn Wren Kennedy# 21f38cb554SJohn Wren Kennedy 22f38cb554SJohn Wren Kennedy# 23f38cb554SJohn Wren Kennedy# Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24f38cb554SJohn Wren Kennedy# Use is subject to license terms. 25f38cb554SJohn Wren Kennedy# 26f38cb554SJohn Wren Kennedy 27f38cb554SJohn Wren Kennedy# 28*1d32ba66SJohn Wren Kennedy# Copyright (c) 2013, 2016 by Delphix. All rights reserved. 29f38cb554SJohn Wren Kennedy# 30f38cb554SJohn Wren Kennedy 31f38cb554SJohn Wren Kennedy. $STF_SUITE/include/libtest.shlib 32f38cb554SJohn Wren Kennedy. $STF_SUITE/tests/functional/redundancy/redundancy.cfg 33f38cb554SJohn Wren Kennedy 34f38cb554SJohn Wren Kennedyfunction cleanup 35f38cb554SJohn Wren Kennedy{ 36f38cb554SJohn Wren Kennedy if poolexists $TESTPOOL; then 37f38cb554SJohn Wren Kennedy destroy_pool $TESTPOOL 38f38cb554SJohn Wren Kennedy fi 39f38cb554SJohn Wren Kennedy typeset dir 40f38cb554SJohn Wren Kennedy for dir in $TESTDIR $BASEDIR; do 41f38cb554SJohn Wren Kennedy if [[ -d $dir ]]; then 42*1d32ba66SJohn Wren Kennedy log_must rm -rf $dir 43f38cb554SJohn Wren Kennedy fi 44f38cb554SJohn Wren Kennedy done 45f38cb554SJohn Wren Kennedy} 46f38cb554SJohn Wren Kennedy 47f38cb554SJohn Wren Kennedy# 48f38cb554SJohn Wren Kennedy# Get random number between min and max number. 49f38cb554SJohn Wren Kennedy# 50f38cb554SJohn Wren Kennedy# $1 Minimal value 51f38cb554SJohn Wren Kennedy# $2 Maximal value 52f38cb554SJohn Wren Kennedy# 53f38cb554SJohn Wren Kennedyfunction random 54f38cb554SJohn Wren Kennedy{ 55f38cb554SJohn Wren Kennedy typeset -i min=$1 56f38cb554SJohn Wren Kennedy typeset -i max=$2 57f38cb554SJohn Wren Kennedy typeset -i value 58f38cb554SJohn Wren Kennedy 59f38cb554SJohn Wren Kennedy while true; do 60f38cb554SJohn Wren Kennedy ((value = RANDOM % (max + 1))) 61f38cb554SJohn Wren Kennedy if ((value >= min)); then 62f38cb554SJohn Wren Kennedy break 63f38cb554SJohn Wren Kennedy fi 64f38cb554SJohn Wren Kennedy done 65f38cb554SJohn Wren Kennedy 66*1d32ba66SJohn Wren Kennedy echo $value 67f38cb554SJohn Wren Kennedy} 68f38cb554SJohn Wren Kennedy 69f38cb554SJohn Wren Kennedy# 70f38cb554SJohn Wren Kennedy# Record the directories construction and checksum all the files which reside 71f38cb554SJohn Wren Kennedy# within the specified pool 72f38cb554SJohn Wren Kennedy# 73f38cb554SJohn Wren Kennedy# $1 The specified pool 74f38cb554SJohn Wren Kennedy# $2 The file which save the record. 75f38cb554SJohn Wren Kennedy# 76f38cb554SJohn Wren Kennedyfunction record_data 77f38cb554SJohn Wren Kennedy{ 78f38cb554SJohn Wren Kennedy typeset pool=$1 79f38cb554SJohn Wren Kennedy typeset recordfile=$2 80f38cb554SJohn Wren Kennedy 81f38cb554SJohn Wren Kennedy [[ -z $pool ]] && log_fail "No specified pool." 82*1d32ba66SJohn Wren Kennedy [[ -f $recordfile ]] && log_must rm -f $recordfile 83f38cb554SJohn Wren Kennedy 84f38cb554SJohn Wren Kennedy typeset mntpnt 85f38cb554SJohn Wren Kennedy mntpnt=$(get_prop mountpoint $pool) 86*1d32ba66SJohn Wren Kennedy log_must eval "du -a $mntpnt > $recordfile 2>&1" 87f38cb554SJohn Wren Kennedy # 88f38cb554SJohn Wren Kennedy # When the data was damaged, checksum is failing and return 1 89f38cb554SJohn Wren Kennedy # So, will not use log_must 90f38cb554SJohn Wren Kennedy # 91*1d32ba66SJohn Wren Kennedy find $mntpnt -type f -exec cksum {} + >> $recordfile 2>&1 92f38cb554SJohn Wren Kennedy} 93f38cb554SJohn Wren Kennedy 94f38cb554SJohn Wren Kennedy# 95f38cb554SJohn Wren Kennedy# Create test pool and fill with files and directories. 96f38cb554SJohn Wren Kennedy# 97f38cb554SJohn Wren Kennedy# $1 pool name 98f38cb554SJohn Wren Kennedy# $2 pool type 99f38cb554SJohn Wren Kennedy# $3 virtual devices number 100f38cb554SJohn Wren Kennedy# 101f38cb554SJohn Wren Kennedyfunction setup_test_env 102f38cb554SJohn Wren Kennedy{ 103f38cb554SJohn Wren Kennedy typeset pool=$1 104f38cb554SJohn Wren Kennedy typeset keyword=$2 105f38cb554SJohn Wren Kennedy typeset -i vdev_cnt=$3 106f38cb554SJohn Wren Kennedy typeset vdevs 107f38cb554SJohn Wren Kennedy 108f38cb554SJohn Wren Kennedy typeset -i i=0 109f38cb554SJohn Wren Kennedy while (( i < vdev_cnt )); do 110f38cb554SJohn Wren Kennedy vdevs="$vdevs $BASEDIR/vdev$i" 111f38cb554SJohn Wren Kennedy ((i += 1)) 112f38cb554SJohn Wren Kennedy done 113f38cb554SJohn Wren Kennedy 114f38cb554SJohn Wren Kennedy if [[ ! -d $BASEDIR ]]; then 115*1d32ba66SJohn Wren Kennedy log_must mkdir $BASEDIR 116f38cb554SJohn Wren Kennedy fi 117f38cb554SJohn Wren Kennedy 118f38cb554SJohn Wren Kennedy if poolexists $pool ; then 119f38cb554SJohn Wren Kennedy destroy_pool $pool 120f38cb554SJohn Wren Kennedy fi 121f38cb554SJohn Wren Kennedy 122*1d32ba66SJohn Wren Kennedy log_must mkfile $MINVDEVSIZE $vdevs 123f38cb554SJohn Wren Kennedy 124*1d32ba66SJohn Wren Kennedy log_must zpool create -m $TESTDIR $pool $keyword $vdevs 125f38cb554SJohn Wren Kennedy 126f38cb554SJohn Wren Kennedy log_note "Filling up the filesystem ..." 127f38cb554SJohn Wren Kennedy typeset -i ret=0 128f38cb554SJohn Wren Kennedy typeset -i i=0 129f38cb554SJohn Wren Kennedy typeset file=$TESTDIR/file 130*1d32ba66SJohn Wren Kennedy while true ; do 131*1d32ba66SJohn Wren Kennedy file_write -o create -f $file.$i \ 132f38cb554SJohn Wren Kennedy -b $BLOCKSZ -c $NUM_WRITES 133f38cb554SJohn Wren Kennedy ret=$? 134f38cb554SJohn Wren Kennedy (( $ret != 0 )) && break 135f38cb554SJohn Wren Kennedy (( i = i + 1 )) 136f38cb554SJohn Wren Kennedy done 137*1d32ba66SJohn Wren Kennedy (($ret != 28 )) && log_note "file_write return value($ret) is unexpected." 138f38cb554SJohn Wren Kennedy 139f38cb554SJohn Wren Kennedy record_data $TESTPOOL $PRE_RECORD_FILE 140f38cb554SJohn Wren Kennedy} 141f38cb554SJohn Wren Kennedy 142f38cb554SJohn Wren Kennedy# 143f38cb554SJohn Wren Kennedy# Check pool status is healthy 144f38cb554SJohn Wren Kennedy# 145f38cb554SJohn Wren Kennedy# $1 pool 146f38cb554SJohn Wren Kennedy# 147f38cb554SJohn Wren Kennedyfunction is_healthy 148f38cb554SJohn Wren Kennedy{ 149f38cb554SJohn Wren Kennedy typeset pool=$1 150f38cb554SJohn Wren Kennedy 151f38cb554SJohn Wren Kennedy typeset healthy_output="pool '$pool' is healthy" 152*1d32ba66SJohn Wren Kennedy typeset real_output=$(zpool status -x $pool) 153f38cb554SJohn Wren Kennedy 154f38cb554SJohn Wren Kennedy if [[ "$real_output" == "$healthy_output" ]]; then 155f38cb554SJohn Wren Kennedy return 0 156f38cb554SJohn Wren Kennedy else 157f38cb554SJohn Wren Kennedy typeset -i ret 158*1d32ba66SJohn Wren Kennedy zpool status -x $pool | grep "state:" | \ 159*1d32ba66SJohn Wren Kennedy grep "FAULTED" >/dev/null 2>&1 160f38cb554SJohn Wren Kennedy ret=$? 161f38cb554SJohn Wren Kennedy (( $ret == 0 )) && return 1 162f38cb554SJohn Wren Kennedy typeset l_scan 163f38cb554SJohn Wren Kennedy typeset errnum 164*1d32ba66SJohn Wren Kennedy l_scan=$(zpool status -x $pool | grep "scan:") 165f38cb554SJohn Wren Kennedy l_scan=${l_scan##*"with"} 166*1d32ba66SJohn Wren Kennedy errnum=$(echo $l_scan | awk '{print $1}') 167f38cb554SJohn Wren Kennedy 168f38cb554SJohn Wren Kennedy return $errnum 169f38cb554SJohn Wren Kennedy fi 170f38cb554SJohn Wren Kennedy} 171f38cb554SJohn Wren Kennedy 172f38cb554SJohn Wren Kennedy# 173f38cb554SJohn Wren Kennedy# Check pool data is valid 174f38cb554SJohn Wren Kennedy# 175f38cb554SJohn Wren Kennedy# $1 pool 176f38cb554SJohn Wren Kennedy# 177f38cb554SJohn Wren Kennedyfunction is_data_valid 178f38cb554SJohn Wren Kennedy{ 179f38cb554SJohn Wren Kennedy typeset pool=$1 180f38cb554SJohn Wren Kennedy 181f38cb554SJohn Wren Kennedy record_data $pool $PST_RECORD_FILE 182*1d32ba66SJohn Wren Kennedy if ! diff $PRE_RECORD_FILE $PST_RECORD_FILE > /dev/null 2>&1; then 183f38cb554SJohn Wren Kennedy return 1 184f38cb554SJohn Wren Kennedy fi 185f38cb554SJohn Wren Kennedy 186f38cb554SJohn Wren Kennedy return 0 187f38cb554SJohn Wren Kennedy} 188f38cb554SJohn Wren Kennedy 189f38cb554SJohn Wren Kennedy# 190f38cb554SJohn Wren Kennedy# Get the specified count devices name 191f38cb554SJohn Wren Kennedy# 192f38cb554SJohn Wren Kennedy# $1 pool name 193f38cb554SJohn Wren Kennedy# $2 devices count 194f38cb554SJohn Wren Kennedy# 195f38cb554SJohn Wren Kennedyfunction get_vdevs #pool cnt 196f38cb554SJohn Wren Kennedy{ 197f38cb554SJohn Wren Kennedy typeset pool=$1 198f38cb554SJohn Wren Kennedy typeset -i cnt=$2 199f38cb554SJohn Wren Kennedy 200*1d32ba66SJohn Wren Kennedy typeset all_devs=$(zpool iostat -v $pool | awk '{print $1}'| \ 201*1d32ba66SJohn Wren Kennedy egrep -v "^pool$|^capacity$|^mirror$|^raidz1$|^raidz2$|---" | \ 202*1d32ba66SJohn Wren Kennedy egrep -v "/old$|^$pool$") 203f38cb554SJohn Wren Kennedy typeset -i i=0 204f38cb554SJohn Wren Kennedy typeset vdevs 205f38cb554SJohn Wren Kennedy while ((i < cnt)); do 206*1d32ba66SJohn Wren Kennedy typeset dev=$(echo $all_devs | awk '{print $1}') 207f38cb554SJohn Wren Kennedy eval all_devs=\${all_devs##*$dev} 208f38cb554SJohn Wren Kennedy 209f38cb554SJohn Wren Kennedy vdevs="$dev $vdevs" 210f38cb554SJohn Wren Kennedy ((i += 1)) 211f38cb554SJohn Wren Kennedy done 212f38cb554SJohn Wren Kennedy 213*1d32ba66SJohn Wren Kennedy echo "$vdevs" 214f38cb554SJohn Wren Kennedy} 215f38cb554SJohn Wren Kennedy 216f38cb554SJohn Wren Kennedy# 217f38cb554SJohn Wren Kennedy# Synchronize all the data in pool 218f38cb554SJohn Wren Kennedy# 219f38cb554SJohn Wren Kennedy# $1 pool name 220f38cb554SJohn Wren Kennedy# 221f38cb554SJohn Wren Kennedyfunction sync_pool #pool 222f38cb554SJohn Wren Kennedy{ 223f38cb554SJohn Wren Kennedy typeset pool=$1 224f38cb554SJohn Wren Kennedy 225*1d32ba66SJohn Wren Kennedy log_must sync 226*1d32ba66SJohn Wren Kennedy log_must sleep 2 227f38cb554SJohn Wren Kennedy # Flush all the pool data. 228f38cb554SJohn Wren Kennedy typeset -i ret 229*1d32ba66SJohn Wren Kennedy zpool scrub $pool >/dev/null 2>&1 230f38cb554SJohn Wren Kennedy ret=$? 231f38cb554SJohn Wren Kennedy (( $ret != 0 )) && \ 232*1d32ba66SJohn Wren Kennedy log_fail "zpool scrub $pool failed." 233f38cb554SJohn Wren Kennedy 234f38cb554SJohn Wren Kennedy while ! is_pool_scrubbed $pool; do 235f38cb554SJohn Wren Kennedy if is_pool_resilvered $pool ; then 236f38cb554SJohn Wren Kennedy log_fail "$pool should not be resilver completed." 237f38cb554SJohn Wren Kennedy fi 238*1d32ba66SJohn Wren Kennedy log_must sleep 2 239f38cb554SJohn Wren Kennedy done 240f38cb554SJohn Wren Kennedy} 241f38cb554SJohn Wren Kennedy 242f38cb554SJohn Wren Kennedy# 243f38cb554SJohn Wren Kennedy# Create and replace the same name virtual device files 244f38cb554SJohn Wren Kennedy# 245f38cb554SJohn Wren Kennedy# $1 pool name 246f38cb554SJohn Wren Kennedy# $2-n virtual device files 247f38cb554SJohn Wren Kennedy# 248f38cb554SJohn Wren Kennedyfunction replace_missing_devs 249f38cb554SJohn Wren Kennedy{ 250f38cb554SJohn Wren Kennedy typeset pool=$1 251f38cb554SJohn Wren Kennedy shift 252f38cb554SJohn Wren Kennedy 253f38cb554SJohn Wren Kennedy typeset vdev 254f38cb554SJohn Wren Kennedy for vdev in $@; do 255*1d32ba66SJohn Wren Kennedy log_must mkfile $MINVDEVSIZE $vdev 256*1d32ba66SJohn Wren Kennedy log_must zpool replace -f $pool $vdev $vdev 257f38cb554SJohn Wren Kennedy while true; do 258f38cb554SJohn Wren Kennedy if ! is_pool_resilvered $pool ; then 259*1d32ba66SJohn Wren Kennedy log_must sleep 2 260f38cb554SJohn Wren Kennedy else 261f38cb554SJohn Wren Kennedy break 262f38cb554SJohn Wren Kennedy fi 263f38cb554SJohn Wren Kennedy done 264f38cb554SJohn Wren Kennedy done 265f38cb554SJohn Wren Kennedy} 266f38cb554SJohn Wren Kennedy 267f38cb554SJohn Wren Kennedy# 268f38cb554SJohn Wren Kennedy# Damage the pool's virtual device files. 269f38cb554SJohn Wren Kennedy# 270f38cb554SJohn Wren Kennedy# $1 pool name 271f38cb554SJohn Wren Kennedy# $2 Failing devices count 272f38cb554SJohn Wren Kennedy# $3 damage vdevs method, if not null, we keep 273f38cb554SJohn Wren Kennedy# the label for the vdevs 274f38cb554SJohn Wren Kennedy# 275f38cb554SJohn Wren Kennedyfunction damage_devs 276f38cb554SJohn Wren Kennedy{ 277f38cb554SJohn Wren Kennedy typeset pool=$1 278f38cb554SJohn Wren Kennedy typeset -i cnt=$2 279f38cb554SJohn Wren Kennedy typeset label="$3" 280f38cb554SJohn Wren Kennedy typeset vdevs 2814b5c8e93SMatthew Ahrens typeset -i bs_count=$((64 * 1024)) 282f38cb554SJohn Wren Kennedy 283f38cb554SJohn Wren Kennedy vdevs=$(get_vdevs $pool $cnt) 2844b5c8e93SMatthew Ahrens typeset dev 285f38cb554SJohn Wren Kennedy if [[ -n $label ]]; then 286f38cb554SJohn Wren Kennedy for dev in $vdevs; do 287*1d32ba66SJohn Wren Kennedy dd if=/dev/zero of=$dev seek=512 bs=1024 \ 2884b5c8e93SMatthew Ahrens count=$bs_count conv=notrunc >/dev/null 2>&1 289f38cb554SJohn Wren Kennedy done 290f38cb554SJohn Wren Kennedy else 2914b5c8e93SMatthew Ahrens for dev in $vdevs; do 292*1d32ba66SJohn Wren Kennedy dd if=/dev/zero of=$dev bs=1024 count=$bs_count \ 2934b5c8e93SMatthew Ahrens conv=notrunc >/dev/null 2>&1 2944b5c8e93SMatthew Ahrens done 295f38cb554SJohn Wren Kennedy fi 296f38cb554SJohn Wren Kennedy 297f38cb554SJohn Wren Kennedy sync_pool $pool 298f38cb554SJohn Wren Kennedy} 299f38cb554SJohn Wren Kennedy 300f38cb554SJohn Wren Kennedy# 301f38cb554SJohn Wren Kennedy# Clear errors in the pool caused by data corruptions 302f38cb554SJohn Wren Kennedy# 303f38cb554SJohn Wren Kennedy# $1 pool name 304f38cb554SJohn Wren Kennedy# 305f38cb554SJohn Wren Kennedyfunction clear_errors 306f38cb554SJohn Wren Kennedy{ 307f38cb554SJohn Wren Kennedy typeset pool=$1 308f38cb554SJohn Wren Kennedy 309*1d32ba66SJohn Wren Kennedy log_must zpool clear $pool 310f38cb554SJohn Wren Kennedy 311f38cb554SJohn Wren Kennedy if ! is_healthy $pool ; then 312f38cb554SJohn Wren Kennedy log_note "$pool should be healthy." 313f38cb554SJohn Wren Kennedy return 1 314f38cb554SJohn Wren Kennedy fi 315f38cb554SJohn Wren Kennedy if ! is_data_valid $pool ; then 316f38cb554SJohn Wren Kennedy log_note "Data should be valid in $pool." 317f38cb554SJohn Wren Kennedy return 1 318f38cb554SJohn Wren Kennedy fi 319f38cb554SJohn Wren Kennedy 320f38cb554SJohn Wren Kennedy return 0 321f38cb554SJohn Wren Kennedy} 322f38cb554SJohn Wren Kennedy 323f38cb554SJohn Wren Kennedy# 324f38cb554SJohn Wren Kennedy# Remove the specified pool's virtual device files 325f38cb554SJohn Wren Kennedy# 326f38cb554SJohn Wren Kennedy# $1 Pool name 327f38cb554SJohn Wren Kennedy# $2 Missing devices count 328f38cb554SJohn Wren Kennedy# 329f38cb554SJohn Wren Kennedyfunction remove_devs 330f38cb554SJohn Wren Kennedy{ 331f38cb554SJohn Wren Kennedy typeset pool=$1 332f38cb554SJohn Wren Kennedy typeset -i cnt=$2 333f38cb554SJohn Wren Kennedy typeset vdevs 334f38cb554SJohn Wren Kennedy 335f38cb554SJohn Wren Kennedy vdevs=$(get_vdevs $pool $cnt) 336*1d32ba66SJohn Wren Kennedy log_must rm -f $vdevs 337f38cb554SJohn Wren Kennedy 338f38cb554SJohn Wren Kennedy sync_pool $pool 339f38cb554SJohn Wren Kennedy} 340f38cb554SJohn Wren Kennedy 341f38cb554SJohn Wren Kennedy# 342f38cb554SJohn Wren Kennedy# Recover the bad or missing device files in the pool 343f38cb554SJohn Wren Kennedy# 344f38cb554SJohn Wren Kennedy# $1 Pool name 345f38cb554SJohn Wren Kennedy# $2 Missing devices count 346f38cb554SJohn Wren Kennedy# 347f38cb554SJohn Wren Kennedyfunction recover_bad_missing_devs 348f38cb554SJohn Wren Kennedy{ 349f38cb554SJohn Wren Kennedy typeset pool=$1 350f38cb554SJohn Wren Kennedy typeset -i cnt=$2 351f38cb554SJohn Wren Kennedy typeset vdevs 352f38cb554SJohn Wren Kennedy 353f38cb554SJohn Wren Kennedy vdevs=$(get_vdevs $pool $cnt) 354f38cb554SJohn Wren Kennedy replace_missing_devs $pool $vdevs 355f38cb554SJohn Wren Kennedy 356f38cb554SJohn Wren Kennedy if ! is_healthy $pool ; then 357f38cb554SJohn Wren Kennedy log_note "$pool should be healthy." 358f38cb554SJohn Wren Kennedy return 1 359f38cb554SJohn Wren Kennedy fi 360f38cb554SJohn Wren Kennedy if ! is_data_valid $pool ; then 361f38cb554SJohn Wren Kennedy log_note "Data should be valid in $pool." 362f38cb554SJohn Wren Kennedy return 1 363f38cb554SJohn Wren Kennedy fi 364f38cb554SJohn Wren Kennedy 365f38cb554SJohn Wren Kennedy return 0 366f38cb554SJohn Wren Kennedy} 367