19f923083SAlexander Pyhalov#!@PYTHON@ 2d583b39bSJohn Wren Kennedy 3d583b39bSJohn Wren Kennedy# 4d583b39bSJohn Wren Kennedy# This file and its contents are supplied under the terms of the 5d583b39bSJohn Wren Kennedy# Common Development and Distribution License ("CDDL"), version 1.0. 6d583b39bSJohn Wren Kennedy# You may only use this file in accordance with the terms of version 7d583b39bSJohn Wren Kennedy# 1.0 of the CDDL. 8d583b39bSJohn Wren Kennedy# 9d583b39bSJohn Wren Kennedy# A full copy of the text of the CDDL should have accompanied this 10d583b39bSJohn Wren Kennedy# source. A copy of the CDDL is also available via the Internet at 11d583b39bSJohn Wren Kennedy# http://www.illumos.org/license/CDDL. 12d583b39bSJohn Wren Kennedy# 13d583b39bSJohn Wren Kennedy 14d583b39bSJohn Wren Kennedy# 15dfc11533SChris Williamson# Copyright (c) 2012, 2016 by Delphix. All rights reserved. 1664ee6612SChris Fraire# Copyright (c) 2017, Chris Fraire <cfraire@me.com>. 172491fc01SJohn Levon# Copyright 2019 Joyent, Inc. 181a2acdcdSAndy Fiddaman# Copyright 2020 OmniOS Community Edition (OmniOSce) Association. 19d583b39bSJohn Wren Kennedy# 20d583b39bSJohn Wren Kennedy 21b0858fdcSAlexander Pyhalovfrom __future__ import print_function 22b0858fdcSAlexander Pyhalovimport sys 23b0858fdcSAlexander PyhalovPY3 = sys.version_info[0] == 3 24b0858fdcSAlexander Pyhalov 25b0858fdcSAlexander Pyhalovif PY3: 26b0858fdcSAlexander Pyhalov import configparser 27b0858fdcSAlexander Pyhalovelse: 28b0858fdcSAlexander Pyhalov import ConfigParser as configparser 29b0858fdcSAlexander Pyhalov 30cdd7a662SAlexander Pyhalovimport io 31d583b39bSJohn Wren Kennedyimport os 32d583b39bSJohn Wren Kennedyimport logging 33a0955b86SJohn Levonimport platform 3478801af7SAndy Fiddamanimport re 3500688ee4SWill Andrewsfrom logging.handlers import WatchedFileHandler 36d583b39bSJohn Wren Kennedyfrom datetime import datetime 37d583b39bSJohn Wren Kennedyfrom optparse import OptionParser 38d583b39bSJohn Wren Kennedyfrom pwd import getpwnam 39d583b39bSJohn Wren Kennedyfrom pwd import getpwuid 40d583b39bSJohn Wren Kennedyfrom select import select 41d583b39bSJohn Wren Kennedyfrom subprocess import PIPE 42d583b39bSJohn Wren Kennedyfrom subprocess import Popen 43d583b39bSJohn Wren Kennedyfrom sys import argv 44b0858fdcSAlexander Pyhalovfrom sys import exit 45b0858fdcSAlexander Pyhalovfrom sys import maxsize 46d583b39bSJohn Wren Kennedyfrom threading import Timer 47d583b39bSJohn Wren Kennedyfrom time import time 48d583b39bSJohn Wren Kennedy 49d583b39bSJohn Wren KennedyBASEDIR = '/var/tmp/test_results' 50b8052df9SRyan MoellerTESTDIR = '/opt/zfs-tests/' 51d583b39bSJohn Wren KennedyKILL = '/usr/bin/kill' 52d583b39bSJohn Wren KennedyTRUE = '/usr/bin/true' 53d583b39bSJohn Wren KennedySUDO = '/usr/bin/sudo' 54d583b39bSJohn Wren Kennedy 552491fc01SJohn Levonretcode = 0 562491fc01SJohn Levon 5700688ee4SWill Andrews# Custom class to reopen the log file in case it is forcibly closed by a test. 5800688ee4SWill Andrewsclass WatchedFileHandlerClosed(WatchedFileHandler): 5900688ee4SWill Andrews """Watch files, including closed files. 6000688ee4SWill Andrews Similar to (and inherits from) logging.handler.WatchedFileHandler, 6100688ee4SWill Andrews except that IOErrors are handled by reopening the stream and retrying. 6200688ee4SWill Andrews This will be retried up to a configurable number of times before 6300688ee4SWill Andrews giving up, default 5. 6400688ee4SWill Andrews """ 6500688ee4SWill Andrews 66cdd7a662SAlexander Pyhalov def __init__(self, filename, mode='a', encoding='utf-8', delay=0, max_tries=5): 6700688ee4SWill Andrews self.max_tries = max_tries 6800688ee4SWill Andrews self.tries = 0 6900688ee4SWill Andrews WatchedFileHandler.__init__(self, filename, mode, encoding, delay) 7000688ee4SWill Andrews 7100688ee4SWill Andrews def emit(self, record): 7200688ee4SWill Andrews while True: 7300688ee4SWill Andrews try: 7400688ee4SWill Andrews WatchedFileHandler.emit(self, record) 7500688ee4SWill Andrews self.tries = 0 7600688ee4SWill Andrews return 7700688ee4SWill Andrews except IOError as err: 7800688ee4SWill Andrews if self.tries == self.max_tries: 7900688ee4SWill Andrews raise 8000688ee4SWill Andrews self.stream.close() 8100688ee4SWill Andrews self.stream = self._open() 8200688ee4SWill Andrews self.tries += 1 83d583b39bSJohn Wren Kennedy 84d583b39bSJohn Wren Kennedyclass Result(object): 85d583b39bSJohn Wren Kennedy total = 0 86d583b39bSJohn Wren Kennedy runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0} 87d583b39bSJohn Wren Kennedy 88d583b39bSJohn Wren Kennedy def __init__(self): 89d583b39bSJohn Wren Kennedy self.starttime = None 90d583b39bSJohn Wren Kennedy self.returncode = None 91d583b39bSJohn Wren Kennedy self.runtime = '' 92d583b39bSJohn Wren Kennedy self.stdout = [] 93d583b39bSJohn Wren Kennedy self.stderr = [] 94d583b39bSJohn Wren Kennedy self.result = '' 95d583b39bSJohn Wren Kennedy 96d583b39bSJohn Wren Kennedy def done(self, proc, killed): 97d583b39bSJohn Wren Kennedy """ 98d583b39bSJohn Wren Kennedy Finalize the results of this Cmd. 9978801af7SAndy Fiddaman Report SKIP for return codes 3,4 (NOTINUSE, UNSUPPORTED) 10078801af7SAndy Fiddaman as defined in ../stf/include/stf.shlib 101d583b39bSJohn Wren Kennedy """ 1022491fc01SJohn Levon global retcode 1032491fc01SJohn Levon 104d583b39bSJohn Wren Kennedy Result.total += 1 105d583b39bSJohn Wren Kennedy m, s = divmod(time() - self.starttime, 60) 106d583b39bSJohn Wren Kennedy self.runtime = '%02d:%02d' % (m, s) 107d583b39bSJohn Wren Kennedy self.returncode = proc.returncode 108d583b39bSJohn Wren Kennedy if killed: 109d583b39bSJohn Wren Kennedy self.result = 'KILLED' 110d583b39bSJohn Wren Kennedy Result.runresults['KILLED'] += 1 1112491fc01SJohn Levon retcode = 2; 1121a2acdcdSAndy Fiddaman elif self.returncode == 0: 113d583b39bSJohn Wren Kennedy self.result = 'PASS' 114d583b39bSJohn Wren Kennedy Result.runresults['PASS'] += 1 1151a2acdcdSAndy Fiddaman elif self.returncode == 3 or self.returncode == 4: 11696c8483aSYuri Pankov self.result = 'SKIP' 11796c8483aSYuri Pankov Result.runresults['SKIP'] += 1 1181a2acdcdSAndy Fiddaman elif self.returncode != 0: 119d583b39bSJohn Wren Kennedy self.result = 'FAIL' 120d583b39bSJohn Wren Kennedy Result.runresults['FAIL'] += 1 1212491fc01SJohn Levon retcode = 1; 122d583b39bSJohn Wren Kennedy 123d583b39bSJohn Wren Kennedy 124d583b39bSJohn Wren Kennedyclass Output(object): 125d583b39bSJohn Wren Kennedy """ 126d583b39bSJohn Wren Kennedy This class is a slightly modified version of the 'Stream' class found 127d583b39bSJohn Wren Kennedy here: http://goo.gl/aSGfv 128d583b39bSJohn Wren Kennedy """ 129d583b39bSJohn Wren Kennedy def __init__(self, stream): 130d583b39bSJohn Wren Kennedy self.stream = stream 131d583b39bSJohn Wren Kennedy self._buf = '' 132d583b39bSJohn Wren Kennedy self.lines = [] 133d583b39bSJohn Wren Kennedy 134d583b39bSJohn Wren Kennedy def fileno(self): 135d583b39bSJohn Wren Kennedy return self.stream.fileno() 136d583b39bSJohn Wren Kennedy 137d583b39bSJohn Wren Kennedy def read(self, drain=0): 138d583b39bSJohn Wren Kennedy """ 139d583b39bSJohn Wren Kennedy Read from the file descriptor. If 'drain' set, read until EOF. 140d583b39bSJohn Wren Kennedy """ 141d583b39bSJohn Wren Kennedy while self._read() is not None: 142d583b39bSJohn Wren Kennedy if not drain: 143d583b39bSJohn Wren Kennedy break 144d583b39bSJohn Wren Kennedy 145d583b39bSJohn Wren Kennedy def _read(self): 146d583b39bSJohn Wren Kennedy """ 147d583b39bSJohn Wren Kennedy Read up to 4k of data from this output stream. Collect the output 148d583b39bSJohn Wren Kennedy up to the last newline, and append it to any leftover data from a 149d583b39bSJohn Wren Kennedy previous call. The lines are stored as a (timestamp, data) tuple 150d583b39bSJohn Wren Kennedy for easy sorting/merging later. 151d583b39bSJohn Wren Kennedy """ 152d583b39bSJohn Wren Kennedy fd = self.fileno() 153cc37296fSAndy Fiddaman buf = os.read(fd, 4096).decode('utf-8', errors='ignore') 154d583b39bSJohn Wren Kennedy if not buf: 155d583b39bSJohn Wren Kennedy return None 156d583b39bSJohn Wren Kennedy if '\n' not in buf: 157d583b39bSJohn Wren Kennedy self._buf += buf 158d583b39bSJohn Wren Kennedy return [] 159d583b39bSJohn Wren Kennedy 160d583b39bSJohn Wren Kennedy buf = self._buf + buf 161d583b39bSJohn Wren Kennedy tmp, rest = buf.rsplit('\n', 1) 162d583b39bSJohn Wren Kennedy self._buf = rest 163d583b39bSJohn Wren Kennedy now = datetime.now() 164d583b39bSJohn Wren Kennedy rows = tmp.split('\n') 165d583b39bSJohn Wren Kennedy self.lines += [(now, r) for r in rows] 166d583b39bSJohn Wren Kennedy 167d583b39bSJohn Wren Kennedy 168d583b39bSJohn Wren Kennedyclass Cmd(object): 169d583b39bSJohn Wren Kennedy verified_users = [] 170d583b39bSJohn Wren Kennedy 171b8052df9SRyan Moeller def __init__(self, pathname, identifier=None, outputdir=None, 172b8052df9SRyan Moeller timeout=None, user=None, tags=None): 173d583b39bSJohn Wren Kennedy self.pathname = pathname 174b8052df9SRyan Moeller self.identifier = identifier 175d583b39bSJohn Wren Kennedy self.outputdir = outputdir or 'BASEDIR' 176dcbf3bd6SGeorge Wilson self.timeout = timeout 177d583b39bSJohn Wren Kennedy self.user = user or '' 178d583b39bSJohn Wren Kennedy self.killed = False 179d583b39bSJohn Wren Kennedy self.result = Result() 180d583b39bSJohn Wren Kennedy 181f9a78bf4SJohn Wren Kennedy if self.timeout is None: 182dcbf3bd6SGeorge Wilson self.timeout = 60 183dcbf3bd6SGeorge Wilson 184d583b39bSJohn Wren Kennedy def __str__(self): 185b8052df9SRyan Moeller return '''\ 186b8052df9SRyan MoellerPathname: %s 187b8052df9SRyan MoellerIdentifier: %s 188b8052df9SRyan MoellerOutputdir: %s 189b8052df9SRyan MoellerTimeout: %d 190b8052df9SRyan MoellerUser: %s 191b8052df9SRyan Moeller''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user) 192d583b39bSJohn Wren Kennedy 193d583b39bSJohn Wren Kennedy def kill_cmd(self, proc): 194d583b39bSJohn Wren Kennedy """ 195d583b39bSJohn Wren Kennedy Kill a running command due to timeout, or ^C from the keyboard. If 196d583b39bSJohn Wren Kennedy sudo is required, this user was verified previously. 197d583b39bSJohn Wren Kennedy """ 198d583b39bSJohn Wren Kennedy self.killed = True 199d583b39bSJohn Wren Kennedy do_sudo = len(self.user) != 0 200d583b39bSJohn Wren Kennedy signal = '-TERM' 201d583b39bSJohn Wren Kennedy 202d583b39bSJohn Wren Kennedy cmd = [SUDO, KILL, signal, str(proc.pid)] 203d583b39bSJohn Wren Kennedy if not do_sudo: 204d583b39bSJohn Wren Kennedy del cmd[0] 205d583b39bSJohn Wren Kennedy 206d583b39bSJohn Wren Kennedy try: 207d583b39bSJohn Wren Kennedy kp = Popen(cmd) 208d583b39bSJohn Wren Kennedy kp.wait() 209d583b39bSJohn Wren Kennedy except: 210d583b39bSJohn Wren Kennedy pass 211d583b39bSJohn Wren Kennedy 212d583b39bSJohn Wren Kennedy def update_cmd_privs(self, cmd, user): 213d583b39bSJohn Wren Kennedy """ 214d583b39bSJohn Wren Kennedy If a user has been specified to run this Cmd and we're not already 215d583b39bSJohn Wren Kennedy running as that user, prepend the appropriate sudo command to run 216d583b39bSJohn Wren Kennedy as that user. 217d583b39bSJohn Wren Kennedy """ 218d583b39bSJohn Wren Kennedy me = getpwuid(os.getuid()) 219d583b39bSJohn Wren Kennedy 2201a2acdcdSAndy Fiddaman if not user or user == me: 221d583b39bSJohn Wren Kennedy return cmd 222d583b39bSJohn Wren Kennedy 223d583b39bSJohn Wren Kennedy ret = '%s -E -u %s %s' % (SUDO, user, cmd) 224d583b39bSJohn Wren Kennedy return ret.split(' ') 225d583b39bSJohn Wren Kennedy 226d583b39bSJohn Wren Kennedy def collect_output(self, proc): 227d583b39bSJohn Wren Kennedy """ 228d583b39bSJohn Wren Kennedy Read from stdout/stderr as data becomes available, until the 229d583b39bSJohn Wren Kennedy process is no longer running. Return the lines from the stdout and 230d583b39bSJohn Wren Kennedy stderr Output objects. 231d583b39bSJohn Wren Kennedy """ 232d583b39bSJohn Wren Kennedy out = Output(proc.stdout) 233d583b39bSJohn Wren Kennedy err = Output(proc.stderr) 234d583b39bSJohn Wren Kennedy res = [] 235d583b39bSJohn Wren Kennedy while proc.returncode is None: 236d583b39bSJohn Wren Kennedy proc.poll() 237d583b39bSJohn Wren Kennedy res = select([out, err], [], [], .1) 238d583b39bSJohn Wren Kennedy for fd in res[0]: 239d583b39bSJohn Wren Kennedy fd.read() 240d583b39bSJohn Wren Kennedy for fd in res[0]: 241d583b39bSJohn Wren Kennedy fd.read(drain=1) 242d583b39bSJohn Wren Kennedy 243d583b39bSJohn Wren Kennedy return out.lines, err.lines 244d583b39bSJohn Wren Kennedy 245d583b39bSJohn Wren Kennedy def run(self, options): 246d583b39bSJohn Wren Kennedy """ 247d583b39bSJohn Wren Kennedy This is the main function that runs each individual test. 248d583b39bSJohn Wren Kennedy Determine whether or not the command requires sudo, and modify it 249d583b39bSJohn Wren Kennedy if needed. Run the command, and update the result object. 250d583b39bSJohn Wren Kennedy """ 251d583b39bSJohn Wren Kennedy if options.dryrun is True: 252b0858fdcSAlexander Pyhalov print(self) 253d583b39bSJohn Wren Kennedy return 254d583b39bSJohn Wren Kennedy 255d583b39bSJohn Wren Kennedy privcmd = self.update_cmd_privs(self.pathname, self.user) 256d583b39bSJohn Wren Kennedy try: 257d583b39bSJohn Wren Kennedy old = os.umask(0) 258d583b39bSJohn Wren Kennedy if not os.path.isdir(self.outputdir): 259b0858fdcSAlexander Pyhalov os.makedirs(self.outputdir, mode=0o777) 260d583b39bSJohn Wren Kennedy os.umask(old) 261b0858fdcSAlexander Pyhalov except OSError as e: 262d583b39bSJohn Wren Kennedy fail('%s' % e) 263d583b39bSJohn Wren Kennedy 264d583b39bSJohn Wren Kennedy try: 265d583b39bSJohn Wren Kennedy self.result.starttime = time() 266b0858fdcSAlexander Pyhalov proc = Popen(privcmd, stdout=PIPE, stderr=PIPE, stdin=PIPE, 267b0858fdcSAlexander Pyhalov universal_newlines=True) 2685cabbc6bSPrashanth Sreenivasa proc.stdin.close() 269dcbf3bd6SGeorge Wilson 270dcbf3bd6SGeorge Wilson # Allow a special timeout value of 0 to mean infinity 271dcbf3bd6SGeorge Wilson if int(self.timeout) == 0: 272b0858fdcSAlexander Pyhalov self.timeout = maxsize 273d583b39bSJohn Wren Kennedy t = Timer(int(self.timeout), self.kill_cmd, [proc]) 274d583b39bSJohn Wren Kennedy t.start() 275d583b39bSJohn Wren Kennedy self.result.stdout, self.result.stderr = self.collect_output(proc) 276d583b39bSJohn Wren Kennedy except KeyboardInterrupt: 277d583b39bSJohn Wren Kennedy self.kill_cmd(proc) 278d583b39bSJohn Wren Kennedy fail('\nRun terminated at user request.') 279d583b39bSJohn Wren Kennedy finally: 280d583b39bSJohn Wren Kennedy t.cancel() 281d583b39bSJohn Wren Kennedy 282d583b39bSJohn Wren Kennedy self.result.done(proc, self.killed) 283d583b39bSJohn Wren Kennedy 284d583b39bSJohn Wren Kennedy def skip(self): 285d583b39bSJohn Wren Kennedy """ 286d583b39bSJohn Wren Kennedy Initialize enough of the test result that we can log a skipped 287d583b39bSJohn Wren Kennedy command. 288d583b39bSJohn Wren Kennedy """ 289d583b39bSJohn Wren Kennedy Result.total += 1 290d583b39bSJohn Wren Kennedy Result.runresults['SKIP'] += 1 291d583b39bSJohn Wren Kennedy self.result.stdout = self.result.stderr = [] 292d583b39bSJohn Wren Kennedy self.result.starttime = time() 293d583b39bSJohn Wren Kennedy m, s = divmod(time() - self.result.starttime, 60) 294d583b39bSJohn Wren Kennedy self.result.runtime = '%02d:%02d' % (m, s) 295d583b39bSJohn Wren Kennedy self.result.result = 'SKIP' 296d583b39bSJohn Wren Kennedy 297d583b39bSJohn Wren Kennedy def log(self, logger, options): 298d583b39bSJohn Wren Kennedy """ 299d583b39bSJohn Wren Kennedy This function is responsible for writing all output. This includes 300d583b39bSJohn Wren Kennedy the console output, the logfile of all results (with timestamped 301d583b39bSJohn Wren Kennedy merged stdout and stderr), and for each test, the unmodified 302d583b39bSJohn Wren Kennedy stdout/stderr/merged in it's own file. 303d583b39bSJohn Wren Kennedy """ 304d583b39bSJohn Wren Kennedy if logger is None: 305d583b39bSJohn Wren Kennedy return 306d583b39bSJohn Wren Kennedy 307f38cb554SJohn Wren Kennedy logname = getpwuid(os.getuid()).pw_name 308f38cb554SJohn Wren Kennedy user = ' (run as %s)' % (self.user if len(self.user) else logname) 309b8052df9SRyan Moeller if self.identifier: 310b8052df9SRyan Moeller msga = 'Test (%s): %s%s ' % (self.identifier, self.pathname, user) 311b8052df9SRyan Moeller else: 312b8052df9SRyan Moeller msga = 'Test: %s%s ' % (self.pathname, user) 313d583b39bSJohn Wren Kennedy msgb = '[%s] [%s]' % (self.result.runtime, self.result.result) 314d583b39bSJohn Wren Kennedy pad = ' ' * (80 - (len(msga) + len(msgb))) 315d583b39bSJohn Wren Kennedy 316d583b39bSJohn Wren Kennedy # If -q is specified, only print a line for tests that didn't pass. 317d583b39bSJohn Wren Kennedy # This means passing tests need to be logged as DEBUG, or the one 318d583b39bSJohn Wren Kennedy # line summary will only be printed in the logfile for failures. 319d583b39bSJohn Wren Kennedy if not options.quiet: 320d583b39bSJohn Wren Kennedy logger.info('%s%s%s' % (msga, pad, msgb)) 3211a2acdcdSAndy Fiddaman elif self.result.result != 'PASS': 322d583b39bSJohn Wren Kennedy logger.info('%s%s%s' % (msga, pad, msgb)) 323d583b39bSJohn Wren Kennedy else: 324d583b39bSJohn Wren Kennedy logger.debug('%s%s%s' % (msga, pad, msgb)) 325d583b39bSJohn Wren Kennedy 326412db4e9SJohn Wren Kennedy lines = sorted(self.result.stdout + self.result.stderr, 327b0858fdcSAlexander Pyhalov key=lambda x: x[0]) 328412db4e9SJohn Wren Kennedy 329412db4e9SJohn Wren Kennedy for dt, line in lines: 330d583b39bSJohn Wren Kennedy logger.debug('%s %s' % (dt.strftime("%H:%M:%S.%f ")[:11], line)) 331d583b39bSJohn Wren Kennedy 332d583b39bSJohn Wren Kennedy if len(self.result.stdout): 333cdd7a662SAlexander Pyhalov with io.open(os.path.join(self.outputdir, 'stdout'), 334cdd7a662SAlexander Pyhalov encoding='utf-8', 335cdd7a662SAlexander Pyhalov errors='surrogateescape', 336cdd7a662SAlexander Pyhalov mode='w') as out: 337d583b39bSJohn Wren Kennedy for _, line in self.result.stdout: 338b0858fdcSAlexander Pyhalov out.write('%s\n' % line) 339d583b39bSJohn Wren Kennedy if len(self.result.stderr): 340cdd7a662SAlexander Pyhalov with io.open(os.path.join(self.outputdir, 'stderr'), 341cdd7a662SAlexander Pyhalov encoding='utf-8', 342cdd7a662SAlexander Pyhalov errors='surrogateescape', 343cdd7a662SAlexander Pyhalov mode='w') as err: 344d583b39bSJohn Wren Kennedy for _, line in self.result.stderr: 345b0858fdcSAlexander Pyhalov err.write('%s\n' % line) 346d583b39bSJohn Wren Kennedy if len(self.result.stdout) and len(self.result.stderr): 347cdd7a662SAlexander Pyhalov with io.open(os.path.join(self.outputdir, 'merged'), 348cdd7a662SAlexander Pyhalov encoding='utf-8', 349cdd7a662SAlexander Pyhalov errors='surrogateescape', 350cdd7a662SAlexander Pyhalov mode='w') as merged: 351412db4e9SJohn Wren Kennedy for _, line in lines: 352b0858fdcSAlexander Pyhalov merged.write('%s\n' % line) 353d583b39bSJohn Wren Kennedy 354d583b39bSJohn Wren Kennedy 355d583b39bSJohn Wren Kennedyclass Test(Cmd): 356d583b39bSJohn Wren Kennedy props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post', 357b8052df9SRyan Moeller 'post_user', 'tags'] 358d583b39bSJohn Wren Kennedy 359b8052df9SRyan Moeller def __init__(self, pathname, 360b8052df9SRyan Moeller pre=None, pre_user=None, post=None, post_user=None, 361b8052df9SRyan Moeller tags=None, **kwargs): 362b8052df9SRyan Moeller super(Test, self).__init__(pathname, **kwargs) 363d583b39bSJohn Wren Kennedy self.pre = pre or '' 364d583b39bSJohn Wren Kennedy self.pre_user = pre_user or '' 365d583b39bSJohn Wren Kennedy self.post = post or '' 366d583b39bSJohn Wren Kennedy self.post_user = post_user or '' 367b8052df9SRyan Moeller self.tags = tags or [] 368d583b39bSJohn Wren Kennedy 369d583b39bSJohn Wren Kennedy def __str__(self): 370d583b39bSJohn Wren Kennedy post_user = pre_user = '' 371d583b39bSJohn Wren Kennedy if len(self.pre_user): 372d583b39bSJohn Wren Kennedy pre_user = ' (as %s)' % (self.pre_user) 373d583b39bSJohn Wren Kennedy if len(self.post_user): 374d583b39bSJohn Wren Kennedy post_user = ' (as %s)' % (self.post_user) 375b8052df9SRyan Moeller return '''\ 376b8052df9SRyan MoellerPathname: %s 377b8052df9SRyan MoellerIdentifier: %s 378b8052df9SRyan MoellerOutputdir: %s 379b8052df9SRyan MoellerTimeout: %d 380b8052df9SRyan MoellerUser: %s 381b8052df9SRyan MoellerPre: %s%s 382b8052df9SRyan MoellerPost: %s%s 383b8052df9SRyan MoellerTags: %s 384b8052df9SRyan Moeller''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user, 385b8052df9SRyan Moeller self.pre, pre_user, self.post, post_user, self.tags) 386d583b39bSJohn Wren Kennedy 387d583b39bSJohn Wren Kennedy def verify(self, logger): 388d583b39bSJohn Wren Kennedy """ 389d583b39bSJohn Wren Kennedy Check the pre/post scripts, user and Test. Omit the Test from this 390d583b39bSJohn Wren Kennedy run if there are any problems. 391d583b39bSJohn Wren Kennedy """ 392d583b39bSJohn Wren Kennedy files = [self.pre, self.pathname, self.post] 393d583b39bSJohn Wren Kennedy users = [self.pre_user, self.user, self.post_user] 394d583b39bSJohn Wren Kennedy 395d583b39bSJohn Wren Kennedy for f in [f for f in files if len(f)]: 396d583b39bSJohn Wren Kennedy if not verify_file(f): 397d583b39bSJohn Wren Kennedy logger.info("Warning: Test '%s' not added to this run because" 398d583b39bSJohn Wren Kennedy " it failed verification." % f) 399d583b39bSJohn Wren Kennedy return False 400d583b39bSJohn Wren Kennedy 401d583b39bSJohn Wren Kennedy for user in [user for user in users if len(user)]: 402d583b39bSJohn Wren Kennedy if not verify_user(user, logger): 403d583b39bSJohn Wren Kennedy logger.info("Not adding Test '%s' to this run." % 404d583b39bSJohn Wren Kennedy self.pathname) 405d583b39bSJohn Wren Kennedy return False 406d583b39bSJohn Wren Kennedy 407d583b39bSJohn Wren Kennedy return True 408d583b39bSJohn Wren Kennedy 409d583b39bSJohn Wren Kennedy def run(self, logger, options): 410d583b39bSJohn Wren Kennedy """ 411d583b39bSJohn Wren Kennedy Create Cmd instances for the pre/post scripts. If the pre script 412d583b39bSJohn Wren Kennedy doesn't pass, skip this Test. Run the post script regardless. 413d583b39bSJohn Wren Kennedy """ 414f9a78bf4SJohn Wren Kennedy odir = os.path.join(self.outputdir, os.path.basename(self.pre)) 415b8052df9SRyan Moeller pretest = Cmd(self.pre, identifier=self.identifier, outputdir=odir, 416b8052df9SRyan Moeller timeout=self.timeout, user=self.pre_user) 417b8052df9SRyan Moeller test = Cmd(self.pathname, identifier=self.identifier, 418b8052df9SRyan Moeller outputdir=self.outputdir, timeout=self.timeout, 419b8052df9SRyan Moeller user=self.user) 420f9a78bf4SJohn Wren Kennedy odir = os.path.join(self.outputdir, os.path.basename(self.post)) 421b8052df9SRyan Moeller posttest = Cmd(self.post, identifier=self.identifier, outputdir=odir, 422b8052df9SRyan Moeller timeout=self.timeout, user=self.post_user) 423d583b39bSJohn Wren Kennedy 424d583b39bSJohn Wren Kennedy cont = True 425d583b39bSJohn Wren Kennedy if len(pretest.pathname): 426d583b39bSJohn Wren Kennedy pretest.run(options) 4271a2acdcdSAndy Fiddaman cont = pretest.result.result == 'PASS' 428d583b39bSJohn Wren Kennedy pretest.log(logger, options) 429d583b39bSJohn Wren Kennedy 430d583b39bSJohn Wren Kennedy if cont: 431d583b39bSJohn Wren Kennedy test.run(options) 432d583b39bSJohn Wren Kennedy else: 433d583b39bSJohn Wren Kennedy test.skip() 434d583b39bSJohn Wren Kennedy 435d583b39bSJohn Wren Kennedy test.log(logger, options) 436d583b39bSJohn Wren Kennedy 437d583b39bSJohn Wren Kennedy if len(posttest.pathname): 438d583b39bSJohn Wren Kennedy posttest.run(options) 439d583b39bSJohn Wren Kennedy posttest.log(logger, options) 440d583b39bSJohn Wren Kennedy 441d583b39bSJohn Wren Kennedy 442d583b39bSJohn Wren Kennedyclass TestGroup(Test): 443d583b39bSJohn Wren Kennedy props = Test.props + ['tests'] 444d583b39bSJohn Wren Kennedy 445b8052df9SRyan Moeller def __init__(self, pathname, tests=None, **kwargs): 446b8052df9SRyan Moeller super(TestGroup, self).__init__(pathname, **kwargs) 447d583b39bSJohn Wren Kennedy self.tests = tests or [] 448d583b39bSJohn Wren Kennedy 449d583b39bSJohn Wren Kennedy def __str__(self): 450d583b39bSJohn Wren Kennedy post_user = pre_user = '' 451d583b39bSJohn Wren Kennedy if len(self.pre_user): 452d583b39bSJohn Wren Kennedy pre_user = ' (as %s)' % (self.pre_user) 453d583b39bSJohn Wren Kennedy if len(self.post_user): 454d583b39bSJohn Wren Kennedy post_user = ' (as %s)' % (self.post_user) 455b8052df9SRyan Moeller return '''\ 456b8052df9SRyan MoellerPathname: %s 457b8052df9SRyan MoellerIdentifier: %s 458b8052df9SRyan MoellerOutputdir: %s 459b8052df9SRyan MoellerTests: %s 460b8052df9SRyan MoellerTimeout: %d 461b8052df9SRyan MoellerUser: %s 462b8052df9SRyan MoellerPre: %s%s 463b8052df9SRyan MoellerPost: %s%s 464b8052df9SRyan MoellerTags: %s 465b8052df9SRyan Moeller''' % (self.pathname, self.identifier, self.outputdir, self.tests, 466b8052df9SRyan Moeller self.timeout, self.user, self.pre, pre_user, self.post, post_user, 467b8052df9SRyan Moeller self.tags) 468d583b39bSJohn Wren Kennedy 46978801af7SAndy Fiddaman def filter(self, keeplist): 47078801af7SAndy Fiddaman self.tests = [ x for x in self.tests if x in keeplist ] 47178801af7SAndy Fiddaman 472d583b39bSJohn Wren Kennedy def verify(self, logger): 473d583b39bSJohn Wren Kennedy """ 474d583b39bSJohn Wren Kennedy Check the pre/post scripts, user and tests in this TestGroup. Omit 475d583b39bSJohn Wren Kennedy the TestGroup entirely, or simply delete the relevant tests in the 476d583b39bSJohn Wren Kennedy group, if that's all that's required. 477d583b39bSJohn Wren Kennedy """ 478d583b39bSJohn Wren Kennedy # If the pre or post scripts are relative pathnames, convert to 479d583b39bSJohn Wren Kennedy # absolute, so they stand a chance of passing verification. 480d583b39bSJohn Wren Kennedy if len(self.pre) and not os.path.isabs(self.pre): 481d583b39bSJohn Wren Kennedy self.pre = os.path.join(self.pathname, self.pre) 482d583b39bSJohn Wren Kennedy if len(self.post) and not os.path.isabs(self.post): 483d583b39bSJohn Wren Kennedy self.post = os.path.join(self.pathname, self.post) 484d583b39bSJohn Wren Kennedy 485d583b39bSJohn Wren Kennedy auxfiles = [self.pre, self.post] 486d583b39bSJohn Wren Kennedy users = [self.pre_user, self.user, self.post_user] 487d583b39bSJohn Wren Kennedy 488d583b39bSJohn Wren Kennedy for f in [f for f in auxfiles if len(f)]: 489d583b39bSJohn Wren Kennedy if self.pathname != os.path.dirname(f): 490d583b39bSJohn Wren Kennedy logger.info("Warning: TestGroup '%s' not added to this run. " 491d583b39bSJohn Wren Kennedy "Auxiliary script '%s' exists in a different " 492d583b39bSJohn Wren Kennedy "directory." % (self.pathname, f)) 493d583b39bSJohn Wren Kennedy return False 494d583b39bSJohn Wren Kennedy 495d583b39bSJohn Wren Kennedy if not verify_file(f): 496d583b39bSJohn Wren Kennedy logger.info("Warning: TestGroup '%s' not added to this run. " 497d583b39bSJohn Wren Kennedy "Auxiliary script '%s' failed verification." % 498d583b39bSJohn Wren Kennedy (self.pathname, f)) 499d583b39bSJohn Wren Kennedy return False 500d583b39bSJohn Wren Kennedy 501d583b39bSJohn Wren Kennedy for user in [user for user in users if len(user)]: 502d583b39bSJohn Wren Kennedy if not verify_user(user, logger): 503d583b39bSJohn Wren Kennedy logger.info("Not adding TestGroup '%s' to this run." % 504d583b39bSJohn Wren Kennedy self.pathname) 505d583b39bSJohn Wren Kennedy return False 506d583b39bSJohn Wren Kennedy 507d583b39bSJohn Wren Kennedy # If one of the tests is invalid, delete it, log it, and drive on. 508dfc11533SChris Williamson self.tests[:] = [f for f in self.tests if 509dfc11533SChris Williamson verify_file(os.path.join(self.pathname, f))] 510d583b39bSJohn Wren Kennedy 5111a2acdcdSAndy Fiddaman return len(self.tests) != 0 512d583b39bSJohn Wren Kennedy 513d583b39bSJohn Wren Kennedy def run(self, logger, options): 514d583b39bSJohn Wren Kennedy """ 515d583b39bSJohn Wren Kennedy Create Cmd instances for the pre/post scripts. If the pre script 516d583b39bSJohn Wren Kennedy doesn't pass, skip all the tests in this TestGroup. Run the post 517d583b39bSJohn Wren Kennedy script regardless. 518d583b39bSJohn Wren Kennedy """ 519b8052df9SRyan Moeller # tags assigned to this test group also include the test names 520b8052df9SRyan Moeller if options.tags and not set(self.tags).intersection(set(options.tags)): 521b8052df9SRyan Moeller return 522b8052df9SRyan Moeller 523f9a78bf4SJohn Wren Kennedy odir = os.path.join(self.outputdir, os.path.basename(self.pre)) 524f9a78bf4SJohn Wren Kennedy pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout, 525b8052df9SRyan Moeller user=self.pre_user, identifier=self.identifier) 526f9a78bf4SJohn Wren Kennedy odir = os.path.join(self.outputdir, os.path.basename(self.post)) 527f9a78bf4SJohn Wren Kennedy posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout, 528b8052df9SRyan Moeller user=self.post_user, identifier=self.identifier) 529d583b39bSJohn Wren Kennedy 530d583b39bSJohn Wren Kennedy cont = True 531d583b39bSJohn Wren Kennedy if len(pretest.pathname): 532d583b39bSJohn Wren Kennedy pretest.run(options) 5331a2acdcdSAndy Fiddaman cont = pretest.result.result == 'PASS' 534d583b39bSJohn Wren Kennedy pretest.log(logger, options) 535d583b39bSJohn Wren Kennedy 536d583b39bSJohn Wren Kennedy for fname in self.tests: 537d583b39bSJohn Wren Kennedy test = Cmd(os.path.join(self.pathname, fname), 538d583b39bSJohn Wren Kennedy outputdir=os.path.join(self.outputdir, fname), 539b8052df9SRyan Moeller timeout=self.timeout, user=self.user, 540b8052df9SRyan Moeller identifier=self.identifier) 541d583b39bSJohn Wren Kennedy if cont: 542d583b39bSJohn Wren Kennedy test.run(options) 543d583b39bSJohn Wren Kennedy else: 544d583b39bSJohn Wren Kennedy test.skip() 545d583b39bSJohn Wren Kennedy 546d583b39bSJohn Wren Kennedy test.log(logger, options) 547d583b39bSJohn Wren Kennedy 548d583b39bSJohn Wren Kennedy if len(posttest.pathname): 549d583b39bSJohn Wren Kennedy posttest.run(options) 550d583b39bSJohn Wren Kennedy posttest.log(logger, options) 551d583b39bSJohn Wren Kennedy 552d583b39bSJohn Wren Kennedy 553d583b39bSJohn Wren Kennedyclass TestRun(object): 554d583b39bSJohn Wren Kennedy props = ['quiet', 'outputdir'] 555d583b39bSJohn Wren Kennedy 556d583b39bSJohn Wren Kennedy def __init__(self, options): 557d583b39bSJohn Wren Kennedy self.tests = {} 558d583b39bSJohn Wren Kennedy self.testgroups = {} 559d583b39bSJohn Wren Kennedy self.starttime = time() 560d583b39bSJohn Wren Kennedy self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S') 561d583b39bSJohn Wren Kennedy self.outputdir = os.path.join(options.outputdir, self.timestamp) 562d583b39bSJohn Wren Kennedy self.logger = self.setup_logging(options) 563d583b39bSJohn Wren Kennedy self.defaults = [ 564d583b39bSJohn Wren Kennedy ('outputdir', BASEDIR), 565d583b39bSJohn Wren Kennedy ('quiet', False), 566d583b39bSJohn Wren Kennedy ('timeout', 60), 567d583b39bSJohn Wren Kennedy ('user', ''), 568d583b39bSJohn Wren Kennedy ('pre', ''), 569d583b39bSJohn Wren Kennedy ('pre_user', ''), 570d583b39bSJohn Wren Kennedy ('post', ''), 571b8052df9SRyan Moeller ('post_user', ''), 572b8052df9SRyan Moeller ('tags', []) 573d583b39bSJohn Wren Kennedy ] 574d583b39bSJohn Wren Kennedy 575d583b39bSJohn Wren Kennedy def __str__(self): 576d583b39bSJohn Wren Kennedy s = 'TestRun:\n outputdir: %s\n' % self.outputdir 577d583b39bSJohn Wren Kennedy s += 'TESTS:\n' 578d583b39bSJohn Wren Kennedy for key in sorted(self.tests.keys()): 579d583b39bSJohn Wren Kennedy s += '%s%s' % (self.tests[key].__str__(), '\n') 580d583b39bSJohn Wren Kennedy s += 'TESTGROUPS:\n' 581d583b39bSJohn Wren Kennedy for key in sorted(self.testgroups.keys()): 582d583b39bSJohn Wren Kennedy s += '%s%s' % (self.testgroups[key].__str__(), '\n') 583d583b39bSJohn Wren Kennedy return s 584d583b39bSJohn Wren Kennedy 585d583b39bSJohn Wren Kennedy def addtest(self, pathname, options): 586d583b39bSJohn Wren Kennedy """ 587d583b39bSJohn Wren Kennedy Create a new Test, and apply any properties that were passed in 588d583b39bSJohn Wren Kennedy from the command line. If it passes verification, add it to the 589d583b39bSJohn Wren Kennedy TestRun. 590d583b39bSJohn Wren Kennedy """ 591d583b39bSJohn Wren Kennedy test = Test(pathname) 592d583b39bSJohn Wren Kennedy for prop in Test.props: 593d583b39bSJohn Wren Kennedy setattr(test, prop, getattr(options, prop)) 594d583b39bSJohn Wren Kennedy 595d583b39bSJohn Wren Kennedy if test.verify(self.logger): 596d583b39bSJohn Wren Kennedy self.tests[pathname] = test 597d583b39bSJohn Wren Kennedy 598d583b39bSJohn Wren Kennedy def addtestgroup(self, dirname, filenames, options): 599d583b39bSJohn Wren Kennedy """ 600d583b39bSJohn Wren Kennedy Create a new TestGroup, and apply any properties that were passed 601d583b39bSJohn Wren Kennedy in from the command line. If it passes verification, add it to the 602d583b39bSJohn Wren Kennedy TestRun. 603d583b39bSJohn Wren Kennedy """ 604d583b39bSJohn Wren Kennedy if dirname not in self.testgroups: 605d583b39bSJohn Wren Kennedy testgroup = TestGroup(dirname) 606d583b39bSJohn Wren Kennedy for prop in Test.props: 607d583b39bSJohn Wren Kennedy setattr(testgroup, prop, getattr(options, prop)) 608d583b39bSJohn Wren Kennedy 609d583b39bSJohn Wren Kennedy # Prevent pre/post scripts from running as regular tests 610d583b39bSJohn Wren Kennedy for f in [testgroup.pre, testgroup.post]: 611d583b39bSJohn Wren Kennedy if f in filenames: 612d583b39bSJohn Wren Kennedy del filenames[filenames.index(f)] 613d583b39bSJohn Wren Kennedy 614d583b39bSJohn Wren Kennedy self.testgroups[dirname] = testgroup 615d583b39bSJohn Wren Kennedy self.testgroups[dirname].tests = sorted(filenames) 616d583b39bSJohn Wren Kennedy 617d583b39bSJohn Wren Kennedy testgroup.verify(self.logger) 618d583b39bSJohn Wren Kennedy 61978801af7SAndy Fiddaman def filter(self, keeplist): 62078801af7SAndy Fiddaman for group in list(self.testgroups.keys()): 62178801af7SAndy Fiddaman if group not in keeplist: 62278801af7SAndy Fiddaman del self.testgroups[group] 62378801af7SAndy Fiddaman continue 62478801af7SAndy Fiddaman 62578801af7SAndy Fiddaman g = self.testgroups[group] 62678801af7SAndy Fiddaman 62778801af7SAndy Fiddaman if g.pre and os.path.basename(g.pre) in keeplist[group]: 62878801af7SAndy Fiddaman continue 62978801af7SAndy Fiddaman 63078801af7SAndy Fiddaman g.filter(keeplist[group]) 63178801af7SAndy Fiddaman 63278801af7SAndy Fiddaman for test in list(self.tests.keys()): 63378801af7SAndy Fiddaman directory, base = os.path.split(test) 63478801af7SAndy Fiddaman if directory not in keeplist or base not in keeplist[directory]: 63578801af7SAndy Fiddaman del self.tests[test] 63678801af7SAndy Fiddaman 637d583b39bSJohn Wren Kennedy def read(self, logger, options): 638d583b39bSJohn Wren Kennedy """ 639d583b39bSJohn Wren Kennedy Read in the specified runfile, and apply the TestRun properties 640d583b39bSJohn Wren Kennedy listed in the 'DEFAULT' section to our TestRun. Then read each 641d583b39bSJohn Wren Kennedy section, and apply the appropriate properties to the Test or 642d583b39bSJohn Wren Kennedy TestGroup. Properties from individual sections override those set 643d583b39bSJohn Wren Kennedy in the 'DEFAULT' section. If the Test or TestGroup passes 644d583b39bSJohn Wren Kennedy verification, add it to the TestRun. 645d583b39bSJohn Wren Kennedy """ 646b0858fdcSAlexander Pyhalov config = configparser.RawConfigParser() 647b8052df9SRyan Moeller parsed = config.read(options.runfiles) 648b8052df9SRyan Moeller failed = options.runfiles - set(parsed) 649b8052df9SRyan Moeller if len(failed): 650b8052df9SRyan Moeller files = ' '.join(sorted(failed)) 651b8052df9SRyan Moeller fail("Couldn't read config files: %s" % files) 652d583b39bSJohn Wren Kennedy 653d583b39bSJohn Wren Kennedy for opt in TestRun.props: 654d583b39bSJohn Wren Kennedy if config.has_option('DEFAULT', opt): 655d583b39bSJohn Wren Kennedy setattr(self, opt, config.get('DEFAULT', opt)) 656d583b39bSJohn Wren Kennedy self.outputdir = os.path.join(self.outputdir, self.timestamp) 657d583b39bSJohn Wren Kennedy 658b8052df9SRyan Moeller testdir = options.testdir 659b8052df9SRyan Moeller 660d583b39bSJohn Wren Kennedy for section in config.sections(): 661a0955b86SJohn Levon if ('arch' in config.options(section) and 662a0955b86SJohn Levon platform.machine() != config.get(section, 'arch')): 663a0955b86SJohn Levon continue 664a0955b86SJohn Levon 665b8052df9SRyan Moeller parts = section.split(':', 1) 666b8052df9SRyan Moeller sectiondir = parts[0] 667b8052df9SRyan Moeller identifier = parts[1] if len(parts) == 2 else None 668b8052df9SRyan Moeller if os.path.isdir(sectiondir): 669b8052df9SRyan Moeller pathname = sectiondir 670b8052df9SRyan Moeller elif os.path.isdir(os.path.join(testdir, sectiondir)): 671b8052df9SRyan Moeller pathname = os.path.join(testdir, sectiondir) 672b8052df9SRyan Moeller else: 673b8052df9SRyan Moeller pathname = sectiondir 674b8052df9SRyan Moeller 675b8052df9SRyan Moeller testgroup = TestGroup(os.path.abspath(pathname), 676b8052df9SRyan Moeller identifier=identifier) 677d583b39bSJohn Wren Kennedy if 'tests' in config.options(section): 678d583b39bSJohn Wren Kennedy for prop in TestGroup.props: 679f9a78bf4SJohn Wren Kennedy for sect in ['DEFAULT', section]: 680f9a78bf4SJohn Wren Kennedy if config.has_option(sect, prop): 681b8052df9SRyan Moeller if prop == 'tags': 682b8052df9SRyan Moeller setattr(testgroup, prop, 683b8052df9SRyan Moeller eval(config.get(sect, prop))) 684b8052df9SRyan Moeller else: 685b8052df9SRyan Moeller setattr(testgroup, prop, 686b8052df9SRyan Moeller config.get(sect, prop)) 687d583b39bSJohn Wren Kennedy 688d583b39bSJohn Wren Kennedy # Repopulate tests using eval to convert the string to a list 689d583b39bSJohn Wren Kennedy testgroup.tests = eval(config.get(section, 'tests')) 690d583b39bSJohn Wren Kennedy 691d583b39bSJohn Wren Kennedy if testgroup.verify(logger): 692d583b39bSJohn Wren Kennedy self.testgroups[section] = testgroup 693dfc11533SChris Williamson 694dfc11533SChris Williamson elif 'autotests' in config.options(section): 695dfc11533SChris Williamson for prop in TestGroup.props: 696dfc11533SChris Williamson for sect in ['DEFAULT', section]: 697dfc11533SChris Williamson if config.has_option(sect, prop): 698dfc11533SChris Williamson setattr(testgroup, prop, config.get(sect, prop)) 699dfc11533SChris Williamson 700b8052df9SRyan Moeller filenames = os.listdir(pathname) 701dfc11533SChris Williamson # only files starting with "tst." are considered tests 702dfc11533SChris Williamson filenames = [f for f in filenames if f.startswith("tst.")] 703dfc11533SChris Williamson testgroup.tests = sorted(filenames) 704dfc11533SChris Williamson 705dfc11533SChris Williamson if testgroup.verify(logger): 706dfc11533SChris Williamson self.testgroups[section] = testgroup 707d583b39bSJohn Wren Kennedy else: 708d583b39bSJohn Wren Kennedy test = Test(section) 709d583b39bSJohn Wren Kennedy for prop in Test.props: 710f9a78bf4SJohn Wren Kennedy for sect in ['DEFAULT', section]: 711f9a78bf4SJohn Wren Kennedy if config.has_option(sect, prop): 712f9a78bf4SJohn Wren Kennedy setattr(test, prop, config.get(sect, prop)) 713f9a78bf4SJohn Wren Kennedy 714d583b39bSJohn Wren Kennedy if test.verify(logger): 715d583b39bSJohn Wren Kennedy self.tests[section] = test 716d583b39bSJohn Wren Kennedy 717d583b39bSJohn Wren Kennedy def write(self, options): 718d583b39bSJohn Wren Kennedy """ 719d583b39bSJohn Wren Kennedy Create a configuration file for editing and later use. The 720d583b39bSJohn Wren Kennedy 'DEFAULT' section of the config file is created from the 721d583b39bSJohn Wren Kennedy properties that were specified on the command line. Tests are 722d583b39bSJohn Wren Kennedy simply added as sections that inherit everything from the 723d583b39bSJohn Wren Kennedy 'DEFAULT' section. TestGroups are the same, except they get an 724d583b39bSJohn Wren Kennedy option including all the tests to run in that directory. 725d583b39bSJohn Wren Kennedy """ 726d583b39bSJohn Wren Kennedy 727d583b39bSJohn Wren Kennedy defaults = dict([(prop, getattr(options, prop)) for prop, _ in 728f9a78bf4SJohn Wren Kennedy self.defaults]) 729b0858fdcSAlexander Pyhalov config = configparser.RawConfigParser(defaults) 730d583b39bSJohn Wren Kennedy 731d583b39bSJohn Wren Kennedy for test in sorted(self.tests.keys()): 732d583b39bSJohn Wren Kennedy config.add_section(test) 73378801af7SAndy Fiddaman for prop in Test.props: 73478801af7SAndy Fiddaman if prop not in self.props: 73578801af7SAndy Fiddaman config.set(testgroup, prop, 73678801af7SAndy Fiddaman getattr(self.testgroups[testgroup], prop)) 737d583b39bSJohn Wren Kennedy 738d583b39bSJohn Wren Kennedy for testgroup in sorted(self.testgroups.keys()): 739d583b39bSJohn Wren Kennedy config.add_section(testgroup) 740d583b39bSJohn Wren Kennedy config.set(testgroup, 'tests', self.testgroups[testgroup].tests) 74178801af7SAndy Fiddaman for prop in TestGroup.props: 74278801af7SAndy Fiddaman if prop not in self.props: 74378801af7SAndy Fiddaman config.set(testgroup, prop, 74478801af7SAndy Fiddaman getattr(self.testgroups[testgroup], prop)) 745d583b39bSJohn Wren Kennedy 746d583b39bSJohn Wren Kennedy try: 747d583b39bSJohn Wren Kennedy with open(options.template, 'w') as f: 748d583b39bSJohn Wren Kennedy return config.write(f) 749d583b39bSJohn Wren Kennedy except IOError: 750d583b39bSJohn Wren Kennedy fail('Could not open \'%s\' for writing.' % options.template) 751d583b39bSJohn Wren Kennedy 752f9a78bf4SJohn Wren Kennedy def complete_outputdirs(self): 753d583b39bSJohn Wren Kennedy """ 754d583b39bSJohn Wren Kennedy Collect all the pathnames for Tests, and TestGroups. Work 755d583b39bSJohn Wren Kennedy backwards one pathname component at a time, to create a unique 756d583b39bSJohn Wren Kennedy directory name in which to deposit test output. Tests will be able 757d583b39bSJohn Wren Kennedy to write output files directly in the newly modified outputdir. 758d583b39bSJohn Wren Kennedy TestGroups will be able to create one subdirectory per test in the 759d583b39bSJohn Wren Kennedy outputdir, and are guaranteed uniqueness because a group can only 760d583b39bSJohn Wren Kennedy contain files in one directory. Pre and post tests will create a 761d583b39bSJohn Wren Kennedy directory rooted at the outputdir of the Test or TestGroup in 762d583b39bSJohn Wren Kennedy question for their output. 763d583b39bSJohn Wren Kennedy """ 764d583b39bSJohn Wren Kennedy done = False 765d583b39bSJohn Wren Kennedy components = 0 766b0858fdcSAlexander Pyhalov tmp_dict = dict(list(self.tests.items()) + list(self.testgroups.items())) 767d583b39bSJohn Wren Kennedy total = len(tmp_dict) 768d583b39bSJohn Wren Kennedy base = self.outputdir 769d583b39bSJohn Wren Kennedy 770d583b39bSJohn Wren Kennedy while not done: 771d583b39bSJohn Wren Kennedy l = [] 772d583b39bSJohn Wren Kennedy components -= 1 773b0858fdcSAlexander Pyhalov for testfile in list(tmp_dict.keys()): 774d583b39bSJohn Wren Kennedy uniq = '/'.join(testfile.split('/')[components:]).lstrip('/') 775f9a78bf4SJohn Wren Kennedy if uniq not in l: 776d583b39bSJohn Wren Kennedy l.append(uniq) 777d583b39bSJohn Wren Kennedy tmp_dict[testfile].outputdir = os.path.join(base, uniq) 778d583b39bSJohn Wren Kennedy else: 779d583b39bSJohn Wren Kennedy break 780d583b39bSJohn Wren Kennedy done = total == len(l) 781d583b39bSJohn Wren Kennedy 782d583b39bSJohn Wren Kennedy def setup_logging(self, options): 783d583b39bSJohn Wren Kennedy """ 784d583b39bSJohn Wren Kennedy Two loggers are set up here. The first is for the logfile which 785d583b39bSJohn Wren Kennedy will contain one line summarizing the test, including the test 786d583b39bSJohn Wren Kennedy name, result, and running time. This logger will also capture the 787d583b39bSJohn Wren Kennedy timestamped combined stdout and stderr of each run. The second 788d583b39bSJohn Wren Kennedy logger is optional console output, which will contain only the one 789d583b39bSJohn Wren Kennedy line summary. The loggers are initialized at two different levels 790d583b39bSJohn Wren Kennedy to facilitate segregating the output. 791d583b39bSJohn Wren Kennedy """ 792d583b39bSJohn Wren Kennedy if options.dryrun is True: 793d583b39bSJohn Wren Kennedy return 794d583b39bSJohn Wren Kennedy 795d583b39bSJohn Wren Kennedy testlogger = logging.getLogger(__name__) 796d583b39bSJohn Wren Kennedy testlogger.setLevel(logging.DEBUG) 797d583b39bSJohn Wren Kennedy 79878801af7SAndy Fiddaman if not options.template: 799d583b39bSJohn Wren Kennedy try: 800d583b39bSJohn Wren Kennedy old = os.umask(0) 801b0858fdcSAlexander Pyhalov os.makedirs(self.outputdir, mode=0o777) 802d583b39bSJohn Wren Kennedy os.umask(old) 803b0858fdcSAlexander Pyhalov except OSError as e: 804d583b39bSJohn Wren Kennedy fail('%s' % e) 805d583b39bSJohn Wren Kennedy filename = os.path.join(self.outputdir, 'log') 806d583b39bSJohn Wren Kennedy 80700688ee4SWill Andrews logfile = WatchedFileHandlerClosed(filename) 808d583b39bSJohn Wren Kennedy logfile.setLevel(logging.DEBUG) 809d583b39bSJohn Wren Kennedy logfilefmt = logging.Formatter('%(message)s') 810d583b39bSJohn Wren Kennedy logfile.setFormatter(logfilefmt) 811d583b39bSJohn Wren Kennedy testlogger.addHandler(logfile) 812d583b39bSJohn Wren Kennedy 813d583b39bSJohn Wren Kennedy cons = logging.StreamHandler() 814d583b39bSJohn Wren Kennedy cons.setLevel(logging.INFO) 815d583b39bSJohn Wren Kennedy consfmt = logging.Formatter('%(message)s') 816d583b39bSJohn Wren Kennedy cons.setFormatter(consfmt) 817d583b39bSJohn Wren Kennedy testlogger.addHandler(cons) 818d583b39bSJohn Wren Kennedy 819d583b39bSJohn Wren Kennedy return testlogger 820d583b39bSJohn Wren Kennedy 821d583b39bSJohn Wren Kennedy def run(self, options): 822d583b39bSJohn Wren Kennedy """ 823d583b39bSJohn Wren Kennedy Walk through all the Tests and TestGroups, calling run(). 824d583b39bSJohn Wren Kennedy """ 82564ee6612SChris Fraire if not options.dryrun: 82664ee6612SChris Fraire try: 82764ee6612SChris Fraire os.chdir(self.outputdir) 82864ee6612SChris Fraire except OSError: 82964ee6612SChris Fraire fail('Could not change to directory %s' % self.outputdir) 830d583b39bSJohn Wren Kennedy for test in sorted(self.tests.keys()): 831d583b39bSJohn Wren Kennedy self.tests[test].run(self.logger, options) 832d583b39bSJohn Wren Kennedy for testgroup in sorted(self.testgroups.keys()): 833d583b39bSJohn Wren Kennedy self.testgroups[testgroup].run(self.logger, options) 834d583b39bSJohn Wren Kennedy 835d583b39bSJohn Wren Kennedy def summary(self): 8361a2acdcdSAndy Fiddaman if Result.total == 0: 83746593baaSToomas Soome print('No tests to run') 838d583b39bSJohn Wren Kennedy return 839d583b39bSJohn Wren Kennedy 840b0858fdcSAlexander Pyhalov print('\nResults Summary') 841b0858fdcSAlexander Pyhalov for key in list(Result.runresults.keys()): 8421a2acdcdSAndy Fiddaman if Result.runresults[key] != 0: 843b0858fdcSAlexander Pyhalov print('%s\t% 4d' % (key, Result.runresults[key])) 844d583b39bSJohn Wren Kennedy 845d583b39bSJohn Wren Kennedy m, s = divmod(time() - self.starttime, 60) 846d583b39bSJohn Wren Kennedy h, m = divmod(m, 60) 847b0858fdcSAlexander Pyhalov print('\nRunning Time:\t%02d:%02d:%02d' % (h, m, s)) 848b0858fdcSAlexander Pyhalov print('Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) / 849b0858fdcSAlexander Pyhalov float(Result.total)) * 100)) 850b0858fdcSAlexander Pyhalov print('Log directory:\t%s' % self.outputdir) 851d583b39bSJohn Wren Kennedy 852d583b39bSJohn Wren Kennedy 853d583b39bSJohn Wren Kennedydef verify_file(pathname): 854d583b39bSJohn Wren Kennedy """ 855d583b39bSJohn Wren Kennedy Verify that the supplied pathname is an executable regular file. 856d583b39bSJohn Wren Kennedy """ 857d583b39bSJohn Wren Kennedy if os.path.isdir(pathname) or os.path.islink(pathname): 858d583b39bSJohn Wren Kennedy return False 859d583b39bSJohn Wren Kennedy 860d583b39bSJohn Wren Kennedy if os.path.isfile(pathname) and os.access(pathname, os.X_OK): 861d583b39bSJohn Wren Kennedy return True 862d583b39bSJohn Wren Kennedy 863d583b39bSJohn Wren Kennedy return False 864d583b39bSJohn Wren Kennedy 865d583b39bSJohn Wren Kennedy 866d583b39bSJohn Wren Kennedydef verify_user(user, logger): 867d583b39bSJohn Wren Kennedy """ 868d583b39bSJohn Wren Kennedy Verify that the specified user exists on this system, and can execute 869d583b39bSJohn Wren Kennedy sudo without being prompted for a password. 870d583b39bSJohn Wren Kennedy """ 871d583b39bSJohn Wren Kennedy testcmd = [SUDO, '-n', '-u', user, TRUE] 872d583b39bSJohn Wren Kennedy 873d583b39bSJohn Wren Kennedy if user in Cmd.verified_users: 874d583b39bSJohn Wren Kennedy return True 875d583b39bSJohn Wren Kennedy 876d583b39bSJohn Wren Kennedy try: 877d583b39bSJohn Wren Kennedy _ = getpwnam(user) 878d583b39bSJohn Wren Kennedy except KeyError: 879d583b39bSJohn Wren Kennedy logger.info("Warning: user '%s' does not exist.", user) 880d583b39bSJohn Wren Kennedy return False 881d583b39bSJohn Wren Kennedy 882d583b39bSJohn Wren Kennedy p = Popen(testcmd) 883d583b39bSJohn Wren Kennedy p.wait() 8841a2acdcdSAndy Fiddaman if p.returncode != 0: 8855e989a96SDavid Höppner logger.info("Warning: user '%s' cannot use passwordless sudo.", user) 886d583b39bSJohn Wren Kennedy return False 887d583b39bSJohn Wren Kennedy else: 888d583b39bSJohn Wren Kennedy Cmd.verified_users.append(user) 889d583b39bSJohn Wren Kennedy 890d583b39bSJohn Wren Kennedy return True 891d583b39bSJohn Wren Kennedy 892d583b39bSJohn Wren Kennedy 893d583b39bSJohn Wren Kennedydef find_tests(testrun, options): 894d583b39bSJohn Wren Kennedy """ 895d583b39bSJohn Wren Kennedy For the given list of pathnames, add files as Tests. For directories, 896d583b39bSJohn Wren Kennedy if do_groups is True, add the directory as a TestGroup. If False, 897d583b39bSJohn Wren Kennedy recursively search for executable files. 898d583b39bSJohn Wren Kennedy """ 899d583b39bSJohn Wren Kennedy 900d583b39bSJohn Wren Kennedy for p in sorted(options.pathnames): 901d583b39bSJohn Wren Kennedy if os.path.isdir(p): 902d583b39bSJohn Wren Kennedy for dirname, _, filenames in os.walk(p): 903d583b39bSJohn Wren Kennedy if options.do_groups: 904d583b39bSJohn Wren Kennedy testrun.addtestgroup(dirname, filenames, options) 905d583b39bSJohn Wren Kennedy else: 906d583b39bSJohn Wren Kennedy for f in sorted(filenames): 907d583b39bSJohn Wren Kennedy testrun.addtest(os.path.join(dirname, f), options) 908d583b39bSJohn Wren Kennedy else: 909d583b39bSJohn Wren Kennedy testrun.addtest(p, options) 910d583b39bSJohn Wren Kennedy 911d583b39bSJohn Wren Kennedy 91278801af7SAndy Fiddamandef filter_tests(testrun, options): 91378801af7SAndy Fiddaman try: 9144947898cSToomas Soome fh = open(options.logfile, "r", errors='replace') 91578801af7SAndy Fiddaman except Exception as e: 91678801af7SAndy Fiddaman fail('%s' % e) 91778801af7SAndy Fiddaman 91878801af7SAndy Fiddaman failed = {} 91978801af7SAndy Fiddaman while True: 92078801af7SAndy Fiddaman line = fh.readline() 92178801af7SAndy Fiddaman if not line: 92278801af7SAndy Fiddaman break 92378801af7SAndy Fiddaman m = re.match(r'Test: (.*)/(\S+).*\[FAIL\]', line) 92478801af7SAndy Fiddaman if not m: 92578801af7SAndy Fiddaman continue 92678801af7SAndy Fiddaman group, test = m.group(1, 2) 927*f4d2cf74SToomas Soome m = re.match(re.escape(options.testdir) + r'(.*)', group) 928*f4d2cf74SToomas Soome if m: 929*f4d2cf74SToomas Soome group = m.group(1) 93078801af7SAndy Fiddaman try: 93178801af7SAndy Fiddaman failed[group].append(test) 93278801af7SAndy Fiddaman except KeyError: 93378801af7SAndy Fiddaman failed[group] = [ test ] 93478801af7SAndy Fiddaman fh.close() 93578801af7SAndy Fiddaman 93678801af7SAndy Fiddaman testrun.filter(failed) 93778801af7SAndy Fiddaman 93878801af7SAndy Fiddaman 939d583b39bSJohn Wren Kennedydef fail(retstr, ret=1): 940b0858fdcSAlexander Pyhalov print('%s: %s' % (argv[0], retstr)) 941d583b39bSJohn Wren Kennedy exit(ret) 942d583b39bSJohn Wren Kennedy 943d583b39bSJohn Wren Kennedy 944d583b39bSJohn Wren Kennedydef options_cb(option, opt_str, value, parser): 945b8052df9SRyan Moeller path_options = ['outputdir', 'template', 'testdir', 'logfile'] 946d583b39bSJohn Wren Kennedy 947d583b39bSJohn Wren Kennedy if opt_str in parser.rargs: 948d583b39bSJohn Wren Kennedy fail('%s may only be specified once.' % opt_str) 949d583b39bSJohn Wren Kennedy 950b8052df9SRyan Moeller if option.dest == 'runfiles': 951b8052df9SRyan Moeller parser.values.cmd = 'rdconfig' 952b8052df9SRyan Moeller value = set(os.path.abspath(p) for p in value.split(',')) 953b8052df9SRyan Moeller if option.dest == 'tags': 954b8052df9SRyan Moeller value = [x.strip() for x in value.split(',')] 955b8052df9SRyan Moeller 956d583b39bSJohn Wren Kennedy if option.dest in path_options: 957d583b39bSJohn Wren Kennedy setattr(parser.values, option.dest, os.path.abspath(value)) 95878801af7SAndy Fiddaman else: 95978801af7SAndy Fiddaman setattr(parser.values, option.dest, value) 960d583b39bSJohn Wren Kennedy 961d583b39bSJohn Wren Kennedy 962d583b39bSJohn Wren Kennedydef parse_args(): 963d583b39bSJohn Wren Kennedy parser = OptionParser() 964d583b39bSJohn Wren Kennedy parser.add_option('-c', action='callback', callback=options_cb, 965b8052df9SRyan Moeller type='string', dest='runfiles', metavar='runfiles', 966b8052df9SRyan Moeller help='Specify tests to run via config files.') 967d583b39bSJohn Wren Kennedy parser.add_option('-d', action='store_true', default=False, dest='dryrun', 968d583b39bSJohn Wren Kennedy help='Dry run. Print tests, but take no other action.') 96978801af7SAndy Fiddaman parser.add_option('-l', action='callback', callback=options_cb, 97078801af7SAndy Fiddaman default=None, dest='logfile', metavar='logfile', 97178801af7SAndy Fiddaman type='string', 97278801af7SAndy Fiddaman help='Read logfile and re-run tests which failed.') 973d583b39bSJohn Wren Kennedy parser.add_option('-g', action='store_true', default=False, 974d583b39bSJohn Wren Kennedy dest='do_groups', help='Make directories TestGroups.') 975d583b39bSJohn Wren Kennedy parser.add_option('-o', action='callback', callback=options_cb, 976d583b39bSJohn Wren Kennedy default=BASEDIR, dest='outputdir', type='string', 977d583b39bSJohn Wren Kennedy metavar='outputdir', help='Specify an output directory.') 978b8052df9SRyan Moeller parser.add_option('-i', action='callback', callback=options_cb, 979b8052df9SRyan Moeller default=TESTDIR, dest='testdir', type='string', 980b8052df9SRyan Moeller metavar='testdir', help='Specify a test directory.') 981d583b39bSJohn Wren Kennedy parser.add_option('-p', action='callback', callback=options_cb, 982d583b39bSJohn Wren Kennedy default='', dest='pre', metavar='script', 983d583b39bSJohn Wren Kennedy type='string', help='Specify a pre script.') 984d583b39bSJohn Wren Kennedy parser.add_option('-P', action='callback', callback=options_cb, 985d583b39bSJohn Wren Kennedy default='', dest='post', metavar='script', 986d583b39bSJohn Wren Kennedy type='string', help='Specify a post script.') 987d583b39bSJohn Wren Kennedy parser.add_option('-q', action='store_true', default=False, dest='quiet', 988d583b39bSJohn Wren Kennedy help='Silence on the console during a test run.') 989d583b39bSJohn Wren Kennedy parser.add_option('-t', action='callback', callback=options_cb, default=60, 990d583b39bSJohn Wren Kennedy dest='timeout', metavar='seconds', type='int', 991d583b39bSJohn Wren Kennedy help='Timeout (in seconds) for an individual test.') 992d583b39bSJohn Wren Kennedy parser.add_option('-u', action='callback', callback=options_cb, 993d583b39bSJohn Wren Kennedy default='', dest='user', metavar='user', type='string', 994d583b39bSJohn Wren Kennedy help='Specify a different user name to run as.') 995d583b39bSJohn Wren Kennedy parser.add_option('-w', action='callback', callback=options_cb, 996d583b39bSJohn Wren Kennedy default=None, dest='template', metavar='template', 997d583b39bSJohn Wren Kennedy type='string', help='Create a new config file.') 998d583b39bSJohn Wren Kennedy parser.add_option('-x', action='callback', callback=options_cb, default='', 999d583b39bSJohn Wren Kennedy dest='pre_user', metavar='pre_user', type='string', 1000d583b39bSJohn Wren Kennedy help='Specify a user to execute the pre script.') 1001d583b39bSJohn Wren Kennedy parser.add_option('-X', action='callback', callback=options_cb, default='', 1002d583b39bSJohn Wren Kennedy dest='post_user', metavar='post_user', type='string', 1003d583b39bSJohn Wren Kennedy help='Specify a user to execute the post script.') 1004b8052df9SRyan Moeller parser.add_option('-T', action='callback', callback=options_cb, default='', 1005b8052df9SRyan Moeller dest='tags', metavar='tags', type='string', 1006b8052df9SRyan Moeller help='Specify tags to execute specific test groups.') 1007d583b39bSJohn Wren Kennedy (options, pathnames) = parser.parse_args() 1008d583b39bSJohn Wren Kennedy 1009b8052df9SRyan Moeller if options.runfiles and len(pathnames): 1010d583b39bSJohn Wren Kennedy fail('Extraneous arguments.') 1011d583b39bSJohn Wren Kennedy 1012d583b39bSJohn Wren Kennedy options.pathnames = [os.path.abspath(path) for path in pathnames] 1013d583b39bSJohn Wren Kennedy 1014d583b39bSJohn Wren Kennedy return options 1015d583b39bSJohn Wren Kennedy 1016d583b39bSJohn Wren Kennedy 1017f9a78bf4SJohn Wren Kennedydef main(): 1018d583b39bSJohn Wren Kennedy options = parse_args() 101978801af7SAndy Fiddaman 1020d583b39bSJohn Wren Kennedy testrun = TestRun(options) 1021d583b39bSJohn Wren Kennedy 1022b8052df9SRyan Moeller if options.runfiles: 1023d583b39bSJohn Wren Kennedy testrun.read(testrun.logger, options) 102478801af7SAndy Fiddaman else: 1025d583b39bSJohn Wren Kennedy find_tests(testrun, options) 102678801af7SAndy Fiddaman 102778801af7SAndy Fiddaman if options.logfile: 102878801af7SAndy Fiddaman filter_tests(testrun, options) 102978801af7SAndy Fiddaman 103078801af7SAndy Fiddaman if options.template: 1031d583b39bSJohn Wren Kennedy testrun.write(options) 1032d583b39bSJohn Wren Kennedy exit(0) 1033d583b39bSJohn Wren Kennedy 1034f9a78bf4SJohn Wren Kennedy testrun.complete_outputdirs() 1035d583b39bSJohn Wren Kennedy testrun.run(options) 1036d583b39bSJohn Wren Kennedy testrun.summary() 10372491fc01SJohn Levon exit(retcode) 1038d583b39bSJohn Wren Kennedy 1039d583b39bSJohn Wren Kennedy 1040d583b39bSJohn Wren Kennedyif __name__ == '__main__': 1041f9a78bf4SJohn Wren Kennedy main() 1042