tests.cpp revision eb1a5f8de9f7ea602c373a710f531abbf81141c4
1//
2// Automated Testing Framework (atf)
3//
4// Copyright (c) 2007 The NetBSD Foundation, Inc.
5// All rights reserved.
6//
7// Redistribution and use in source and binary forms, with or without
8// modification, are permitted provided that the following conditions
9// are met:
10// 1. Redistributions of source code must retain the above copyright
11//    notice, this list of conditions and the following disclaimer.
12// 2. Redistributions in binary form must reproduce the above copyright
13//    notice, this list of conditions and the following disclaimer in the
14//    documentation and/or other materials provided with the distribution.
15//
16// THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
17// CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
18// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
19// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20// IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
21// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
23// GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
25// IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
26// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
27// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28//
29
30extern "C" {
31#include <sys/types.h>
32#include <sys/stat.h>
33#include <sys/time.h>
34#include <sys/wait.h>
35#include <signal.h>
36#include <unistd.h>
37}
38
39#include <algorithm>
40#include <cctype>
41#include <cerrno>
42#include <cstdlib>
43#include <cstring>
44#include <fstream>
45#include <iostream>
46#include <map>
47#include <memory>
48#include <sstream>
49#include <stdexcept>
50#include <vector>
51
52extern "C" {
53#include "atf-c/error.h"
54#include "atf-c/tc.h"
55#include "atf-c/utils.h"
56}
57
58#include "tests.hpp"
59
60#include "detail/application.hpp"
61#include "detail/auto_array.hpp"
62#include "detail/env.hpp"
63#include "detail/exceptions.hpp"
64#include "detail/fs.hpp"
65#include "detail/sanity.hpp"
66#include "detail/text.hpp"
67
68namespace impl = atf::tests;
69namespace detail = atf::tests::detail;
70#define IMPL_NAME "atf::tests"
71
72// ------------------------------------------------------------------------
73// The "atf_tp_writer" class.
74// ------------------------------------------------------------------------
75
76detail::atf_tp_writer::atf_tp_writer(std::ostream& os) :
77    m_os(os),
78    m_is_first(true)
79{
80    m_os << "Content-Type: application/X-atf-tp; version=\"1\"\n\n";
81}
82
83void
84detail::atf_tp_writer::start_tc(const std::string& ident)
85{
86    if (!m_is_first)
87        m_os << "\n";
88    m_os << "ident: " << ident << "\n";
89    m_os.flush();
90}
91
92void
93detail::atf_tp_writer::end_tc(void)
94{
95    if (m_is_first)
96        m_is_first = false;
97}
98
99void
100detail::atf_tp_writer::tc_meta_data(const std::string& name,
101                                    const std::string& value)
102{
103    PRE(name != "ident");
104    m_os << name << ": " << value << "\n";
105    m_os.flush();
106}
107
108// ------------------------------------------------------------------------
109// Free helper functions.
110// ------------------------------------------------------------------------
111
112bool
113detail::match(const std::string& regexp, const std::string& str)
114{
115    return atf::text::match(str, regexp);
116}
117
118// ------------------------------------------------------------------------
119// The "tc" class.
120// ------------------------------------------------------------------------
121
122static std::map< atf_tc_t*, impl::tc* > wraps;
123static std::map< const atf_tc_t*, const impl::tc* > cwraps;
124
125struct impl::tc_impl {
126private:
127    // Non-copyable.
128    tc_impl(const tc_impl&);
129    tc_impl& operator=(const tc_impl&);
130
131public:
132    std::string m_ident;
133    atf_tc_t m_tc;
134    bool m_has_cleanup;
135
136    tc_impl(const std::string& ident, const bool has_cleanup) :
137        m_ident(ident),
138        m_has_cleanup(has_cleanup)
139    {
140    }
141
142    static void
143    wrap_head(atf_tc_t *tc)
144    {
145        std::map< atf_tc_t*, impl::tc* >::iterator iter = wraps.find(tc);
146        INV(iter != wraps.end());
147        (*iter).second->head();
148    }
149
150    static void
151    wrap_body(const atf_tc_t *tc)
152    {
153        std::map< const atf_tc_t*, const impl::tc* >::const_iterator iter =
154            cwraps.find(tc);
155        INV(iter != cwraps.end());
156        try {
157            (*iter).second->body();
158        } catch (const std::exception& e) {
159            (*iter).second->fail("Caught unhandled exception: " + std::string(
160                                     e.what()));
161        } catch (...) {
162            (*iter).second->fail("Caught unknown exception");
163        }
164    }
165
166    static void
167    wrap_cleanup(const atf_tc_t *tc)
168    {
169        std::map< const atf_tc_t*, const impl::tc* >::const_iterator iter =
170            cwraps.find(tc);
171        INV(iter != cwraps.end());
172        (*iter).second->cleanup();
173    }
174};
175
176impl::tc::tc(const std::string& ident, const bool has_cleanup) :
177    pimpl(new tc_impl(ident, has_cleanup))
178{
179}
180
181impl::tc::~tc(void)
182{
183    cwraps.erase(&pimpl->m_tc);
184    wraps.erase(&pimpl->m_tc);
185
186    atf_tc_fini(&pimpl->m_tc);
187}
188
189void
190impl::tc::init(const vars_map& config)
191{
192    atf_error_t err;
193
194    auto_array< const char * > array(new const char*[(config.size() * 2) + 1]);
195    const char **ptr = array.get();
196    for (vars_map::const_iterator iter = config.begin();
197         iter != config.end(); iter++) {
198         *ptr = (*iter).first.c_str();
199         *(ptr + 1) = (*iter).second.c_str();
200         ptr += 2;
201    }
202    *ptr = NULL;
203
204    wraps[&pimpl->m_tc] = this;
205    cwraps[&pimpl->m_tc] = this;
206
207    err = atf_tc_init(&pimpl->m_tc, pimpl->m_ident.c_str(), pimpl->wrap_head,
208        pimpl->wrap_body, pimpl->m_has_cleanup ? pimpl->wrap_cleanup : NULL,
209        array.get());
210    if (atf_is_error(err))
211        throw_atf_error(err);
212}
213
214bool
215impl::tc::has_config_var(const std::string& var)
216    const
217{
218    return atf_tc_has_config_var(&pimpl->m_tc, var.c_str());
219}
220
221bool
222impl::tc::has_md_var(const std::string& var)
223    const
224{
225    return atf_tc_has_md_var(&pimpl->m_tc, var.c_str());
226}
227
228const std::string
229impl::tc::get_config_var(const std::string& var)
230    const
231{
232    return atf_tc_get_config_var(&pimpl->m_tc, var.c_str());
233}
234
235const std::string
236impl::tc::get_config_var(const std::string& var, const std::string& defval)
237    const
238{
239    return atf_tc_get_config_var_wd(&pimpl->m_tc, var.c_str(), defval.c_str());
240}
241
242const std::string
243impl::tc::get_md_var(const std::string& var)
244    const
245{
246    return atf_tc_get_md_var(&pimpl->m_tc, var.c_str());
247}
248
249const impl::vars_map
250impl::tc::get_md_vars(void)
251    const
252{
253    vars_map vars;
254
255    char **array = atf_tc_get_md_vars(&pimpl->m_tc);
256    try {
257        char **ptr;
258        for (ptr = array; *ptr != NULL; ptr += 2)
259            vars[*ptr] = *(ptr + 1);
260    } catch (...) {
261        atf_utils_free_charpp(array);
262        throw;
263    }
264
265    return vars;
266}
267
268void
269impl::tc::set_md_var(const std::string& var, const std::string& val)
270{
271    atf_error_t err = atf_tc_set_md_var(&pimpl->m_tc, var.c_str(), val.c_str());
272    if (atf_is_error(err))
273        throw_atf_error(err);
274}
275
276void
277impl::tc::run(const std::string& resfile)
278    const
279{
280    atf_error_t err = atf_tc_run(&pimpl->m_tc, resfile.c_str());
281    if (atf_is_error(err))
282        throw_atf_error(err);
283}
284
285void
286impl::tc::run_cleanup(void)
287    const
288{
289    atf_error_t err = atf_tc_cleanup(&pimpl->m_tc);
290    if (atf_is_error(err))
291        throw_atf_error(err);
292}
293
294void
295impl::tc::head(void)
296{
297}
298
299void
300impl::tc::cleanup(void)
301    const
302{
303}
304
305void
306impl::tc::require_prog(const std::string& prog)
307    const
308{
309    atf_tc_require_prog(prog.c_str());
310}
311
312void
313impl::tc::pass(void)
314{
315    atf_tc_pass();
316}
317
318void
319impl::tc::fail(const std::string& reason)
320{
321    atf_tc_fail("%s", reason.c_str());
322}
323
324void
325impl::tc::fail_nonfatal(const std::string& reason)
326{
327    atf_tc_fail_nonfatal("%s", reason.c_str());
328}
329
330void
331impl::tc::skip(const std::string& reason)
332{
333    atf_tc_skip("%s", reason.c_str());
334}
335
336void
337impl::tc::check_errno(const char* file, const int line, const int exp_errno,
338                      const char* expr_str, const bool result)
339{
340    atf_tc_check_errno(file, line, exp_errno, expr_str, result);
341}
342
343void
344impl::tc::require_errno(const char* file, const int line, const int exp_errno,
345                        const char* expr_str, const bool result)
346{
347    atf_tc_require_errno(file, line, exp_errno, expr_str, result);
348}
349
350void
351impl::tc::expect_pass(void)
352{
353    atf_tc_expect_pass();
354}
355
356void
357impl::tc::expect_fail(const std::string& reason)
358{
359    atf_tc_expect_fail("%s", reason.c_str());
360}
361
362void
363impl::tc::expect_exit(const int exitcode, const std::string& reason)
364{
365    atf_tc_expect_exit(exitcode, "%s", reason.c_str());
366}
367
368void
369impl::tc::expect_signal(const int signo, const std::string& reason)
370{
371    atf_tc_expect_signal(signo, "%s", reason.c_str());
372}
373
374void
375impl::tc::expect_death(const std::string& reason)
376{
377    atf_tc_expect_death("%s", reason.c_str());
378}
379
380void
381impl::tc::expect_timeout(const std::string& reason)
382{
383    atf_tc_expect_timeout("%s", reason.c_str());
384}
385
386// ------------------------------------------------------------------------
387// The "tp" class.
388// ------------------------------------------------------------------------
389
390class tp : public atf::application::app {
391public:
392    typedef std::vector< impl::tc * > tc_vector;
393
394private:
395    static const char* m_description;
396
397    bool m_lflag;
398    atf::fs::path m_resfile;
399    std::string m_srcdir_arg;
400    atf::fs::path m_srcdir;
401
402    atf::tests::vars_map m_vars;
403
404    std::string specific_args(void) const;
405    options_set specific_options(void) const;
406    void process_option(int, const char*);
407
408    void (*m_add_tcs)(tc_vector&);
409    tc_vector m_tcs;
410
411    void parse_vflag(const std::string&);
412    void handle_srcdir(void);
413
414    tc_vector init_tcs(void);
415
416    enum tc_part {
417        BODY,
418        CLEANUP,
419    };
420
421    void list_tcs(void);
422    impl::tc* find_tc(tc_vector, const std::string&);
423    static std::pair< std::string, tc_part > process_tcarg(const std::string&);
424    int run_tc(const std::string&);
425
426public:
427    tp(void (*)(tc_vector&));
428    ~tp(void);
429
430    int main(void);
431};
432
433const char* tp::m_description =
434    "This is an independent atf test program.";
435
436tp::tp(void (*add_tcs)(tc_vector&)) :
437    app(m_description, "atf-test-program(1)"),
438    m_lflag(false),
439    m_resfile("/dev/stdout"),
440    m_srcdir("."),
441    m_add_tcs(add_tcs)
442{
443}
444
445tp::~tp(void)
446{
447    for (tc_vector::iterator iter = m_tcs.begin();
448         iter != m_tcs.end(); iter++) {
449        impl::tc* tc = *iter;
450
451        delete tc;
452    }
453}
454
455std::string
456tp::specific_args(void)
457    const
458{
459    return "test_case";
460}
461
462tp::options_set
463tp::specific_options(void)
464    const
465{
466    using atf::application::option;
467    options_set opts;
468    opts.insert(option('l', "", "List test cases and their purpose"));
469    opts.insert(option('r', "resfile", "The file to which the test program "
470                                       "will write the results of the "
471                                       "executed test case"));
472    opts.insert(option('s', "srcdir", "Directory where the test's data "
473                                      "files are located"));
474    opts.insert(option('v', "var=value", "Sets the configuration variable "
475                                         "`var' to `value'"));
476    return opts;
477}
478
479void
480tp::process_option(int ch, const char* arg)
481{
482    switch (ch) {
483    case 'l':
484        m_lflag = true;
485        break;
486
487    case 'r':
488        m_resfile = atf::fs::path(arg);
489        break;
490
491    case 's':
492        m_srcdir_arg = arg;
493        break;
494
495    case 'v':
496        parse_vflag(arg);
497        break;
498
499    default:
500        UNREACHABLE;
501    }
502}
503
504void
505tp::parse_vflag(const std::string& str)
506{
507    if (str.empty())
508        throw std::runtime_error("-v requires a non-empty argument");
509
510    std::vector< std::string > ws = atf::text::split(str, "=");
511    if (ws.size() == 1 && str[str.length() - 1] == '=') {
512        m_vars[ws[0]] = "";
513    } else {
514        if (ws.size() != 2)
515            throw std::runtime_error("-v requires an argument of the form "
516                                     "var=value");
517
518        m_vars[ws[0]] = ws[1];
519    }
520}
521
522void
523tp::handle_srcdir(void)
524{
525    if (m_srcdir_arg.empty()) {
526        m_srcdir = atf::fs::path(m_argv0).branch_path();
527        if (m_srcdir.leaf_name() == ".libs")
528            m_srcdir = m_srcdir.branch_path();
529    } else
530        m_srcdir = atf::fs::path(m_srcdir_arg);
531
532    if (!atf::fs::exists(m_srcdir / m_prog_name))
533        throw std::runtime_error("Cannot find the test program in the "
534                                 "source directory `" + m_srcdir.str() + "'");
535
536    if (!m_srcdir.is_absolute())
537        m_srcdir = m_srcdir.to_absolute();
538
539    m_vars["srcdir"] = m_srcdir.str();
540}
541
542tp::tc_vector
543tp::init_tcs(void)
544{
545    m_add_tcs(m_tcs);
546    for (tc_vector::iterator iter = m_tcs.begin();
547         iter != m_tcs.end(); iter++) {
548        impl::tc* tc = *iter;
549
550        tc->init(m_vars);
551    }
552    return m_tcs;
553}
554
555//
556// An auxiliary unary predicate that compares the given test case's
557// identifier to the identifier stored in it.
558//
559class tc_equal_to_ident {
560    const std::string& m_ident;
561
562public:
563    tc_equal_to_ident(const std::string& i) :
564        m_ident(i)
565    {
566    }
567
568    bool operator()(const impl::tc* tc)
569    {
570        return tc->get_md_var("ident") == m_ident;
571    }
572};
573
574void
575tp::list_tcs(void)
576{
577    tc_vector tcs = init_tcs();
578    detail::atf_tp_writer writer(std::cout);
579
580    for (tc_vector::const_iterator iter = tcs.begin();
581         iter != tcs.end(); iter++) {
582        const impl::vars_map vars = (*iter)->get_md_vars();
583
584        {
585            impl::vars_map::const_iterator iter2 = vars.find("ident");
586            INV(iter2 != vars.end());
587            writer.start_tc((*iter2).second);
588        }
589
590        for (impl::vars_map::const_iterator iter2 = vars.begin();
591             iter2 != vars.end(); iter2++) {
592            const std::string& key = (*iter2).first;
593            if (key != "ident")
594                writer.tc_meta_data(key, (*iter2).second);
595        }
596
597        writer.end_tc();
598    }
599}
600
601impl::tc*
602tp::find_tc(tc_vector tcs, const std::string& name)
603{
604    std::vector< std::string > ids;
605    for (tc_vector::iterator iter = tcs.begin();
606         iter != tcs.end(); iter++) {
607        impl::tc* tc = *iter;
608
609        if (tc->get_md_var("ident") == name)
610            return tc;
611    }
612    throw atf::application::usage_error("Unknown test case `%s'",
613                                        name.c_str());
614}
615
616std::pair< std::string, tp::tc_part >
617tp::process_tcarg(const std::string& tcarg)
618{
619    const std::string::size_type pos = tcarg.find(':');
620    if (pos == std::string::npos) {
621        return std::make_pair(tcarg, BODY);
622    } else {
623        const std::string tcname = tcarg.substr(0, pos);
624
625        const std::string partname = tcarg.substr(pos + 1);
626        if (partname == "body")
627            return std::make_pair(tcname, BODY);
628        else if (partname == "cleanup")
629            return std::make_pair(tcname, CLEANUP);
630        else {
631            using atf::application::usage_error;
632            throw usage_error("Invalid test case part `%s'", partname.c_str());
633        }
634    }
635}
636
637int
638tp::run_tc(const std::string& tcarg)
639{
640    const std::pair< std::string, tc_part > fields = process_tcarg(tcarg);
641
642    impl::tc* tc = find_tc(init_tcs(), fields.first);
643
644    if (!atf::env::has("__RUNNING_INSIDE_ATF_RUN") || atf::env::get(
645        "__RUNNING_INSIDE_ATF_RUN") != "internal-yes-value")
646    {
647        std::cerr << m_prog_name << ": WARNING: Running test cases without "
648            "atf-run(1) is unsupported\n";
649        std::cerr << m_prog_name << ": WARNING: No isolation nor timeout "
650            "control is being applied; you may get unexpected failures; see "
651            "atf-test-case(4)\n";
652    }
653
654    try {
655        switch (fields.second) {
656        case BODY:
657            tc->run(m_resfile.str());
658            break;
659        case CLEANUP:
660            tc->run_cleanup();
661            break;
662        default:
663            UNREACHABLE;
664        }
665        return EXIT_SUCCESS;
666    } catch (const std::runtime_error& e) {
667        std::cerr << "ERROR: " << e.what() << "\n";
668        return EXIT_FAILURE;
669    }
670}
671
672int
673tp::main(void)
674{
675    using atf::application::usage_error;
676
677    int errcode;
678
679    handle_srcdir();
680
681    if (m_lflag) {
682        if (m_argc > 0)
683            throw usage_error("Cannot provide test case names with -l");
684
685        list_tcs();
686        errcode = EXIT_SUCCESS;
687    } else {
688        if (m_argc == 0)
689            throw usage_error("Must provide a test case name");
690        else if (m_argc > 1)
691            throw usage_error("Cannot provide more than one test case name");
692        INV(m_argc == 1);
693
694        errcode = run_tc(m_argv[0]);
695    }
696
697    return errcode;
698}
699
700namespace atf {
701    namespace tests {
702        int run_tp(int, char* const*, void (*)(tp::tc_vector&));
703    }
704}
705
706int
707impl::run_tp(int argc, char* const* argv, void (*add_tcs)(tp::tc_vector&))
708{
709    return tp(add_tcs).run(argc, argv);
710}
711