tests.cpp revision 2cc168fb9fc9be682ff4c984b6306ddfe22c033d
1//
2// Automated Testing Framework (atf)
3//
4// Copyright (c) 2007 The NetBSD Foundation, Inc.
5// All rights reserved.
6//
7// Redistribution and use in source and binary forms, with or without
8// modification, are permitted provided that the following conditions
9// are met:
10// 1. Redistributions of source code must retain the above copyright
11//    notice, this list of conditions and the following disclaimer.
12// 2. Redistributions in binary form must reproduce the above copyright
13//    notice, this list of conditions and the following disclaimer in the
14//    documentation and/or other materials provided with the distribution.
15//
16// THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
17// CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
18// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
19// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20// IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
21// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
23// GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
25// IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
26// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
27// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28//
29
30extern "C" {
31#include <sys/types.h>
32#include <sys/stat.h>
33#include <sys/time.h>
34#include <sys/wait.h>
35#include <signal.h>
36#include <unistd.h>
37}
38
39#include <algorithm>
40#include <cctype>
41#include <cerrno>
42#include <cstdlib>
43#include <cstring>
44#include <fstream>
45#include <iostream>
46#include <map>
47#include <memory>
48#include <sstream>
49#include <stdexcept>
50#include <vector>
51
52extern "C" {
53#include "atf-c/error.h"
54#include "atf-c/tc.h"
55#include "atf-c/utils.h"
56}
57
58#include "noncopyable.hpp"
59#include "tests.hpp"
60
61#include "detail/application.hpp"
62#include "detail/auto_array.hpp"
63#include "detail/env.hpp"
64#include "detail/exceptions.hpp"
65#include "detail/fs.hpp"
66#include "detail/parser.hpp"
67#include "detail/sanity.hpp"
68#include "detail/text.hpp"
69
70namespace impl = atf::tests;
71namespace detail = atf::tests::detail;
72#define IMPL_NAME "atf::tests"
73
74// ------------------------------------------------------------------------
75// The "atf_tp_writer" class.
76// ------------------------------------------------------------------------
77
78detail::atf_tp_writer::atf_tp_writer(std::ostream& os) :
79    m_os(os),
80    m_is_first(true)
81{
82    atf::parser::headers_map hm;
83    atf::parser::attrs_map ct_attrs;
84    ct_attrs["version"] = "1";
85    hm["Content-Type"] = atf::parser::header_entry("Content-Type",
86        "application/X-atf-tp", ct_attrs);
87    atf::parser::write_headers(hm, m_os);
88}
89
90void
91detail::atf_tp_writer::start_tc(const std::string& ident)
92{
93    if (!m_is_first)
94        m_os << "\n";
95    m_os << "ident: " << ident << "\n";
96    m_os.flush();
97}
98
99void
100detail::atf_tp_writer::end_tc(void)
101{
102    if (m_is_first)
103        m_is_first = false;
104}
105
106void
107detail::atf_tp_writer::tc_meta_data(const std::string& name,
108                                    const std::string& value)
109{
110    PRE(name != "ident");
111    m_os << name << ": " << value << "\n";
112    m_os.flush();
113}
114
115// ------------------------------------------------------------------------
116// Free helper functions.
117// ------------------------------------------------------------------------
118
119bool
120detail::match(const std::string& regexp, const std::string& str)
121{
122    return atf::text::match(str, regexp);
123}
124
125// ------------------------------------------------------------------------
126// The "tc" class.
127// ------------------------------------------------------------------------
128
129static std::map< atf_tc_t*, impl::tc* > wraps;
130static std::map< const atf_tc_t*, const impl::tc* > cwraps;
131
132struct impl::tc_impl : atf::noncopyable {
133    std::string m_ident;
134    atf_tc_t m_tc;
135    bool m_has_cleanup;
136
137    tc_impl(const std::string& ident, const bool has_cleanup) :
138        m_ident(ident),
139        m_has_cleanup(has_cleanup)
140    {
141    }
142
143    static void
144    wrap_head(atf_tc_t *tc)
145    {
146        std::map< atf_tc_t*, impl::tc* >::iterator iter = wraps.find(tc);
147        INV(iter != wraps.end());
148        (*iter).second->head();
149    }
150
151    static void
152    wrap_body(const atf_tc_t *tc)
153    {
154        std::map< const atf_tc_t*, const impl::tc* >::const_iterator iter =
155            cwraps.find(tc);
156        INV(iter != cwraps.end());
157        try {
158            (*iter).second->body();
159        } catch (const std::exception& e) {
160            (*iter).second->fail("Caught unhandled exception: " + std::string(
161                                     e.what()));
162        } catch (...) {
163            (*iter).second->fail("Caught unknown exception");
164        }
165    }
166
167    static void
168    wrap_cleanup(const atf_tc_t *tc)
169    {
170        std::map< const atf_tc_t*, const impl::tc* >::const_iterator iter =
171            cwraps.find(tc);
172        INV(iter != cwraps.end());
173        (*iter).second->cleanup();
174    }
175};
176
177impl::tc::tc(const std::string& ident, const bool has_cleanup) :
178    pimpl(new tc_impl(ident, has_cleanup))
179{
180}
181
182impl::tc::~tc(void)
183{
184    cwraps.erase(&pimpl->m_tc);
185    wraps.erase(&pimpl->m_tc);
186
187    atf_tc_fini(&pimpl->m_tc);
188}
189
190void
191impl::tc::init(const vars_map& config)
192{
193    atf_error_t err;
194
195    auto_array< const char * > array(new const char*[(config.size() * 2) + 1]);
196    const char **ptr = array.get();
197    for (vars_map::const_iterator iter = config.begin();
198         iter != config.end(); iter++) {
199         *ptr = (*iter).first.c_str();
200         *(ptr + 1) = (*iter).second.c_str();
201         ptr += 2;
202    }
203    *ptr = NULL;
204
205    wraps[&pimpl->m_tc] = this;
206    cwraps[&pimpl->m_tc] = this;
207
208    err = atf_tc_init(&pimpl->m_tc, pimpl->m_ident.c_str(), pimpl->wrap_head,
209        pimpl->wrap_body, pimpl->m_has_cleanup ? pimpl->wrap_cleanup : NULL,
210        array.get());
211    if (atf_is_error(err))
212        throw_atf_error(err);
213}
214
215bool
216impl::tc::has_config_var(const std::string& var)
217    const
218{
219    return atf_tc_has_config_var(&pimpl->m_tc, var.c_str());
220}
221
222bool
223impl::tc::has_md_var(const std::string& var)
224    const
225{
226    return atf_tc_has_md_var(&pimpl->m_tc, var.c_str());
227}
228
229const std::string
230impl::tc::get_config_var(const std::string& var)
231    const
232{
233    return atf_tc_get_config_var(&pimpl->m_tc, var.c_str());
234}
235
236const std::string
237impl::tc::get_config_var(const std::string& var, const std::string& defval)
238    const
239{
240    return atf_tc_get_config_var_wd(&pimpl->m_tc, var.c_str(), defval.c_str());
241}
242
243const std::string
244impl::tc::get_md_var(const std::string& var)
245    const
246{
247    return atf_tc_get_md_var(&pimpl->m_tc, var.c_str());
248}
249
250const impl::vars_map
251impl::tc::get_md_vars(void)
252    const
253{
254    vars_map vars;
255
256    char **array = atf_tc_get_md_vars(&pimpl->m_tc);
257    try {
258        char **ptr;
259        for (ptr = array; *ptr != NULL; ptr += 2)
260            vars[*ptr] = *(ptr + 1);
261    } catch (...) {
262        atf_utils_free_charpp(array);
263        throw;
264    }
265
266    return vars;
267}
268
269void
270impl::tc::set_md_var(const std::string& var, const std::string& val)
271{
272    atf_error_t err = atf_tc_set_md_var(&pimpl->m_tc, var.c_str(), val.c_str());
273    if (atf_is_error(err))
274        throw_atf_error(err);
275}
276
277void
278impl::tc::run(const std::string& resfile)
279    const
280{
281    atf_error_t err = atf_tc_run(&pimpl->m_tc, resfile.c_str());
282    if (atf_is_error(err))
283        throw_atf_error(err);
284}
285
286void
287impl::tc::run_cleanup(void)
288    const
289{
290    atf_error_t err = atf_tc_cleanup(&pimpl->m_tc);
291    if (atf_is_error(err))
292        throw_atf_error(err);
293}
294
295void
296impl::tc::head(void)
297{
298}
299
300void
301impl::tc::cleanup(void)
302    const
303{
304}
305
306void
307impl::tc::require_prog(const std::string& prog)
308    const
309{
310    atf_tc_require_prog(prog.c_str());
311}
312
313void
314impl::tc::pass(void)
315{
316    atf_tc_pass();
317}
318
319void
320impl::tc::fail(const std::string& reason)
321{
322    atf_tc_fail("%s", reason.c_str());
323}
324
325void
326impl::tc::fail_nonfatal(const std::string& reason)
327{
328    atf_tc_fail_nonfatal("%s", reason.c_str());
329}
330
331void
332impl::tc::skip(const std::string& reason)
333{
334    atf_tc_skip("%s", reason.c_str());
335}
336
337void
338impl::tc::check_errno(const char* file, const int line, const int exp_errno,
339                      const char* expr_str, const bool result)
340{
341    atf_tc_check_errno(file, line, exp_errno, expr_str, result);
342}
343
344void
345impl::tc::require_errno(const char* file, const int line, const int exp_errno,
346                        const char* expr_str, const bool result)
347{
348    atf_tc_require_errno(file, line, exp_errno, expr_str, result);
349}
350
351void
352impl::tc::expect_pass(void)
353{
354    atf_tc_expect_pass();
355}
356
357void
358impl::tc::expect_fail(const std::string& reason)
359{
360    atf_tc_expect_fail("%s", reason.c_str());
361}
362
363void
364impl::tc::expect_exit(const int exitcode, const std::string& reason)
365{
366    atf_tc_expect_exit(exitcode, "%s", reason.c_str());
367}
368
369void
370impl::tc::expect_signal(const int signo, const std::string& reason)
371{
372    atf_tc_expect_signal(signo, "%s", reason.c_str());
373}
374
375void
376impl::tc::expect_death(const std::string& reason)
377{
378    atf_tc_expect_death("%s", reason.c_str());
379}
380
381void
382impl::tc::expect_timeout(const std::string& reason)
383{
384    atf_tc_expect_timeout("%s", reason.c_str());
385}
386
387// ------------------------------------------------------------------------
388// The "tp" class.
389// ------------------------------------------------------------------------
390
391class tp : public atf::application::app {
392public:
393    typedef std::vector< impl::tc * > tc_vector;
394
395private:
396    static const char* m_description;
397
398    bool m_lflag;
399    atf::fs::path m_resfile;
400    std::string m_srcdir_arg;
401    atf::fs::path m_srcdir;
402
403    atf::tests::vars_map m_vars;
404
405    std::string specific_args(void) const;
406    options_set specific_options(void) const;
407    void process_option(int, const char*);
408
409    void (*m_add_tcs)(tc_vector&);
410    tc_vector m_tcs;
411
412    void parse_vflag(const std::string&);
413    void handle_srcdir(void);
414
415    tc_vector init_tcs(void);
416
417    enum tc_part {
418        BODY,
419        CLEANUP,
420    };
421
422    void list_tcs(void);
423    impl::tc* find_tc(tc_vector, const std::string&);
424    static std::pair< std::string, tc_part > process_tcarg(const std::string&);
425    int run_tc(const std::string&);
426
427public:
428    tp(void (*)(tc_vector&));
429    ~tp(void);
430
431    int main(void);
432};
433
434const char* tp::m_description =
435    "This is an independent atf test program.";
436
437tp::tp(void (*add_tcs)(tc_vector&)) :
438    app(m_description, "atf-test-program(1)", "atf(7)", false),
439    m_lflag(false),
440    m_resfile("/dev/stdout"),
441    m_srcdir("."),
442    m_add_tcs(add_tcs)
443{
444}
445
446tp::~tp(void)
447{
448    for (tc_vector::iterator iter = m_tcs.begin();
449         iter != m_tcs.end(); iter++) {
450        impl::tc* tc = *iter;
451
452        delete tc;
453    }
454}
455
456std::string
457tp::specific_args(void)
458    const
459{
460    return "test_case";
461}
462
463tp::options_set
464tp::specific_options(void)
465    const
466{
467    using atf::application::option;
468    options_set opts;
469    opts.insert(option('l', "", "List test cases and their purpose"));
470    opts.insert(option('r', "resfile", "The file to which the test program "
471                                       "will write the results of the "
472                                       "executed test case"));
473    opts.insert(option('s', "srcdir", "Directory where the test's data "
474                                      "files are located"));
475    opts.insert(option('v', "var=value", "Sets the configuration variable "
476                                         "`var' to `value'"));
477    return opts;
478}
479
480void
481tp::process_option(int ch, const char* arg)
482{
483    switch (ch) {
484    case 'l':
485        m_lflag = true;
486        break;
487
488    case 'r':
489        m_resfile = atf::fs::path(arg);
490        break;
491
492    case 's':
493        m_srcdir_arg = arg;
494        break;
495
496    case 'v':
497        parse_vflag(arg);
498        break;
499
500    default:
501        UNREACHABLE;
502    }
503}
504
505void
506tp::parse_vflag(const std::string& str)
507{
508    if (str.empty())
509        throw std::runtime_error("-v requires a non-empty argument");
510
511    std::vector< std::string > ws = atf::text::split(str, "=");
512    if (ws.size() == 1 && str[str.length() - 1] == '=') {
513        m_vars[ws[0]] = "";
514    } else {
515        if (ws.size() != 2)
516            throw std::runtime_error("-v requires an argument of the form "
517                                     "var=value");
518
519        m_vars[ws[0]] = ws[1];
520    }
521}
522
523void
524tp::handle_srcdir(void)
525{
526    if (m_srcdir_arg.empty()) {
527        m_srcdir = atf::fs::path(m_argv0).branch_path();
528        if (m_srcdir.leaf_name() == ".libs")
529            m_srcdir = m_srcdir.branch_path();
530    } else
531        m_srcdir = atf::fs::path(m_srcdir_arg);
532
533    if (!atf::fs::exists(m_srcdir / m_prog_name))
534        throw std::runtime_error("Cannot find the test program in the "
535                                 "source directory `" + m_srcdir.str() + "'");
536
537    if (!m_srcdir.is_absolute())
538        m_srcdir = m_srcdir.to_absolute();
539
540    m_vars["srcdir"] = m_srcdir.str();
541}
542
543tp::tc_vector
544tp::init_tcs(void)
545{
546    m_add_tcs(m_tcs);
547    for (tc_vector::iterator iter = m_tcs.begin();
548         iter != m_tcs.end(); iter++) {
549        impl::tc* tc = *iter;
550
551        tc->init(m_vars);
552    }
553    return m_tcs;
554}
555
556//
557// An auxiliary unary predicate that compares the given test case's
558// identifier to the identifier stored in it.
559//
560class tc_equal_to_ident {
561    const std::string& m_ident;
562
563public:
564    tc_equal_to_ident(const std::string& i) :
565        m_ident(i)
566    {
567    }
568
569    bool operator()(const impl::tc* tc)
570    {
571        return tc->get_md_var("ident") == m_ident;
572    }
573};
574
575void
576tp::list_tcs(void)
577{
578    tc_vector tcs = init_tcs();
579    detail::atf_tp_writer writer(std::cout);
580
581    for (tc_vector::const_iterator iter = tcs.begin();
582         iter != tcs.end(); iter++) {
583        const impl::vars_map vars = (*iter)->get_md_vars();
584
585        {
586            impl::vars_map::const_iterator iter2 = vars.find("ident");
587            INV(iter2 != vars.end());
588            writer.start_tc((*iter2).second);
589        }
590
591        for (impl::vars_map::const_iterator iter2 = vars.begin();
592             iter2 != vars.end(); iter2++) {
593            const std::string& key = (*iter2).first;
594            if (key != "ident")
595                writer.tc_meta_data(key, (*iter2).second);
596        }
597
598        writer.end_tc();
599    }
600}
601
602impl::tc*
603tp::find_tc(tc_vector tcs, const std::string& name)
604{
605    std::vector< std::string > ids;
606    for (tc_vector::iterator iter = tcs.begin();
607         iter != tcs.end(); iter++) {
608        impl::tc* tc = *iter;
609
610        if (tc->get_md_var("ident") == name)
611            return tc;
612    }
613    throw atf::application::usage_error("Unknown test case `%s'",
614                                        name.c_str());
615}
616
617std::pair< std::string, tp::tc_part >
618tp::process_tcarg(const std::string& tcarg)
619{
620    const std::string::size_type pos = tcarg.find(':');
621    if (pos == std::string::npos) {
622        return std::make_pair(tcarg, BODY);
623    } else {
624        const std::string tcname = tcarg.substr(0, pos);
625
626        const std::string partname = tcarg.substr(pos + 1);
627        if (partname == "body")
628            return std::make_pair(tcname, BODY);
629        else if (partname == "cleanup")
630            return std::make_pair(tcname, CLEANUP);
631        else {
632            using atf::application::usage_error;
633            throw usage_error("Invalid test case part `%s'", partname.c_str());
634        }
635    }
636}
637
638int
639tp::run_tc(const std::string& tcarg)
640{
641    const std::pair< std::string, tc_part > fields = process_tcarg(tcarg);
642
643    impl::tc* tc = find_tc(init_tcs(), fields.first);
644
645    if (!atf::env::has("__RUNNING_INSIDE_ATF_RUN") || atf::env::get(
646        "__RUNNING_INSIDE_ATF_RUN") != "internal-yes-value")
647    {
648        std::cerr << m_prog_name << ": WARNING: Running test cases without "
649            "atf-run(1) is unsupported\n";
650        std::cerr << m_prog_name << ": WARNING: No isolation nor timeout "
651            "control is being applied; you may get unexpected failures; see "
652            "atf-test-case(4)\n";
653    }
654
655    try {
656        switch (fields.second) {
657        case BODY:
658            tc->run(m_resfile.str());
659            break;
660        case CLEANUP:
661            tc->run_cleanup();
662            break;
663        default:
664            UNREACHABLE;
665        }
666        return EXIT_SUCCESS;
667    } catch (const std::runtime_error& e) {
668        std::cerr << "ERROR: " << e.what() << "\n";
669        return EXIT_FAILURE;
670    }
671}
672
673int
674tp::main(void)
675{
676    using atf::application::usage_error;
677
678    int errcode;
679
680    handle_srcdir();
681
682    if (m_lflag) {
683        if (m_argc > 0)
684            throw usage_error("Cannot provide test case names with -l");
685
686        list_tcs();
687        errcode = EXIT_SUCCESS;
688    } else {
689        if (m_argc == 0)
690            throw usage_error("Must provide a test case name");
691        else if (m_argc > 1)
692            throw usage_error("Cannot provide more than one test case name");
693        INV(m_argc == 1);
694
695        errcode = run_tc(m_argv[0]);
696    }
697
698    return errcode;
699}
700
701namespace atf {
702    namespace tests {
703        int run_tp(int, char* const*, void (*)(tp::tc_vector&));
704    }
705}
706
707int
708impl::run_tp(int argc, char* const* argv, void (*add_tcs)(tp::tc_vector&))
709{
710    return tp(add_tcs).run(argc, argv);
711}
712