tap.cpp revision 2af35c15f0b6443224e3bf1c396805a387f42410
1// Copyright 2015 The Kyua Authors.
2// All rights reserved.
3//
4// Redistribution and use in source and binary forms, with or without
5// modification, are permitted provided that the following conditions are
6// met:
7//
8// * Redistributions of source code must retain the above copyright
9//   notice, this list of conditions and the following disclaimer.
10// * Redistributions in binary form must reproduce the above copyright
11//   notice, this list of conditions and the following disclaimer in the
12//   documentation and/or other materials provided with the distribution.
13// * Neither the name of Google Inc. nor the names of its contributors
14//   may be used to endorse or promote products derived from this software
15//   without specific prior written permission.
16//
17// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29#include "engine/tap.hpp"
30
31extern "C" {
32#include <unistd.h>
33}
34
35#include <cstdlib>
36
37#include "engine/exceptions.hpp"
38#include "engine/tap_parser.hpp"
39#include "model/test_case.hpp"
40#include "model/test_program.hpp"
41#include "model/test_result.hpp"
42#include "utils/defs.hpp"
43#include "utils/env.hpp"
44#include "utils/format/macros.hpp"
45#include "utils/optional.ipp"
46#include "utils/process/operations.hpp"
47#include "utils/process/status.hpp"
48#include "utils/sanity.hpp"
49
50namespace config = utils::config;
51namespace fs = utils::fs;
52namespace process = utils::process;
53
54using utils::optional;
55
56
57namespace {
58
59
60/// Computes the result of a TAP test program termination.
61///
62/// Timeouts and bad TAP data must be handled by the caller.  Here we assume
63/// that we have been able to successfully parse the TAP output.
64///
65/// \param summary Parsed TAP data for the test program.
66/// \param status Exit status of the test program.
67///
68/// \return A test result.
69static model::test_result
70tap_to_result(const engine::tap_summary& summary,
71              const process::status& status)
72{
73    if (summary.bailed_out()) {
74        return model::test_result(model::test_result_failed, "Bailed out");
75    }
76
77    if (summary.plan() == engine::all_skipped_plan) {
78        return model::test_result(model::test_result_skipped,
79                                  summary.all_skipped_reason());
80    }
81
82    if (summary.not_ok_count() == 0) {
83        if (status.exitstatus() == EXIT_SUCCESS) {
84            return model::test_result(model::test_result_passed);
85        } else {
86            return model::test_result(
87                model::test_result_broken,
88                F("Dubious test program: reported all tests as passed "
89                  "but returned exit code %s") % status.exitstatus());
90        }
91    } else {
92        const std::size_t total = summary.ok_count() + summary.not_ok_count();
93        return model::test_result(model::test_result_failed,
94                                  F("%s of %s tests failed") %
95                                  summary.not_ok_count() % total);
96    }
97}
98
99
100}  // anonymous namespace
101
102
103/// Executes a test program's list operation.
104///
105/// This method is intended to be called within a subprocess and is expected
106/// to terminate execution either by exec(2)ing the test program or by
107/// exiting with a failure.
108void
109engine::tap_interface::exec_list(
110    const model::test_program& /* test_program */,
111    const config::properties_map& /* vars */) const
112{
113    ::_exit(EXIT_SUCCESS);
114}
115
116
117/// Computes the test cases list of a test program.
118///
119/// \return A list of test cases.
120model::test_cases_map
121engine::tap_interface::parse_list(
122    const optional< process::status >& /* status */,
123    const fs::path& /* stdout_path */,
124    const fs::path& /* stderr_path */) const
125{
126    return model::test_cases_map_builder().add("main").build();
127}
128
129
130/// Executes a test case of the test program.
131///
132/// This method is intended to be called within a subprocess and is expected
133/// to terminate execution either by exec(2)ing the test program or by
134/// exiting with a failure.
135///
136/// \param test_program The test program to execute.
137/// \param test_case_name Name of the test case to invoke.
138/// \param vars User-provided variables to pass to the test program.
139void
140engine::tap_interface::exec_test(
141    const model::test_program& test_program,
142    const std::string& test_case_name,
143    const config::properties_map& vars,
144    const fs::path& /* control_directory */) const
145{
146    PRE(test_case_name == "main");
147
148    for (config::properties_map::const_iterator iter = vars.begin();
149         iter != vars.end(); ++iter) {
150        utils::setenv(F("TEST_ENV_%s") % (*iter).first, (*iter).second);
151    }
152
153    process::args_vector args;
154    process::exec(test_program.absolute_path(), args);
155}
156
157
158/// Computes the result of a test case based on its termination status.
159///
160/// \param status The termination status of the subprocess used to execute
161///     the exec_test() method or none if the test timed out.
162/// \param stdout_path Path to the file containing the stdout of the test.
163///
164/// \return A test result.
165model::test_result
166engine::tap_interface::compute_result(
167    const optional< process::status >& status,
168    const fs::path& /* control_directory */,
169    const fs::path& stdout_path,
170    const fs::path& /* stderr_path */) const
171{
172    if (!status) {
173        return model::test_result(model::test_result_broken,
174                                  "Test case timed out");
175    } else {
176        if (status.get().signaled()) {
177            return model::test_result(
178                model::test_result_broken,
179                F("Received signal %s") % status.get().termsig());
180        } else {
181            try {
182                const tap_summary summary = parse_tap_output(stdout_path);
183                return tap_to_result(summary, status.get());
184            } catch (const load_error& e) {
185                return model::test_result(
186                    model::test_result_broken,
187                    F("TAP test program yielded invalid data: %s") % e.what());
188            }
189        }
190    }
191}
192