| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | //====================================================================================================================== | ||
| 2 | //! @file | ||
| 3 | /* | ||
| 4 | TTS - Tiny Test System | ||
| 5 | Copyright : TTS Contributors & Maintainers | ||
| 6 | SPDX-License-Identifier: BSL-1.0 | ||
| 7 | */ | ||
| 8 | //====================================================================================================================== | ||
| 9 | #pragma once | ||
| 10 | |||
| 11 | #include <tts/engine/abort.hpp> | ||
| 12 | #include <tts/engine/logger.hpp> | ||
| 13 | #include <tts/engine/test.hpp> | ||
| 14 | #include <tts/engine/environment.hpp> | ||
| 15 | #include <tts/tools/clock.hpp> | ||
| 16 | #include <tts/tools/options.hpp> | ||
| 17 | #include <tts/tools/random.hpp> | ||
| 18 | |||
| 19 | #if defined(TTS_DOXYGEN_INVOKED) | ||
| 20 | //====================================================================================================================== | ||
| 21 | /** | ||
| 22 | @defgroup customization-points Customization Points | ||
| 23 | @{ | ||
| 24 | **/ | ||
| 25 | //====================================================================================================================== | ||
| 26 | |||
| 27 | //====================================================================================================================== | ||
| 28 | /*! | ||
| 29 | @def TTS_CUSTOM_DRIVER_FUNCTION | ||
| 30 | @brief Test application entry-point customization | ||
| 31 | |||
| 32 | In case where the `main` function is to be provided externally, one can redefine | ||
| 33 | @ref TTS_CUSTOM_DRIVER_FUNCTION in order to specify the name of the test executable entry point. | ||
| 34 | |||
| 35 | @see TTS_MAIN | ||
| 36 | @see tts::initialize | ||
| 37 | @see tts::report | ||
| 38 | |||
| 39 | @groupheader{Example} | ||
| 40 | @snippet doc/custom_driver.cpp snippet | ||
| 41 | **/ | ||
| 42 | //====================================================================================================================== | ||
| 43 | #define TTS_CUSTOM_DRIVER_FUNCTION | ||
| 44 | |||
| 45 | //====================================================================================================================== | ||
| 46 | /*! | ||
| 47 | @def TTS_MAIN | ||
| 48 | @brief Identify the file containing tests entry point | ||
| 49 | |||
| 50 | Users can dispatch tests in multiple Translation Units to be linked afterward. In this scenario, | ||
| 51 | one single Translation Unit must define @ref TTS_MAIN to generate the test entry point function. | ||
| 52 | **/ | ||
| 53 | //====================================================================================================================== | ||
| 54 | #define TTS_MAIN | ||
| 55 | |||
| 56 | //====================================================================================================================== | ||
| 57 | /// @} | ||
| 58 | //====================================================================================================================== | ||
| 59 | #endif | ||
| 60 | |||
| 61 | #if !defined(TTS_CUSTOM_DRIVER_FUNCTION) | ||
| 62 | #define TTS_CUSTOM_DRIVER_FUNCTION main | ||
| 63 | namespace tts::_ | ||
| 64 | { | ||
| 65 | inline constexpr bool use_main = true; | ||
| 66 | } | ||
| 67 | #else | ||
| 68 | namespace tts::_ | ||
| 69 | { | ||
| 70 | inline constexpr bool use_main = false; | ||
| 71 | } | ||
| 72 | #endif | ||
| 73 | |||
| 74 | #if defined(TTS_MAIN) | ||
| 75 | #include <tts/engine/guard.hpp> // NOSONAR - the driver is the only unit that needs the signals | ||
| 76 | #include <tts/engine/watchdog.hpp> // NOSONAR - it needs the timers, and only the driver does | ||
| 77 | #include <tts/engine/shard.hpp> // NOSONAR - only the driver reads it, so only its unit parses it | ||
| 78 | #include <tts/engine/sink_selection.hpp> // NOSONAR - only the driver reads it, so only its unit parses it | ||
| 79 | #include <tts/engine/usage.hpp> // NOSONAR - only the driver reads it, so only its unit parses it | ||
| 80 | #include <tts/sinks/sinks.hpp> // NOSONAR - only the driver reads it, so only its unit parses it | ||
| 81 | #include <tts/tools/file.hpp> // NOSONAR - only the driver reads it, so only its unit parses it | ||
| 82 | |||
| 83 | TTS_DISABLE_WARNING_PUSH | ||
| 84 | TTS_DISABLE_WARNING_CRT_SECURE | ||
| 85 | 101 | int TTS_CUSTOM_DRIVER_FUNCTION([[maybe_unused]] int argc, [[maybe_unused]] char const** argv) | |
| 86 | { | ||
| 87 | 101 | ::tts::initialize(argc, argv); | |
| 88 |
2/4✓ Branch 2 taken 101 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 101 times.
✗ Branch 7 not taken.
|
101 | if(::tts::arguments()("-h", "--help")) return ::tts::_::usage(argv[ 0 ]); |
| 89 | |||
| 90 | 101 | bool shard_ok = true; | |
| 91 |
1/1✓ Branch 1 taken 101 times.
|
101 | auto shard = ::tts::_::parse_shard(shard_ok); |
| 92 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 101 times.
|
101 | if(!shard_ok) |
| 93 | { | ||
| 94 | ✗ | ::tts::output().writeln("Invalid --shard value, expected i/n with 0 <= i < n"); | |
| 95 | ✗ | return 1; | |
| 96 | } | ||
| 97 | |||
| 98 | // No-op for a custom driver: tts::report(fails, invalids) expects the whole suite's count. | ||
| 99 | 30 | if constexpr(!::tts::_::use_main) shard.active = false; | |
| 100 | |||
| 101 |
2/2✓ Branch 2 taken 1 times.
✓ Branch 3 taken 100 times.
|
101 | if(::tts::arguments()("--dry")) |
| 102 | { | ||
| 103 | 1 | std::size_t position = 0; | |
| 104 |
3/3✓ Branch 1 taken 1 times.
✓ Branch 5 taken 3 times.
✓ Branch 6 taken 1 times.
|
4 | for(auto const& t: ::tts::_::suite()) |
| 105 | { | ||
| 106 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 3 times.
|
3 | if(!shard.selects(position++)) continue; |
| 107 | |||
| 108 |
3/3✓ Branch 1 taken 2 times.
✓ Branch 2 taken 1 times.
✓ Branch 5 taken 2 times.
|
3 | if(t.types.is_empty()) ::tts::output().writeln(t.name); |
| 109 |
1/1✓ Branch 3 taken 1 times.
|
1 | else ::tts::output().writeln("%s <%s>", t.name, t.types.data()); |
| 110 | } | ||
| 111 | 1 | return 0; | |
| 112 | } | ||
| 113 | |||
| 114 |
1/1✓ Branch 2 taken 100 times.
|
100 | ::tts::_::set_verbose(::tts::arguments()("-v", "--verbose")); |
| 115 |
1/1✓ Branch 2 taken 100 times.
|
100 | ::tts::_::set_quiet(::tts::arguments()("-q", "--quiet")); |
| 116 | |||
| 117 |
1/1✓ Branch 2 taken 100 times.
|
100 | ::tts::text capture_path = ::tts::arguments().value<::tts::text>("--capture"); |
| 118 | 100 | ::tts::_::file_guard capture_file; | |
| 119 | |||
| 120 |
2/2✓ Branch 1 taken 2 times.
✓ Branch 2 taken 98 times.
|
100 | if(!capture_path.is_empty()) |
| 121 | { | ||
| 122 |
1/1✓ Branch 2 taken 2 times.
|
2 | capture_file = ::tts::_::file_guard {fopen(capture_path.data(), "w")}; // NOSONAR |
| 123 |
2/2✓ Branch 1 taken 1 times.
✓ Branch 2 taken 1 times.
|
2 | if(!capture_file) |
| 124 | { | ||
| 125 |
1/1✓ Branch 3 taken 1 times.
|
1 | ::tts::output().writeln("Unable to open '%s' for writing (--capture)", capture_path.data()); |
| 126 | 1 | return 1; | |
| 127 | } | ||
| 128 | } | ||
| 129 | |||
| 130 | // --sink and --capture are orthogonal - every candidate sink below targets capture_target. | ||
| 131 | 99 | ::tts::gathering_sink capture_sink; | |
| 132 | 99 | ::tts::output_sink& capture_target = capture_file ? static_cast<::tts::output_sink&>(capture_sink) | |
| 133 |
2/2✓ Branch 0 taken 1 times.
✓ Branch 1 taken 98 times.
|
99 | : ::tts::output_handler::default_sink(); |
| 134 | |||
| 135 | // Like --shard, no-op for a custom driver: it already manages its own sink lifecycle. | ||
| 136 |
1/1✓ Branch 2 taken 99 times.
|
99 | ::tts::text sink_name = ::tts::arguments().value<::tts::text>("--sink"); |
| 137 | 28 | if constexpr(!::tts::_::use_main) sink_name = ::tts::text {}; | |
| 138 | |||
| 139 | 99 | bool sink_ok = true; | |
| 140 |
1/1✓ Branch 1 taken 99 times.
|
99 | ::tts::text sink_error = ::tts::_::validate_sink_name(sink_name, sink_ok); |
| 141 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 99 times.
|
99 | if(!sink_ok) |
| 142 | { | ||
| 143 | ✗ | ::tts::output().writeln(sink_error); | |
| 144 | ✗ | return 1; | |
| 145 | } | ||
| 146 | |||
| 147 | 99 | ::tts::colorized_sink colorized_candidate {capture_target}; | |
| 148 | 99 | ::tts::tap_sink tap_candidate {capture_target}; | |
| 149 | 99 | ::tts::diagnostics_sink diagnostics_candidate {capture_target}; | |
| 150 | 99 | ::tts::json_sink json_candidate {capture_target}; | |
| 151 | 99 | ::tts::junit_sink junit_candidate {capture_target}; | |
| 152 | |||
| 153 | // Neither flag given: leave whatever sink is already installed alone. | ||
| 154 |
5/6✓ Branch 1 taken 99 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1 times.
✓ Branch 5 taken 98 times.
✓ Branch 6 taken 1 times.
✓ Branch 7 taken 98 times.
|
99 | if(sink_name.is_empty() && capture_file) ::tts::output().sink(capture_sink); |
| 155 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 98 times.
|
98 | else if(sink_name == "colored") ::tts::output().sink(colorized_candidate); |
| 156 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 98 times.
|
98 | else if(sink_name == "tap") ::tts::output().sink(tap_candidate); |
| 157 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 98 times.
|
98 | else if(sink_name == "diagnostics") ::tts::output().sink(diagnostics_candidate); |
| 158 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 98 times.
|
98 | else if(sink_name == "json") ::tts::output().sink(json_candidate); |
| 159 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 98 times.
|
98 | else if(sink_name == "junit") ::tts::output().sink(junit_candidate); |
| 160 | |||
| 161 |
1/1✓ Branch 1 taken 99 times.
|
99 | auto nb_tests = shard.count(::tts::_::suite().size()); |
| 162 | 99 | std::size_t done_tests = 0; | |
| 163 |
1/1✓ Branch 1 taken 99 times.
|
99 | auto seed = ::tts::random_seed(); |
| 164 | 99 | ::tts::set_random_seed(static_cast<std::uint64_t>(seed)); | |
| 165 |
1/1✓ Branch 2 taken 99 times.
|
99 | ::tts::output().writeln( |
| 166 | "Random seed: %d (rerun with --seed=%d to reproduce this run)", seed, seed); | ||
| 167 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 99 times.
|
99 | if(shard.active) |
| 168 | ✗ | ::tts::output().writeln("Shard: %u/%u (%zu test%s selected)", | |
| 169 | shard.index, | ||
| 170 | shard.total, | ||
| 171 | nb_tests, | ||
| 172 | ✗ | nb_tests > 1 ? "s" : ""); | |
| 173 | |||
| 174 | // perform_abort() never returns here, so the capture file is flushed from there. | ||
| 175 | ::tts::_::abort_epilogue = | ||
| 176 | 99 | ::tts::_::callable {[ &capture_file, &capture_sink ]() | |
| 177 | { | ||
| 178 | ✗ | if(capture_file) | |
| 179 | { | ||
| 180 | ✗ | ::tts::output().sink(::tts::output_handler::default_sink()); | |
| 181 | ✗ | fputs(capture_sink.content().data(), | |
| 182 | capture_file.get()); // NOSONAR | ||
| 183 | } | ||
| 184 |
1/1✓ Branch 1 taken 99 times.
|
99 | }}; |
| 185 | |||
| 186 | try | ||
| 187 | { | ||
| 188 | 99 | std::size_t position = 0; | |
| 189 |
3/3✓ Branch 1 taken 99 times.
✓ Branch 5 taken 282 times.
✓ Branch 6 taken 97 times.
|
379 | for(auto& t: ::tts::_::suite()) |
| 190 | { | ||
| 191 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 282 times.
|
282 | if(!shard.selects(position++)) continue; |
| 192 | |||
| 193 | 282 | auto test_count = ::tts::global_runtime.test_count; | |
| 194 | 282 | auto failure_count = ::tts::global_runtime.failure_count; | |
| 195 | 282 | ::tts::global_runtime.fail_status = false; | |
| 196 | |||
| 197 |
1/1✓ Branch 3 taken 282 times.
|
282 | ::tts::output().test_started(::tts::text {t.name}); |
| 198 | |||
| 199 |
2/3✓ Branch 1 taken 282 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 282 times.
|
282 | if(!::tts::is_quiet()) ::tts::output().writeln("TEST: '%s'", t.name); |
| 200 |
1/1✓ Branch 2 taken 282 times.
|
282 | ::tts::output().flush(); |
| 201 | |||
| 202 | 282 | ::tts::_::remaining_tests = nb_tests - done_tests - 1; | |
| 203 | |||
| 204 | // Always measured regardless of --verbose - sinks and the Total Time: line need it either way. | ||
| 205 | 282 | auto start_ns = ::tts::_::now_ns(); | |
| 206 | { | ||
| 207 |
1/1✓ Branch 1 taken 282 times.
|
282 | [[maybe_unused]] ::tts::_::crash_guard guard {}; |
| 208 | [[maybe_unused]] ::tts::_::watchdog deadline { | ||
| 209 |
2/3✗ Branch 0 not taken.
✓ Branch 1 taken 282 times.
✓ Branch 3 taken 282 times.
|
282 | t.timeout_set ? t.timeout_ns : ::tts::_::default_timeout_ns()}; |
| 210 |
1/1✓ Branch 1 taken 280 times.
|
282 | t(); |
| 211 | 284 | } | |
| 212 | 280 | auto duration_ns = ::tts::_::now_ns() - start_ns; | |
| 213 | 280 | done_tests++; | |
| 214 | |||
| 215 | 280 | ::tts::global_runtime.total_duration_ns += duration_ns; | |
| 216 | |||
| 217 | 280 | bool invalid = (test_count == ::tts::global_runtime.test_count); | |
| 218 |
4/4✓ Branch 0 taken 272 times.
✓ Branch 1 taken 8 times.
✓ Branch 2 taken 244 times.
✓ Branch 3 taken 28 times.
|
280 | bool passed = !invalid && (failure_count == ::tts::global_runtime.failure_count); |
| 219 | |||
| 220 | // invalid/passed reflect what happened, not the tag - only the comparison below does. | ||
| 221 | 280 | bool matches_expectation = false; | |
| 222 | using enum ::tts::expected_outcome; | ||
| 223 |
4/5✓ Branch 0 taken 273 times.
✓ Branch 1 taken 3 times.
✓ Branch 2 taken 2 times.
✓ Branch 3 taken 2 times.
✗ Branch 4 not taken.
|
280 | switch(t.tag) |
| 224 | { | ||
| 225 | 273 | case pass: matches_expectation = passed; break; | |
| 226 |
2/4✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 3 times.
✗ Branch 3 not taken.
|
3 | case xfail: matches_expectation = !invalid && !passed; break; |
| 227 | 2 | case may_fail: matches_expectation = !invalid; break; | |
| 228 | 2 | case xinvalid: matches_expectation = invalid; break; | |
| 229 | } | ||
| 230 | |||
| 231 |
2/2✓ Branch 0 taken 8 times.
✓ Branch 1 taken 272 times.
|
280 | if(invalid) ::tts::global_runtime.invalid(); |
| 232 |
2/2✓ Branch 0 taken 30 times.
✓ Branch 1 taken 250 times.
|
280 | if(!matches_expectation) ::tts::global_runtime.unexpected(); |
| 233 | |||
| 234 |
1/1✓ Branch 3 taken 280 times.
|
280 | ::tts::output().test_finished(::tts::text {t.name}, passed, invalid, duration_ns); |
| 235 | |||
| 236 |
1/1✓ Branch 1 taken 280 times.
|
560 | ::tts::text duration_txt = ::tts::_::format_duration(static_cast<double>(duration_ns)); |
| 237 | |||
| 238 |
3/4✓ Branch 0 taken 7 times.
✓ Branch 1 taken 273 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 7 times.
|
280 | if(t.tag != pass && !matches_expectation) |
| 239 | { | ||
| 240 | ✗ | if(!::tts::is_quiet()) | |
| 241 | ✗ | ::tts::output().writeln("TEST: '%s' - ** UNEXPECTED ** (tagged %s) (%s)", | |
| 242 | ✗ | t.name, | |
| 243 | ✗ | ::tts::_::tag_name(t.tag), | |
| 244 | ✗ | duration_txt.data()); | |
| 245 | ✗ | ::tts::output().flush(); | |
| 246 | } | ||
| 247 |
2/2✓ Branch 0 taken 8 times.
✓ Branch 1 taken 272 times.
|
280 | else if(invalid) |
| 248 | { | ||
| 249 |
1/2✓ Branch 1 taken 8 times.
✗ Branch 2 not taken.
|
8 | if(!::tts::is_quiet()) |
| 250 | { | ||
| 251 | 8 | ::tts::text line = ::tts::is_verbose() | |
| 252 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 8 times.
|
8 | ? ::tts::text {" [!!]: EMPTY TEST CASE (%s)", duration_txt.data()} |
| 253 | 8 | : ::tts::text {" [!!]: EMPTY TEST CASE"}; | |
| 254 |
1/1✓ Branch 2 taken 8 times.
|
8 | ::tts::output().writeln(line); |
| 255 | 8 | } | |
| 256 |
1/1✓ Branch 2 taken 8 times.
|
8 | ::tts::output().flush(); |
| 257 | } | ||
| 258 |
2/2✓ Branch 0 taken 244 times.
✓ Branch 1 taken 28 times.
|
272 | else if(passed) |
| 259 | { | ||
| 260 |
1/2✓ Branch 1 taken 244 times.
✗ Branch 2 not taken.
|
244 | if(!::tts::is_quiet()) |
| 261 | { | ||
| 262 | ::tts::text line = | ||
| 263 | 244 | ::tts::is_verbose() | |
| 264 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 244 times.
|
244 | ? ::tts::text {"TEST: '%s' - [PASSED] (%s)", t.name, duration_txt.data()} |
| 265 | 244 | : ::tts::text {"TEST: '%s' - [PASSED]", t.name}; | |
| 266 |
1/1✓ Branch 2 taken 244 times.
|
244 | ::tts::output().writeln(line); |
| 267 | 244 | } | |
| 268 |
1/1✓ Branch 2 taken 244 times.
|
244 | ::tts::output().flush(); |
| 269 | } | ||
| 270 |
2/6✗ Branch 1 not taken.
✓ Branch 2 taken 28 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 28 times.
|
28 | else if(::tts::is_verbose() && !::tts::is_quiet()) |
| 271 | { | ||
| 272 | ✗ | ::tts::output().writeln("TEST: '%s' - (%s)", t.name, duration_txt.data()); | |
| 273 | ✗ | ::tts::output().flush(); | |
| 274 | } | ||
| 275 | } | ||
| 276 | } | ||
| 277 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
|
2 | catch(::tts::_::fatal_signal&) |
| 278 | { | ||
| 279 | // t() never returned, so per-case bookkeeping above never ran - count the abort here instead. | ||
| 280 | 2 | ::tts::global_runtime.unexpected(); | |
| 281 | |||
| 282 |
1/1✓ Branch 2 taken 2 times.
|
2 | ::tts::output().suite_aborted(); |
| 283 | |||
| 284 |
1/2✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
|
2 | if(!::tts::is_quiet()) |
| 285 | 2 | ::tts::output().writeln("@@ ABORTING DUE TO EARLY FAILURE @@ - %d Tests not run", | |
| 286 |
1/1✓ Branch 1 taken 2 times.
|
2 | static_cast<int>(nb_tests - done_tests - 1)); |
| 287 | 2 | } | |
| 288 | |||
| 289 | // finish() is TTS_MAIN-only, like --sink - a custom driver decides for itself when to dump its own sink. | ||
| 290 | 99 | int exit_code = 0; | |
| 291 | if constexpr(::tts::_::use_main) | ||
| 292 | { | ||
| 293 |
1/1✓ Branch 1 taken 71 times.
|
71 | exit_code = ::tts::report(0, 0); |
| 294 |
1/1✓ Branch 2 taken 71 times.
|
71 | ::tts::output().finish(); |
| 295 | } | ||
| 296 | |||
| 297 |
2/2✓ Branch 1 taken 1 times.
✓ Branch 2 taken 98 times.
|
99 | if(capture_file) |
| 298 | { | ||
| 299 | 1 | ::tts::output().sink(::tts::output_handler::default_sink()); | |
| 300 |
1/1✓ Branch 4 taken 1 times.
|
1 | fputs(capture_sink.content().data(), capture_file.get()); // NOSONAR |
| 301 | } | ||
| 302 | |||
| 303 | 99 | return exit_code; | |
| 304 | 100 | } | |
| 305 | TTS_DISABLE_WARNING_POP | ||
| 306 | |||
| 307 | #endif | ||
| 308 |