| // Copyright 2013 The Chromium Authors. All rights reserved. | |
| // Use of this source code is governed by a BSD-style license that can be | |
| // found in the LICENSE file. | |
| #ifndef BASE_TEST_LAUNCHER_TEST_LAUNCHER_H_ | |
| #define BASE_TEST_LAUNCHER_TEST_LAUNCHER_H_ | |
| #include <stddef.h> | |
| #include <stdint.h> | |
| #include <memory> | |
| #include <set> | |
| #include <string> | |
| #include <vector> | |
| #include "base/compiler_specific.h" | |
| #include "base/macros.h" | |
| #include "base/process/launch.h" | |
| #include "base/test/gtest_util.h" | |
| #include "base/test/launcher/test_result.h" | |
| #include "base/test/launcher/test_results_tracker.h" | |
| #include "base/threading/thread_checker.h" | |
| #include "base/time/time.h" | |
| #include "base/timer/timer.h" | |
| #include "build/build_config.h" | |
| namespace base { | |
| class CommandLine; | |
| struct LaunchOptions; | |
| class TestLauncher; | |
| // Constants for GTest command-line flags. | |
| extern const char kGTestFilterFlag[]; | |
| extern const char kGTestFlagfileFlag[]; | |
| extern const char kGTestHelpFlag[]; | |
| extern const char kGTestListTestsFlag[]; | |
| extern const char kGTestRepeatFlag[]; | |
| extern const char kGTestRunDisabledTestsFlag[]; | |
| extern const char kGTestOutputFlag[]; | |
| // Interface for use with LaunchTests that abstracts away exact details | |
| // which tests and how are run. | |
| class TestLauncherDelegate { | |
| public: | |
| // Called to get names of tests available for running. The delegate | |
| // must put the result in |output| and return true on success. | |
| virtual bool GetTests(std::vector<TestIdentifier>* output) = 0; | |
| // Called before a test is considered for running. If it returns false, | |
| // the test is not run. If it returns true, the test will be run provided | |
| // it is part of the current shard. | |
| virtual bool ShouldRunTest(const std::string& test_case_name, | |
| const std::string& test_name) = 0; | |
| // Called to make the delegate run the specified tests. The delegate must | |
| // return the number of actual tests it's going to run (can be smaller, | |
| // equal to, or larger than size of |test_names|). It must also call | |
| // |test_launcher|'s OnTestFinished method once per every run test, | |
| // regardless of its success. | |
| virtual size_t RunTests(TestLauncher* test_launcher, | |
| const std::vector<std::string>& test_names) = 0; | |
| // Called to make the delegate retry the specified tests. The delegate must | |
| // return the number of actual tests it's going to retry (can be smaller, | |
| // equal to, or larger than size of |test_names|). It must also call | |
| // |test_launcher|'s OnTestFinished method once per every retried test, | |
| // regardless of its success. | |
| virtual size_t RetryTests(TestLauncher* test_launcher, | |
| const std::vector<std::string>& test_names) = 0; | |
| protected: | |
| virtual ~TestLauncherDelegate(); | |
| }; | |
| // An observer of child process lifetime events generated by | |
| // LaunchChildGTestProcess. | |
| class ProcessLifetimeObserver { | |
| public: | |
| virtual ~ProcessLifetimeObserver() = default; | |
| // Invoked once the child process is started. |handle| is a handle to the | |
| // child process and |id| is its pid. NOTE: this method is invoked on the | |
| // thread the process is launched on immediately after it is launched. The | |
| // caller owns the ProcessHandle. | |
| virtual void OnLaunched(ProcessHandle handle, ProcessId id) {} | |
| // Invoked when a test process exceeds its runtime, immediately before it is | |
| // terminated. |command_line| is the command line used to launch the process. | |
| // NOTE: this method is invoked on the thread the process is launched on. | |
| virtual void OnTimedOut(const CommandLine& command_line) {} | |
| // Invoked after a child process finishes, reporting the process |exit_code|, | |
| // child process |elapsed_time|, whether or not the process was terminated as | |
| // a result of a timeout, and the output of the child (stdout and stderr | |
| // together). NOTE: this method is invoked on the same thread as | |
| // LaunchChildGTestProcess. | |
| virtual void OnCompleted(int exit_code, | |
| TimeDelta elapsed_time, | |
| bool was_timeout, | |
| const std::string& output) {} | |
| protected: | |
| ProcessLifetimeObserver() = default; | |
| private: | |
| DISALLOW_COPY_AND_ASSIGN(ProcessLifetimeObserver); | |
| }; | |
| // Launches tests using a TestLauncherDelegate. | |
| class TestLauncher { | |
| public: | |
| // Flags controlling behavior of LaunchChildGTestProcess. | |
| enum LaunchChildGTestProcessFlags { | |
| // Allows usage of job objects on Windows. Helps properly clean up child | |
| // processes. | |
| USE_JOB_OBJECTS = (1 << 0), | |
| // Allows breakaway from job on Windows. May result in some child processes | |
| // not being properly terminated after launcher dies if these processes | |
| // fail to cooperate. | |
| ALLOW_BREAKAWAY_FROM_JOB = (1 << 1), | |
| }; | |
| struct LaunchOptions { | |
| LaunchOptions(); | |
| LaunchOptions(const LaunchOptions& other); | |
| ~LaunchOptions(); | |
| int flags = 0; | |
| // These mirror values in base::LaunchOptions, see it for details. | |
| #if defined(OS_WIN) | |
| base::LaunchOptions::Inherit inherit_mode = | |
| base::LaunchOptions::Inherit::kSpecific; | |
| base::HandlesToInheritVector handles_to_inherit; | |
| #elif defined(OS_POSIX) | |
| FileHandleMappingVector fds_to_remap; | |
| #endif | |
| }; | |
| // Constructor. |parallel_jobs| is the limit of simultaneous parallel test | |
| // jobs. | |
| TestLauncher(TestLauncherDelegate* launcher_delegate, size_t parallel_jobs); | |
| ~TestLauncher(); | |
| // Runs the launcher. Must be called at most once. | |
| bool Run() WARN_UNUSED_RESULT; | |
| // Launches a child process (assumed to be gtest-based binary) using | |
| // |command_line|. If |wrapper| is not empty, it is prepended to the final | |
| // command line. |observer|, if not null, is used to convey process lifetime | |
| // events to the caller. |observer| is destroyed after its OnCompleted | |
| // method is invoked. | |
| void LaunchChildGTestProcess( | |
| const CommandLine& command_line, | |
| const std::string& wrapper, | |
| TimeDelta timeout, | |
| const LaunchOptions& options, | |
| std::unique_ptr<ProcessLifetimeObserver> observer); | |
| // Called when a test has finished running. | |
| void OnTestFinished(const TestResult& result); | |
| private: | |
| bool Init() WARN_UNUSED_RESULT; | |
| // Runs all tests in current iteration. | |
| void RunTests(); | |
| void CombinePositiveTestFilters(std::vector<std::string> filter_a, | |
| std::vector<std::string> filter_b); | |
| void RunTestIteration(); | |
| #if defined(OS_POSIX) | |
| void OnShutdownPipeReadable(); | |
| #endif | |
| // Saves test results summary as JSON if requested from command line. | |
| void MaybeSaveSummaryAsJSON(const std::vector<std::string>& additional_tags); | |
| // Called when a test iteration is finished. | |
| void OnTestIterationFinished(); | |
| // Called by the delay timer when no output was made for a while. | |
| void OnOutputTimeout(); | |
| // Make sure we don't accidentally call the wrong methods e.g. on the worker | |
| // pool thread. Should be the first member so that it's destroyed last: when | |
| // destroying other members, especially the worker pool, we may check the code | |
| // is running on the correct thread. | |
| ThreadChecker thread_checker_; | |
| TestLauncherDelegate* launcher_delegate_; | |
| // Support for outer sharding, just like gtest does. | |
| int32_t total_shards_; // Total number of outer shards, at least one. | |
| int32_t shard_index_; // Index of shard the launcher is to run. | |
| int cycles_; // Number of remaining test itreations, or -1 for infinite. | |
| // Test filters (empty means no filter). | |
| bool has_at_least_one_positive_filter_; | |
| std::vector<std::string> positive_test_filter_; | |
| std::vector<std::string> negative_test_filter_; | |
| // Tests to use (cached result of TestLauncherDelegate::GetTests). | |
| std::vector<TestIdentifier> tests_; | |
| // Number of tests found in this binary. | |
| size_t test_found_count_; | |
| // Number of tests started in this iteration. | |
| size_t test_started_count_; | |
| // Number of tests finished in this iteration. | |
| size_t test_finished_count_; | |
| // Number of tests successfully finished in this iteration. | |
| size_t test_success_count_; | |
| // Number of tests either timing out or having an unknown result, | |
| // likely indicating a more systemic problem if widespread. | |
| size_t test_broken_count_; | |
| // Number of retries in this iteration. | |
| size_t retry_count_; | |
| // Maximum number of retries per iteration. | |
| size_t retry_limit_; | |
| // If true will not early exit nor skip retries even if too many tests are | |
| // broken. | |
| bool force_run_broken_tests_; | |
| // Tests to retry in this iteration. | |
| std::set<std::string> tests_to_retry_; | |
| // Result to be returned from Run. | |
| bool run_result_; | |
| TestResultsTracker results_tracker_; | |
| // Watchdog timer to make sure we do not go without output for too long. | |
| DelayTimer watchdog_timer_; | |
| // Number of jobs to run in parallel. | |
| size_t parallel_jobs_; | |
| DISALLOW_COPY_AND_ASSIGN(TestLauncher); | |
| }; | |
| // Return the number of parallel jobs to use, or 0U in case of error. | |
| size_t NumParallelJobs(); | |
| // Extract part from |full_output| that applies to |result|. | |
| std::string GetTestOutputSnippet(const TestResult& result, | |
| const std::string& full_output); | |
| } // namespace base | |
| #endif // BASE_TEST_LAUNCHER_TEST_LAUNCHER_H_ |