Add export compile commands .gn setting. Adds a "export_compile_commands" setting to the .gn file. This can be used as a consistent way to generate a compilation database for projects that require this (the existing --export-compile-commands switch is intended for end-users to use with editor integration). The new variable uses label patterns for better consistency with the rest of the system. The switch value takes precendence if present. Bug: 111 Change-Id: Ieec41f55e9a140066e8b3e3f69115ab087f52172 Reviewed-on: https://gn-review.googlesource.com/c/gn/+/14520 Commit-Queue: Brett Wilson <brettw@chromium.org> Reviewed-by: Takuto Ikuta <tikuta@google.com>
diff --git a/src/gn/command_gen.cc b/src/gn/command_gen.cc index e212fe9..e6ea5d7 100644 --- a/src/gn/command_gen.cc +++ b/src/gn/command_gen.cc
@@ -351,26 +351,49 @@ return res; } -bool RunCompileCommandsWriter(const BuildSettings* build_settings, - const Builder& builder, - Err* err) { +bool RunCompileCommandsWriter(Setup& setup, Err* err) { + // The compilation database is written if either the .gn setting is set or if + // the command line flag is set. The command line flag takes precedence. const base::CommandLine* command_line = base::CommandLine::ForCurrentProcess(); + bool has_switch = command_line->HasSwitch(kSwitchExportCompileCommands); + + bool has_patterns = !setup.export_compile_commands().empty(); + if (!has_switch && !has_patterns) + return true; // No compilation database needs to be written. + bool quiet = command_line->HasSwitch(switches::kQuiet); base::ElapsedTimer timer; - std::string file_name = "compile_commands.json"; - std::string target_filters = - command_line->GetSwitchValueASCII(kSwitchExportCompileCommands); + // The compilation database file goes in the build directory. + SourceFile output_file = + setup.build_settings().build_dir().ResolveRelativeFile( + Value(nullptr, "compile_commands.json"), err); + if (output_file.is_null()) + return false; + base::FilePath output_path = setup.build_settings().GetFullPath(output_file); - bool res = CompileCommandsWriter::RunAndWriteFiles( - build_settings, builder, file_name, target_filters, quiet, err); - if (res && !quiet) { + bool ok = true; + if (has_switch) { + // Legacy format using the command-line switch. + std::string target_filters = + command_line->GetSwitchValueASCII(kSwitchExportCompileCommands); + ok = CompileCommandsWriter::RunAndWriteFilesLegacyFilters( + &setup.build_settings(), setup.builder(), output_path, target_filters, + err); + } else { + // Use the patterns from the .gn file. + ok = CompileCommandsWriter::RunAndWriteFiles( + &setup.build_settings(), setup.builder(), output_path, + setup.export_compile_commands(), err); + } + + if (ok && !quiet) { OutputString("Generating compile_commands took " + base::Int64ToString(timer.Elapsed().InMilliseconds()) + "ms\n"); } - return res; + return ok; } bool RunNinjaPostProcessTools(const BuildSettings* build_settings, @@ -610,20 +633,17 @@ This is an unstable format and likely to change without warning. --export-compile-commands[=<target_name1,target_name2...>] - Produces a compile_commands.json file in the root of the build directory - containing an array of “command objects”, where each command object - specifies one way a translation unit is compiled in the project. If a list - of target_name is supplied, only targets that are reachable from any - target in any build file whose name is target_name will be used for - “command objects” generation, otherwise all available targets will be used. - This is used for various Clang-based tooling, allowing for the replay of - individual compilations independent of the build system. - e.g. "foo" will match: - - "//path/to/src:foo" - - "//other/path:foo" - - "//foo:foo" + Overrides the value of the export_compile_commands in the .gn file (see + "gn help dotfile"). + + Unlike the .gn setting, this switch takes a legacy format which is a list + of target names that are matched in any directory. For example, "foo" will + match: + - "//path/to/src:foo" + - "//other/path:foo" + - "//foo:foo" and not match: - - "//foo:bar" + - "//foo:bar" )"; int RunGen(const std::vector<std::string>& args) { @@ -718,9 +738,7 @@ return 1; } - if (command_line->HasSwitch(kSwitchExportCompileCommands) && - !RunCompileCommandsWriter(&setup->build_settings(), setup->builder(), - &err)) { + if (!RunCompileCommandsWriter(*setup, &err)) { err.PrintToStdout(); return 1; }
diff --git a/src/gn/compile_commands_writer.cc b/src/gn/compile_commands_writer.cc index a2ee3a0..10c5bf5 100644 --- a/src/gn/compile_commands_writer.cc +++ b/src/gn/compile_commands_writer.cc
@@ -300,17 +300,25 @@ bool CompileCommandsWriter::RunAndWriteFiles( const BuildSettings* build_settings, const Builder& builder, - const std::string& file_name, - const std::string& target_filters, - bool quiet, + const base::FilePath& output_path, + const std::vector<LabelPattern>& patterns, Err* err) { - SourceFile output_file = build_settings->build_dir().ResolveRelativeFile( - Value(nullptr, file_name), err); - if (output_file.is_null()) - return false; + std::vector<const Target*> to_write = + CollectDepsOfMatches(builder.GetAllResolvedTargets(), patterns); - base::FilePath output_path = build_settings->GetFullPath(output_file); + StringOutputBuffer json; + std::ostream output_to_json(&json); + OutputJSON(build_settings, to_write, output_to_json); + return json.WriteToFileIfChanged(output_path, err); +} + +bool CompileCommandsWriter::RunAndWriteFilesLegacyFilters( + const BuildSettings* build_settings, + const Builder& builder, + const base::FilePath& output_path, + const std::string& target_filters, + Err* err) { std::vector<const Target*> all_targets = builder.GetAllResolvedTargets(); std::set<std::string> target_filters_set; @@ -333,6 +341,59 @@ return json.WriteToFileIfChanged(output_path, err); } +std::vector<const Target*> CompileCommandsWriter::CollectDepsOfMatches( + const std::vector<const Target*>& all_targets, + const std::vector<LabelPattern>& patterns) { + // The current set of matched targets. + TargetSet collected; + + // Represents the next layer of the breadth-first seach. These are all targets + // that we haven't checked so far. + std::vector<const Target*> frontier; + + // Collect the first level of target matches. These are the ones that the + // patterns match directly. + for (const Target* target : all_targets) { + if (LabelPattern::VectorMatches(patterns, target->label())) { + collected.add(target); + frontier.push_back(target); + } + } + + // Collects the dependencies for the next level of iteration. This could be + // inside the loop but is kept outside to avoid reallocating in every + // iteration. + std::vector<const Target*> next_frontier; + + // Loop for each level of the search. + while (!frontier.empty()) { + for (const Target* target : frontier) { + // Check the target's dependencies. + for (const auto& pair : target->GetDeps(Target::DEPS_ALL)) { + if (!collected.contains(pair.ptr)) { + // New dependency found. + collected.add(pair.ptr); + next_frontier.push_back(pair.ptr); + } + } + } + + // Swap to the new level and clear out the next one without deallocating the + // buffer (in most STL implementations, clear() doesn't free the existing + // buffer). + std::swap(frontier, next_frontier); + next_frontier.clear(); + } + + // Convert to vector for output. + std::vector<const Target*> output; + output.reserve(collected.size()); + for (const Target* target : collected) { + output.push_back(target); + } + return output; +} + std::vector<const Target*> CompileCommandsWriter::FilterTargets( const std::vector<const Target*>& all_targets, const std::set<std::string>& target_filters_set) {
diff --git a/src/gn/compile_commands_writer.h b/src/gn/compile_commands_writer.h index 03006c8..3193f50 100644 --- a/src/gn/compile_commands_writer.h +++ b/src/gn/compile_commands_writer.h
@@ -5,7 +5,10 @@ #ifndef TOOLS_GN_COMPILE_COMMANDS_WRITER_H_ #define TOOLS_GN_COMPILE_COMMANDS_WRITER_H_ +#include <vector> + #include "gn/err.h" +#include "gn/label_pattern.h" #include "gn/target.h" class Builder; @@ -13,23 +16,39 @@ class CompileCommandsWriter { public: - // Write compile commands into a json file located by parameter file_name. - // - // Parameter target_filters should be in "target_name1,target_name2..." - // format. If it is not empty, only targets that are reachable from targets - // in target_filters are used to generate compile commands. - // - // Parameter quiet is not used. + // Writes a compilation database to the given file name consisting of the + // recursive dependencies of all targets that match or are dependencies of + // targets that match any given pattern. static bool RunAndWriteFiles(const BuildSettings* build_setting, const Builder& builder, - const std::string& file_name, - const std::string& target_filters, - bool quiet, + const base::FilePath& output_path, + const std::vector<LabelPattern>& patterns, Err* err); + // Writes a compilation database using the legacy way of specifying which + // targets to output. This format uses a comma-separated list of target names + // ("target_name1,target_name2...") which are matched against targets in any + // directory. Then the recursive dependencies of these deps are collected. + // + // TODO: Remove this legacy target_name behavior and use vector<LabelPattern> + // version consistently. + static bool RunAndWriteFilesLegacyFilters(const BuildSettings* build_setting, + const Builder& builder, + const base::FilePath& output_path, + const std::string& target_filters, + Err* err); + static std::string RenderJSON(const BuildSettings* build_settings, std::vector<const Target*>& all_targets); + // Does a depth-first search of the graph starting at each target that matches + // the given pattern, and collects all recursive dependencies of those + // targets. + static std::vector<const Target*> CollectDepsOfMatches( + const std::vector<const Target*>& all_targets, + const std::vector<LabelPattern>& patterns); + + // Performs the legacy target_name filtering. static std::vector<const Target*> FilterTargets( const std::vector<const Target*>& all_targets, const std::set<std::string>& target_filters_set);
diff --git a/src/gn/compile_commands_writer_unittest.cc b/src/gn/compile_commands_writer_unittest.cc index 29e6a57..f2582c5 100644 --- a/src/gn/compile_commands_writer_unittest.cc +++ b/src/gn/compile_commands_writer_unittest.cc
@@ -586,6 +586,102 @@ EXPECT_EQ(expected, out); } +TEST_F(CompileCommandsTest, CollectDepsOfMatches) { + // Contruct the dependency tree: + // + // //foo:bar1 + // //base:base + // //foo:bar2 + // //base:i18n + // //base + // //third_party:icu + // //random:random + Err err; + std::vector<const Target*> targets; + + Target icu_target(settings(), Label(SourceDir("//third_party/"), "icu")); + icu_target.set_output_type(Target::SOURCE_SET); + icu_target.visibility().SetPublic(); + icu_target.SetToolchain(toolchain()); + ASSERT_TRUE(icu_target.OnResolved(&err)); + targets.push_back(&icu_target); + + Target base_target(settings(), Label(SourceDir("//base/"), "base")); + base_target.set_output_type(Target::SOURCE_SET); + base_target.visibility().SetPublic(); + base_target.SetToolchain(toolchain()); + ASSERT_TRUE(base_target.OnResolved(&err)); + targets.push_back(&base_target); + + Target base_i18n(settings(), Label(SourceDir("//base/"), "i18n")); + base_i18n.set_output_type(Target::SOURCE_SET); + base_i18n.visibility().SetPublic(); + base_i18n.private_deps().push_back(LabelTargetPair(&icu_target)); + base_i18n.public_deps().push_back(LabelTargetPair(&base_target)); + base_i18n.SetToolchain(toolchain()); + ASSERT_TRUE(base_i18n.OnResolved(&err)) + << err.message() << " " << err.help_text(); + targets.push_back(&base_i18n); + + Target target1(settings(), Label(SourceDir("//foo/"), "bar1")); + target1.set_output_type(Target::SOURCE_SET); + target1.public_deps().push_back(LabelTargetPair(&base_target)); + target1.SetToolchain(toolchain()); + ASSERT_TRUE(target1.OnResolved(&err)); + targets.push_back(&target1); + + Target target2(settings(), Label(SourceDir("//foo/"), "bar2")); + target2.set_output_type(Target::SOURCE_SET); + target2.public_deps().push_back(LabelTargetPair(&base_i18n)); + target2.SetToolchain(toolchain()); + ASSERT_TRUE(target2.OnResolved(&err)); + targets.push_back(&target2); + + Target random_target(settings(), Label(SourceDir("//random/"), "random")); + random_target.set_output_type(Target::SOURCE_SET); + random_target.SetToolchain(toolchain()); + ASSERT_TRUE(random_target.OnResolved(&err)); + targets.push_back(&random_target); + + // Sort for stability in comparisons below. + auto compare_label = [](const Target* a, const Target* b) -> bool { + return a->label() < b->label(); + }; + std::sort(targets.begin(), targets.end(), compare_label); + + // Collect everything, the result should match the input. + const std::string source_root("/home/me/build/"); + LabelPattern wildcard_pattern = LabelPattern::GetPattern( + SourceDir(), source_root, Value(nullptr, "//*"), &err); + ASSERT_FALSE(err.has_error()); + std::vector<const Target*> output = + CompileCommandsWriter::CollectDepsOfMatches( + targets, std::vector<LabelPattern>{wildcard_pattern}); + std::sort(output.begin(), output.end(), compare_label); + EXPECT_EQ(output, targets); + + // Collect nothing. + output = CompileCommandsWriter::CollectDepsOfMatches( + targets, std::vector<LabelPattern>()); + EXPECT_TRUE(output.empty()); + + // Collect all deps of "//foo/*". + LabelPattern foo_wildcard = LabelPattern::GetPattern( + SourceDir(), source_root, Value(nullptr, "//foo/*"), &err); + ASSERT_FALSE(err.has_error()); + output = CompileCommandsWriter::CollectDepsOfMatches( + targets, std::vector<LabelPattern>{foo_wildcard}); + + // The result should be everything except "random". + std::sort(output.begin(), output.end(), compare_label); + ASSERT_EQ(5u, output.size()); + EXPECT_EQ(&base_target, output[0]); // Note: order sorted by label. + EXPECT_EQ(&base_i18n, output[1]); + EXPECT_EQ(&target1, output[2]); + EXPECT_EQ(&target2, output[3]); + EXPECT_EQ(&icu_target, output[4]); +} + TEST_F(CompileCommandsTest, CompDBFilter) { Err err;
diff --git a/src/gn/setup.cc b/src/gn/setup.cc index 68d42b7..41bd96d 100644 --- a/src/gn/setup.cc +++ b/src/gn/setup.cc
@@ -23,6 +23,7 @@ #include "gn/exec_process.h" #include "gn/filesystem_utils.h" #include "gn/input_file.h" +#include "gn/label_pattern.h" #include "gn/parse_tree.h" #include "gn/parser.h" #include "gn/source_dir.h" @@ -116,6 +117,25 @@ "//build/my_config.gni", ] + export_compile_commands [optional] + A list of label patterns for which to generate a Clang compilation + database (see "gn help label_pattern" for the string format). + + When specified, GN will generate a compile_commands.json file in the root + of the build directory containing information on how to compile each + source file reachable from any label matching any pattern in the list. + This is used for Clang-based tooling and some editor integration. See + https://clang.llvm.org/docs/JSONCompilationDatabase.html + + The switch --export-compile-commands to "gn gen" (see "gn help gen") + overwrites this value. + + Example: + export_compile_commands = [ + "//base/*", + "//tools:doom_melon", + ] + root [optional] Label of the root build target. The GN build will start by loading the build file containing this target name. This defaults to "//:" which will @@ -1045,7 +1065,19 @@ return false; } build_settings_.set_no_stamp_files(no_stamp_files_value->boolean_value()); - CHECK(!build_settings_.no_stamp_files()) << "no_stamp_files does not work yet!"; + CHECK(!build_settings_.no_stamp_files()) + << "no_stamp_files does not work yet!"; + } + + // Export compile commands. + const Value* export_cc_value = + dotfile_scope_.GetValue("export_compile_commands", true); + if (export_cc_value) { + if (!ExtractListOfLabelPatterns(&build_settings_, *export_cc_value, + SourceDir("//"), &export_compile_commands_, + err)) { + return false; + } } return true;
diff --git a/src/gn/setup.h b/src/gn/setup.h index 2809a7d..91aca82 100644 --- a/src/gn/setup.h +++ b/src/gn/setup.h
@@ -113,6 +113,10 @@ return no_check_patterns_.get(); } + const std::vector<LabelPattern>& export_compile_commands() const { + return export_compile_commands_; + } + BuildSettings& build_settings() { return build_settings_; } Builder& builder() { return builder_; } LoaderImpl* loader() { return loader_.get(); } @@ -207,6 +211,8 @@ std::vector<Token> args_tokens_; std::unique_ptr<ParseNode> args_root_; + std::vector<LabelPattern> export_compile_commands_; + Setup(const Setup&) = delete; Setup& operator=(const Setup&) = delete; };