GN: Add write_runtime_deps variable Allows targets to use runtime_deps at build-time. BUG=593416 Review URL: https://codereview.chromium.org/1804303004 Cr-Original-Commit-Position: refs/heads/master@{#384411} Cr-Mirrored-From: https://chromium.googlesource.com/chromium/src Cr-Mirrored-Commit: 0f8ad42ac1a6b159655c51bf7095ff2d21ae540f
diff --git a/tools/gn/BUILD.gn b/tools/gn/BUILD.gn index bb36b1f..2728c0d 100644 --- a/tools/gn/BUILD.gn +++ b/tools/gn/BUILD.gn
@@ -277,6 +277,7 @@ "function_write_file_unittest.cc", "functions_target_unittest.cc", "functions_unittest.cc", + "group_target_generator_unittest.cc", "header_checker_unittest.cc", "inherited_libraries_unittest.cc", "input_conversion_unittest.cc",
diff --git a/tools/gn/group_target_generator_unittest.cc b/tools/gn/group_target_generator_unittest.cc new file mode 100644 index 0000000..3b0c824 --- /dev/null +++ b/tools/gn/group_target_generator_unittest.cc
@@ -0,0 +1,46 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "testing/gtest/include/gtest/gtest.h" +#include "tools/gn/group_target_generator.h" +#include "tools/gn/scheduler.h" +#include "tools/gn/test_with_scope.h" + +namespace { + +// Returns true on success, false if write_file signaled an error. +bool ParseWriteRuntimeDeps(Scope* scope, const std::string& value) { + TestParseInput input( + "group(\"foo\") { write_runtime_deps = " + value + "}"); + if (input.has_error()) + return false; + + Err err; + input.parsed()->Execute(scope, &err); + return !err.has_error(); +} + +} // namespace + + +// Tests that actions can't have output substitutions. +TEST(GroupTargetGenerator, WriteRuntimeDeps) { + Scheduler scheduler; + TestWithScope setup; + Scope::ItemVector items_; + setup.scope()->set_item_collector(&items_); + + // Should refuse to write files outside of the output dir. + EXPECT_FALSE(ParseWriteRuntimeDeps(setup.scope(), "\"//foo.txt\"")); + EXPECT_EQ(0U, scheduler.GetWriteRuntimeDepsTargets().size()); + + // Should fail for garbage inputs. + EXPECT_FALSE(ParseWriteRuntimeDeps(setup.scope(), "0")); + EXPECT_EQ(0U, scheduler.GetWriteRuntimeDepsTargets().size()); + + // Should be able to write inside the out dir. + EXPECT_TRUE(ParseWriteRuntimeDeps(setup.scope(), "\"//out/Debug/foo.txt\"")); + EXPECT_EQ(1U, scheduler.GetWriteRuntimeDepsTargets().size()); +} +
diff --git a/tools/gn/runtime_deps.cc b/tools/gn/runtime_deps.cc index 078bebe..17b9c07 100644 --- a/tools/gn/runtime_deps.cc +++ b/tools/gn/runtime_deps.cc
@@ -17,6 +17,7 @@ #include "tools/gn/filesystem_utils.h" #include "tools/gn/loader.h" #include "tools/gn/output_file.h" +#include "tools/gn/scheduler.h" #include "tools/gn/settings.h" #include "tools/gn/switches.h" #include "tools/gn/target.h" @@ -118,25 +119,72 @@ } } -bool WriteRuntimeDepsFile(const Target* target) { - SourceFile target_output_as_source = - GetMainOutput(target).AsSourceFile(target->settings()->build_settings()); - std::string data_deps_file_as_str = target_output_as_source.value(); - data_deps_file_as_str.append(".runtime_deps"); +bool CollectRuntimeDepsFromFlag(const Builder& builder, + RuntimeDepsVector* files_to_write, + Err* err) { + std::string deps_target_list_file = + base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII( + switches::kRuntimeDepsListFile); + + if (deps_target_list_file.empty()) + return true; + + std::string list_contents; + ScopedTrace load_trace(TraceItem::TRACE_FILE_LOAD, deps_target_list_file); + if (!base::ReadFileToString(UTF8ToFilePath(deps_target_list_file), + &list_contents)) { + *err = Err(Location(), + std::string("File for --") + switches::kRuntimeDepsListFile + + " doesn't exist.", + "The file given was \"" + deps_target_list_file + "\""); + return false; + } + load_trace.Done(); + + SourceDir root_dir("//"); + Label default_toolchain_label = builder.loader()->GetDefaultToolchain(); + for (const auto& line : + base::SplitString(list_contents, "\n", base::TRIM_WHITESPACE, + base::SPLIT_WANT_ALL)) { + if (line.empty()) + continue; + Label label = Label::Resolve(root_dir, default_toolchain_label, + Value(nullptr, line), err); + if (err->has_error()) + return false; + + const Item* item = builder.GetItem(label); + const Target* target = item ? item->AsTarget() : nullptr; + if (!target) { + *err = Err(Location(), "The label \"" + label.GetUserVisibleName(true) + + "\" isn't a target.", + "When reading the line:\n " + line + "\n" + "from the --" + switches::kRuntimeDepsListFile + "=" + + deps_target_list_file); + return false; + } + + OutputFile output_file = + OutputFile(GetMainOutput(target).value() + ".runtime_deps"); + files_to_write->push_back(std::make_pair(output_file, target)); + } + return true; +} + +bool WriteRuntimeDepsFile(const OutputFile& output_file, + const Target* target, + Err* err) { + SourceFile output_as_source = + output_file.AsSourceFile(target->settings()->build_settings()); base::FilePath data_deps_file = - target->settings()->build_settings()->GetFullPath( - SourceFile(SourceFile::SwapIn(), &data_deps_file_as_str)); + target->settings()->build_settings()->GetFullPath(output_as_source); std::stringstream contents; for (const auto& pair : ComputeRuntimeDeps(target)) contents << pair.first.value() << std::endl; - ScopedTrace trace(TraceItem::TRACE_FILE_WRITE, data_deps_file_as_str); - base::CreateDirectory(data_deps_file.DirName()); - - std::string contents_str = contents.str(); - return base::WriteFile(data_deps_file, contents_str.c_str(), - static_cast<int>(contents_str.size())) > -1; + ScopedTrace trace(TraceItem::TRACE_FILE_WRITE, output_as_source.value()); + return WriteFileIfChanged(data_deps_file, contents.str(), err); } } // namespace @@ -146,8 +194,8 @@ "\n" " Runtime dependencies of a target are exposed via the \"runtime_deps\"\n" " category of \"gn desc\" (see \"gn help desc\") or they can be written\n" - " at build generation time via \"--runtime-deps-list-file\"\n" - " (see \"gn help --runtime-deps-list-file\").\n" + " at build generation time via write_runtime_deps(), or\n" + " --runtime-deps-list-file (see \"gn help --runtime-deps-list-file\").\n" "\n" " To a first approximation, the runtime dependencies of a target are\n" " the set of \"data\" files, data directories, and the shared libraries\n" @@ -225,50 +273,22 @@ } bool WriteRuntimeDepsFilesIfNecessary(const Builder& builder, Err* err) { - std::string deps_target_list_file = - base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII( - switches::kRuntimeDepsListFile); - if (deps_target_list_file.empty()) - return true; // Nothing to do. - - std::string list_contents; - ScopedTrace load_trace(TraceItem::TRACE_FILE_LOAD, deps_target_list_file); - if (!base::ReadFileToString(UTF8ToFilePath(deps_target_list_file), - &list_contents)) { - *err = Err(Location(), - std::string("File for --") + switches::kRuntimeDepsListFile + - " doesn't exist.", - "The file given was \"" + deps_target_list_file + "\""); + RuntimeDepsVector files_to_write; + if (!CollectRuntimeDepsFromFlag(builder, &files_to_write, err)) return false; + + // Files scheduled by write_runtime_deps. + for (const Target* target : g_scheduler->GetWriteRuntimeDepsTargets()) { + files_to_write.push_back( + std::make_pair(target->write_runtime_deps_output(), target)); } - load_trace.Done(); - SourceDir root_dir("//"); - Label default_toolchain_label = builder.loader()->GetDefaultToolchain(); - for (const auto& line : base::SplitString( - list_contents, "\n", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL)) { - if (line.empty()) - continue; - Label label = Label::Resolve(root_dir, default_toolchain_label, - Value(nullptr, line), err); - if (err->has_error()) - return false; - - const Item* item = builder.GetItem(label); - const Target* target = item ? item->AsTarget() : nullptr; - if (!target) { - *err = Err(Location(), "The label \"" + label.GetUserVisibleName(true) + - "\" isn't a target.", - "When reading the line:\n " + line + "\n" - "from the --" + switches::kRuntimeDepsListFile + "=" + - deps_target_list_file); - return false; - } - + for (const auto& entry : files_to_write) { // Currently this writes all runtime deps files sequentially. We generally // expect few of these. We can run this on the worker pool if it looks // like it's talking a long time. - WriteRuntimeDepsFile(target); + if (!WriteRuntimeDepsFile(entry.first, entry.second, err)) + return false; } return true; }
diff --git a/tools/gn/scheduler.cc b/tools/gn/scheduler.cc index dfc877d..a711df1 100644 --- a/tools/gn/scheduler.cc +++ b/tools/gn/scheduler.cc
@@ -12,6 +12,7 @@ #include "build/build_config.h" #include "tools/gn/standard_out.h" #include "tools/gn/switches.h" +#include "tools/gn/target.h" #if defined(OS_WIN) #include <windows.h> @@ -153,6 +154,28 @@ unknown_generated_inputs_.insert(std::make_pair(file, target)); } +void Scheduler::AddWriteRuntimeDepsTarget(const Target* target) { + base::AutoLock lock(lock_); + write_runtime_deps_targets_.push_back(target); +} + +std::vector<const Target*> Scheduler::GetWriteRuntimeDepsTargets() const { + base::AutoLock lock(lock_); + return write_runtime_deps_targets_; +} + +bool Scheduler::IsFileGeneratedByWriteRuntimeDeps( + const OutputFile& file) const { + base::AutoLock lock(lock_); + // Number of targets should be quite small, so brute-force search is fine. + for (const Target* target : write_runtime_deps_targets_) { + if (file == target->write_runtime_deps_output()) { + return true; + } + } + return false; +} + std::multimap<SourceFile, const Target*> Scheduler::GetUnknownGeneratedInputs() const { base::AutoLock lock(lock_);
diff --git a/tools/gn/scheduler.h b/tools/gn/scheduler.h index 45225ce..a9bb036 100644 --- a/tools/gn/scheduler.h +++ b/tools/gn/scheduler.h
@@ -15,6 +15,9 @@ #include "base/synchronization/lock.h" #include "base/threading/sequenced_worker_pool.h" #include "tools/gn/input_file_manager.h" +#include "tools/gn/label.h" +#include "tools/gn/source_file.h" +#include "tools/gn/token.h" class Target; @@ -57,6 +60,11 @@ // inputs (see AddUnknownGeneratedInput below). void AddWrittenFile(const SourceFile& file); + // Schedules a file to be written due to a target setting write_runtime_deps. + void AddWriteRuntimeDepsTarget(const Target* entry); + std::vector<const Target*> GetWriteRuntimeDepsTargets() const; + bool IsFileGeneratedByWriteRuntimeDeps(const OutputFile& file) const; + // Unknown generated inputs are files that a target declares as an input // in the output directory, but which aren't generated by any dependency. // @@ -113,6 +121,7 @@ // Protected by the lock. See the corresponding Add/Get functions above. std::vector<base::FilePath> gen_dependencies_; std::vector<SourceFile> written_files_; + std::vector<const Target*> write_runtime_deps_targets_; std::multimap<SourceFile, const Target*> unknown_generated_inputs_; DISALLOW_COPY_AND_ASSIGN(Scheduler);
diff --git a/tools/gn/target.cc b/tools/gn/target.cc index 2386870..005dd97 100644 --- a/tools/gn/target.cc +++ b/tools/gn/target.cc
@@ -83,6 +83,7 @@ const OutputFile& file, bool check_private_deps, bool consider_object_files, + bool check_data_deps, std::set<const Target*>* seen_targets) { if (seen_targets->find(target) != seen_targets->end()) return false; // Already checked this one and it's not found. @@ -96,6 +97,9 @@ return true; } + if (file == target->write_runtime_deps_output()) + return true; + // Check binary target intermediate files if requested. if (consider_object_files && target->IsBinary()) { std::vector<OutputFile> source_outputs; @@ -109,11 +113,22 @@ } } + if (check_data_deps) { + check_data_deps = false; // Consider only direct data_deps. + for (const auto& pair : target->data_deps()) { + if (EnsureFileIsGeneratedByDependency(pair.ptr, file, false, + consider_object_files, + check_data_deps, seen_targets)) + return true; // Found a path. + } + } + // Check all public dependencies (don't do data ones since those are // runtime-only). for (const auto& pair : target->public_deps()) { if (EnsureFileIsGeneratedByDependency(pair.ptr, file, false, - consider_object_files, seen_targets)) + consider_object_files, + check_data_deps, seen_targets)) return true; // Found a path. } @@ -122,14 +137,14 @@ for (const auto& pair : target->private_deps()) { if (EnsureFileIsGeneratedByDependency(pair.ptr, file, false, consider_object_files, - seen_targets)) + check_data_deps, seen_targets)) return true; // Found a path. } if (target->output_type() == Target::CREATE_BUNDLE) { for (const auto& dep : target->bundle_data().bundle_deps()) { if (EnsureFileIsGeneratedByDependency(dep, file, false, consider_object_files, - seen_targets)) + check_data_deps, seen_targets)) return true; // Found a path. } } @@ -752,13 +767,20 @@ // can be filtered out of this list. OutputFile out_file(settings()->build_settings(), source); std::set<const Target*> seen_targets; - if (!EnsureFileIsGeneratedByDependency(this, out_file, true, false, + bool check_data_deps = false; + bool consider_object_files = false; + if (!EnsureFileIsGeneratedByDependency(this, out_file, true, + consider_object_files, check_data_deps, &seen_targets)) { + seen_targets.clear(); + // Allow dependency to be through data_deps for files generated by gn. + check_data_deps = g_scheduler->IsFileGeneratedByWriteRuntimeDeps(out_file); // Check object files (much slower and very rare) only if the "normal" // output check failed. - seen_targets.clear(); - if (!EnsureFileIsGeneratedByDependency(this, out_file, true, true, - &seen_targets)) + consider_object_files = !check_data_deps; + if (!EnsureFileIsGeneratedByDependency(this, out_file, true, + consider_object_files, + check_data_deps, &seen_targets)) g_scheduler->AddUnknownGeneratedInput(this, source); } }
diff --git a/tools/gn/target.h b/tools/gn/target.h index 95e682f..c3fed01 100644 --- a/tools/gn/target.h +++ b/tools/gn/target.h
@@ -128,6 +128,13 @@ bool testonly() const { return testonly_; } void set_testonly(bool value) { testonly_ = value; } + OutputFile write_runtime_deps_output() const { + return write_runtime_deps_output_; + } + void set_write_runtime_deps_output(const OutputFile& value) { + write_runtime_deps_output_ = value; + } + // Compile-time extra dependencies. const FileList& inputs() const { return inputs_; } FileList& inputs() { return inputs_; } @@ -322,6 +329,7 @@ FileList inputs_; std::vector<std::string> data_; BundleData bundle_data_; + OutputFile write_runtime_deps_output_; LabelTargetVector private_deps_; LabelTargetVector public_deps_;
diff --git a/tools/gn/target_generator.cc b/tools/gn/target_generator.cc index f5b4ec8..4a84bcd 100644 --- a/tools/gn/target_generator.cc +++ b/tools/gn/target_generator.cc
@@ -58,6 +58,9 @@ if (!Visibility::FillItemVisibility(target_, scope_, err_)) return; + if (!FillWriteRuntimeDeps()) + return; + // Do type-specific generation. DoRun(); } @@ -383,3 +386,23 @@ } return !err_->has_error(); } + +bool TargetGenerator::FillWriteRuntimeDeps() { + const Value* value = scope_->GetValue(variables::kWriteRuntimeDeps, true); + if (!value) + return true; + + // Compute the file name and make sure it's in the output dir. + SourceFile source_file = scope_->GetSourceDir().ResolveRelativeFile( + *value, err_, GetBuildSettings()->root_path_utf8()); + if (err_->has_error()) + return false; + if (!EnsureStringIsInOutputDir(GetBuildSettings()->build_dir(), + source_file.value(), value->origin(), err_)) + return false; + OutputFile output_file(GetBuildSettings(), source_file); + target_->set_write_runtime_deps_output(output_file); + + g_scheduler->AddWriteRuntimeDepsTarget(target_); + return true; +}
diff --git a/tools/gn/target_generator.h b/tools/gn/target_generator.h index 4fa51f6..a3bdd58 100644 --- a/tools/gn/target_generator.h +++ b/tools/gn/target_generator.h
@@ -71,6 +71,7 @@ bool FillDependencies(); // Includes data dependencies. bool FillTestonly(); bool FillAssertNoDeps(); + bool FillWriteRuntimeDeps(); // Reads configs/deps from the given var name, and uses the given setting on // the target to save them.
diff --git a/tools/gn/target_unittest.cc b/tools/gn/target_unittest.cc index 876c2e3..b3a85b7 100644 --- a/tools/gn/target_unittest.cc +++ b/tools/gn/target_unittest.cc
@@ -627,6 +627,53 @@ EXPECT_TRUE(scheduler.GetUnknownGeneratedInputs().empty()); } +TEST(Target, WriteRuntimeDepsGeneratedInputs) { + Scheduler scheduler; + TestWithScope setup; + Err err; + + SourceFile source_file("//out/Debug/generated.runtime_deps"); + OutputFile output_file(setup.build_settings(), source_file); + + TestTarget generator(setup, "//foo:generator", Target::EXECUTABLE); + generator.set_write_runtime_deps_output(output_file); + g_scheduler->AddWriteRuntimeDepsTarget(&generator); + + TestTarget middle_data_dep(setup, "//foo:middle", Target::EXECUTABLE); + middle_data_dep.data_deps().push_back(LabelTargetPair(&generator)); + + // This target has a generated input and no dependency makes it. + TestTarget dep_missing(setup, "//foo:no_dep", Target::EXECUTABLE); + dep_missing.sources().push_back(source_file); + EXPECT_TRUE(dep_missing.OnResolved(&err)); + AssertSchedulerHasOneUnknownFileMatching(&dep_missing, source_file); + scheduler.ClearUnknownGeneratedInputsAndWrittenFiles(); + + // This target has a generated file and we've directly dependended on it. + TestTarget dep_present(setup, "//foo:with_dep", Target::EXECUTABLE); + dep_present.sources().push_back(source_file); + dep_present.private_deps().push_back(LabelTargetPair(&generator)); + EXPECT_TRUE(dep_present.OnResolved(&err)); + EXPECT_TRUE(scheduler.GetUnknownGeneratedInputs().empty()); + + // This target has a generated file and we've indirectly dependended on it + // via data_deps. + TestTarget dep_indirect(setup, "//foo:with_dep", Target::EXECUTABLE); + dep_indirect.sources().push_back(source_file); + dep_indirect.data_deps().push_back(LabelTargetPair(&middle_data_dep)); + EXPECT_TRUE(dep_indirect.OnResolved(&err)); + AssertSchedulerHasOneUnknownFileMatching(&dep_indirect, source_file); + scheduler.ClearUnknownGeneratedInputsAndWrittenFiles(); + + // This target has a generated file and we've directly dependended on it + // via data_deps. + TestTarget data_dep_present(setup, "//foo:with_dep", Target::EXECUTABLE); + data_dep_present.sources().push_back(source_file); + data_dep_present.data_deps().push_back(LabelTargetPair(&generator)); + EXPECT_TRUE(data_dep_present.OnResolved(&err)); + EXPECT_TRUE(scheduler.GetUnknownGeneratedInputs().empty()); +} + // Tests that intermediate object files generated by binary targets are also // considered generated for the purposes of input checking. Above, we tested // the failure cases for generated inputs, so here only test .o files that are
diff --git a/tools/gn/variables.cc b/tools/gn/variables.cc index 81bb7e3..88e0ed1 100644 --- a/tools/gn/variables.cc +++ b/tools/gn/variables.cc
@@ -1516,6 +1516,30 @@ " any targets in \"//bar/\" and any subdirectory thereof.\n" " visibility = [ \"./*\", \"//bar/*\" ]\n"; +const char kWriteRuntimeDeps[] = "write_runtime_deps"; +const char kWriteRuntimeDeps_HelpShort[] = + "write_runtime_deps: Writes the target's runtime_deps to the given path."; +const char kWriteRuntimeDeps_Help[] = + "write_runtime_deps: Writes the target's runtime_deps to the given path.\n" + "\n" + " Does not synchronously write the file, but rather schedules it\n" + " to be written at the end of generation.\n" + "\n" + " If the file exists and the contents are identical to that being\n" + " written, the file will not be updated. This will prevent unnecessary\n" + " rebuilds of targets that depend on this file.\n" + "\n" + " Path must be within the output directory.\n" + "\n" + " See \"gn help runtime_deps\" for how the runtime dependencies are\n" + " computed.\n" + "\n" + " The format of this file will list one file per line with no escaping.\n" + " The files will be relative to the root_build_dir. The first line of\n" + " the file will be the main output file of the target itself. The file\n" + " contents will be the same as requesting the runtime deps be written on\n" + " the command line (see \"gn help --runtime-deps-list-file\").\n"; + // ----------------------------------------------------------------------------- VariableInfo::VariableInfo() @@ -1596,6 +1620,7 @@ INSERT_VARIABLE(Sources) INSERT_VARIABLE(Testonly) INSERT_VARIABLE(Visibility) + INSERT_VARIABLE(WriteRuntimeDeps) } return info_map; }
diff --git a/tools/gn/variables.h b/tools/gn/variables.h index 45fa0eb..27bde5a 100644 --- a/tools/gn/variables.h +++ b/tools/gn/variables.h
@@ -235,6 +235,10 @@ extern const char kVisibility_HelpShort[]; extern const char kVisibility_Help[]; +extern const char kWriteRuntimeDeps[]; +extern const char kWriteRuntimeDeps_HelpShort[]; +extern const char kWriteRuntimeDeps_Help[]; + // ----------------------------------------------------------------------------- struct VariableInfo {