Delay the whole Ninja writer of delayed generated_file() targets generated_file() targets that set collect_validations_metadata must run their metadata walk after all targets are resolved. This was done by having NinjaGeneratedFileTargetWriter report through an out-param that it skipped writing the file, and then writing only the file from the post-resolution callback. Instead, decide whether a target is delayed in ItemResolvedAndGeneratedCallback() (which runs on the main thread), and run the regular BackgroundDoWrite() for delayed targets from the post-resolution callback. This removes the delayed_generated_file_write out-param from NinjaTargetWriter::RunAndWriteFile() and NinjaGeneratedFileTargetWriter. The set of delayed targets and the generated files are unchanged. This is in preparation for letting the Ninja build statement of a generated_file() depend on the result of its metadata walk. Bug: 579603584 Change-Id: I5f5f0940dba3c5d2e0a719dfd3d79b2e95fbd7a3 Agent-Generated: true Reviewed-on: https://gn-review.googlesource.com/c/gn/+/28700 Reviewed-by: David Turner <digit@google.com> Commit-Queue: David Turner <digit@google.com>
diff --git a/src/gn/builder_record.h b/src/gn/builder_record.h index fbb4eab..26bdd9c 100644 --- a/src/gn/builder_record.h +++ b/src/gn/builder_record.h
@@ -132,9 +132,8 @@ // (because D is not loaded or resolved yet). This results in a // runtime crash. // -// To avoid this, these metadata walks MUST be delayed *after* all -// items in the graph have been resolved, and cannot be performed -// in the NinjaGeneratedFileTargetWriter::Run() function itself. +// To avoid this, these generated_file() targets MUST be written *after* all +// items in the graph have been resolved. // class BuilderRecord { public:
diff --git a/src/gn/command_gen.cc b/src/gn/command_gen.cc index 4e991c3..1d6668a 100644 --- a/src/gn/command_gen.cc +++ b/src/gn/command_gen.cc
@@ -18,7 +18,6 @@ #include "gn/compile_commands_writer.h" #include "gn/eclipse_writer.h" #include "gn/filesystem_utils.h" -#include "gn/generated_file_writer.h" #include "gn/json_project_writer.h" #include "gn/label_pattern.h" #include "gn/ninja_outputs_writer.h" @@ -92,8 +91,9 @@ NinjaOutputsMap ninja_outputs_map; - // The list of generated_file() targets whose file writes have been - // delayed because they have collect_validations_metadata == true. + // The list of generated_file() targets whose writes have been delayed + // because they require a fully resolved graph. Only accessed from the main + // thread, so it is not protected by the lock. std::vector<const Target*> delayed_generated_file_targets; std::unique_ptr<ResolvedTargetData> resolved = @@ -109,10 +109,8 @@ std::vector<OutputFile>* ninja_outputs = write_info->want_ninja_outputs ? &target_ninja_outputs : nullptr; - bool delayed_generated_file_write = false; - - std::string rule = NinjaTargetWriter::RunAndWriteFile( - target, resolved, ninja_outputs, &delayed_generated_file_write); + std::string rule = + NinjaTargetWriter::RunAndWriteFile(target, resolved, ninja_outputs); { std::lock_guard<std::mutex> lock(write_info->lock); @@ -127,9 +125,6 @@ write_info->ninja_outputs_map.emplace(target, std::move(target_ninja_outputs)); } - - if (delayed_generated_file_write) - write_info->delayed_generated_file_targets.push_back(target); } } @@ -139,6 +134,13 @@ const Item* item = record->item(); const Target* target = item->AsTarget(); if (target) { + // Do not write these targets yet because some of the targets visited by + // their metadata walks might not be resolved, which would cause runtime + // crashes. See the comment in builder_record.h. + if (target->collect_validations_metadata()) { + write_info->delayed_generated_file_targets.push_back(target); + return; + } g_scheduler->ScheduleWork( [write_info, target]() { BackgroundDoWrite(write_info, target); }); } @@ -827,7 +829,7 @@ }); // Set a post-resolution callback that will be used to schedule - // the writes of delayed generated_file() outputs which require + // the writes of delayed generated_file() targets which require // the full resolution of all targets. // See https://issuetracker.google.com/566346002 setup->SetPostResolutionCallback([&write_info](Scheduler& scheduler) -> bool { @@ -836,12 +838,8 @@ scheduler.IncrementWorkCount(); for (const Target* target : write_info.delayed_generated_file_targets) { - scheduler.ScheduleWork([target, &scheduler]() { - Err err; - if (!WriteGeneratedFileToDisk(target, &err)) { - scheduler.FailWithError(err); - } - }); + scheduler.ScheduleWork( + [target, &write_info]() { BackgroundDoWrite(&write_info, target); }); } scheduler.DecrementWorkCount(); return scheduler.Run();
diff --git a/src/gn/ninja_generated_file_target_writer.cc b/src/gn/ninja_generated_file_target_writer.cc index cafc5eb..7314e72 100644 --- a/src/gn/ninja_generated_file_target_writer.cc +++ b/src/gn/ninja_generated_file_target_writer.cc
@@ -22,19 +22,11 @@ NinjaGeneratedFileTargetWriter::~NinjaGeneratedFileTargetWriter() = default; void NinjaGeneratedFileTargetWriter::Run() { - // Write the file. - if (target_->collect_validations_metadata()) { - // Do not write the file yet because some transitive dependencies - // might not be fully resolved, causing runtime crashes during the - // write. The |delayed_generated_file_write_| pointer must be set - // when validations metadata is collected. - CHECK(delayed_generated_file_write_ptr_); - *delayed_generated_file_write_ptr_ = true; - } else { - Err err; - if (!WriteGeneratedFileToDisk(target_, &err)) { - g_scheduler->FailWithError(err); - } + // Write the file. Targets whose metadata walks require a fully resolved + // graph are only written after all targets are resolved (see command_gen.cc). + Err err; + if (!WriteGeneratedFileToDisk(target_, &err)) { + g_scheduler->FailWithError(err); } // A generated_file target should generate a phony target with dependencies
diff --git a/src/gn/ninja_generated_file_target_writer.h b/src/gn/ninja_generated_file_target_writer.h index 8dbaa43..1e58ffe 100644 --- a/src/gn/ninja_generated_file_target_writer.h +++ b/src/gn/ninja_generated_file_target_writer.h
@@ -13,15 +13,9 @@ NinjaGeneratedFileTargetWriter(const Target* target, StringOutputBuffer& out); ~NinjaGeneratedFileTargetWriter() override; - void SetDelayedGeneratedFileWritePtr(bool* flag_ptr) { - delayed_generated_file_write_ptr_ = flag_ptr; - } - void Run() override; private: - bool* delayed_generated_file_write_ptr_ = nullptr; - NinjaGeneratedFileTargetWriter(const NinjaGeneratedFileTargetWriter&) = delete; NinjaGeneratedFileTargetWriter& operator=(
diff --git a/src/gn/ninja_generated_file_target_writer_unittest.cc b/src/gn/ninja_generated_file_target_writer_unittest.cc index 8a16cf1..f9bb03c 100644 --- a/src/gn/ninja_generated_file_target_writer_unittest.cc +++ b/src/gn/ninja_generated_file_target_writer_unittest.cc
@@ -60,9 +60,7 @@ ASSERT_TRUE(target.OnResolved(&err)) << err.message(); StringOutputBuffer out; - bool delayed_write = false; NinjaGeneratedFileTargetWriter writer(&target, out); - writer.SetDelayedGeneratedFileWritePtr(&delayed_write); writer.Run(); const char expected[] = @@ -70,66 +68,4 @@ "phony/foo/dep2 || " "phony/foo/bundle_data_dep phony/foo/datadep\n"; EXPECT_EQ(expected, out); - EXPECT_FALSE(delayed_write); -} - -TEST_F(NinjaGeneratedFileTargetWriterTest, RunWithDelayedGeneratedFileWrite) { - Err err; - TestWithScope setup; - - Target target(setup.settings(), Label(SourceDir("//foo/"), "bar")); - target.set_output_type(Target::GENERATED_FILE); - target.visibility().SetPublic(); - target.action_values().outputs() = - SubstitutionList::MakeForTest("//out/Debug/foo.json"); - target.set_contents(Value(nullptr, true)); - target.set_output_conversion(Value(nullptr, "json")); - target.set_collect_validations_metadata(true); - - Target dep(setup.settings(), Label(SourceDir("//foo/"), "dep")); - dep.set_output_type(Target::ACTION); - dep.visibility().SetPublic(); - dep.SetToolchain(setup.toolchain()); - ASSERT_TRUE(dep.OnResolved(&err)); - - Target dep2(setup.settings(), Label(SourceDir("//foo/"), "dep2")); - dep2.set_output_type(Target::ACTION); - dep2.visibility().SetPublic(); - dep2.SetToolchain(setup.toolchain()); - ASSERT_TRUE(dep2.OnResolved(&err)); - - Target bundle_data_dep(setup.settings(), - Label(SourceDir("//foo/"), "bundle_data_dep")); - bundle_data_dep.sources().push_back(SourceFile("//foo/some_data.txt")); - bundle_data_dep.set_output_type(Target::BUNDLE_DATA); - bundle_data_dep.visibility().SetPublic(); - bundle_data_dep.SetToolchain(setup.toolchain()); - ASSERT_TRUE(bundle_data_dep.OnResolved(&err)); - - Target datadep(setup.settings(), Label(SourceDir("//foo/"), "datadep")); - datadep.set_output_type(Target::ACTION); - datadep.visibility().SetPublic(); - datadep.SetToolchain(setup.toolchain()); - ASSERT_TRUE(datadep.OnResolved(&err)); - - target.public_deps().push_back(LabelTargetPair(&dep)); - target.public_deps().push_back(LabelTargetPair(&dep2)); - target.public_deps().push_back(LabelTargetPair(&bundle_data_dep)); - target.data_deps().push_back(LabelTargetPair(&datadep)); - - target.SetToolchain(setup.toolchain()); - ASSERT_TRUE(target.OnResolved(&err)) << err.message(); - - StringOutputBuffer out; - NinjaGeneratedFileTargetWriter writer(&target, out); - bool delayed_write = false; - writer.SetDelayedGeneratedFileWritePtr(&delayed_write); - writer.Run(); - - const char expected[] = - "build phony/foo/bar: phony foo.json phony/foo/dep " - "phony/foo/dep2 || " - "phony/foo/bundle_data_dep phony/foo/datadep\n"; - EXPECT_EQ(expected, out); - EXPECT_TRUE(delayed_write); }
diff --git a/src/gn/ninja_target_writer.cc b/src/gn/ninja_target_writer.cc index 4994433..0bdcfa5 100644 --- a/src/gn/ninja_target_writer.cc +++ b/src/gn/ninja_target_writer.cc
@@ -99,8 +99,7 @@ std::string NinjaTargetWriter::RunAndWriteFile( const Target* target, ResolvedTargetData* resolved, - std::vector<OutputFile>* ninja_outputs, - bool* delayed_generated_file_write) { + std::vector<OutputFile>* ninja_outputs) { const Settings* settings = target->settings(); ScopedTrace trace(TraceItem::TRACE_FILE_WRITE_NINJA, @@ -110,9 +109,6 @@ if (g_scheduler->verbose_logging()) g_scheduler->Log("Computing", target->label().GetUserVisibleName(true)); - if (delayed_generated_file_write) - *delayed_generated_file_write = false; - // It's ridiculously faster to write to a string and then write that to // disk in one operation than to use an fstream here. StringOutputBuffer storage; @@ -163,7 +159,6 @@ NinjaGeneratedFileTargetWriter writer(target, storage); writer.SetResolvedTargetData(resolved); writer.SetNinjaOutputs(ninja_outputs); - writer.SetDelayedGeneratedFileWritePtr(delayed_generated_file_write); writer.Run(); } else if (target->IsBinary()) { needs_file_write = true;
diff --git a/src/gn/ninja_target_writer.h b/src/gn/ninja_target_writer.h index fff7c97..4f5c0d1 100644 --- a/src/gn/ninja_target_writer.h +++ b/src/gn/ninja_target_writer.h
@@ -48,17 +48,10 @@ // // If |ninja_outputs| is not nullptr, it will be set with the list of // Ninja output paths generated by the corresponding writer. - // - // If |delayed_generated_file_write| is not null, and the target is - // a GENERATED_FILE with collect_validations_metadata enabled, then - // this function will not write the collected result to a file, but instead - // write true to |*delayed_generated_file_write|. This is to avoid runtime - // crashes (see https://issuetracker.google.com/566346002). static std::string RunAndWriteFile( const Target* target, ResolvedTargetData* resolved = nullptr, - std::vector<OutputFile>* ninja_outputs = nullptr, - bool* delayed_generated_file_write = nullptr); + std::vector<OutputFile>* ninja_outputs = nullptr); // Writes a stamp or phony rule for the target's public inputs if it exports // any. This does nothing if resolved->ExportsPublicInputs(target) returns