GN: Makes GN output deterministic
Sorts various output just prior to printing to ensure that GN output
is deterministic.
The overall effect on performance is relatively small, with
runtime over 50 runs varying as follows on my Z840:
+-------+-------+-------+
| | old | new |
+-------+-------+-------+
| mean | 1415 | 1430 |
+-------+-------+-------+
| stdev | 58.0 | 70.5 |
+-------+-------+-------+
To verify results:
$ gn gen out/Default; mv out out-1
$ gn gen out/Default; mv out out-2
$ diff -qr out-1 out-2
The diff should be empty.
Initial discussion can be seen at https://groups.google.com/a/chromium.org/forum/#!topic/gn-dev/8mOLgM4r3PI.
BUG=565075
Review URL: https://codereview.chromium.org/1494883002
Cr-Original-Commit-Position: refs/heads/master@{#364954}
Cr-Mirrored-From: https://chromium.googlesource.com/chromium/src
Cr-Mirrored-Commit: 98ec25a8fb3f710d43d60b1bcdc5a1475aab136f
diff --git a/tools/gn/ninja_build_writer.cc b/tools/gn/ninja_build_writer.cc
index 07d42e1..a6bea91 100644
--- a/tools/gn/ninja_build_writer.cc
+++ b/tools/gn/ninja_build_writer.cc
@@ -215,12 +215,16 @@
dep_out_ << "build.ninja:";
std::vector<base::FilePath> input_files;
g_scheduler->input_file_manager()->GetAllPhysicalInputFileNames(&input_files);
- for (const auto& input_file : input_files)
- dep_out_ << " " << FilePathToUTF8(input_file);
// Other files read by the build.
std::vector<base::FilePath> other_files = g_scheduler->GetGenDependencies();
- for (const auto& other_file : other_files)
+
+ // Sort the input files to order them deterministically.
+ // Additionally, remove duplicate filepaths that seem to creep in.
+ std::set<base::FilePath> fileset(input_files.begin(), input_files.end());
+ fileset.insert(other_files.begin(), other_files.end());
+
+ for (const auto& other_file : fileset)
dep_out_ << " " << FilePathToUTF8(other_file);
out_ << std::endl;
diff --git a/tools/gn/ninja_target_writer.cc b/tools/gn/ninja_target_writer.cc
index 7641307..d3902c5 100644
--- a/tools/gn/ninja_target_writer.cc
+++ b/tools/gn/ninja_target_writer.cc
@@ -207,26 +207,42 @@
}
// The different souces of input deps may duplicate some targets, so uniquify
- // them (ordering doesn't matter for this case).
- std::set<const Target*> unique_deps;
+ // them. These are sorted so the generated files are deterministic.
+ std::vector<const Target*> sorted_deps;
// Hard dependencies that are direct or indirect dependencies.
- const std::set<const Target*>& hard_deps = target_->recursive_hard_deps();
- for (const auto& dep : hard_deps)
- unique_deps.insert(dep);
+ // These are large (up to 100s), hence why we check other
+ const std::set<const Target*>& hard_deps(target_->recursive_hard_deps());
- // Extra hard dependencies passed in.
- unique_deps.insert(extra_hard_deps.begin(), extra_hard_deps.end());
+ // Extra hard dependencies passed in. Note that these are usually empty/small.
+ for (const Target* target : extra_hard_deps) {
+ if (!hard_deps.count(target))
+ sorted_deps.push_back(target);
+ }
// Toolchain dependencies. These must be resolved before doing anything.
- // This just writs all toolchain deps for simplicity. If we find that
+ // This just writes all toolchain deps for simplicity. If we find that
// toolchains often have more than one dependency, we could consider writing
// a toolchain-specific stamp file and only include the stamp here.
+ // Note that these are usually empty/small.
const LabelTargetVector& toolchain_deps = target_->toolchain()->deps();
- for (const auto& toolchain_dep : toolchain_deps)
- unique_deps.insert(toolchain_dep.ptr);
+ for (const auto& toolchain_dep : toolchain_deps) {
+ // This could theoretically duplicate dependencies already in the list,
+ // but shouldn't happen in practice, is inconvenient to check for,
+ // and only results in harmless redundant dependencies listed.
+ if (!hard_deps.count(toolchain_dep.ptr))
+ sorted_deps.push_back(toolchain_dep.ptr);
+ }
- for (const auto& dep : unique_deps) {
+ for (const Target* target : hard_deps) {
+ sorted_deps.push_back(target);
+ }
+
+ std::sort(
+ sorted_deps.begin(), sorted_deps.end(),
+ [](const Target* a, const Target* b) { return a->label() < b->label(); });
+
+ for (const auto& dep : sorted_deps) {
DCHECK(!dep->dependency_output_file().value().empty());
out_ << " ";
path_output_.WriteFile(out_, dep->dependency_output_file());
diff --git a/tools/gn/ninja_writer.cc b/tools/gn/ninja_writer.cc
index 23b9a72..154f017 100644
--- a/tools/gn/ninja_writer.cc
+++ b/tools/gn/ninja_writer.cc
@@ -10,6 +10,7 @@
#include "tools/gn/ninja_build_writer.h"
#include "tools/gn/ninja_toolchain_writer.h"
#include "tools/gn/settings.h"
+#include "tools/gn/target.h"
NinjaWriter::NinjaWriter(const BuildSettings* build_settings,
Builder* builder)
@@ -66,19 +67,26 @@
return false;
}
+ for (auto& i : categorized) {
+ // Sort targets so that they are in a deterministic order.
+ std::sort(i.second.begin(), i.second.end(),
+ [](const Target* a, const Target* b) {
+ return a->label() < b->label();
+ });
+ }
+
Label default_label = builder_->loader()->GetDefaultToolchain();
// Write out the toolchain buildfiles, and also accumulate the set of
// all settings and find the list of targets in the default toolchain.
- for (CategorizedMap::const_iterator i = categorized.begin();
- i != categorized.end(); ++i) {
+ for (const auto& i : categorized) {
const Settings* settings =
- builder_->loader()->GetToolchainSettings(i->first);
- const Toolchain* toolchain = builder_->GetToolchain(i->first);
+ builder_->loader()->GetToolchainSettings(i.first);
+ const Toolchain* toolchain = builder_->GetToolchain(i.first);
all_settings->push_back(settings);
- if (!NinjaToolchainWriter::RunAndWriteFile(settings, toolchain,
- i->second)) {
+
+ if (!NinjaToolchainWriter::RunAndWriteFile(settings, toolchain, i.second)) {
Err(Location(),
"Couldn't open toolchain buildfile(s) for writing").PrintToStdout();
return false;
