Restat build.ninja after gen As of version 1.8, ninja caches mtimes in the .ninja_log file and will not restat a file if it has an entry in the .ninja_log. This becomes problematic for generator files such as build.ninja. In a scenario where the .ninja_log has an entry for build.ninja (ie, after ninja has had to trigger a re-gen) and a manual run of of `gn gen` has updated the build.ninja file, a subsequent invocation of ninja will trigger yet another re-gen of build.ninja. This can be reproduced like this: # Initial build gn gen out && ninja -C out # Trigger ninja re-gen so there's a build.ninja log entry touch BUILD.gn && ninja -C out # Manual re-gen gn gen out # The following command will re-gen again before building ninja -C out To work around this, the recommended approach from ninja is to use the ninja restat tool to get ninja to update the mtime in the .ninja_log. This CL does that by attempting to invoke `ninja -t restat build.ninja` immediately after the build.ninja is generated by `gn gen`. Because this tool was only introduced in 1.10, the usage is gated on having a ninja binary of at least that version. The ninja binary can be provided via the --ninja-executable switch. Bug: 136 Change-Id: If40391aa13e6ef71c4e8ab26aff57d66a8137b8e Reviewed-on: https://gn-review.googlesource.com/c/gn/+/10400 Commit-Queue: RJ Ascani <rjascani@google.com> Reviewed-by: Brett Wilson <brettw@chromium.org>
diff --git a/build/gen.py b/build/gen.py index 486354b..0f40b75 100755 --- a/build/gen.py +++ b/build/gen.py
@@ -586,6 +586,7 @@ 'src/gn/ninja_target_command_util.cc', 'src/gn/ninja_target_writer.cc', 'src/gn/ninja_toolchain_writer.cc', + 'src/gn/ninja_tools.cc', 'src/gn/ninja_utils.cc', 'src/gn/ninja_writer.cc', 'src/gn/operators.cc',
diff --git a/docs/reference.md b/docs/reference.md index 06ab5a0..8eb1510 100644 --- a/docs/reference.md +++ b/docs/reference.md
@@ -718,6 +718,16 @@ See "gn help switches" for the common command-line switches. ``` +#### **General options** + +``` + --ninja-executable=<string> + Can be used to specify the ninja executable to use. This executable will + be used as an IDE option to indicate which ninja to use for building. This + executable will also be used as part of the gen process for triggering a + restat on generated ninja files. +``` + #### **IDE options** ``` @@ -7760,6 +7770,7 @@ * --dotfile: Override the name of the ".gn" file. * --fail-on-unused-args: Treat unused build args as fatal errors. * --markdown: Write help output in the Markdown format. + * --ninja-executable: Set the Ninja executable. * --nocolor: Force non-colored output. * -q: Quiet mode. Don't print output on success. * --root: Explicitly specify source root.
diff --git a/src/gn/command_gen.cc b/src/gn/command_gen.cc index 7c7fd01..4437236 100644 --- a/src/gn/command_gen.cc +++ b/src/gn/command_gen.cc
@@ -12,8 +12,10 @@ #include "gn/commands.h" #include "gn/compile_commands_writer.h" #include "gn/eclipse_writer.h" +#include "gn/filesystem_utils.h" #include "gn/json_project_writer.h" #include "gn/ninja_target_writer.h" +#include "gn/ninja_tools.h" #include "gn/ninja_writer.h" #include "gn/qt_creator_writer.h" #include "gn/runtime_deps.h" @@ -352,6 +354,38 @@ return res; } +bool RunNinjaPostProcessTools(const BuildSettings* build_settings, + base::FilePath ninja_executable, + bool is_regeneration, + Err* err) { + // If the user did not specify an executable, skip running the post processing + // tools. Since these tools can re-write ninja build log and dep logs, it is + // really important that ninja executable used for tools matches the + // executable that is used for builds. + if (ninja_executable.empty()) { + return true; + } + + base::FilePath build_dir = + build_settings->GetFullPath(build_settings->build_dir()); + + // If we have a ninja version that supports restat, we should restat the + // build.ninja file so the next ninja invocation will use the right mtime. If + // gen is being invoked as part of a re-gen (ie, ninja is invoking gn gen), + // then we can elide this restat, as ninja will restat build.ninja anyways + // after it is complete. + if (!is_regeneration && + build_settings->ninja_required_version() >= Version{1, 10, 0}) { + std::vector<base::FilePath> files_to_restat{ + base::FilePath(FILE_PATH_LITERAL("build.ninja"))}; + if (!InvokeNinjaRestatTool(ninja_executable, build_dir, files_to_restat, + err)) { + return false; + } + } + return true; +} + } // namespace const char kGen[] = "gen"; @@ -373,6 +407,14 @@ See "gn help switches" for the common command-line switches. +General options + + --ninja-executable=<string> + Can be used to specify the ninja executable to use. This executable will + be used as an IDE option to indicate which ninja to use for building. This + executable will also be used as part of the gen process for triggering a + restat on generated ninja files. + IDE options GN optionally generates files for IDE. Files won't be overwritten if their @@ -558,6 +600,14 @@ return 1; } + if (!RunNinjaPostProcessTools( + &setup->build_settings(), + command_line->GetSwitchValuePath(switches::kNinjaExecutable), + command_line->HasSwitch(switches::kRegeneration), &err)) { + err.PrintToStdout(); + return 1; + } + if (!WriteRuntimeDepsFilesIfNecessary(&setup->build_settings(), setup->builder(), &err)) { err.PrintToStdout();
diff --git a/src/gn/ninja_build_writer.cc b/src/gn/ninja_build_writer.cc index 164c1d5..04e0754 100644 --- a/src/gn/ninja_build_writer.cc +++ b/src/gn/ninja_build_writer.cc
@@ -113,6 +113,15 @@ } } + // Add the regeneration switch if not already present. This is so that when + // the regeneration is invoked by ninja, the gen command is aware that it is a + // regeneration invocation and not an user invocation. This allows the gen + // command to elide ninja post processing steps that ninja will perform + // itself. + if (!cmdline.HasSwitch(switches::kRegeneration)) { + cmdline.AppendSwitch(switches::kRegeneration); + } + return cmdline; }
diff --git a/src/gn/ninja_tools.cc b/src/gn/ninja_tools.cc new file mode 100644 index 0000000..ed45544 --- /dev/null +++ b/src/gn/ninja_tools.cc
@@ -0,0 +1,64 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "gn/ninja_tools.h" + +#include <vector> + +#include "base/command_line.h" +#include "base/files/file_path.h" +#include "base/strings/string_number_conversions.h" +#include "gn/err.h" +#include "gn/exec_process.h" +#include "gn/filesystem_utils.h" + +namespace { + +base::CommandLine CreateNinjaToolCommandLine(const base::FilePath& ninja_executable, + const std::string& tool) { + base::CommandLine cmdline(ninja_executable); + cmdline.SetParseSwitches(false); + cmdline.AppendArg("-t"); + cmdline.AppendArg(tool); + return cmdline; +} + +bool RunNinja(const base::CommandLine& cmdline, + const base::FilePath& startup_dir, + std::string* output, + Err* err) { + std::string stderr_output; + + int exit_code = 0; + if (!internal::ExecProcess(cmdline, startup_dir, output, &stderr_output, + &exit_code)) { + *err = Err(Location(), "Could not execute Ninja.", + "I was trying to execute \"" + + FilePathToUTF8(cmdline.GetProgram()) + "\"."); + return false; + } + + if (exit_code != 0) { + *err = Err(Location(), "Ninja has quit with exit code " + + base::IntToString(exit_code) + "."); + return false; + } + + return true; +} + +} // namespace + +bool InvokeNinjaRestatTool(const base::FilePath& ninja_executable, + const base::FilePath& build_dir, + const std::vector<base::FilePath>& files_to_restat, + Err* err) { + base::CommandLine cmdline = + CreateNinjaToolCommandLine(ninja_executable, "restat"); + for (const base::FilePath& file : files_to_restat) { + cmdline.AppendArgPath(file); + } + std::string output; + return RunNinja(cmdline, build_dir, &output, err); +}
diff --git a/src/gn/ninja_tools.h b/src/gn/ninja_tools.h new file mode 100644 index 0000000..4eda1e1 --- /dev/null +++ b/src/gn/ninja_tools.h
@@ -0,0 +1,30 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef TOOLS_GN_NINJA_TOOLS_H_ +#define TOOLS_GN_NINJA_TOOLS_H_ + +#include <vector> + +#include "base/files/file_path.h" +#include "gn/err.h" + +// Invokes the ninja restat tool (ie, ninja -C build_dir -t restat). This tool +// tells ninja that it should check the mtime of the provided files and update +// the .ninja_log accordingly. This is useful when GN knows that an output file +// in the ninja graph has been updated without invoking ninja. +// +// The best example of this is after gn gen runs, we know that build.ninja has +// been potentially updated, but ninja will still use the mtime from the +// .ninja_log and could trigger another re-gen. By telling ninja to restat +// build.ninja, we can eliminate the extra re-gen. +// +// If files_to_restat is empty, ninja will restat all files that have an entry +// in the .ninja_log. +bool InvokeNinjaRestatTool(const base::FilePath& ninja_executable, + const base::FilePath& build_dir, + const std::vector<base::FilePath>& files_to_restat, + Err* err); + +#endif // TOOLS_GN_NINJA_TOOLS_H_
diff --git a/src/gn/switches.cc b/src/gn/switches.cc index 0ad3a6b..fb63b30 100644 --- a/src/gn/switches.cc +++ b/src/gn/switches.cc
@@ -99,6 +99,16 @@ const char kNoColor_HelpShort[] = "--nocolor: Force non-colored output."; const char kNoColor_Help[] = COLOR_HELP_LONG; +const char kNinjaExecutable[] = "ninja-executable"; +const char kNinjaExecutable_HelpShort[] = + "--ninja-executable: Set the Ninja executable."; +const char kNinjaExecutable_Help[] = + R"(--ninja-executable: Set the Ninja executable. + + When set specifies the Ninja executable that will be used to perform some + post-processing on the generated files for more consistent builds. +)"; + const char kScriptExecutable[] = "script-executable"; const char kScriptExecutable_HelpShort[] = "--script-executable: Set the executable used to execute scripts."; @@ -306,6 +316,7 @@ const char kDefaultToolchain[] = "default-toolchain"; +const char kRegeneration[] = "regeneration"; // ----------------------------------------------------------------------------- SwitchInfo::SwitchInfo() : short_help(""), long_help("") {} @@ -324,6 +335,7 @@ INSERT_VARIABLE(Dotfile) INSERT_VARIABLE(FailOnUnusedArgs) INSERT_VARIABLE(Markdown) + INSERT_VARIABLE(NinjaExecutable) INSERT_VARIABLE(NoColor) INSERT_VARIABLE(Root) INSERT_VARIABLE(RootTarget)
diff --git a/src/gn/switches.h b/src/gn/switches.h index 2f89e50..2382bb7 100644 --- a/src/gn/switches.h +++ b/src/gn/switches.h
@@ -58,6 +58,10 @@ extern const char kMetaRebaseFiles_HelpShort[]; extern const char kMetaRebaseFiles_Help[]; +extern const char kNinjaExecutable[]; +extern const char kNinjaExecutable_HelpShort[]; +extern const char kNinjaExecutable_Help[]; + extern const char kNoColor[]; extern const char kNoColor_HelpShort[]; extern const char kNoColor_Help[]; @@ -115,6 +119,13 @@ " Non-wildcard inputs with no explicit toolchain specification will\n" \ " always match only a target in the default toolchain if one exists.\n" +// This switch is used to signal to the gen command that it is being invoked on +// a regeneration step. Ie, ninja has realized that build.ninja needs to be +// generated again and has invoked gn gen. There is no help associated with it +// because users should not be setting this switch. It is located in this file +// so it can be shared between command_gen and ninja_build_writer. +extern const char kRegeneration[]; + } // namespace switches #endif // TOOLS_GN_SWITCHES_H_