[apple] Optimise enumeration of additional files for Xcode project Instead of enumerating the files by traversing the filesystem for each pattern, first perform the enumeration and filter the file names using the patterns. This avoid traversing the filesystem multiple time which is the slowest operation. Additionally add a --enumerate-files-with-git command-line argument to use `git ls-files ...` to enumerate the files without traversing the filesystem which is even faster (but will omit untracked files). Bug: none Change-Id: Ie9719fe33f8ab1ad79efa74553e5194f72eaaaf4 Reviewed-on: https://gn-review.googlesource.com/c/gn/+/22300 Reviewed-by: David Turner <digit@google.com> Commit-Queue: Sylvain Defresne <sdefresne@chromium.org>
diff --git a/build/gen.py b/build/gen.py index 2a7e731..44f92b9 100755 --- a/build/gen.py +++ b/build/gen.py
@@ -931,6 +931,7 @@ if platform.is_posix() or platform.is_zos(): static_libraries['base']['sources'].extend([ 'src/base/files/file_enumerator_posix.cc', + 'src/base/files/file_path_posix.cc', 'src/base/files/file_posix.cc', 'src/base/files/file_util_posix.cc', 'src/base/posix/file_descriptor_shuffle.cc', @@ -943,6 +944,7 @@ if platform.is_windows(): static_libraries['base']['sources'].extend([ 'src/base/files/file_enumerator_win.cc', + 'src/base/files/file_path_win.cc', 'src/base/files/file_util_win.cc', 'src/base/files/file_win.cc', 'src/base/win/registry.cc',
diff --git a/src/base/files/file_enumerator_posix.cc b/src/base/files/file_enumerator_posix.cc index 41d52b8..73ba861 100644 --- a/src/base/files/file_enumerator_posix.cc +++ b/src/base/files/file_enumerator_posix.cc
@@ -163,8 +163,7 @@ } bool FileEnumerator::IsPatternMatched(const FilePath& path) const { - return pattern_.empty() || - !fnmatch(pattern_.c_str(), path.value().c_str(), FNM_NOESCAPE); + return pattern_.empty() || path.IsMatchingPattern(pattern_); } } // namespace base
diff --git a/src/base/files/file_enumerator_win.cc b/src/base/files/file_enumerator_win.cc index 0e8f6dc..f8ab469 100644 --- a/src/base/files/file_enumerator_win.cc +++ b/src/base/files/file_enumerator_win.cc
@@ -186,7 +186,7 @@ case FolderSearchPolicy::ALL: // ALL policy enumerates all files, we need to check pattern match // manually. - return PathMatchSpec(ToWCharT(&src.value()), ToWCharT(&pattern_)) == TRUE; + return pattern_.empty() || src.IsMatchingPattern(pattern_); } NOTREACHED(); return false;
diff --git a/src/base/files/file_path.h b/src/base/files/file_path.h index 26bf011..f18c227 100644 --- a/src/base/files/file_path.h +++ b/src/base/files/file_path.h
@@ -330,6 +330,14 @@ // separator. [[nodiscard]] FilePath StripTrailingSeparators() const; + // Returns whether the current FilePath matches `pattern`. This works like + // shell globbing. For example, "*.txt" or "Foo???.doc". However, be careful + // in specifying patterns that aren't cross platform since the underlying + // code uses OS-specific matching routines. In general, Windows matching + // is less featureful than others, so test there first. If unspecified, + // this will match all files. + bool IsMatchingPattern(const StringType& pattern) const; + // Returns true if this FilePath contains an attempt to reference a parent // directory (e.g. has a path component that is ".."). bool ReferencesParent() const;
diff --git a/src/base/files/file_path_posix.cc b/src/base/files/file_path_posix.cc new file mode 100644 index 0000000..3583737 --- /dev/null +++ b/src/base/files/file_path_posix.cc
@@ -0,0 +1,15 @@ +// Copyright 2026 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "base/files/file_path.h" + +#include <fnmatch.h> + +namespace base { + +bool FilePath::IsMatchingPattern(const FilePath::StringType& pattern) const { + return fnmatch(pattern.c_str(), value().c_str(), FNM_NOESCAPE) == 0; +} + +} // namespace base
diff --git a/src/base/files/file_path_win.cc b/src/base/files/file_path_win.cc new file mode 100644 index 0000000..64a7b7e --- /dev/null +++ b/src/base/files/file_path_win.cc
@@ -0,0 +1,17 @@ +// Copyright 2026 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "base/files/file_path.h" + +#include <shlwapi.h> + +#include "base/win/win_util.h" + +namespace base { + +bool FilePath::IsMatchingPattern(const FilePath::StringType& pattern) const { + return PathMatchSpec(ToWCharT(&value()), ToWCharT(&pattern)) == TRUE; +} + +} // namespace base
diff --git a/src/gn/switches.cc b/src/gn/switches.cc index 4081695..5442776 100644 --- a/src/gn/switches.cc +++ b/src/gn/switches.cc
@@ -65,6 +65,18 @@ use a different file. )"; +const char kEnumerateFilesWithGit[] = "enumerate-files-with-git"; +const char kEnumerateFilesWithGit_HelpShort[] = + "--enumerate-files-with-git: Use git to list files."; +const char kEnumerateFilesWithGit_Help[] = + R"(--enumerate-files-with-git: Use git to list files. + + Make the Xcode project writer use `git ls-files` to find source files, + instead of a much slower recursive file system walk. This will collect + both tracked and untracked files. This is significantly faster for large + repositories. +)"; + const char kErrorLimit[] = "error-limit"; const char kErrorLimit_HelpShort[] = "--error-limit: Limit the number of errors or warnings to print."; @@ -344,6 +356,7 @@ INSERT_VARIABLE(Args) INSERT_VARIABLE(Color) INSERT_VARIABLE(Dotfile) + INSERT_VARIABLE(EnumerateFilesWithGit) INSERT_VARIABLE(ErrorLimit) INSERT_VARIABLE(FailOnUnusedArgs) INSERT_VARIABLE(Markdown)
diff --git a/src/gn/switches.h b/src/gn/switches.h index 5fe34e7..2c7c42b 100644 --- a/src/gn/switches.h +++ b/src/gn/switches.h
@@ -38,6 +38,10 @@ extern const char kDotfile_HelpShort[]; extern const char kDotfile_Help[]; +extern const char kEnumerateFilesWithGit[]; +extern const char kEnumerateFilesWithGit_HelpShort[]; +extern const char kEnumerateFilesWithGit_Help[]; + extern const char kErrorLimit[]; extern const char kErrorLimit_HelpShort[]; extern const char kErrorLimit_Help[];
diff --git a/src/gn/xcode_writer.cc b/src/gn/xcode_writer.cc index e834b87..4895057 100644 --- a/src/gn/xcode_writer.cc +++ b/src/gn/xcode_writer.cc
@@ -4,16 +4,20 @@ #include "gn/xcode_writer.h" +#include <algorithm> +#include <condition_variable> #include <iomanip> #include <iterator> #include <map> #include <memory> +#include <mutex> #include <optional> #include <sstream> #include <string> #include <string_view> #include <utility> +#include "base/command_line.h" #include "base/environment.h" #include "base/files/file_enumerator.h" #include "base/logging.h" @@ -28,6 +32,7 @@ #include "gn/bundle_data.h" #include "gn/commands.h" #include "gn/deps_iterator.h" +#include "gn/exec_process.h" #include "gn/filesystem_utils.h" #include "gn/item.h" #include "gn/loader.h" @@ -36,6 +41,7 @@ #include "gn/source_file.h" #include "gn/string_output_buffer.h" #include "gn/substitution_writer.h" +#include "gn/switches.h" #include "gn/target.h" #include "gn/value.h" #include "gn/variables.h" @@ -251,7 +257,7 @@ const XcodeWriter::Options& options) { return base::SplitString(options.additional_files_patterns, FILE_PATH_LITERAL(";"), base::TRIM_WHITESPACE, - base::SPLIT_WANT_ALL); + base::SPLIT_WANT_NONEMPTY); } // Returns the list of roots to use when looking for additional files @@ -280,6 +286,69 @@ return root_paths; } +// Returns the path of files in `root`. If `use_git` is true, then it will +// first try to use `git ls-files`. If false or if the git command fails, +// will fall-back to traversing the filesystem. +std::vector<base::FilePath> FindFilesInRoot(const base::FilePath& root, + bool use_git) { + if (use_git) { + base::CommandLine cmdline(base::FilePath(FILE_PATH_LITERAL("git"))); + cmdline.AppendArg("ls-files"); + cmdline.AppendArg("--cached"); + cmdline.AppendArg("--others"); + cmdline.AppendArg("--exclude-standard"); + cmdline.AppendArg("-z"); // Use NUL as separator. + + // A string view that is a single NUL terminator (which is used by + // git ls-files to seperate the file paths while using -z option). + constexpr std::string_view kSeparator = std::string_view("\0", 1); + + int exit = 0; + std::string std_out; + std::string std_err; + if (::internal::ExecProcess(cmdline, root, &std_out, &std_err, &exit)) { + if (exit == 0) { + std::vector<std::string_view> lines = + base::SplitStringPiece(std_out, kSeparator, base::KEEP_WHITESPACE, + base::SPLIT_WANT_NONEMPTY); + + std::vector<base::FilePath> files; + for (std::string_view line : lines) { + base::FilePath path = root.Append(UTF8ToFilePath(std::string(line))); + files.push_back(std::move(path)); + } + return files; + } + } + } + + std::vector<base::FilePath> files; + base::FileEnumerator e(root, /*recursive=*/true, base::FileEnumerator::FILES); + for (base::FilePath path = e.Next(); !path.empty(); path = e.Next()) { + files.push_back(std::move(path)); + } + return files; +} + +// Filters `paths` to return only the files matching `patterns`. +std::vector<base::FilePath> FilterPathsMatchingPatterns( + std::vector<base::FilePath> paths, + const std::vector<base::FilePath::StringType>& patterns) { + std::vector<base::FilePath> filtered_paths; + for (base::FilePath& path : paths) { + const base::FilePath filename = path.BaseName(); + const bool is_matching_any_pattern = + std::ranges::any_of(patterns, [&](const auto& pattern) { + return filename.IsMatchingPattern(pattern); + }); + + if (is_matching_any_pattern) { + filtered_paths.push_back(std::move(path)); + } + } + return filtered_paths; +} + // Helper class to resolve list of XCTest files per target. // // Uses a cache of file found per intermediate targets to reduce the need @@ -799,17 +868,37 @@ const std::vector<base::FilePath> roots = GetAdditionalFilesRoots(build_settings_, options_); - for (const base::FilePath& root : roots) { - for (const base::FilePath::StringType& pattern : patterns) { - base::FileEnumerator it(root, /*recursive*/ true, - base::FileEnumerator::FILES, pattern, - base::FileEnumerator::FolderSearchPolicy::ALL); + std::mutex lock; + std::condition_variable cv; + size_t pending_tasks = roots.size(); - for (base::FilePath path = it.Next(); !path.empty(); path = it.Next()) { - const SourceFile source = FilePathToSourceFile(build_settings_, path); - sources.AddSourceFile(source); + const bool use_git = base::CommandLine::ForCurrentProcess()->HasSwitch( + switches::kEnumerateFilesWithGit); + + std::set<base::FilePath> collected_paths; + for (const base::FilePath& root : roots) { + g_scheduler->ScheduleWork([&, root]() { + std::vector<base::FilePath> local_paths = FilterPathsMatchingPatterns( + FindFilesInRoot(root, use_git), patterns); + + std::lock_guard<std::mutex> l(lock); + if (!local_paths.empty()) { + for (base::FilePath& path : local_paths) { + collected_paths.insert(std::move(path)); + } } - } + + if (--pending_tasks == 0) { + cv.notify_one(); + } + }); + } + + std::unique_lock<std::mutex> l(lock); + cv.wait(l, [&] { return pending_tasks == 0; }); + + for (const base::FilePath& path : collected_paths) { + sources.AddSourceFile(FilePathToSourceFile(build_settings_, path)); } }