gn recipe: Link GN with rpmalloc to speed it up. This CL changes the GN recipe to link the executable with the rpmalloc allocator from https://github.com/mjansson/rpmalloc This speeds up the 'gn gen' operation of both Chromium and Fuchsia operations by 28% and 12% respectively [1], and has slightly better performance than using tcmalloc from gperftools in the Fuchsia case. Note that: - The sources are grabbed directly from github.com, but it might be desirable to use a Chromium or Fuchsia supported mirror instead. - Only Linux has been tested so far, so only this platform enables this build mode. [1] Median measurements from 10 runs of 'gn gen' using the default Linux allocator (glibc), gperftools' tcmalloc, and rpmalloc: Fuchsia: glibc (default): 9917 ms gperftools: 7858 ms (26% faster) rpmalloc: 7752 ms (28% faster) Chromium: glibc (default): 4742 ms gperftools: 4186 ms (13% faster) rpmalloc: 4235 ms (12% faster) Change-Id: Ia52134ef602f546d6e8b33bef0146c620d214b58 Reviewed-on: https://gn-review.googlesource.com/c/gn/+/9920 Commit-Queue: David Turner <digit@google.com> Reviewed-by: Scott Graham <scottmg@chromium.org> Reviewed-by: Petr Hosek <phosek@google.com>
diff --git a/infra/README.recipes.md b/infra/README.recipes.md index 869f775..e045932 100644 --- a/infra/README.recipes.md +++ b/infra/README.recipes.md
@@ -91,7 +91,7 @@ Recipe for building GN. -— **def [RunSteps](/infra/recipes/gn.py#30)(api, repository):** +— **def [RunSteps](/infra/recipes/gn.py#72)(api, repository):** ### *recipes* / [macos\_sdk:examples/full](/infra/recipe_modules/macos_sdk/examples/full.py) [DEPS](/infra/recipe_modules/macos_sdk/examples/full.py#5): [macos\_sdk](#recipe_modules-macos_sdk), [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/step][recipe_engine/recipe_modules/step]
diff --git a/infra/recipes/gn.expected/ci_linux.json b/infra/recipes/gn.expected/ci_linux.json index 4a301f2..46d945b 100644 --- a/infra/recipes/gn.expected/ci_linux.json +++ b/infra/recipes/gn.expected/ci_linux.json
@@ -96,6 +96,213 @@ }, { "cmd": [], + "name": "rpmalloc" + }, + { + "cmd": [ + "git", + "init", + "[START_DIR]/rpmalloc" + ], + "name": "rpmalloc.init", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "git", + "fetch", + "--tags", + "https://fuchsia.googlesource.com/third_party/github.com/mjansson/rpmalloc", + "9c030f7a5efed87e3d036440e753c31342f9f909" + ], + "cwd": "[START_DIR]/rpmalloc", + "infra_step": true, + "name": "rpmalloc.fetch", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "git", + "checkout", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/rpmalloc", + "infra_step": true, + "name": "rpmalloc.checkout", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "rmtree", + "[CLEANUP]/rpmalloc-linux-amd64" + ], + "infra_step": true, + "name": "rpmalloc.remove sources linux-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copytree", + "[START_DIR]/rpmalloc", + "[CLEANUP]/rpmalloc-linux-amd64" + ], + "infra_step": true, + "name": "rpmalloc.copy sources linux-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [], + "name": "rpmalloc.build rpmalloc-linux-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[CLEANUP]/rpmalloc-linux-amd64/configure.py", + "-c", + "release", + "-a", + "x86-64", + "--lto" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-amd64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-amd64.configure", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [ + "[START_DIR]/cipd/ninja", + "lib/linux/release/x86-64/librpmallocwrap.a" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-amd64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-amd64.ninja", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "rmtree", + "[CLEANUP]/rpmalloc-linux-arm64" + ], + "infra_step": true, + "name": "rpmalloc.remove sources linux-arm64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copytree", + "[START_DIR]/rpmalloc", + "[CLEANUP]/rpmalloc-linux-arm64" + ], + "infra_step": true, + "name": "rpmalloc.copy sources linux-arm64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [], + "name": "rpmalloc.build rpmalloc-linux-arm64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[CLEANUP]/rpmalloc-linux-arm64/configure.py", + "-c", + "release", + "-a", + "arm64", + "--lto" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-arm64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-arm64.configure", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [ + "[START_DIR]/cipd/ninja", + "lib/linux/release/arm64/librpmallocwrap.a" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-arm64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-arm64.ninja", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [], "name": "debug" }, { @@ -170,7 +377,8 @@ "-u", "[START_DIR]/gn/build/gen.py", "--use-lto", - "--use-icf" + "--use-icf", + "--link-lib=[CLEANUP]/rpmalloc-linux-amd64/lib/linux/release/x86-64/librpmallocwrap.a" ], "cwd": "[START_DIR]/gn", "env": { @@ -258,7 +466,8 @@ "-u", "[START_DIR]/gn/build/gen.py", "--use-lto", - "--use-icf" + "--use-icf", + "--link-lib=[CLEANUP]/rpmalloc-linux-arm64/lib/linux/release/arm64/librpmallocwrap.a" ], "cwd": "[START_DIR]/gn", "env": {
diff --git a/infra/recipes/gn.expected/ci_mac.json b/infra/recipes/gn.expected/ci_mac.json index 03d5da6..a2db672 100644 --- a/infra/recipes/gn.expected/ci_mac.json +++ b/infra/recipes/gn.expected/ci_mac.json
@@ -144,26 +144,26 @@ "name": "debug" }, { - "cmd": [], - "name": "debug.mac-amd64", - "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@" - ] - }, - { "cmd": [ "xcrun", "--show-sdk-path" ], - "name": "debug.mac-amd64.xcrun", + "name": "debug.xcrun", "stdout": "/path/to/tmp/", "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@2@@@", + "@@@STEP_NEST_LEVEL@1@@@", "@@@STEP_LOG_LINE@raw_io.output[sdk-path]@/some/xcode/path@@@", "@@@STEP_LOG_END@raw_io.output[sdk-path]@@@" ] }, { + "cmd": [], + "name": "debug.mac-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { "cmd": [ "python", "-u", @@ -216,26 +216,26 @@ "name": "release" }, { - "cmd": [], - "name": "release.mac-amd64", - "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@" - ] - }, - { "cmd": [ "xcrun", "--show-sdk-path" ], - "name": "release.mac-amd64.xcrun", + "name": "release.xcrun", "stdout": "/path/to/tmp/", "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@2@@@", + "@@@STEP_NEST_LEVEL@1@@@", "@@@STEP_LOG_LINE@raw_io.output[sdk-path]@/some/xcode/path@@@", "@@@STEP_LOG_END@raw_io.output[sdk-path]@@@" ] }, { + "cmd": [], + "name": "release.mac-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { "cmd": [ "python", "-u",
diff --git a/infra/recipes/gn.expected/cipd_exists.json b/infra/recipes/gn.expected/cipd_exists.json index d7127c2..f76eb84 100644 --- a/infra/recipes/gn.expected/cipd_exists.json +++ b/infra/recipes/gn.expected/cipd_exists.json
@@ -96,6 +96,213 @@ }, { "cmd": [], + "name": "rpmalloc" + }, + { + "cmd": [ + "git", + "init", + "[START_DIR]/rpmalloc" + ], + "name": "rpmalloc.init", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "git", + "fetch", + "--tags", + "https://fuchsia.googlesource.com/third_party/github.com/mjansson/rpmalloc", + "9c030f7a5efed87e3d036440e753c31342f9f909" + ], + "cwd": "[START_DIR]/rpmalloc", + "infra_step": true, + "name": "rpmalloc.fetch", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "git", + "checkout", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/rpmalloc", + "infra_step": true, + "name": "rpmalloc.checkout", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "rmtree", + "[CLEANUP]/rpmalloc-linux-amd64" + ], + "infra_step": true, + "name": "rpmalloc.remove sources linux-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copytree", + "[START_DIR]/rpmalloc", + "[CLEANUP]/rpmalloc-linux-amd64" + ], + "infra_step": true, + "name": "rpmalloc.copy sources linux-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [], + "name": "rpmalloc.build rpmalloc-linux-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[CLEANUP]/rpmalloc-linux-amd64/configure.py", + "-c", + "release", + "-a", + "x86-64", + "--lto" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-amd64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-amd64.configure", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [ + "[START_DIR]/cipd/ninja", + "lib/linux/release/x86-64/librpmallocwrap.a" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-amd64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-amd64.ninja", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "rmtree", + "[CLEANUP]/rpmalloc-linux-arm64" + ], + "infra_step": true, + "name": "rpmalloc.remove sources linux-arm64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copytree", + "[START_DIR]/rpmalloc", + "[CLEANUP]/rpmalloc-linux-arm64" + ], + "infra_step": true, + "name": "rpmalloc.copy sources linux-arm64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [], + "name": "rpmalloc.build rpmalloc-linux-arm64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[CLEANUP]/rpmalloc-linux-arm64/configure.py", + "-c", + "release", + "-a", + "arm64", + "--lto" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-arm64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-arm64.configure", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [ + "[START_DIR]/cipd/ninja", + "lib/linux/release/arm64/librpmallocwrap.a" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-arm64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-arm64.ninja", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [], "name": "debug" }, { @@ -170,7 +377,8 @@ "-u", "[START_DIR]/gn/build/gen.py", "--use-lto", - "--use-icf" + "--use-icf", + "--link-lib=[CLEANUP]/rpmalloc-linux-amd64/lib/linux/release/x86-64/librpmallocwrap.a" ], "cwd": "[START_DIR]/gn", "env": { @@ -289,7 +497,8 @@ "-u", "[START_DIR]/gn/build/gen.py", "--use-lto", - "--use-icf" + "--use-icf", + "--link-lib=[CLEANUP]/rpmalloc-linux-arm64/lib/linux/release/arm64/librpmallocwrap.a" ], "cwd": "[START_DIR]/gn", "env": {
diff --git a/infra/recipes/gn.expected/cipd_register.json b/infra/recipes/gn.expected/cipd_register.json index 2e09f95..94900d0 100644 --- a/infra/recipes/gn.expected/cipd_register.json +++ b/infra/recipes/gn.expected/cipd_register.json
@@ -96,6 +96,213 @@ }, { "cmd": [], + "name": "rpmalloc" + }, + { + "cmd": [ + "git", + "init", + "[START_DIR]/rpmalloc" + ], + "name": "rpmalloc.init", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "git", + "fetch", + "--tags", + "https://fuchsia.googlesource.com/third_party/github.com/mjansson/rpmalloc", + "9c030f7a5efed87e3d036440e753c31342f9f909" + ], + "cwd": "[START_DIR]/rpmalloc", + "infra_step": true, + "name": "rpmalloc.fetch", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "git", + "checkout", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/rpmalloc", + "infra_step": true, + "name": "rpmalloc.checkout", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "rmtree", + "[CLEANUP]/rpmalloc-linux-amd64" + ], + "infra_step": true, + "name": "rpmalloc.remove sources linux-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copytree", + "[START_DIR]/rpmalloc", + "[CLEANUP]/rpmalloc-linux-amd64" + ], + "infra_step": true, + "name": "rpmalloc.copy sources linux-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [], + "name": "rpmalloc.build rpmalloc-linux-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[CLEANUP]/rpmalloc-linux-amd64/configure.py", + "-c", + "release", + "-a", + "x86-64", + "--lto" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-amd64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-amd64.configure", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [ + "[START_DIR]/cipd/ninja", + "lib/linux/release/x86-64/librpmallocwrap.a" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-amd64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-amd64.ninja", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "rmtree", + "[CLEANUP]/rpmalloc-linux-arm64" + ], + "infra_step": true, + "name": "rpmalloc.remove sources linux-arm64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copytree", + "[START_DIR]/rpmalloc", + "[CLEANUP]/rpmalloc-linux-arm64" + ], + "infra_step": true, + "name": "rpmalloc.copy sources linux-arm64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [], + "name": "rpmalloc.build rpmalloc-linux-arm64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[CLEANUP]/rpmalloc-linux-arm64/configure.py", + "-c", + "release", + "-a", + "arm64", + "--lto" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-arm64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-arm64.configure", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [ + "[START_DIR]/cipd/ninja", + "lib/linux/release/arm64/librpmallocwrap.a" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-arm64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-arm64.ninja", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [], "name": "debug" }, { @@ -170,7 +377,8 @@ "-u", "[START_DIR]/gn/build/gen.py", "--use-lto", - "--use-icf" + "--use-icf", + "--link-lib=[CLEANUP]/rpmalloc-linux-amd64/lib/linux/release/x86-64/librpmallocwrap.a" ], "cwd": "[START_DIR]/gn", "env": { @@ -303,7 +511,8 @@ "-u", "[START_DIR]/gn/build/gen.py", "--use-lto", - "--use-icf" + "--use-icf", + "--link-lib=[CLEANUP]/rpmalloc-linux-arm64/lib/linux/release/arm64/librpmallocwrap.a" ], "cwd": "[START_DIR]/gn", "env": {
diff --git a/infra/recipes/gn.expected/cq_linux.json b/infra/recipes/gn.expected/cq_linux.json index 75b64a4..82ae5dd 100644 --- a/infra/recipes/gn.expected/cq_linux.json +++ b/infra/recipes/gn.expected/cq_linux.json
@@ -74,12 +74,12 @@ { "cmd": [ "git", - "checkout", + "cherry-pick", "FETCH_HEAD" ], "cwd": "[START_DIR]/gn", "infra_step": true, - "name": "git.checkout 123456/7", + "name": "git.cherry-pick 123456/7", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@1@@@" ] @@ -123,6 +123,213 @@ }, { "cmd": [], + "name": "rpmalloc" + }, + { + "cmd": [ + "git", + "init", + "[START_DIR]/rpmalloc" + ], + "name": "rpmalloc.init", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "git", + "fetch", + "--tags", + "https://fuchsia.googlesource.com/third_party/github.com/mjansson/rpmalloc", + "9c030f7a5efed87e3d036440e753c31342f9f909" + ], + "cwd": "[START_DIR]/rpmalloc", + "infra_step": true, + "name": "rpmalloc.fetch", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "git", + "checkout", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/rpmalloc", + "infra_step": true, + "name": "rpmalloc.checkout", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "rmtree", + "[CLEANUP]/rpmalloc-linux-amd64" + ], + "infra_step": true, + "name": "rpmalloc.remove sources linux-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copytree", + "[START_DIR]/rpmalloc", + "[CLEANUP]/rpmalloc-linux-amd64" + ], + "infra_step": true, + "name": "rpmalloc.copy sources linux-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [], + "name": "rpmalloc.build rpmalloc-linux-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[CLEANUP]/rpmalloc-linux-amd64/configure.py", + "-c", + "release", + "-a", + "x86-64", + "--lto" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-amd64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-amd64.configure", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [ + "[START_DIR]/cipd/ninja", + "lib/linux/release/x86-64/librpmallocwrap.a" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-amd64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-amd64.ninja", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "rmtree", + "[CLEANUP]/rpmalloc-linux-arm64" + ], + "infra_step": true, + "name": "rpmalloc.remove sources linux-arm64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copytree", + "[START_DIR]/rpmalloc", + "[CLEANUP]/rpmalloc-linux-arm64" + ], + "infra_step": true, + "name": "rpmalloc.copy sources linux-arm64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [], + "name": "rpmalloc.build rpmalloc-linux-arm64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[CLEANUP]/rpmalloc-linux-arm64/configure.py", + "-c", + "release", + "-a", + "arm64", + "--lto" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-arm64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-arm64.configure", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [ + "[START_DIR]/cipd/ninja", + "lib/linux/release/arm64/librpmallocwrap.a" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-arm64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-arm64.ninja", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [], "name": "debug" }, { @@ -197,7 +404,8 @@ "-u", "[START_DIR]/gn/build/gen.py", "--use-lto", - "--use-icf" + "--use-icf", + "--link-lib=[CLEANUP]/rpmalloc-linux-amd64/lib/linux/release/x86-64/librpmallocwrap.a" ], "cwd": "[START_DIR]/gn", "env": { @@ -253,7 +461,8 @@ "-u", "[START_DIR]/gn/build/gen.py", "--use-lto", - "--use-icf" + "--use-icf", + "--link-lib=[CLEANUP]/rpmalloc-linux-arm64/lib/linux/release/arm64/librpmallocwrap.a" ], "cwd": "[START_DIR]/gn", "env": {
diff --git a/infra/recipes/gn.expected/cq_mac.json b/infra/recipes/gn.expected/cq_mac.json index 0244003..76ceca4 100644 --- a/infra/recipes/gn.expected/cq_mac.json +++ b/infra/recipes/gn.expected/cq_mac.json
@@ -74,12 +74,12 @@ { "cmd": [ "git", - "checkout", + "cherry-pick", "FETCH_HEAD" ], "cwd": "[START_DIR]/gn", "infra_step": true, - "name": "git.checkout 123456/7", + "name": "git.cherry-pick 123456/7", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@1@@@" ] @@ -171,26 +171,26 @@ "name": "debug" }, { - "cmd": [], - "name": "debug.mac-amd64", - "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@" - ] - }, - { "cmd": [ "xcrun", "--show-sdk-path" ], - "name": "debug.mac-amd64.xcrun", + "name": "debug.xcrun", "stdout": "/path/to/tmp/", "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@2@@@", + "@@@STEP_NEST_LEVEL@1@@@", "@@@STEP_LOG_LINE@raw_io.output[sdk-path]@/some/xcode/path@@@", "@@@STEP_LOG_END@raw_io.output[sdk-path]@@@" ] }, { + "cmd": [], + "name": "debug.mac-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { "cmd": [ "python", "-u", @@ -243,26 +243,26 @@ "name": "release" }, { - "cmd": [], - "name": "release.mac-amd64", - "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@" - ] - }, - { "cmd": [ "xcrun", "--show-sdk-path" ], - "name": "release.mac-amd64.xcrun", + "name": "release.xcrun", "stdout": "/path/to/tmp/", "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@2@@@", + "@@@STEP_NEST_LEVEL@1@@@", "@@@STEP_LOG_LINE@raw_io.output[sdk-path]@/some/xcode/path@@@", "@@@STEP_LOG_END@raw_io.output[sdk-path]@@@" ] }, { + "cmd": [], + "name": "release.mac-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { "cmd": [ "python", "-u",
diff --git a/infra/recipes/gn.expected/cq_win.json b/infra/recipes/gn.expected/cq_win.json index 164b245..8e352de 100644 --- a/infra/recipes/gn.expected/cq_win.json +++ b/infra/recipes/gn.expected/cq_win.json
@@ -74,12 +74,12 @@ { "cmd": [ "git", - "checkout", + "cherry-pick", "FETCH_HEAD" ], "cwd": "[START_DIR]\\gn", "infra_step": true, - "name": "git.checkout 123456/7", + "name": "git.cherry-pick 123456/7", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@1@@@" ]
diff --git a/infra/recipes/gn.py b/infra/recipes/gn.py index 68e721c..298e870 100644 --- a/infra/recipes/gn.py +++ b/infra/recipes/gn.py
@@ -26,10 +26,55 @@ 'repository': Property(kind=str, default='https://gn.googlesource.com/gn'), } +# On select platforms, link the GN executable against rpmalloc for a small 10% speed boost. +_RPMALLOC_GIT_URL = 'https://fuchsia.googlesource.com/third_party/github.com/mjansson/rpmalloc' +_RPMALLOC_REVISION = '9c030f7a5efed87e3d036440e753c31342f9f909' + +# Used to convert os and arch strings to rpmalloc format +_RPMALLOC_MAP = { + 'amd64': 'x86-64', + 'mac': 'macos', +} + + +def _get_compilation_environment(api, target, cipd_dir): + if target.is_linux: + triple = '--target=%s' % target.triple + sysroot = '--sysroot=%s' % cipd_dir.join('sysroot') + env = { + 'CC': cipd_dir.join('bin', 'clang'), + 'CXX': cipd_dir.join('bin', 'clang++'), + 'AR': cipd_dir.join('bin', 'llvm-ar'), + 'CFLAGS': '%s %s' % (triple, sysroot), + 'LDFLAGS': '%s %s -static-libstdc++' % (triple, sysroot), + } + elif target.is_mac: + triple = '--target=%s' % target.triple + sysroot = '--sysroot=%s' % api.step( + 'xcrun', ['xcrun', '--show-sdk-path'], + stdout=api.raw_io.output(name='sdk-path', add_output_log=True), + step_test_data=lambda: api.raw_io.test_api.stream_output( + '/some/xcode/path')).stdout.strip() + stdlib = cipd_dir.join('lib', 'libc++.a') + env = { + 'CC': cipd_dir.join('bin', 'clang'), + 'CXX': cipd_dir.join('bin', 'clang++'), + 'AR': cipd_dir.join('bin', 'llvm-ar'), + 'CFLAGS': '%s %s' % (triple, sysroot), + 'LDFLAGS': '%s %s -nostdlib++ %s' % (triple, sysroot, stdlib), + } + else: + env = {} + + return env + def RunSteps(api, repository): src_dir = api.path['start_dir'].join('gn') + # TODO: Verify that building and linking rpmalloc works on OS X and Windows as + # well. + use_rpmalloc = api.platform.is_linux with api.step.nest('git'), api.context(infra_steps=True): api.step('init', ['git', 'init', src_dir]) @@ -50,8 +95,8 @@ 'refs/changes/%s/%s/%s' % (str(change.change)[-2:], change.change, change.patchset) ]) - api.step('checkout %s/%s' % (change.change, change.patchset), - ['git', 'checkout', 'FETCH_HEAD']) + api.step('cherry-pick %s/%s' % (change.change, change.patchset), + ['git', 'cherry-pick', 'FETCH_HEAD']) with api.context(infra_steps=True): cipd_dir = api.path['start_dir'].join('cipd') @@ -73,7 +118,8 @@ 'targets': [api.target.host], }, { - 'name': 'release', + 'name': + 'release', 'args': ['--use-lto', '--use-icf'], 'targets': [api.target('linux-amd64'), api.target('linux-arm64')] @@ -82,44 +128,83 @@ ] with api.macos_sdk(), api.windows_sdk(): + # Build the rpmalloc stastic libraries if needed. + if use_rpmalloc: + # Maps a target.platform string to the location of the corresponding + # rpmalloc static library. + rpmalloc_static_libs = {} + + # Get the list of all target platforms that are listed in `configs` + # above. Note that this is a list of Target instances, some of them + # may refer to the same platform string (e.g. linux-amd64). + # + # For each platform, a version of rpmalloc will be built if necessary, + # but doing this properly requires having a valid target instance to + # call _get_compilation_environment. So create a { platform -> Target } + # map to do that later. + all_config_platforms = {} + for c in configs: + for t in c['targets']: + if t.platform not in all_config_platforms: + all_config_platforms[t.platform] = t + + rpmalloc_src_dir = api.path['start_dir'].join('rpmalloc') + with api.step.nest('rpmalloc'): + api.step('init', ['git', 'init', rpmalloc_src_dir]) + with api.context(cwd=rpmalloc_src_dir, infra_steps=True): + api.step( + 'fetch', + ['git', 'fetch', '--tags', _RPMALLOC_GIT_URL, _RPMALLOC_REVISION]) + api.step('checkout', ['git', 'checkout', 'FETCH_HEAD']) + + for platform in all_config_platforms: + # Convert target architecture and os to rpmalloc format. + rpmalloc_os, rpmalloc_arch = platform.split('-') + rpmalloc_os = _RPMALLOC_MAP.get(rpmalloc_os, rpmalloc_os) + rpmalloc_arch = _RPMALLOC_MAP.get(rpmalloc_arch, rpmalloc_arch) + + # The rpmalloc build system doesn't support out-of-tree builds. + # So copy the source tree into a new clean directory. + rpmalloc_build_dir = api.path['cleanup'].join('rpmalloc-' + platform) + api.file.rmtree('remove sources ' + platform, rpmalloc_build_dir) + api.file.copytree('copy sources ' + platform, rpmalloc_src_dir, + rpmalloc_build_dir) + + env = _get_compilation_environment(api, + all_config_platforms[platform], + cipd_dir) + with api.step.nest('build rpmalloc-' + platform), api.context( + env=env, cwd=rpmalloc_build_dir): + api.python( + 'configure', + rpmalloc_build_dir.join('configure.py'), + args=['-c', 'release', '-a', rpmalloc_arch, '--lto']) + + # NOTE: Only build the static library. + rpmalloc_static_lib = api.path.join('lib', rpmalloc_os, 'release', + rpmalloc_arch, + 'librpmallocwrap.a') + api.step('ninja', [cipd_dir.join('ninja'), rpmalloc_static_lib]) + + rpmalloc_static_libs[platform] = rpmalloc_build_dir.join( + rpmalloc_static_lib) + for config in configs: with api.step.nest(config['name']): for target in config['targets']: + env = _get_compilation_environment(api, target, cipd_dir) with api.step.nest(target.platform): - if target.is_linux: - triple = '--target=%s' % target.triple - sysroot = '--sysroot=%s' % cipd_dir.join('sysroot') - env = { - 'CC': cipd_dir.join('bin', 'clang'), - 'CXX': cipd_dir.join('bin', 'clang++'), - 'AR': cipd_dir.join('bin', 'llvm-ar'), - 'CFLAGS': '%s %s' % (triple, sysroot), - 'LDFLAGS': '%s %s -static-libstdc++' % (triple, sysroot), - } - elif target.is_mac: - triple = '--target=%s' % target.triple - sysroot = '--sysroot=%s' % api.step( - 'xcrun', ['xcrun', '--show-sdk-path'], - stdout=api.raw_io.output( - name='sdk-path', add_output_log=True), - step_test_data=lambda: api.raw_io.test_api.stream_output( - '/some/xcode/path')).stdout.strip() - stdlib = cipd_dir.join('lib', 'libc++.a') - env = { - 'CC': cipd_dir.join('bin', 'clang'), - 'CXX': cipd_dir.join('bin', 'clang++'), - 'AR': cipd_dir.join('bin', 'llvm-ar'), - 'CFLAGS': '%s %s' % (triple, sysroot), - 'LDFLAGS': '%s %s -nostdlib++ %s' % (triple, sysroot, stdlib), - } - else: - env = {} - with api.context(env=env, cwd=src_dir): - api.python( - 'generate', - src_dir.join('build', 'gen.py'), - args=config['args']) + args = config['args'] + + # Ensure that the release `gn` binary will be linked to the rpmalloc static + # library when needed. + if use_rpmalloc and config['name'] == 'release': + args = args[:] + [ + '--link-lib=%s' % rpmalloc_static_libs[target.platform] + ] + + api.python('generate', src_dir.join('build', 'gen.py'), args=args) # Windows requires the environment modifications when building too. api.step('build',