gn recipe: Link GN with rpmalloc to speed it up. NOTE: This is a reland of [1] with an updated rpmalloc revision that includes fixes for the -Wconversion errors that required reverting the original CL. This CL changes the GN recipe to link the executable with the rpmalloc allocator from https://github.com/mjansson/rpmalloc This speeds up the 'gn gen' operation of both Chromium and Fuchsia operations by 28% and 12% respectively [1], and has slightly better performance than using tcmalloc from gperftools in the Fuchsia case. Note that: - The sources are grabbed directly from github.com, but it might be desirable to use a Chromium or Fuchsia supported mirror instead. - Only Linux has been tested so far, so only this platform enables this build mode. [1] Median measurements from 10 runs of 'gn gen' using the default Linux allocator (glibc), gperftools' tcmalloc, and rpmalloc: Fuchsia: glibc (default): 9917 ms gperftools: 7858 ms (26% faster) rpmalloc: 7752 ms (28% faster) Chromium: glibc (default): 4742 ms gperftools: 4186 ms (13% faster) rpmalloc: 4235 ms (12% faster) [1] https://gn-review.googlesource.com/c/gn/+/9920 Change-Id: I0860c34437faee7c1f97e01f831cbf25f49c819e Reviewed-on: https://gn-review.googlesource.com/c/gn/+/10420 Commit-Queue: David Turner <digit@google.com> Reviewed-by: Petr Hosek <phosek@google.com>
diff --git a/infra/README.recipes.md b/infra/README.recipes.md index c99a1bd..6fa4190 100644 --- a/infra/README.recipes.md +++ b/infra/README.recipes.md
@@ -93,7 +93,7 @@ Recipe for building GN. -— **def [RunSteps](/infra/recipes/gn.py#30)(api, repository):** +— **def [RunSteps](/infra/recipes/gn.py#101)(api, repository):** ### *recipes* / [macos\_sdk:examples/full](/infra/recipe_modules/macos_sdk/examples/full.py) [DEPS](/infra/recipe_modules/macos_sdk/examples/full.py#5): [macos\_sdk](#recipe_modules-macos_sdk), [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/step][recipe_engine/recipe_modules/step]
diff --git a/infra/recipes/gn.expected/ci_linux.json b/infra/recipes/gn.expected/ci_linux.json index 4a301f2..54e55c4 100644 --- a/infra/recipes/gn.expected/ci_linux.json +++ b/infra/recipes/gn.expected/ci_linux.json
@@ -96,6 +96,213 @@ }, { "cmd": [], + "name": "rpmalloc" + }, + { + "cmd": [ + "git", + "init", + "[START_DIR]/rpmalloc" + ], + "name": "rpmalloc.init", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "git", + "fetch", + "--tags", + "https://fuchsia.googlesource.com/third_party/github.com/mjansson/rpmalloc", + "6bb6ca97a8d6a72d626153fd8431ef8477a21145" + ], + "cwd": "[START_DIR]/rpmalloc", + "infra_step": true, + "name": "rpmalloc.fetch", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "git", + "checkout", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/rpmalloc", + "infra_step": true, + "name": "rpmalloc.checkout", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "rmtree", + "[CLEANUP]/rpmalloc-linux-amd64" + ], + "infra_step": true, + "name": "rpmalloc.remove sources linux-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copytree", + "[START_DIR]/rpmalloc", + "[CLEANUP]/rpmalloc-linux-amd64" + ], + "infra_step": true, + "name": "rpmalloc.copy sources linux-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [], + "name": "rpmalloc.build rpmalloc-linux-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[CLEANUP]/rpmalloc-linux-amd64/configure.py", + "-c", + "release", + "-a", + "x86-64", + "--lto" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-amd64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-amd64.configure", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [ + "[START_DIR]/cipd/ninja", + "lib/linux/release/x86-64/librpmallocwrap.a" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-amd64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-amd64.ninja", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "rmtree", + "[CLEANUP]/rpmalloc-linux-arm64" + ], + "infra_step": true, + "name": "rpmalloc.remove sources linux-arm64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copytree", + "[START_DIR]/rpmalloc", + "[CLEANUP]/rpmalloc-linux-arm64" + ], + "infra_step": true, + "name": "rpmalloc.copy sources linux-arm64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [], + "name": "rpmalloc.build rpmalloc-linux-arm64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[CLEANUP]/rpmalloc-linux-arm64/configure.py", + "-c", + "release", + "-a", + "arm64", + "--lto" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-arm64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-arm64.configure", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [ + "[START_DIR]/cipd/ninja", + "lib/linux/release/arm64/librpmallocwrap.a" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-arm64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-arm64.ninja", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [], "name": "debug" }, { @@ -148,6 +355,14 @@ "cmd": [ "[START_DIR]/gn/out/gn_unittests" ], + "cwd": "[START_DIR]/gn", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, "name": "debug.linux-amd64.test", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@2@@@" @@ -170,7 +385,8 @@ "-u", "[START_DIR]/gn/build/gen.py", "--use-lto", - "--use-icf" + "--use-icf", + "--link-lib=[CLEANUP]/rpmalloc-linux-amd64/lib/linux/release/x86-64/librpmallocwrap.a" ], "cwd": "[START_DIR]/gn", "env": { @@ -208,6 +424,14 @@ "cmd": [ "[START_DIR]/gn/out/gn_unittests" ], + "cwd": "[START_DIR]/gn", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, "name": "release.linux-amd64.test", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@2@@@" @@ -233,6 +457,14 @@ "-json-output", "/path/to/tmp/json" ], + "cwd": "[START_DIR]/gn", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, "name": "release.linux-amd64.upload.build gn/gn/linux-amd64", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@3@@@", @@ -258,7 +490,8 @@ "-u", "[START_DIR]/gn/build/gen.py", "--use-lto", - "--use-icf" + "--use-icf", + "--link-lib=[CLEANUP]/rpmalloc-linux-arm64/lib/linux/release/arm64/librpmallocwrap.a" ], "cwd": "[START_DIR]/gn", "env": { @@ -312,6 +545,14 @@ "-json-output", "/path/to/tmp/json" ], + "cwd": "[START_DIR]/gn", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, "name": "release.linux-arm64.upload.build gn/gn/linux-arm64", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@3@@@",
diff --git a/infra/recipes/gn.expected/ci_mac.json b/infra/recipes/gn.expected/ci_mac.json index fa8164d..5816d9c 100644 --- a/infra/recipes/gn.expected/ci_mac.json +++ b/infra/recipes/gn.expected/ci_mac.json
@@ -144,21 +144,14 @@ "name": "debug" }, { - "cmd": [], - "name": "debug.mac-amd64", - "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@" - ] - }, - { "cmd": [ "xcrun", "--show-sdk-path" ], - "name": "debug.mac-amd64.xcrun sdk-path", + "name": "debug.xcrun sdk-path", "stdout": "/path/to/tmp/", "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@2@@@", + "@@@STEP_NEST_LEVEL@1@@@", "@@@STEP_LOG_LINE@raw_io.output[sdk-path]@/some/xcode/path@@@", "@@@STEP_LOG_END@raw_io.output[sdk-path]@@@" ] @@ -174,15 +167,22 @@ "-Wp,-v", "-" ], - "name": "debug.mac-amd64.xcrun toolchain", + "name": "debug.xcrun toolchain", "stderr": "/path/to/tmp/", "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@2@@@", + "@@@STEP_NEST_LEVEL@1@@@", "@@@STEP_LOG_LINE@raw_io.output[toolchain]@[CACHE]/macos_sdk/XCode.app/include/c++/v1@@@", "@@@STEP_LOG_END@raw_io.output[toolchain]@@@" ] }, { + "cmd": [], + "name": "debug.mac-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { "cmd": [ "python", "-u", @@ -225,6 +225,14 @@ "cmd": [ "[START_DIR]/gn/out/gn_unittests" ], + "cwd": "[START_DIR]/gn", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-apple-darwin --sysroot=/some/xcode/path -nostdinc++ -cxx-isystem [CACHE]/macos_sdk/XCode.app/include/c++/v1", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-apple-darwin --sysroot=/some/xcode/path" + }, "name": "debug.mac-amd64.test", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@2@@@" @@ -235,21 +243,14 @@ "name": "release" }, { - "cmd": [], - "name": "release.mac-amd64", - "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@" - ] - }, - { "cmd": [ "xcrun", "--show-sdk-path" ], - "name": "release.mac-amd64.xcrun sdk-path", + "name": "release.xcrun sdk-path", "stdout": "/path/to/tmp/", "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@2@@@", + "@@@STEP_NEST_LEVEL@1@@@", "@@@STEP_LOG_LINE@raw_io.output[sdk-path]@/some/xcode/path@@@", "@@@STEP_LOG_END@raw_io.output[sdk-path]@@@" ] @@ -265,15 +266,22 @@ "-Wp,-v", "-" ], - "name": "release.mac-amd64.xcrun toolchain", + "name": "release.xcrun toolchain", "stderr": "/path/to/tmp/", "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@2@@@", + "@@@STEP_NEST_LEVEL@1@@@", "@@@STEP_LOG_LINE@raw_io.output[toolchain]@[CACHE]/macos_sdk/XCode.app/include/c++/v1@@@", "@@@STEP_LOG_END@raw_io.output[toolchain]@@@" ] }, { + "cmd": [], + "name": "release.mac-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { "cmd": [ "python", "-u", @@ -317,6 +325,14 @@ "cmd": [ "[START_DIR]/gn/out/gn_unittests" ], + "cwd": "[START_DIR]/gn", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-apple-darwin --sysroot=/some/xcode/path -nostdinc++ -cxx-isystem [CACHE]/macos_sdk/XCode.app/include/c++/v1", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-apple-darwin --sysroot=/some/xcode/path" + }, "name": "release.mac-amd64.test", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@2@@@" @@ -342,6 +358,14 @@ "-json-output", "/path/to/tmp/json" ], + "cwd": "[START_DIR]/gn", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-apple-darwin --sysroot=/some/xcode/path -nostdinc++ -cxx-isystem [CACHE]/macos_sdk/XCode.app/include/c++/v1", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-apple-darwin --sysroot=/some/xcode/path" + }, "name": "release.mac-amd64.upload.build gn/gn/mac-amd64", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@3@@@", @@ -355,21 +379,14 @@ ] }, { - "cmd": [], - "name": "release.mac-arm64", - "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@" - ] - }, - { "cmd": [ "xcrun", "--show-sdk-path" ], - "name": "release.mac-arm64.xcrun sdk-path", + "name": "release.xcrun sdk-path (2)", "stdout": "/path/to/tmp/", "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@2@@@", + "@@@STEP_NEST_LEVEL@1@@@", "@@@STEP_LOG_LINE@raw_io.output[sdk-path]@/some/xcode/path@@@", "@@@STEP_LOG_END@raw_io.output[sdk-path]@@@" ] @@ -385,15 +402,22 @@ "-Wp,-v", "-" ], - "name": "release.mac-arm64.xcrun toolchain", + "name": "release.xcrun toolchain (2)", "stderr": "/path/to/tmp/", "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@2@@@", + "@@@STEP_NEST_LEVEL@1@@@", "@@@STEP_LOG_LINE@raw_io.output[toolchain]@[CACHE]/macos_sdk/XCode.app/include/c++/v1@@@", "@@@STEP_LOG_END@raw_io.output[toolchain]@@@" ] }, { + "cmd": [], + "name": "release.mac-arm64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { "cmd": [ "python", "-u", @@ -453,6 +477,14 @@ "-json-output", "/path/to/tmp/json" ], + "cwd": "[START_DIR]/gn", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=arm64-apple-darwin --sysroot=/some/xcode/path -nostdinc++ -cxx-isystem [CACHE]/macos_sdk/XCode.app/include/c++/v1", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=arm64-apple-darwin --sysroot=/some/xcode/path" + }, "name": "release.mac-arm64.upload.build gn/gn/mac-arm64", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@3@@@",
diff --git a/infra/recipes/gn.expected/ci_win.json b/infra/recipes/gn.expected/ci_win.json index 88e8edb..842f015 100644 --- a/infra/recipes/gn.expected/ci_win.json +++ b/infra/recipes/gn.expected/ci_win.json
@@ -199,6 +199,7 @@ "cmd": [ "[START_DIR]\\gn\\out\\gn_unittests" ], + "cwd": "[START_DIR]\\gn", "env": { "VSINSTALLDIR": "[CACHE]\\windows_sdk" }, @@ -269,6 +270,7 @@ "cmd": [ "[START_DIR]\\gn\\out\\gn_unittests" ], + "cwd": "[START_DIR]\\gn", "env": { "VSINSTALLDIR": "[CACHE]\\windows_sdk" }, @@ -302,6 +304,7 @@ "-json-output", "/path/to/tmp/json" ], + "cwd": "[START_DIR]\\gn", "env": { "VSINSTALLDIR": "[CACHE]\\windows_sdk" },
diff --git a/infra/recipes/gn.expected/cipd_exists.json b/infra/recipes/gn.expected/cipd_exists.json index d7127c2..b93e7c4 100644 --- a/infra/recipes/gn.expected/cipd_exists.json +++ b/infra/recipes/gn.expected/cipd_exists.json
@@ -96,6 +96,213 @@ }, { "cmd": [], + "name": "rpmalloc" + }, + { + "cmd": [ + "git", + "init", + "[START_DIR]/rpmalloc" + ], + "name": "rpmalloc.init", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "git", + "fetch", + "--tags", + "https://fuchsia.googlesource.com/third_party/github.com/mjansson/rpmalloc", + "6bb6ca97a8d6a72d626153fd8431ef8477a21145" + ], + "cwd": "[START_DIR]/rpmalloc", + "infra_step": true, + "name": "rpmalloc.fetch", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "git", + "checkout", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/rpmalloc", + "infra_step": true, + "name": "rpmalloc.checkout", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "rmtree", + "[CLEANUP]/rpmalloc-linux-amd64" + ], + "infra_step": true, + "name": "rpmalloc.remove sources linux-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copytree", + "[START_DIR]/rpmalloc", + "[CLEANUP]/rpmalloc-linux-amd64" + ], + "infra_step": true, + "name": "rpmalloc.copy sources linux-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [], + "name": "rpmalloc.build rpmalloc-linux-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[CLEANUP]/rpmalloc-linux-amd64/configure.py", + "-c", + "release", + "-a", + "x86-64", + "--lto" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-amd64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-amd64.configure", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [ + "[START_DIR]/cipd/ninja", + "lib/linux/release/x86-64/librpmallocwrap.a" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-amd64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-amd64.ninja", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "rmtree", + "[CLEANUP]/rpmalloc-linux-arm64" + ], + "infra_step": true, + "name": "rpmalloc.remove sources linux-arm64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copytree", + "[START_DIR]/rpmalloc", + "[CLEANUP]/rpmalloc-linux-arm64" + ], + "infra_step": true, + "name": "rpmalloc.copy sources linux-arm64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [], + "name": "rpmalloc.build rpmalloc-linux-arm64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[CLEANUP]/rpmalloc-linux-arm64/configure.py", + "-c", + "release", + "-a", + "arm64", + "--lto" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-arm64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-arm64.configure", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [ + "[START_DIR]/cipd/ninja", + "lib/linux/release/arm64/librpmallocwrap.a" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-arm64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-arm64.ninja", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [], "name": "debug" }, { @@ -148,6 +355,14 @@ "cmd": [ "[START_DIR]/gn/out/gn_unittests" ], + "cwd": "[START_DIR]/gn", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, "name": "debug.linux-amd64.test", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@2@@@" @@ -170,7 +385,8 @@ "-u", "[START_DIR]/gn/build/gen.py", "--use-lto", - "--use-icf" + "--use-icf", + "--link-lib=[CLEANUP]/rpmalloc-linux-amd64/lib/linux/release/x86-64/librpmallocwrap.a" ], "cwd": "[START_DIR]/gn", "env": { @@ -208,6 +424,14 @@ "cmd": [ "[START_DIR]/gn/out/gn_unittests" ], + "cwd": "[START_DIR]/gn", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, "name": "release.linux-amd64.test", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@2@@@" @@ -233,6 +457,14 @@ "-json-output", "/path/to/tmp/json" ], + "cwd": "[START_DIR]/gn", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, "name": "release.linux-amd64.upload.build gn/gn/linux-amd64", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@3@@@", @@ -255,6 +487,14 @@ "-json-output", "/path/to/tmp/json" ], + "cwd": "[START_DIR]/gn", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, "name": "release.linux-amd64.upload.cipd search gn/gn/linux-amd64 git_revision:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@3@@@", @@ -289,7 +529,8 @@ "-u", "[START_DIR]/gn/build/gen.py", "--use-lto", - "--use-icf" + "--use-icf", + "--link-lib=[CLEANUP]/rpmalloc-linux-arm64/lib/linux/release/arm64/librpmallocwrap.a" ], "cwd": "[START_DIR]/gn", "env": { @@ -343,6 +584,14 @@ "-json-output", "/path/to/tmp/json" ], + "cwd": "[START_DIR]/gn", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, "name": "release.linux-arm64.upload.build gn/gn/linux-arm64", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@3@@@", @@ -365,6 +614,14 @@ "-json-output", "/path/to/tmp/json" ], + "cwd": "[START_DIR]/gn", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, "name": "release.linux-arm64.upload.cipd search gn/gn/linux-arm64 git_revision:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@3@@@",
diff --git a/infra/recipes/gn.expected/cipd_register.json b/infra/recipes/gn.expected/cipd_register.json index 2e09f95..75d38d2 100644 --- a/infra/recipes/gn.expected/cipd_register.json +++ b/infra/recipes/gn.expected/cipd_register.json
@@ -96,6 +96,213 @@ }, { "cmd": [], + "name": "rpmalloc" + }, + { + "cmd": [ + "git", + "init", + "[START_DIR]/rpmalloc" + ], + "name": "rpmalloc.init", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "git", + "fetch", + "--tags", + "https://fuchsia.googlesource.com/third_party/github.com/mjansson/rpmalloc", + "6bb6ca97a8d6a72d626153fd8431ef8477a21145" + ], + "cwd": "[START_DIR]/rpmalloc", + "infra_step": true, + "name": "rpmalloc.fetch", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "git", + "checkout", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/rpmalloc", + "infra_step": true, + "name": "rpmalloc.checkout", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "rmtree", + "[CLEANUP]/rpmalloc-linux-amd64" + ], + "infra_step": true, + "name": "rpmalloc.remove sources linux-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copytree", + "[START_DIR]/rpmalloc", + "[CLEANUP]/rpmalloc-linux-amd64" + ], + "infra_step": true, + "name": "rpmalloc.copy sources linux-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [], + "name": "rpmalloc.build rpmalloc-linux-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[CLEANUP]/rpmalloc-linux-amd64/configure.py", + "-c", + "release", + "-a", + "x86-64", + "--lto" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-amd64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-amd64.configure", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [ + "[START_DIR]/cipd/ninja", + "lib/linux/release/x86-64/librpmallocwrap.a" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-amd64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-amd64.ninja", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "rmtree", + "[CLEANUP]/rpmalloc-linux-arm64" + ], + "infra_step": true, + "name": "rpmalloc.remove sources linux-arm64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copytree", + "[START_DIR]/rpmalloc", + "[CLEANUP]/rpmalloc-linux-arm64" + ], + "infra_step": true, + "name": "rpmalloc.copy sources linux-arm64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [], + "name": "rpmalloc.build rpmalloc-linux-arm64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[CLEANUP]/rpmalloc-linux-arm64/configure.py", + "-c", + "release", + "-a", + "arm64", + "--lto" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-arm64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-arm64.configure", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [ + "[START_DIR]/cipd/ninja", + "lib/linux/release/arm64/librpmallocwrap.a" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-arm64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-arm64.ninja", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [], "name": "debug" }, { @@ -148,6 +355,14 @@ "cmd": [ "[START_DIR]/gn/out/gn_unittests" ], + "cwd": "[START_DIR]/gn", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, "name": "debug.linux-amd64.test", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@2@@@" @@ -170,7 +385,8 @@ "-u", "[START_DIR]/gn/build/gen.py", "--use-lto", - "--use-icf" + "--use-icf", + "--link-lib=[CLEANUP]/rpmalloc-linux-amd64/lib/linux/release/x86-64/librpmallocwrap.a" ], "cwd": "[START_DIR]/gn", "env": { @@ -208,6 +424,14 @@ "cmd": [ "[START_DIR]/gn/out/gn_unittests" ], + "cwd": "[START_DIR]/gn", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, "name": "release.linux-amd64.test", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@2@@@" @@ -233,6 +457,14 @@ "-json-output", "/path/to/tmp/json" ], + "cwd": "[START_DIR]/gn", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, "name": "release.linux-amd64.upload.build gn/gn/linux-amd64", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@3@@@", @@ -255,6 +487,14 @@ "-json-output", "/path/to/tmp/json" ], + "cwd": "[START_DIR]/gn", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, "name": "release.linux-amd64.upload.cipd search gn/gn/linux-amd64 git_revision:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@3@@@", @@ -278,6 +518,14 @@ "-json-output", "/path/to/tmp/json" ], + "cwd": "[START_DIR]/gn", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, "name": "release.linux-amd64.upload.register gn/gn/linux-amd64", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@3@@@", @@ -303,7 +551,8 @@ "-u", "[START_DIR]/gn/build/gen.py", "--use-lto", - "--use-icf" + "--use-icf", + "--link-lib=[CLEANUP]/rpmalloc-linux-arm64/lib/linux/release/arm64/librpmallocwrap.a" ], "cwd": "[START_DIR]/gn", "env": { @@ -357,6 +606,14 @@ "-json-output", "/path/to/tmp/json" ], + "cwd": "[START_DIR]/gn", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, "name": "release.linux-arm64.upload.build gn/gn/linux-arm64", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@3@@@", @@ -379,6 +636,14 @@ "-json-output", "/path/to/tmp/json" ], + "cwd": "[START_DIR]/gn", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, "name": "release.linux-arm64.upload.cipd search gn/gn/linux-arm64 git_revision:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@3@@@",
diff --git a/infra/recipes/gn.expected/cq_linux.json b/infra/recipes/gn.expected/cq_linux.json index 75b64a4..2593ed3 100644 --- a/infra/recipes/gn.expected/cq_linux.json +++ b/infra/recipes/gn.expected/cq_linux.json
@@ -123,6 +123,213 @@ }, { "cmd": [], + "name": "rpmalloc" + }, + { + "cmd": [ + "git", + "init", + "[START_DIR]/rpmalloc" + ], + "name": "rpmalloc.init", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "git", + "fetch", + "--tags", + "https://fuchsia.googlesource.com/third_party/github.com/mjansson/rpmalloc", + "6bb6ca97a8d6a72d626153fd8431ef8477a21145" + ], + "cwd": "[START_DIR]/rpmalloc", + "infra_step": true, + "name": "rpmalloc.fetch", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "git", + "checkout", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/rpmalloc", + "infra_step": true, + "name": "rpmalloc.checkout", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "rmtree", + "[CLEANUP]/rpmalloc-linux-amd64" + ], + "infra_step": true, + "name": "rpmalloc.remove sources linux-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copytree", + "[START_DIR]/rpmalloc", + "[CLEANUP]/rpmalloc-linux-amd64" + ], + "infra_step": true, + "name": "rpmalloc.copy sources linux-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [], + "name": "rpmalloc.build rpmalloc-linux-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[CLEANUP]/rpmalloc-linux-amd64/configure.py", + "-c", + "release", + "-a", + "x86-64", + "--lto" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-amd64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-amd64.configure", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [ + "[START_DIR]/cipd/ninja", + "lib/linux/release/x86-64/librpmallocwrap.a" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-amd64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-amd64.ninja", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "rmtree", + "[CLEANUP]/rpmalloc-linux-arm64" + ], + "infra_step": true, + "name": "rpmalloc.remove sources linux-arm64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copytree", + "[START_DIR]/rpmalloc", + "[CLEANUP]/rpmalloc-linux-arm64" + ], + "infra_step": true, + "name": "rpmalloc.copy sources linux-arm64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [], + "name": "rpmalloc.build rpmalloc-linux-arm64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[CLEANUP]/rpmalloc-linux-arm64/configure.py", + "-c", + "release", + "-a", + "arm64", + "--lto" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-arm64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-arm64.configure", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [ + "[START_DIR]/cipd/ninja", + "lib/linux/release/arm64/librpmallocwrap.a" + ], + "cwd": "[CLEANUP]/rpmalloc-linux-arm64", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, + "name": "rpmalloc.build rpmalloc-linux-arm64.ninja", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [], "name": "debug" }, { @@ -175,6 +382,14 @@ "cmd": [ "[START_DIR]/gn/out/gn_unittests" ], + "cwd": "[START_DIR]/gn", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, "name": "debug.linux-amd64.test", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@2@@@" @@ -197,7 +412,8 @@ "-u", "[START_DIR]/gn/build/gen.py", "--use-lto", - "--use-icf" + "--use-icf", + "--link-lib=[CLEANUP]/rpmalloc-linux-amd64/lib/linux/release/x86-64/librpmallocwrap.a" ], "cwd": "[START_DIR]/gn", "env": { @@ -235,6 +451,14 @@ "cmd": [ "[START_DIR]/gn/out/gn_unittests" ], + "cwd": "[START_DIR]/gn", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" + }, "name": "release.linux-amd64.test", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@2@@@" @@ -253,7 +477,8 @@ "-u", "[START_DIR]/gn/build/gen.py", "--use-lto", - "--use-icf" + "--use-icf", + "--link-lib=[CLEANUP]/rpmalloc-linux-arm64/lib/linux/release/arm64/librpmallocwrap.a" ], "cwd": "[START_DIR]/gn", "env": {
diff --git a/infra/recipes/gn.expected/cq_mac.json b/infra/recipes/gn.expected/cq_mac.json index c1d51f5..3965438 100644 --- a/infra/recipes/gn.expected/cq_mac.json +++ b/infra/recipes/gn.expected/cq_mac.json
@@ -171,21 +171,14 @@ "name": "debug" }, { - "cmd": [], - "name": "debug.mac-amd64", - "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@" - ] - }, - { "cmd": [ "xcrun", "--show-sdk-path" ], - "name": "debug.mac-amd64.xcrun sdk-path", + "name": "debug.xcrun sdk-path", "stdout": "/path/to/tmp/", "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@2@@@", + "@@@STEP_NEST_LEVEL@1@@@", "@@@STEP_LOG_LINE@raw_io.output[sdk-path]@/some/xcode/path@@@", "@@@STEP_LOG_END@raw_io.output[sdk-path]@@@" ] @@ -201,15 +194,22 @@ "-Wp,-v", "-" ], - "name": "debug.mac-amd64.xcrun toolchain", + "name": "debug.xcrun toolchain", "stderr": "/path/to/tmp/", "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@2@@@", + "@@@STEP_NEST_LEVEL@1@@@", "@@@STEP_LOG_LINE@raw_io.output[toolchain]@[CACHE]/macos_sdk/XCode.app/include/c++/v1@@@", "@@@STEP_LOG_END@raw_io.output[toolchain]@@@" ] }, { + "cmd": [], + "name": "debug.mac-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { "cmd": [ "python", "-u", @@ -252,6 +252,14 @@ "cmd": [ "[START_DIR]/gn/out/gn_unittests" ], + "cwd": "[START_DIR]/gn", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-apple-darwin --sysroot=/some/xcode/path -nostdinc++ -cxx-isystem [CACHE]/macos_sdk/XCode.app/include/c++/v1", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-apple-darwin --sysroot=/some/xcode/path" + }, "name": "debug.mac-amd64.test", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@2@@@" @@ -262,21 +270,14 @@ "name": "release" }, { - "cmd": [], - "name": "release.mac-amd64", - "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@" - ] - }, - { "cmd": [ "xcrun", "--show-sdk-path" ], - "name": "release.mac-amd64.xcrun sdk-path", + "name": "release.xcrun sdk-path", "stdout": "/path/to/tmp/", "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@2@@@", + "@@@STEP_NEST_LEVEL@1@@@", "@@@STEP_LOG_LINE@raw_io.output[sdk-path]@/some/xcode/path@@@", "@@@STEP_LOG_END@raw_io.output[sdk-path]@@@" ] @@ -292,15 +293,22 @@ "-Wp,-v", "-" ], - "name": "release.mac-amd64.xcrun toolchain", + "name": "release.xcrun toolchain", "stderr": "/path/to/tmp/", "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@2@@@", + "@@@STEP_NEST_LEVEL@1@@@", "@@@STEP_LOG_LINE@raw_io.output[toolchain]@[CACHE]/macos_sdk/XCode.app/include/c++/v1@@@", "@@@STEP_LOG_END@raw_io.output[toolchain]@@@" ] }, { + "cmd": [], + "name": "release.mac-amd64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { "cmd": [ "python", "-u", @@ -344,27 +352,28 @@ "cmd": [ "[START_DIR]/gn/out/gn_unittests" ], + "cwd": "[START_DIR]/gn", + "env": { + "AR": "[START_DIR]/cipd/bin/llvm-ar", + "CC": "[START_DIR]/cipd/bin/clang", + "CFLAGS": "--target=x86_64-apple-darwin --sysroot=/some/xcode/path -nostdinc++ -cxx-isystem [CACHE]/macos_sdk/XCode.app/include/c++/v1", + "CXX": "[START_DIR]/cipd/bin/clang++", + "LDFLAGS": "--target=x86_64-apple-darwin --sysroot=/some/xcode/path" + }, "name": "release.mac-amd64.test", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@2@@@" ] }, { - "cmd": [], - "name": "release.mac-arm64", - "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@" - ] - }, - { "cmd": [ "xcrun", "--show-sdk-path" ], - "name": "release.mac-arm64.xcrun sdk-path", + "name": "release.xcrun sdk-path (2)", "stdout": "/path/to/tmp/", "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@2@@@", + "@@@STEP_NEST_LEVEL@1@@@", "@@@STEP_LOG_LINE@raw_io.output[sdk-path]@/some/xcode/path@@@", "@@@STEP_LOG_END@raw_io.output[sdk-path]@@@" ] @@ -380,15 +389,22 @@ "-Wp,-v", "-" ], - "name": "release.mac-arm64.xcrun toolchain", + "name": "release.xcrun toolchain (2)", "stderr": "/path/to/tmp/", "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@2@@@", + "@@@STEP_NEST_LEVEL@1@@@", "@@@STEP_LOG_LINE@raw_io.output[toolchain]@[CACHE]/macos_sdk/XCode.app/include/c++/v1@@@", "@@@STEP_LOG_END@raw_io.output[toolchain]@@@" ] }, { + "cmd": [], + "name": "release.mac-arm64", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { "cmd": [ "python", "-u",
diff --git a/infra/recipes/gn.expected/cq_win.json b/infra/recipes/gn.expected/cq_win.json index 164b245..7e4424b 100644 --- a/infra/recipes/gn.expected/cq_win.json +++ b/infra/recipes/gn.expected/cq_win.json
@@ -226,6 +226,7 @@ "cmd": [ "[START_DIR]\\gn\\out\\gn_unittests" ], + "cwd": "[START_DIR]\\gn", "env": { "VSINSTALLDIR": "[CACHE]\\windows_sdk" }, @@ -296,6 +297,7 @@ "cmd": [ "[START_DIR]\\gn\\out\\gn_unittests" ], + "cwd": "[START_DIR]\\gn", "env": { "VSINSTALLDIR": "[CACHE]\\windows_sdk" },
diff --git a/infra/recipes/gn.py b/infra/recipes/gn.py index 5c767b1..bac3233 100644 --- a/infra/recipes/gn.py +++ b/infra/recipes/gn.py
@@ -26,10 +26,84 @@ 'repository': Property(kind=str, default='https://gn.googlesource.com/gn'), } +# On select platforms, link the GN executable against rpmalloc for a small 10% speed boost. +RPMALLOC_GIT_URL = 'https://fuchsia.googlesource.com/third_party/github.com/mjansson/rpmalloc' +RPMALLOC_REVISION = '6bb6ca97a8d6a72d626153fd8431ef8477a21145' + +# Used to convert os and arch strings to rpmalloc format +RPMALLOC_MAP = { + 'amd64': 'x86-64', + 'mac': 'macos', +} + + +def _get_libcxx_include_path(api): + # Run the preprocessor with an empty input and print all include paths. + lines = api.step( + 'xcrun toolchain', [ + 'xcrun', '--toolchain', 'clang', 'clang++', '-xc++', '-fsyntax-only', + '-Wp,-v', '-' + ], + stderr=api.raw_io.output(name='toolchain', add_output_log=True), + step_test_data=lambda: api.raw_io.test_api.stream_output( + str(api.macos_sdk.sdk_dir.join('include', 'c++', 'v1')), + stream='stderr')).stderr.splitlines() + # Iterate over all include paths and look for the SDK libc++ one. + sdk_dir = str(api.macos_sdk.sdk_dir) + for line in lines: + line = line.strip() + if line.startswith(sdk_dir) and 'include/c++/v1' in line: + return line + return None # pragma: no cover + + +def _get_compilation_environment(api, target, cipd_dir): + if target.is_linux: + triple = '--target=%s' % target.triple + sysroot = '--sysroot=%s' % cipd_dir.join('sysroot') + env = { + 'CC': cipd_dir.join('bin', 'clang'), + 'CXX': cipd_dir.join('bin', 'clang++'), + 'AR': cipd_dir.join('bin', 'llvm-ar'), + 'CFLAGS': '%s %s' % (triple, sysroot), + 'LDFLAGS': '%s %s -static-libstdc++' % (triple, sysroot), + } + elif target.is_mac: + triple = '--target=%s' % target.triple + sysroot = '--sysroot=%s' % api.step( + 'xcrun sdk-path', ['xcrun', '--show-sdk-path'], + stdout=api.raw_io.output(name='sdk-path', add_output_log=True), + step_test_data=lambda: api.raw_io.test_api.stream_output( + '/some/xcode/path')).stdout.strip() + stdlib = cipd_dir.join('lib', 'libc++.a') + cxx_include = _get_libcxx_include_path(api) + env = { + 'CC': + cipd_dir.join('bin', 'clang'), + 'CXX': + cipd_dir.join('bin', 'clang++'), + 'AR': + cipd_dir.join('bin', 'llvm-ar'), + 'CFLAGS': + '%s %s -nostdinc++ -cxx-isystem %s' % + (triple, sysroot, cxx_include), + # TODO(phosek): Use the system libc++ temporarily until we + # have universal libc++.a for macOS that supports both x86_64 + # and arm64. + 'LDFLAGS': + '%s %s' % (triple, sysroot), + } + else: + env = {} + + return env + def RunSteps(api, repository): src_dir = api.path['start_dir'].join('gn') + # TODO: Verify that building and linking rpmalloc works on OS X and Windows as + # well. with api.step.nest('git'), api.context(infra_steps=True): api.step('init', ['git', 'init', src_dir]) @@ -84,85 +158,98 @@ 'name': 'release', 'args': ['--use-lto', '--use-icf'], 'targets': release_targets(), + # TODO: Enable this for OS X and Windows. + 'use_rpmalloc': api.platform.is_linux }, ] - def get_libcxx_include_path(): - # Run the preprocessor with an empty input and print all include paths. - lines = api.step( - 'xcrun toolchain', [ - 'xcrun', '--toolchain', 'clang', 'clang++', '-xc++', - '-fsyntax-only', '-Wp,-v', '-' - ], - stderr=api.raw_io.output(name='toolchain', add_output_log=True), - step_test_data=lambda: api.raw_io.test_api.stream_output( - str(api.macos_sdk.sdk_dir.join('include', 'c++', 'v1')), - stream='stderr')).stderr.splitlines() - # Iterate over all include paths and look for the SDK libc++ one. - sdk_dir = str(api.macos_sdk.sdk_dir) - for line in lines: - line = line.strip() - if line.startswith(sdk_dir) and 'include/c++/v1' in line: - return line - return None # pragma: no cover + # True if any config uses rpmalloc. + use_rpmalloc = any(c.get('use_rpmalloc', False) for c in configs) with api.macos_sdk(), api.windows_sdk(): + # Build the rpmalloc static libraries if needed. + if use_rpmalloc: + # Maps a target.platform string to the location of the corresponding + # rpmalloc static library. + rpmalloc_static_libs = {} + + # Get the list of all target platforms that are listed in `configs` + # above. Note that this is a list of Target instances, some of them + # may refer to the same platform string (e.g. linux-amd64). + # + # For each platform, a version of rpmalloc will be built if necessary, + # but doing this properly requires having a valid target instance to + # call _get_compilation_environment. So create a { platform -> Target } + # map to do that later. + all_config_platforms = {} + for c in configs: + if not c.get('use_rpmalloc', False): + continue + for t in c['targets']: + if t.platform not in all_config_platforms: + all_config_platforms[t.platform] = t + + rpmalloc_src_dir = api.path['start_dir'].join('rpmalloc') + with api.step.nest('rpmalloc'): + api.step('init', ['git', 'init', rpmalloc_src_dir]) + with api.context(cwd=rpmalloc_src_dir, infra_steps=True): + api.step( + 'fetch', + ['git', 'fetch', '--tags', RPMALLOC_GIT_URL, RPMALLOC_REVISION]) + api.step('checkout', ['git', 'checkout', 'FETCH_HEAD']) + + for platform in all_config_platforms: + # Convert target architecture and os to rpmalloc format. + rpmalloc_os, rpmalloc_arch = platform.split('-') + rpmalloc_os = RPMALLOC_MAP.get(rpmalloc_os, rpmalloc_os) + rpmalloc_arch = RPMALLOC_MAP.get(rpmalloc_arch, rpmalloc_arch) + + # The rpmalloc build system doesn't support out-of-tree builds, and + # trying to build binaries for multiple target architectures doesn't + # work unless one cleans the build. To work around these issues, + # simply copy the source tree into a new clean directory. + rpmalloc_build_dir = api.path['cleanup'].join('rpmalloc-' + platform) + api.file.rmtree('remove sources ' + platform, rpmalloc_build_dir) + api.file.copytree('copy sources ' + platform, rpmalloc_src_dir, + rpmalloc_build_dir) + + env = _get_compilation_environment(api, + all_config_platforms[platform], + cipd_dir) + with api.step.nest('build rpmalloc-' + platform), api.context( + env=env, cwd=rpmalloc_build_dir): + api.python( + 'configure', + rpmalloc_build_dir.join('configure.py'), + args=['-c', 'release', '-a', rpmalloc_arch, '--lto']) + + # NOTE: Only build the static library. + rpmalloc_static_lib = api.path.join('lib', rpmalloc_os, 'release', + rpmalloc_arch, + 'librpmallocwrap.a') + api.step('ninja', [cipd_dir.join('ninja'), rpmalloc_static_lib]) + + rpmalloc_static_libs[platform] = rpmalloc_build_dir.join( + rpmalloc_static_lib) + for config in configs: with api.step.nest(config['name']): for target in config['targets']: - with api.step.nest(target.platform): - if target.is_linux: - triple = '--target=%s' % target.triple - sysroot = '--sysroot=%s' % cipd_dir.join('sysroot') - env = { - 'CC': cipd_dir.join('bin', 'clang'), - 'CXX': cipd_dir.join('bin', 'clang++'), - 'AR': cipd_dir.join('bin', 'llvm-ar'), - 'CFLAGS': '%s %s' % (triple, sysroot), - 'LDFLAGS': '%s %s -static-libstdc++' % (triple, sysroot), - } - elif target.is_mac: - triple = '--target=%s' % target.triple - sysroot = '--sysroot=%s' % api.step( - 'xcrun sdk-path', ['xcrun', '--show-sdk-path'], - stdout=api.raw_io.output( - name='sdk-path', add_output_log=True), - step_test_data=lambda: api.raw_io.test_api.stream_output( - '/some/xcode/path')).stdout.strip() - stdlib = cipd_dir.join('lib', 'libc++.a') - cxx_include = get_libcxx_include_path() - assert cxx_include, 'cannot find C++ header directory' - env = { - 'CC': - cipd_dir.join('bin', 'clang'), - 'CXX': - cipd_dir.join('bin', 'clang++'), - 'AR': - cipd_dir.join('bin', 'llvm-ar'), - # TODO(phosek): temporarily use XCode C++ headers to make - # sure these are compatible with the system libc++. - 'CFLAGS': - '%s %s -nostdinc++ -cxx-isystem %s' % - (triple, sysroot, cxx_include), - # TODO(phosek): Use the system libc++ temporarily until we - # have universal libc++.a for macOS that supports both x86_64 - # and arm64. - 'LDFLAGS': - '%s %s' % (triple, sysroot), - } - else: - env = {} + env = _get_compilation_environment(api, target, cipd_dir) + with api.step.nest(target.platform), api.context( + env=env, cwd=src_dir): + args = config['args'] + if config.get('use_rpmalloc', False): + args = args[:] + [ + '--link-lib=%s' % rpmalloc_static_libs[target.platform] + ] - with api.context(env=env, cwd=src_dir): - api.python( - 'generate', - src_dir.join('build', 'gen.py'), - args=config['args']) + api.python('generate', src_dir.join('build', 'gen.py'), args=args) - # Windows requires the environment modifications when building too. - api.step('build', - [cipd_dir.join('ninja'), '-C', - src_dir.join('out')]) + # Windows requires the environment modifications when building too. + api.step('build', + [cipd_dir.join('ninja'), '-C', + src_dir.join('out')]) if target.is_host: api.step('test', [src_dir.join('out', 'gn_unittests')])