[infra] Link to jemalloc instead of rpmalloc Change the recipe to link to the jemalloc allocator instead of rpmalloc, since the latter has a nasty race condition that triggers segmentation faults at runtime when the machine is under heavy load (see associated bug for details). Profiling shows no difference in `gn gen` time when compared to the rpmalloc-linked binary. Bug=b:273830995 Change-Id: I83e8987fee1dbffc1d2e5e23e61651fce342fdaa Reviewed-on: https://gn-review.googlesource.com/c/gn/+/15700 Reviewed-by: Takuto Ikuta <tikuta@google.com> Commit-Queue: David Turner <digit@google.com>
diff --git a/infra/README.recipes.md b/infra/README.recipes.md index 0196775..c92400b 100644 --- a/infra/README.recipes.md +++ b/infra/README.recipes.md
@@ -101,7 +101,7 @@ Recipe for building GN. -— **def [RunSteps](/infra/recipes/gn.py#103)(api, repository):** +— **def [RunSteps](/infra/recipes/gn.py#95)(api, repository):** ### *recipes* / [macos\_sdk:examples/full](/infra/recipe_modules/macos_sdk/examples/full.py) [DEPS](/infra/recipe_modules/macos_sdk/examples/full.py#5): [macos\_sdk](#recipe_modules-macos_sdk), [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/step][recipe_engine/recipe_modules/step]
diff --git a/infra/recipes/gn.expected/ci_linux.json b/infra/recipes/gn.expected/ci_linux.json index 27e7b3f..8d83c61 100644 --- a/infra/recipes/gn.expected/ci_linux.json +++ b/infra/recipes/gn.expected/ci_linux.json
@@ -157,13 +157,13 @@ }, { "cmd": [], - "name": "rpmalloc" + "name": "jemalloc" }, { "cmd": [ "git", "init", - "[START_DIR]/rpmalloc" + "[START_DIR]/jemalloc" ], "luci_context": { "realm": { @@ -177,7 +177,7 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.init", + "name": "jemalloc.init", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@1@@@" ] @@ -186,11 +186,11 @@ "cmd": [ "git", "fetch", - "--tags", - "https://fuchsia.googlesource.com/third_party/github.com/mjansson/rpmalloc", - "+upstream/develop" + "https://fuchsia.googlesource.com/third_party/github.com/jemalloc/jemalloc.git", + "refs/tags/5.3.0", + "--depth=1" ], - "cwd": "[START_DIR]/rpmalloc", + "cwd": "[START_DIR]/jemalloc", "infra_step": true, "luci_context": { "realm": { @@ -204,7 +204,7 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.fetch", + "name": "jemalloc.fetch", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@1@@@" ] @@ -213,9 +213,9 @@ "cmd": [ "git", "checkout", - "668a7f81b588a985c6528b70674dbcc005d9cb75" + "FETCH_HEAD" ], - "cwd": "[START_DIR]/rpmalloc", + "cwd": "[START_DIR]/jemalloc", "infra_step": true, "luci_context": { "realm": { @@ -229,22 +229,16 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.checkout", + "name": "jemalloc.checkout", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@1@@@" ] }, { "cmd": [ - "vpython3", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "[START_DIR]/rpmalloc/build/ninja/clang.py", - "/path/to/tmp/" + "autoconf" ], + "cwd": "[START_DIR]/jemalloc", "infra_step": true, "luci_context": { "realm": { @@ -258,25 +252,17 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.read [START_DIR]/rpmalloc/build/ninja/clang.py", + "name": "jemalloc.autoconf", "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@", - "@@@STEP_LOG_LINE@clang.py@CXXFLAGS = ['-Wall', '-Weverything', '-Wfoo']@@@", - "@@@STEP_LOG_END@clang.py@@@" + "@@@STEP_NEST_LEVEL@1@@@" ] }, { "cmd": [ - "vpython3", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "CXXFLAGS = ['-Wall', '-Weverything', '-Wfoo']", - "[START_DIR]/rpmalloc/build/ninja/clang.py" + "mkdir", + "-p", + "[START_DIR]/jemalloc/build-linux-amd64" ], - "infra_step": true, "luci_context": { "realm": { "name": "gn:ci" @@ -289,37 +275,35 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.write [START_DIR]/rpmalloc/build/ninja/clang.py", + "name": "jemalloc.prepare linux-amd64 build", "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@", - "@@@STEP_LOG_LINE@clang.py@CXXFLAGS = ['-Wall', '-Weverything', '-Wfoo']@@@", - "@@@STEP_LOG_END@clang.py@@@" + "@@@STEP_NEST_LEVEL@1@@@" ] }, { "cmd": [], - "name": "rpmalloc.build rpmalloc-linux-amd64", + "name": "jemalloc.build jemalloc-linux-amd64", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@1@@@" ] }, { "cmd": [ - "python3", - "-u", - "[START_DIR]/rpmalloc/configure.py", - "-c", - "release", - "-a", - "x86-64", - "--lto" + "../configure", + "--build=x86_64-linux-gnu", + "--host=x86_64-linux-gnu", + "--disable-shared", + "--enable-static", + "--disable-syscall", + "--disable-stats" ], - "cwd": "[START_DIR]/rpmalloc", + "cwd": "[START_DIR]/jemalloc/build-linux-amd64", "env": { "AR": "[START_DIR]/cipd/bin/llvm-ar", "CC": "[START_DIR]/cipd/bin/clang", - "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "CXX": "[START_DIR]/cipd/bin/clang++", + "CXXFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" }, "luci_context": { @@ -334,22 +318,24 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.build rpmalloc-linux-amd64.configure", + "name": "jemalloc.build jemalloc-linux-amd64.configure", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@2@@@" ] }, { "cmd": [ - "[START_DIR]/cipd/ninja", - "lib/linux/release/x86-64/librpmallocwrap.a" + "make", + "-j8", + "build_lib_static" ], - "cwd": "[START_DIR]/rpmalloc", + "cwd": "[START_DIR]/jemalloc/build-linux-amd64", "env": { "AR": "[START_DIR]/cipd/bin/llvm-ar", "CC": "[START_DIR]/cipd/bin/clang", - "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "CXX": "[START_DIR]/cipd/bin/clang++", + "CXXFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" }, "luci_context": { @@ -364,35 +350,58 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.build rpmalloc-linux-amd64.ninja", + "name": "jemalloc.build jemalloc-linux-amd64.build", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@2@@@" ] }, { + "cmd": [ + "mkdir", + "-p", + "[START_DIR]/jemalloc/build-linux-arm64" + ], + "luci_context": { + "realm": { + "name": "gn:ci" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:8945511751514863184", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "jemalloc.prepare linux-arm64 build", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { "cmd": [], - "name": "rpmalloc.build rpmalloc-linux-arm64", + "name": "jemalloc.build jemalloc-linux-arm64", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@1@@@" ] }, { "cmd": [ - "python3", - "-u", - "[START_DIR]/rpmalloc/configure.py", - "-c", - "release", - "-a", - "arm64", - "--lto" + "../configure", + "--build=x86_64-linux-gnu", + "--host=aarch64-linux-gnu", + "--disable-shared", + "--enable-static", + "--disable-syscall", + "--disable-stats" ], - "cwd": "[START_DIR]/rpmalloc", + "cwd": "[START_DIR]/jemalloc/build-linux-arm64", "env": { "AR": "[START_DIR]/cipd/bin/llvm-ar", "CC": "[START_DIR]/cipd/bin/clang", - "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "CXX": "[START_DIR]/cipd/bin/clang++", + "CXXFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "LDFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" }, "luci_context": { @@ -407,22 +416,24 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.build rpmalloc-linux-arm64.configure", + "name": "jemalloc.build jemalloc-linux-arm64.configure", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@2@@@" ] }, { "cmd": [ - "[START_DIR]/cipd/ninja", - "lib/linux/release/arm64/librpmallocwrap.a" + "make", + "-j8", + "build_lib_static" ], - "cwd": "[START_DIR]/rpmalloc", + "cwd": "[START_DIR]/jemalloc/build-linux-arm64", "env": { "AR": "[START_DIR]/cipd/bin/llvm-ar", "CC": "[START_DIR]/cipd/bin/clang", - "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "CXX": "[START_DIR]/cipd/bin/clang++", + "CXXFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "LDFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" }, "luci_context": { @@ -437,7 +448,7 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.build rpmalloc-linux-arm64.ninja", + "name": "jemalloc.build jemalloc-linux-arm64.build", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@2@@@" ] @@ -563,7 +574,7 @@ "[START_DIR]/gn/build/gen.py", "--use-lto", "--use-icf", - "--link-lib=[START_DIR]/rpmalloc/lib/linux/release/x86-64/librpmallocwrap.a" + "--link-lib=[START_DIR]/jemalloc/build-linux-amd64/lib/libjemalloc.a" ], "cwd": "[START_DIR]/gn", "env": { @@ -716,7 +727,7 @@ "[START_DIR]/gn/build/gen.py", "--use-lto", "--use-icf", - "--link-lib=[START_DIR]/rpmalloc/lib/linux/release/arm64/librpmallocwrap.a" + "--link-lib=[START_DIR]/jemalloc/build-linux-arm64/lib/libjemalloc.a" ], "cwd": "[START_DIR]/gn", "env": {
diff --git a/infra/recipes/gn.expected/cipd_exists.json b/infra/recipes/gn.expected/cipd_exists.json index 64c1fb5..ba2b9f2 100644 --- a/infra/recipes/gn.expected/cipd_exists.json +++ b/infra/recipes/gn.expected/cipd_exists.json
@@ -157,13 +157,13 @@ }, { "cmd": [], - "name": "rpmalloc" + "name": "jemalloc" }, { "cmd": [ "git", "init", - "[START_DIR]/rpmalloc" + "[START_DIR]/jemalloc" ], "luci_context": { "realm": { @@ -177,7 +177,7 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.init", + "name": "jemalloc.init", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@1@@@" ] @@ -186,11 +186,11 @@ "cmd": [ "git", "fetch", - "--tags", - "https://fuchsia.googlesource.com/third_party/github.com/mjansson/rpmalloc", - "+upstream/develop" + "https://fuchsia.googlesource.com/third_party/github.com/jemalloc/jemalloc.git", + "refs/tags/5.3.0", + "--depth=1" ], - "cwd": "[START_DIR]/rpmalloc", + "cwd": "[START_DIR]/jemalloc", "infra_step": true, "luci_context": { "realm": { @@ -204,7 +204,7 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.fetch", + "name": "jemalloc.fetch", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@1@@@" ] @@ -213,9 +213,9 @@ "cmd": [ "git", "checkout", - "668a7f81b588a985c6528b70674dbcc005d9cb75" + "FETCH_HEAD" ], - "cwd": "[START_DIR]/rpmalloc", + "cwd": "[START_DIR]/jemalloc", "infra_step": true, "luci_context": { "realm": { @@ -229,22 +229,16 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.checkout", + "name": "jemalloc.checkout", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@1@@@" ] }, { "cmd": [ - "vpython3", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "[START_DIR]/rpmalloc/build/ninja/clang.py", - "/path/to/tmp/" + "autoconf" ], + "cwd": "[START_DIR]/jemalloc", "infra_step": true, "luci_context": { "realm": { @@ -258,25 +252,17 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.read [START_DIR]/rpmalloc/build/ninja/clang.py", + "name": "jemalloc.autoconf", "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@", - "@@@STEP_LOG_LINE@clang.py@CXXFLAGS = ['-Wall', '-Weverything', '-Wfoo']@@@", - "@@@STEP_LOG_END@clang.py@@@" + "@@@STEP_NEST_LEVEL@1@@@" ] }, { "cmd": [ - "vpython3", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "CXXFLAGS = ['-Wall', '-Weverything', '-Wfoo']", - "[START_DIR]/rpmalloc/build/ninja/clang.py" + "mkdir", + "-p", + "[START_DIR]/jemalloc/build-linux-amd64" ], - "infra_step": true, "luci_context": { "realm": { "name": "infra-internal:ci" @@ -289,37 +275,35 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.write [START_DIR]/rpmalloc/build/ninja/clang.py", + "name": "jemalloc.prepare linux-amd64 build", "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@", - "@@@STEP_LOG_LINE@clang.py@CXXFLAGS = ['-Wall', '-Weverything', '-Wfoo']@@@", - "@@@STEP_LOG_END@clang.py@@@" + "@@@STEP_NEST_LEVEL@1@@@" ] }, { "cmd": [], - "name": "rpmalloc.build rpmalloc-linux-amd64", + "name": "jemalloc.build jemalloc-linux-amd64", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@1@@@" ] }, { "cmd": [ - "python3", - "-u", - "[START_DIR]/rpmalloc/configure.py", - "-c", - "release", - "-a", - "x86-64", - "--lto" + "../configure", + "--build=x86_64-linux-gnu", + "--host=x86_64-linux-gnu", + "--disable-shared", + "--enable-static", + "--disable-syscall", + "--disable-stats" ], - "cwd": "[START_DIR]/rpmalloc", + "cwd": "[START_DIR]/jemalloc/build-linux-amd64", "env": { "AR": "[START_DIR]/cipd/bin/llvm-ar", "CC": "[START_DIR]/cipd/bin/clang", - "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "CXX": "[START_DIR]/cipd/bin/clang++", + "CXXFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" }, "luci_context": { @@ -334,22 +318,24 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.build rpmalloc-linux-amd64.configure", + "name": "jemalloc.build jemalloc-linux-amd64.configure", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@2@@@" ] }, { "cmd": [ - "[START_DIR]/cipd/ninja", - "lib/linux/release/x86-64/librpmallocwrap.a" + "make", + "-j8", + "build_lib_static" ], - "cwd": "[START_DIR]/rpmalloc", + "cwd": "[START_DIR]/jemalloc/build-linux-amd64", "env": { "AR": "[START_DIR]/cipd/bin/llvm-ar", "CC": "[START_DIR]/cipd/bin/clang", - "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "CXX": "[START_DIR]/cipd/bin/clang++", + "CXXFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" }, "luci_context": { @@ -364,35 +350,58 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.build rpmalloc-linux-amd64.ninja", + "name": "jemalloc.build jemalloc-linux-amd64.build", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@2@@@" ] }, { + "cmd": [ + "mkdir", + "-p", + "[START_DIR]/jemalloc/build-linux-arm64" + ], + "luci_context": { + "realm": { + "name": "infra-internal:ci" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:8945511751514863184", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "jemalloc.prepare linux-arm64 build", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { "cmd": [], - "name": "rpmalloc.build rpmalloc-linux-arm64", + "name": "jemalloc.build jemalloc-linux-arm64", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@1@@@" ] }, { "cmd": [ - "python3", - "-u", - "[START_DIR]/rpmalloc/configure.py", - "-c", - "release", - "-a", - "arm64", - "--lto" + "../configure", + "--build=x86_64-linux-gnu", + "--host=aarch64-linux-gnu", + "--disable-shared", + "--enable-static", + "--disable-syscall", + "--disable-stats" ], - "cwd": "[START_DIR]/rpmalloc", + "cwd": "[START_DIR]/jemalloc/build-linux-arm64", "env": { "AR": "[START_DIR]/cipd/bin/llvm-ar", "CC": "[START_DIR]/cipd/bin/clang", - "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "CXX": "[START_DIR]/cipd/bin/clang++", + "CXXFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "LDFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" }, "luci_context": { @@ -407,22 +416,24 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.build rpmalloc-linux-arm64.configure", + "name": "jemalloc.build jemalloc-linux-arm64.configure", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@2@@@" ] }, { "cmd": [ - "[START_DIR]/cipd/ninja", - "lib/linux/release/arm64/librpmallocwrap.a" + "make", + "-j8", + "build_lib_static" ], - "cwd": "[START_DIR]/rpmalloc", + "cwd": "[START_DIR]/jemalloc/build-linux-arm64", "env": { "AR": "[START_DIR]/cipd/bin/llvm-ar", "CC": "[START_DIR]/cipd/bin/clang", - "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "CXX": "[START_DIR]/cipd/bin/clang++", + "CXXFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "LDFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" }, "luci_context": { @@ -437,7 +448,7 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.build rpmalloc-linux-arm64.ninja", + "name": "jemalloc.build jemalloc-linux-arm64.build", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@2@@@" ] @@ -563,7 +574,7 @@ "[START_DIR]/gn/build/gen.py", "--use-lto", "--use-icf", - "--link-lib=[START_DIR]/rpmalloc/lib/linux/release/x86-64/librpmallocwrap.a" + "--link-lib=[START_DIR]/jemalloc/build-linux-amd64/lib/libjemalloc.a" ], "cwd": "[START_DIR]/gn", "env": { @@ -767,7 +778,7 @@ "[START_DIR]/gn/build/gen.py", "--use-lto", "--use-icf", - "--link-lib=[START_DIR]/rpmalloc/lib/linux/release/arm64/librpmallocwrap.a" + "--link-lib=[START_DIR]/jemalloc/build-linux-arm64/lib/libjemalloc.a" ], "cwd": "[START_DIR]/gn", "env": {
diff --git a/infra/recipes/gn.expected/cipd_register.json b/infra/recipes/gn.expected/cipd_register.json index 650b9da..f692406 100644 --- a/infra/recipes/gn.expected/cipd_register.json +++ b/infra/recipes/gn.expected/cipd_register.json
@@ -157,13 +157,13 @@ }, { "cmd": [], - "name": "rpmalloc" + "name": "jemalloc" }, { "cmd": [ "git", "init", - "[START_DIR]/rpmalloc" + "[START_DIR]/jemalloc" ], "luci_context": { "realm": { @@ -177,7 +177,7 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.init", + "name": "jemalloc.init", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@1@@@" ] @@ -186,11 +186,11 @@ "cmd": [ "git", "fetch", - "--tags", - "https://fuchsia.googlesource.com/third_party/github.com/mjansson/rpmalloc", - "+upstream/develop" + "https://fuchsia.googlesource.com/third_party/github.com/jemalloc/jemalloc.git", + "refs/tags/5.3.0", + "--depth=1" ], - "cwd": "[START_DIR]/rpmalloc", + "cwd": "[START_DIR]/jemalloc", "infra_step": true, "luci_context": { "realm": { @@ -204,7 +204,7 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.fetch", + "name": "jemalloc.fetch", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@1@@@" ] @@ -213,9 +213,9 @@ "cmd": [ "git", "checkout", - "668a7f81b588a985c6528b70674dbcc005d9cb75" + "FETCH_HEAD" ], - "cwd": "[START_DIR]/rpmalloc", + "cwd": "[START_DIR]/jemalloc", "infra_step": true, "luci_context": { "realm": { @@ -229,22 +229,16 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.checkout", + "name": "jemalloc.checkout", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@1@@@" ] }, { "cmd": [ - "vpython3", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "[START_DIR]/rpmalloc/build/ninja/clang.py", - "/path/to/tmp/" + "autoconf" ], + "cwd": "[START_DIR]/jemalloc", "infra_step": true, "luci_context": { "realm": { @@ -258,25 +252,17 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.read [START_DIR]/rpmalloc/build/ninja/clang.py", + "name": "jemalloc.autoconf", "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@", - "@@@STEP_LOG_LINE@clang.py@CXXFLAGS = ['-Wall', '-Weverything', '-Wfoo']@@@", - "@@@STEP_LOG_END@clang.py@@@" + "@@@STEP_NEST_LEVEL@1@@@" ] }, { "cmd": [ - "vpython3", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "CXXFLAGS = ['-Wall', '-Weverything', '-Wfoo']", - "[START_DIR]/rpmalloc/build/ninja/clang.py" + "mkdir", + "-p", + "[START_DIR]/jemalloc/build-linux-amd64" ], - "infra_step": true, "luci_context": { "realm": { "name": "infra-internal:ci" @@ -289,37 +275,35 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.write [START_DIR]/rpmalloc/build/ninja/clang.py", + "name": "jemalloc.prepare linux-amd64 build", "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@", - "@@@STEP_LOG_LINE@clang.py@CXXFLAGS = ['-Wall', '-Weverything', '-Wfoo']@@@", - "@@@STEP_LOG_END@clang.py@@@" + "@@@STEP_NEST_LEVEL@1@@@" ] }, { "cmd": [], - "name": "rpmalloc.build rpmalloc-linux-amd64", + "name": "jemalloc.build jemalloc-linux-amd64", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@1@@@" ] }, { "cmd": [ - "python3", - "-u", - "[START_DIR]/rpmalloc/configure.py", - "-c", - "release", - "-a", - "x86-64", - "--lto" + "../configure", + "--build=x86_64-linux-gnu", + "--host=x86_64-linux-gnu", + "--disable-shared", + "--enable-static", + "--disable-syscall", + "--disable-stats" ], - "cwd": "[START_DIR]/rpmalloc", + "cwd": "[START_DIR]/jemalloc/build-linux-amd64", "env": { "AR": "[START_DIR]/cipd/bin/llvm-ar", "CC": "[START_DIR]/cipd/bin/clang", - "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "CXX": "[START_DIR]/cipd/bin/clang++", + "CXXFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" }, "luci_context": { @@ -334,22 +318,24 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.build rpmalloc-linux-amd64.configure", + "name": "jemalloc.build jemalloc-linux-amd64.configure", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@2@@@" ] }, { "cmd": [ - "[START_DIR]/cipd/ninja", - "lib/linux/release/x86-64/librpmallocwrap.a" + "make", + "-j8", + "build_lib_static" ], - "cwd": "[START_DIR]/rpmalloc", + "cwd": "[START_DIR]/jemalloc/build-linux-amd64", "env": { "AR": "[START_DIR]/cipd/bin/llvm-ar", "CC": "[START_DIR]/cipd/bin/clang", - "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "CXX": "[START_DIR]/cipd/bin/clang++", + "CXXFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" }, "luci_context": { @@ -364,35 +350,58 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.build rpmalloc-linux-amd64.ninja", + "name": "jemalloc.build jemalloc-linux-amd64.build", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@2@@@" ] }, { + "cmd": [ + "mkdir", + "-p", + "[START_DIR]/jemalloc/build-linux-arm64" + ], + "luci_context": { + "realm": { + "name": "infra-internal:ci" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:8945511751514863184", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "jemalloc.prepare linux-arm64 build", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { "cmd": [], - "name": "rpmalloc.build rpmalloc-linux-arm64", + "name": "jemalloc.build jemalloc-linux-arm64", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@1@@@" ] }, { "cmd": [ - "python3", - "-u", - "[START_DIR]/rpmalloc/configure.py", - "-c", - "release", - "-a", - "arm64", - "--lto" + "../configure", + "--build=x86_64-linux-gnu", + "--host=aarch64-linux-gnu", + "--disable-shared", + "--enable-static", + "--disable-syscall", + "--disable-stats" ], - "cwd": "[START_DIR]/rpmalloc", + "cwd": "[START_DIR]/jemalloc/build-linux-arm64", "env": { "AR": "[START_DIR]/cipd/bin/llvm-ar", "CC": "[START_DIR]/cipd/bin/clang", - "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "CXX": "[START_DIR]/cipd/bin/clang++", + "CXXFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "LDFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" }, "luci_context": { @@ -407,22 +416,24 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.build rpmalloc-linux-arm64.configure", + "name": "jemalloc.build jemalloc-linux-arm64.configure", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@2@@@" ] }, { "cmd": [ - "[START_DIR]/cipd/ninja", - "lib/linux/release/arm64/librpmallocwrap.a" + "make", + "-j8", + "build_lib_static" ], - "cwd": "[START_DIR]/rpmalloc", + "cwd": "[START_DIR]/jemalloc/build-linux-arm64", "env": { "AR": "[START_DIR]/cipd/bin/llvm-ar", "CC": "[START_DIR]/cipd/bin/clang", - "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "CXX": "[START_DIR]/cipd/bin/clang++", + "CXXFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "LDFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" }, "luci_context": { @@ -437,7 +448,7 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.build rpmalloc-linux-arm64.ninja", + "name": "jemalloc.build jemalloc-linux-arm64.build", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@2@@@" ] @@ -563,7 +574,7 @@ "[START_DIR]/gn/build/gen.py", "--use-lto", "--use-icf", - "--link-lib=[START_DIR]/rpmalloc/lib/linux/release/x86-64/librpmallocwrap.a" + "--link-lib=[START_DIR]/jemalloc/build-linux-amd64/lib/libjemalloc.a" ], "cwd": "[START_DIR]/gn", "env": { @@ -802,7 +813,7 @@ "[START_DIR]/gn/build/gen.py", "--use-lto", "--use-icf", - "--link-lib=[START_DIR]/rpmalloc/lib/linux/release/arm64/librpmallocwrap.a" + "--link-lib=[START_DIR]/jemalloc/build-linux-arm64/lib/libjemalloc.a" ], "cwd": "[START_DIR]/gn", "env": {
diff --git a/infra/recipes/gn.expected/cq_linux.json b/infra/recipes/gn.expected/cq_linux.json index 1259594..9f674da 100644 --- a/infra/recipes/gn.expected/cq_linux.json +++ b/infra/recipes/gn.expected/cq_linux.json
@@ -208,13 +208,13 @@ }, { "cmd": [], - "name": "rpmalloc" + "name": "jemalloc" }, { "cmd": [ "git", "init", - "[START_DIR]/rpmalloc" + "[START_DIR]/jemalloc" ], "luci_context": { "realm": { @@ -228,7 +228,7 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.init", + "name": "jemalloc.init", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@1@@@" ] @@ -237,11 +237,11 @@ "cmd": [ "git", "fetch", - "--tags", - "https://fuchsia.googlesource.com/third_party/github.com/mjansson/rpmalloc", - "+upstream/develop" + "https://fuchsia.googlesource.com/third_party/github.com/jemalloc/jemalloc.git", + "refs/tags/5.3.0", + "--depth=1" ], - "cwd": "[START_DIR]/rpmalloc", + "cwd": "[START_DIR]/jemalloc", "infra_step": true, "luci_context": { "realm": { @@ -255,7 +255,7 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.fetch", + "name": "jemalloc.fetch", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@1@@@" ] @@ -264,9 +264,9 @@ "cmd": [ "git", "checkout", - "668a7f81b588a985c6528b70674dbcc005d9cb75" + "FETCH_HEAD" ], - "cwd": "[START_DIR]/rpmalloc", + "cwd": "[START_DIR]/jemalloc", "infra_step": true, "luci_context": { "realm": { @@ -280,22 +280,16 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.checkout", + "name": "jemalloc.checkout", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@1@@@" ] }, { "cmd": [ - "vpython3", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "[START_DIR]/rpmalloc/build/ninja/clang.py", - "/path/to/tmp/" + "autoconf" ], + "cwd": "[START_DIR]/jemalloc", "infra_step": true, "luci_context": { "realm": { @@ -309,25 +303,17 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.read [START_DIR]/rpmalloc/build/ninja/clang.py", + "name": "jemalloc.autoconf", "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@", - "@@@STEP_LOG_LINE@clang.py@CXXFLAGS = ['-Wall', '-Weverything', '-Wfoo']@@@", - "@@@STEP_LOG_END@clang.py@@@" + "@@@STEP_NEST_LEVEL@1@@@" ] }, { "cmd": [ - "vpython3", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "CXXFLAGS = ['-Wall', '-Weverything', '-Wfoo']", - "[START_DIR]/rpmalloc/build/ninja/clang.py" + "mkdir", + "-p", + "[START_DIR]/jemalloc/build-linux-amd64" ], - "infra_step": true, "luci_context": { "realm": { "name": "gn:try" @@ -340,37 +326,35 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.write [START_DIR]/rpmalloc/build/ninja/clang.py", + "name": "jemalloc.prepare linux-amd64 build", "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@", - "@@@STEP_LOG_LINE@clang.py@CXXFLAGS = ['-Wall', '-Weverything', '-Wfoo']@@@", - "@@@STEP_LOG_END@clang.py@@@" + "@@@STEP_NEST_LEVEL@1@@@" ] }, { "cmd": [], - "name": "rpmalloc.build rpmalloc-linux-amd64", + "name": "jemalloc.build jemalloc-linux-amd64", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@1@@@" ] }, { "cmd": [ - "python3", - "-u", - "[START_DIR]/rpmalloc/configure.py", - "-c", - "release", - "-a", - "x86-64", - "--lto" + "../configure", + "--build=x86_64-linux-gnu", + "--host=x86_64-linux-gnu", + "--disable-shared", + "--enable-static", + "--disable-syscall", + "--disable-stats" ], - "cwd": "[START_DIR]/rpmalloc", + "cwd": "[START_DIR]/jemalloc/build-linux-amd64", "env": { "AR": "[START_DIR]/cipd/bin/llvm-ar", "CC": "[START_DIR]/cipd/bin/clang", - "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "CXX": "[START_DIR]/cipd/bin/clang++", + "CXXFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" }, "luci_context": { @@ -385,22 +369,24 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.build rpmalloc-linux-amd64.configure", + "name": "jemalloc.build jemalloc-linux-amd64.configure", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@2@@@" ] }, { "cmd": [ - "[START_DIR]/cipd/ninja", - "lib/linux/release/x86-64/librpmallocwrap.a" + "make", + "-j8", + "build_lib_static" ], - "cwd": "[START_DIR]/rpmalloc", + "cwd": "[START_DIR]/jemalloc/build-linux-amd64", "env": { "AR": "[START_DIR]/cipd/bin/llvm-ar", "CC": "[START_DIR]/cipd/bin/clang", - "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "CXX": "[START_DIR]/cipd/bin/clang++", + "CXXFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "LDFLAGS": "--target=x86_64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" }, "luci_context": { @@ -415,35 +401,58 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.build rpmalloc-linux-amd64.ninja", + "name": "jemalloc.build jemalloc-linux-amd64.build", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@2@@@" ] }, { + "cmd": [ + "mkdir", + "-p", + "[START_DIR]/jemalloc/build-linux-arm64" + ], + "luci_context": { + "realm": { + "name": "gn:try" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:8945511751514863184", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "jemalloc.prepare linux-arm64 build", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { "cmd": [], - "name": "rpmalloc.build rpmalloc-linux-arm64", + "name": "jemalloc.build jemalloc-linux-arm64", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@1@@@" ] }, { "cmd": [ - "python3", - "-u", - "[START_DIR]/rpmalloc/configure.py", - "-c", - "release", - "-a", - "arm64", - "--lto" + "../configure", + "--build=x86_64-linux-gnu", + "--host=aarch64-linux-gnu", + "--disable-shared", + "--enable-static", + "--disable-syscall", + "--disable-stats" ], - "cwd": "[START_DIR]/rpmalloc", + "cwd": "[START_DIR]/jemalloc/build-linux-arm64", "env": { "AR": "[START_DIR]/cipd/bin/llvm-ar", "CC": "[START_DIR]/cipd/bin/clang", - "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "CXX": "[START_DIR]/cipd/bin/clang++", + "CXXFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "LDFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" }, "luci_context": { @@ -458,22 +467,24 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.build rpmalloc-linux-arm64.configure", + "name": "jemalloc.build jemalloc-linux-arm64.configure", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@2@@@" ] }, { "cmd": [ - "[START_DIR]/cipd/ninja", - "lib/linux/release/arm64/librpmallocwrap.a" + "make", + "-j8", + "build_lib_static" ], - "cwd": "[START_DIR]/rpmalloc", + "cwd": "[START_DIR]/jemalloc/build-linux-arm64", "env": { "AR": "[START_DIR]/cipd/bin/llvm-ar", "CC": "[START_DIR]/cipd/bin/clang", - "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot", + "CFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "CXX": "[START_DIR]/cipd/bin/clang++", + "CXXFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -Wno-error", "LDFLAGS": "--target=aarch64-linux-gnu --sysroot=[START_DIR]/cipd/sysroot -static-libstdc++" }, "luci_context": { @@ -488,7 +499,7 @@ "hostname": "rdbhost" } }, - "name": "rpmalloc.build rpmalloc-linux-arm64.ninja", + "name": "jemalloc.build jemalloc-linux-arm64.build", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@2@@@" ] @@ -614,7 +625,7 @@ "[START_DIR]/gn/build/gen.py", "--use-lto", "--use-icf", - "--link-lib=[START_DIR]/rpmalloc/lib/linux/release/x86-64/librpmallocwrap.a" + "--link-lib=[START_DIR]/jemalloc/build-linux-amd64/lib/libjemalloc.a" ], "cwd": "[START_DIR]/gn", "env": { @@ -894,7 +905,7 @@ "[START_DIR]/gn/build/gen.py", "--use-lto", "--use-icf", - "--link-lib=[START_DIR]/rpmalloc/lib/linux/release/arm64/librpmallocwrap.a" + "--link-lib=[START_DIR]/jemalloc/build-linux-arm64/lib/libjemalloc.a" ], "cwd": "[START_DIR]/gn", "env": {
diff --git a/infra/recipes/gn.py b/infra/recipes/gn.py index 7cb6229..87a9f47 100644 --- a/infra/recipes/gn.py +++ b/infra/recipes/gn.py
@@ -26,17 +26,9 @@ 'repository': Property(kind=str, default='https://gn.googlesource.com/gn'), } -# On select platforms, link the GN executable against rpmalloc for a small 10% speed boost. -RPMALLOC_GIT_URL = 'https://fuchsia.googlesource.com/third_party/github.com/mjansson/rpmalloc' -RPMALLOC_BRANCH = '+upstream/develop' -RPMALLOC_REVISION = '668a7f81b588a985c6528b70674dbcc005d9cb75' - -# Used to convert os and arch strings to rpmalloc format -RPMALLOC_MAP = { - 'amd64': 'x86-64', - 'mac': 'macos', -} - +# On select platforms, link the GN executable against jemalloc for a drastic speed boost. +JEMALLOC_GIT_URL = 'https://fuchsia.googlesource.com/third_party/github.com/jemalloc/jemalloc.git' +JEMALLOC_TAG = '5.3.0' def _get_libcxx_include_path(api): # Run the preprocessor with an empty input and print all include paths. @@ -103,7 +95,7 @@ def RunSteps(api, repository): src_dir = api.path['start_dir'].join('gn') - # TODO: Verify that building and linking rpmalloc works on OS X and Windows as + # TODO: Verify that building and linking jemalloc works on OS X and Windows as # well. with api.step.nest('git'), api.context(infra_steps=True): api.step('init', ['git', 'init', src_dir]) @@ -160,83 +152,88 @@ 'args': ['--use-lto', '--use-icf'], 'targets': release_targets(), # TODO: Enable this for OS X and Windows. - 'use_rpmalloc': api.platform.is_linux, + 'use_jemalloc': api.platform.is_linux, }, ] - # True if any config uses rpmalloc. - use_rpmalloc = any(c.get('use_rpmalloc', False) for c in configs) + use_jemalloc = any(c.get('use_jemalloc', False) for c in configs) with api.macos_sdk(), api.windows_sdk(): - # Build the rpmalloc static libraries if needed. - if use_rpmalloc: + # Build the jemalloc static library if needed. + if use_jemalloc: # Maps a target.platform string to the location of the corresponding - # rpmalloc static library. - rpmalloc_static_libs = {} + # jemalloc static library. + jemalloc_static_libs = {} # Get the list of all target platforms that are listed in `configs` # above. Note that this is a list of Target instances, some of them # may refer to the same platform string (e.g. linux-amd64). # - # For each platform, a version of rpmalloc will be built if necessary, + # For each platform, a version of jemalloc will be built if necessary, # but doing this properly requires having a valid target instance to # call _get_compilation_environment. So create a { platform -> Target } # map to do that later. all_config_platforms = {} for c in configs: - if not c.get('use_rpmalloc', False): + if not c.get('use_jemalloc', False): continue for t in c['targets']: if t.platform not in all_config_platforms: all_config_platforms[t.platform] = t - rpmalloc_src_dir = api.path['start_dir'].join('rpmalloc') - with api.step.nest('rpmalloc'): - api.step('init', ['git', 'init', rpmalloc_src_dir]) - with api.context(cwd=rpmalloc_src_dir, infra_steps=True): + jemalloc_src_dir = api.path['start_dir'].join('jemalloc') + with api.step.nest('jemalloc'): + api.step('init', ['git', 'init', jemalloc_src_dir]) + with api.context(cwd=jemalloc_src_dir, infra_steps=True): api.step( 'fetch', - ['git', 'fetch', '--tags', RPMALLOC_GIT_URL, RPMALLOC_BRANCH]) - api.step('checkout', ['git', 'checkout', RPMALLOC_REVISION]) - - # Patch configure.py since to add -Wno-unsafe-buffer-usage since Clang-16 will - # now complain about this when building rpmalloc (latest version only - # supports clang-15). - build_ninja_clang_path = api.path.join(rpmalloc_src_dir, 'build/ninja/clang.py') - build_ninja_clang_py = api.file.read_text('read %s' % build_ninja_clang_path, - build_ninja_clang_path, - "CXXFLAGS = ['-Wall', '-Weverything', '-Wfoo']") - build_ninja_clang_py = build_ninja_clang_py.replace( - "'-Wno-disabled-macro-expansion'", - "'-Wno-disabled-macro-expansion', '-Wno-unsafe-buffer-usage'") - api.file.write_text('write %s' % build_ninja_clang_path, - build_ninja_clang_path, - build_ninja_clang_py) + ['git', 'fetch', JEMALLOC_GIT_URL, 'refs/tags/' + JEMALLOC_TAG, '--depth=1']) + api.step('checkout', ['git', 'checkout', 'FETCH_HEAD']) + api.step('autoconf', ['autoconf']) for platform in all_config_platforms: - # Convert target architecture and os to rpmalloc format. - rpmalloc_os, rpmalloc_arch = platform.split('-') - rpmalloc_os = RPMALLOC_MAP.get(rpmalloc_os, rpmalloc_os) - rpmalloc_arch = RPMALLOC_MAP.get(rpmalloc_arch, rpmalloc_arch) + # Convert target architecture and os to jemalloc format. + jemalloc_build_dir = jemalloc_src_dir.join('build-' + platform) - env = _get_compilation_environment(api, - all_config_platforms[platform], - cipd_dir) - with api.step.nest('build rpmalloc-' + platform), api.context( - env=env, cwd=rpmalloc_src_dir): + target = all_config_platforms[platform] + host_target = api.target.host + env = _get_compilation_environment(api, target, cipd_dir) + + # Prepare environment for configuring and building jemalloc + # + # CFLAGS: -Wno-error is required to avoid warnings that break + # compilation with Clang 17. + # + # CXXFLAGS: Required to pass the right --target=<target> argument + # to the C++ compiler when building jemalloc_cpp.o, + # otherwise that file will not be properly cross-compiled. + # + # Note that -flto is never passed to CFLAGS/CXXFLAGS/LDFLAGS, since + # benchmarking shows that this leads to no performance difference + # for the resulting GN binary. + # + env['CFLAGS'] += " -Wno-error" + env['CXXFLAGS'] = env['CFLAGS'] + + api.step('prepare %s build' % platform, ['mkdir', '-p', jemalloc_build_dir]) + + with api.step.nest('build jemalloc-' + platform), api.context( + env=env, cwd=jemalloc_build_dir): api.step( 'configure', - ['python3', '-u', rpmalloc_src_dir.join('configure.py')] + - ['-c', 'release', '-a', rpmalloc_arch, '--lto']) + [ + '../configure', + '--build=' + host_target.triple, + '--host=' + target.triple, + '--disable-shared', + '--enable-static', + '--disable-syscall', + '--disable-stats', + ]) + api.step('build', ['make', '-j%d' % api.platform.cpu_count, 'build_lib_static']) + jemalloc_static_lib = jemalloc_build_dir.join('lib', 'libjemalloc.a') - # NOTE: Only build the static library. - rpmalloc_static_lib = api.path.join('lib', rpmalloc_os, 'release', - rpmalloc_arch, - 'librpmallocwrap.a') - api.step('ninja', [cipd_dir.join('ninja'), rpmalloc_static_lib]) - - rpmalloc_static_libs[platform] = rpmalloc_src_dir.join( - rpmalloc_static_lib) + jemalloc_static_libs[platform] = jemalloc_static_lib for config in configs: with api.step.nest(config['name']): @@ -245,9 +242,9 @@ with api.step.nest(target.platform), api.context( env=env, cwd=src_dir): args = config['args'] - if config.get('use_rpmalloc', False): + if config.get('use_jemalloc', False): args = args[:] + [ - '--link-lib=%s' % rpmalloc_static_libs[target.platform] + '--link-lib=%s' % jemalloc_static_libs[target.platform] ] api.step('generate',