blob: 011d4b069d5e35f5eb0efd3666f7095692c12f2c [file] [log] [blame]
# Copyright 2018 The Chromium Authors. All rights reserved.
# Use of this source code is governed under the Apache License, Version 2.0
# that can be found in the LICENSE file.
"""Recipe for building GN."""
from recipe_engine.recipe_api import Property
PYTHON_VERSION_COMPATIBILITY = 'PY3'
DEPS = [
'recipe_engine/buildbucket',
'recipe_engine/cas',
'recipe_engine/cipd',
'recipe_engine/context',
'recipe_engine/file',
'recipe_engine/json',
'recipe_engine/path',
'recipe_engine/platform',
'recipe_engine/properties',
'recipe_engine/python',
'recipe_engine/raw_io',
'recipe_engine/step',
'target',
'macos_sdk',
'windows_sdk',
]
PROPERTIES = {
'repository': Property(kind=str, default='https://gn.googlesource.com/gn'),
}
# On select platforms, link the GN executable against rpmalloc for a small 10% speed boost.
RPMALLOC_GIT_URL = 'https://fuchsia.googlesource.com/third_party/github.com/mjansson/rpmalloc'
RPMALLOC_BRANCH = '+upstream/develop'
RPMALLOC_REVISION = 'b097fd0916500439721a114bb9cd8d14bd998683'
# Used to convert os and arch strings to rpmalloc format
RPMALLOC_MAP = {
'amd64': 'x86-64',
'mac': 'macos',
}
def _get_libcxx_include_path(api):
# Run the preprocessor with an empty input and print all include paths.
lines = api.step(
'xcrun toolchain', [
'xcrun', '--toolchain', 'clang', 'clang++', '-xc++', '-fsyntax-only',
'-Wp,-v', '/dev/null'
],
stderr=api.raw_io.output_text(name='toolchain', add_output_log=True),
step_test_data=lambda: api.raw_io.test_api.stream_output_text(
str(api.macos_sdk.sdk_dir.join('include', 'c++', 'v1')),
stream='stderr')).stderr.splitlines()
# Iterate over all include paths and look for the SDK libc++ one.
sdk_dir = str(api.macos_sdk.sdk_dir)
for line in lines:
line = line.strip()
if line.startswith(sdk_dir) and 'include/c++/v1' in line:
return line
return None # pragma: no cover
def _get_compilation_environment(api, target, cipd_dir):
if target.is_linux:
triple = '--target=%s' % target.triple
sysroot = '--sysroot=%s' % cipd_dir.join('sysroot')
env = {
'CC': cipd_dir.join('bin', 'clang'),
'CXX': cipd_dir.join('bin', 'clang++'),
'AR': cipd_dir.join('bin', 'llvm-ar'),
'CFLAGS': '%s %s' % (triple, sysroot),
'LDFLAGS': '%s %s -static-libstdc++' % (triple, sysroot),
}
elif target.is_mac:
triple = '--target=%s' % target.triple
sysroot = '--sysroot=%s' % api.step(
'xcrun sdk-path', ['xcrun', '--show-sdk-path'],
stdout=api.raw_io.output_text(name='sdk-path', add_output_log=True),
step_test_data=lambda: api.raw_io.test_api.stream_output_text(
'/some/xcode/path')).stdout.strip()
stdlib = cipd_dir.join('lib', 'libc++.a')
cxx_include = _get_libcxx_include_path(api)
env = {
'CC':
cipd_dir.join('bin', 'clang'),
'CXX':
cipd_dir.join('bin', 'clang++'),
'AR':
cipd_dir.join('bin', 'llvm-ar'),
'CFLAGS':
'%s %s -nostdinc++ -cxx-isystem %s' %
(triple, sysroot, cxx_include),
# TODO(phosek): Use the system libc++ temporarily until we
# have universal libc++.a for macOS that supports both x86_64
# and arm64.
'LDFLAGS':
'%s %s' % (triple, sysroot),
}
else:
env = {}
return env
def RunSteps(api, repository):
src_dir = api.path['start_dir'].join('gn')
# TODO: Verify that building and linking rpmalloc works on OS X and Windows as
# well.
with api.step.nest('git'), api.context(infra_steps=True):
api.step('init', ['git', 'init', src_dir])
with api.context(cwd=src_dir):
build_input = api.buildbucket.build_input
ref = (
build_input.gitiles_commit.id
if build_input.gitiles_commit else 'refs/heads/master')
# Fetch tags so `git describe` works.
api.step('fetch', ['git', 'fetch', '--tags', repository, ref])
api.step('checkout', ['git', 'checkout', 'FETCH_HEAD'])
revision = api.step(
'rev-parse', ['git', 'rev-parse', 'HEAD'],
stdout=api.raw_io.output_text()).stdout.strip()
for change in build_input.gerrit_changes:
api.step('fetch %s/%s' % (change.change, change.patchset), [
'git', 'fetch', repository,
'refs/changes/%s/%s/%s' %
(str(change.change)[-2:], change.change, change.patchset)
])
api.step('checkout %s/%s' % (change.change, change.patchset),
['git', 'checkout', 'FETCH_HEAD'])
with api.context(infra_steps=True):
cipd_dir = api.path['start_dir'].join('cipd')
pkgs = api.cipd.EnsureFile()
pkgs.add_package('infra/ninja/${platform}', 'version:1.8.2')
if api.platform.is_linux or api.platform.is_mac:
pkgs.add_package('fuchsia/third_party/clang/${platform}', 'integration')
if api.platform.is_linux:
pkgs.add_package('fuchsia/third_party/sysroot/linux',
'git_revision:c912d089c3d46d8982fdef76a50514cca79b6132',
'sysroot')
api.cipd.ensure(cipd_dir, pkgs)
def release_targets():
if api.platform.is_linux:
return [api.target('linux-amd64'), api.target('linux-arm64')]
elif api.platform.is_mac:
return [api.target('mac-amd64'), api.target('mac-arm64')]
else:
return [api.target.host]
# The order is important since release build will get uploaded to CIPD.
configs = [
{
'name': 'debug',
'args': ['-d'],
'targets': [api.target.host],
},
{
'name': 'release',
'args': ['--use-lto', '--use-icf'],
'targets': release_targets(),
# TODO: Enable this for OS X and Windows.
'use_rpmalloc': api.platform.is_linux,
},
]
# True if any config uses rpmalloc.
use_rpmalloc = any(c.get('use_rpmalloc', False) for c in configs)
with api.macos_sdk(), api.windows_sdk():
# Build the rpmalloc static libraries if needed.
if use_rpmalloc:
# Maps a target.platform string to the location of the corresponding
# rpmalloc static library.
rpmalloc_static_libs = {}
# Get the list of all target platforms that are listed in `configs`
# above. Note that this is a list of Target instances, some of them
# may refer to the same platform string (e.g. linux-amd64).
#
# For each platform, a version of rpmalloc will be built if necessary,
# but doing this properly requires having a valid target instance to
# call _get_compilation_environment. So create a { platform -> Target }
# map to do that later.
all_config_platforms = {}
for c in configs:
if not c.get('use_rpmalloc', False):
continue
for t in c['targets']:
if t.platform not in all_config_platforms:
all_config_platforms[t.platform] = t
rpmalloc_src_dir = api.path['start_dir'].join('rpmalloc')
with api.step.nest('rpmalloc'):
api.step('init', ['git', 'init', rpmalloc_src_dir])
with api.context(cwd=rpmalloc_src_dir, infra_steps=True):
api.step(
'fetch',
['git', 'fetch', '--tags', RPMALLOC_GIT_URL, RPMALLOC_BRANCH])
api.step('checkout', ['git', 'checkout', RPMALLOC_REVISION])
# Patch configure.py since to add -Wno-ignored-optimization-flag since
# Clang will now complain when `-funit-at-a-time` is being used.
build_ninja_clang_path = api.path.join(rpmalloc_src_dir, 'build/ninja/clang.py')
build_ninja_clang_py = api.file.read_text('read %s' % build_ninja_clang_path,
build_ninja_clang_path,
"CXXFLAGS = ['-Wall', '-Weverything', '-Wfoo']")
build_ninja_clang_py = build_ninja_clang_py.replace(
"'-Wno-disabled-macro-expansion'",
"'-Wno-disabled-macro-expansion', '-Wno-ignored-optimization-argument'")
api.file.write_text('write %s' % build_ninja_clang_path,
build_ninja_clang_path,
build_ninja_clang_py)
for platform in all_config_platforms:
# Convert target architecture and os to rpmalloc format.
rpmalloc_os, rpmalloc_arch = platform.split('-')
rpmalloc_os = RPMALLOC_MAP.get(rpmalloc_os, rpmalloc_os)
rpmalloc_arch = RPMALLOC_MAP.get(rpmalloc_arch, rpmalloc_arch)
env = _get_compilation_environment(api,
all_config_platforms[platform],
cipd_dir)
with api.step.nest('build rpmalloc-' + platform), api.context(
env=env, cwd=rpmalloc_src_dir):
api.python(
'configure',
rpmalloc_src_dir.join('configure.py'),
args=['-c', 'release', '-a', rpmalloc_arch, '--lto'])
# NOTE: Only build the static library.
rpmalloc_static_lib = api.path.join('lib', rpmalloc_os, 'release',
rpmalloc_arch,
'librpmallocwrap.a')
api.step('ninja', [cipd_dir.join('ninja'), rpmalloc_static_lib])
rpmalloc_static_libs[platform] = rpmalloc_src_dir.join(
rpmalloc_static_lib)
for config in configs:
with api.step.nest(config['name']):
for target in config['targets']:
env = _get_compilation_environment(api, target, cipd_dir)
with api.step.nest(target.platform), api.context(
env=env, cwd=src_dir):
args = config['args']
if config.get('use_rpmalloc', False):
args = args[:] + [
'--link-lib=%s' % rpmalloc_static_libs[target.platform]
]
api.python('generate', src_dir.join('build', 'gen.py'), args=args)
# Windows requires the environment modifications when building too.
api.step('build',
[cipd_dir.join('ninja'), '-C',
src_dir.join('out')])
if target.is_host:
api.step('test', [src_dir.join('out', 'gn_unittests')])
if config['name'] != 'release':
continue
with api.step.nest('upload'):
gn = 'gn' + ('.exe' if target.is_win else '')
if build_input.gerrit_changes:
# Upload to CAS from CQ.
api.cas.archive('upload binary to CAS', src_dir.join('out'),
src_dir.join('out', gn))
continue
cipd_pkg_name = 'gn/gn/%s' % target.platform
pkg_def = api.cipd.PackageDefinition(
package_name=cipd_pkg_name,
package_root=src_dir.join('out'),
install_mode='copy')
pkg_def.add_file(src_dir.join('out', gn))
pkg_def.add_version_file('.versions/%s.cipd_version' % gn)
cipd_pkg_file = api.path['cleanup'].join('gn.cipd')
api.cipd.build_from_pkg(
pkg_def=pkg_def,
output_package=cipd_pkg_file,
)
if api.buildbucket.builder_id.project == 'infra-internal':
cipd_pin = api.cipd.search(cipd_pkg_name,
'git_revision:' + revision)
if cipd_pin:
api.step('Package is up-to-date', cmd=None)
continue
api.cipd.register(
package_name=cipd_pkg_name,
package_path=cipd_pkg_file,
refs=['latest'],
tags={
'git_repository': repository,
'git_revision': revision,
},
)
def GenTests(api):
for platform in ('linux', 'mac', 'win'):
yield (api.test('ci_' + platform) + api.platform.name(platform) +
api.buildbucket.ci_build(
project='gn',
git_repo='gn.googlesource.com/gn',
))
yield (api.test('cq_' + platform) + api.platform.name(platform) +
api.buildbucket.try_build(
project='gn',
git_repo='gn.googlesource.com/gn',
))
yield (api.test('cipd_exists') + api.buildbucket.ci_build(
project='infra-internal',
git_repo='gn.googlesource.com/gn',
revision='a' * 40,
) + api.step_data(
'git.rev-parse', api.raw_io.stream_output_text('a' * 40)
) + api.step_data(
'release.linux-amd64.upload.cipd search gn/gn/linux-amd64 git_revision:' +
'a' * 40,
api.cipd.example_search('gn/gn/linux-amd64',
['git_revision:' + 'a' * 40])))
yield (api.test('cipd_register') + api.buildbucket.ci_build(
project='infra-internal',
git_repo='gn.googlesource.com/gn',
revision='a' * 40,
) + api.step_data(
'git.rev-parse', api.raw_io.stream_output_text('a' * 40)
) + api.step_data(
'release.linux-amd64.upload.cipd search gn/gn/linux-amd64 git_revision:' +
'a' * 40, api.cipd.example_search('gn/gn/linux-amd64', [])))