CQ/CI configuration and recipe Change-Id: Ib94c9710ed181409c1c8e7300e1b8a9d0dd385c6 Reviewed-on: https://gn-review.googlesource.com/1040 Reviewed-by: Vadim Shtayura <vadimsh@chromium.org> Reviewed-by: Andrii Shyshkalov <tandrii@google.com> Reviewed-by: Petr Hosek <phosek@google.com>
diff --git a/.gitignore b/.gitignore index 9447ff9..8c85b91 100644 --- a/.gitignore +++ b/.gitignore
@@ -53,6 +53,7 @@ Thumbs.db # Settings directories for eclipse /.externalToolBuilders/ +/.recipe_deps/ /.settings/ /.vs/ # Visual Studio Code
diff --git a/.style.yapf b/.style.yapf new file mode 100644 index 0000000..de0c6a7 --- /dev/null +++ b/.style.yapf
@@ -0,0 +1,2 @@ +[style] +based_on_style = chromium
diff --git a/infra/README.recipes.md b/infra/README.recipes.md new file mode 100644 index 0000000..72fb3b3 --- /dev/null +++ b/infra/README.recipes.md
@@ -0,0 +1,19 @@ +<!--- AUTOGENERATED BY `./recipes.py test train` --> +# Package documentation for [gn]() +## Table of Contents + +**[Recipes](#Recipes)** + * [gn](#recipes-gn) — Recipe for building GN. +## Recipes + +### *recipes* / [gn](/infra/recipes/gn.py) + +[DEPS](/infra/recipes/gn.py#6): [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/step][recipe_engine/recipe_modules/step] + +Recipe for building GN. + +— **def [RunSteps](/infra/recipes/gn.py#13)(api):** + +[recipe_engine/recipe_modules/path]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/10f67fe223a7973a95bc5fff219d1a7a3d95a326/README.recipes.md#recipe_modules-path +[recipe_engine/recipe_modules/platform]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/10f67fe223a7973a95bc5fff219d1a7a3d95a326/README.recipes.md#recipe_modules-platform +[recipe_engine/recipe_modules/step]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/10f67fe223a7973a95bc5fff219d1a7a3d95a326/README.recipes.md#recipe_modules-step
diff --git a/infra/config/cq.cfg b/infra/config/cq.cfg new file mode 100644 index 0000000..bb3a274 --- /dev/null +++ b/infra/config/cq.cfg
@@ -0,0 +1,27 @@ +# See http://luci-config.appspot.com/schemas/projects/refs:cq.cfg for the +# documentation of this file format. + +version: 1 +cq_name: "gn" +git_repo_url: "https://gn.googlesource.com/gn" +gerrit {} +verifiers { + try_job { + buckets { + name: "luci.gn.try", + builders { + name: "linux" + } + builders { + name: "mac" + } + builders { + name: "win" + } + } + } + gerrit_cq_ability { + committer_list: "project-gn-committers" + dry_run_access_list: "project-gn-tryjob-access" + } +}
diff --git a/infra/config/recipes.cfg b/infra/config/recipes.cfg new file mode 100644 index 0000000..1e725fe --- /dev/null +++ b/infra/config/recipes.cfg
@@ -0,0 +1,12 @@ +{ + "api_version": 2, + "deps": { + "recipe_engine": { + "branch": "master", + "revision": "10f67fe223a7973a95bc5fff219d1a7a3d95a326", + "url": "https://chromium.googlesource.com/infra/luci/recipes-py" + } + }, + "project_id": "gn", + "recipes_path": "infra" +}
diff --git a/infra/config/refs.cfg b/infra/config/refs.cfg new file mode 100644 index 0000000..465672f --- /dev/null +++ b/infra/config/refs.cfg
@@ -0,0 +1,8 @@ +# See RefsCfg message in +# http://luci-config.appspot.com/schemas/projects:refs.cfg for the +# documentation of this file format. + +refs { + name: "refs/heads/master" + config_path: "infra/config" +}
diff --git a/infra/recipes.py b/infra/recipes.py new file mode 100755 index 0000000..fe6589d --- /dev/null +++ b/infra/recipes.py
@@ -0,0 +1,216 @@ +#!/usr/bin/env python + +# Copyright 2017 The LUCI Authors. All rights reserved. +# Use of this source code is governed under the Apache License, Version 2.0 +# that can be found in the LICENSE file. + +"""Bootstrap script to clone and forward to the recipe engine tool. + +******************* +** DO NOT MODIFY ** +******************* + +This is a copy of https://chromium.googlesource.com/infra/luci/recipes-py/+/master/doc/recipes.py. +To fix bugs, fix in the googlesource repo then run the autoroller. +""" + +import argparse +import json +import logging +import os +import random +import subprocess +import sys +import time +import urlparse + +from collections import namedtuple + +from cStringIO import StringIO + +# The dependency entry for the recipe_engine in the client repo's recipes.cfg +# +# url (str) - the url to the engine repo we want to use. +# revision (str) - the git revision for the engine to get. +# path_override (str) - the subdirectory in the engine repo we should use to +# find it's recipes.py entrypoint. This is here for completeness, but will +# essentially always be empty. It would be used if the recipes-py repo was +# merged as a subdirectory of some other repo and you depended on that +# subdirectory. +# branch (str) - the branch to fetch for the engine as an absolute ref (e.g. +# refs/heads/master) +# repo_type ("GIT"|"GITILES") - An ignored enum which will be removed soon. +EngineDep = namedtuple('EngineDep', + 'url revision path_override branch repo_type') + + +class MalformedRecipesCfg(Exception): + def __init__(self, msg, path): + super(MalformedRecipesCfg, self).__init__('malformed recipes.cfg: %s: %r' + % (msg, path)) + + +def parse(repo_root, recipes_cfg_path): + """Parse is a lightweight a recipes.cfg file parser. + + Args: + repo_root (str) - native path to the root of the repo we're trying to run + recipes for. + recipes_cfg_path (str) - native path to the recipes.cfg file to process. + + Returns (as tuple): + engine_dep (EngineDep|None): The recipe_engine dependency, or None, if the + current repo IS the recipe_engine. + recipes_path (str) - native path to where the recipes live inside of the + current repo (i.e. the folder containing `recipes/` and/or + `recipe_modules`) + """ + with open(recipes_cfg_path, 'rU') as fh: + pb = json.load(fh) + + try: + if pb['api_version'] != 2: + raise MalformedRecipesCfg('unknown version %d' % pb['api_version'], + recipes_cfg_path) + + # If we're running ./doc/recipes.py from the recipe_engine repo itself, then + # return None to signal that there's no EngineDep. + if pb['project_id'] == 'recipe_engine': + return None, pb.get('recipes_path', '') + + engine = pb['deps']['recipe_engine'] + + if 'url' not in engine: + raise MalformedRecipesCfg( + 'Required field "url" in dependency "recipe_engine" not found', + recipes_cfg_path) + + engine.setdefault('revision', '') + engine.setdefault('path_override', '') + engine.setdefault('branch', 'refs/heads/master') + recipes_path = pb.get('recipes_path', '') + + # TODO(iannucci): only support absolute refs + if not engine['branch'].startswith('refs/'): + engine['branch'] = 'refs/heads/' + engine['branch'] + + engine.setdefault('repo_type', 'GIT') + if engine['repo_type'] not in ('GIT', 'GITILES'): + raise MalformedRecipesCfg( + 'Unsupported "repo_type" value in dependency "recipe_engine"', + recipes_cfg_path) + + recipes_path = os.path.join( + repo_root, recipes_path.replace('/', os.path.sep)) + return EngineDep(**engine), recipes_path + except KeyError as ex: + raise MalformedRecipesCfg(ex.message, recipes_cfg_path) + + +GIT = 'git.bat' if sys.platform.startswith(('win', 'cygwin')) else 'git' + + +def _subprocess_call(argv, **kwargs): + logging.info('Running %r', argv) + return subprocess.call(argv, **kwargs) + + +def _git_check_call(argv, **kwargs): + argv = [GIT]+argv + logging.info('Running %r', argv) + subprocess.check_call(argv, **kwargs) + + +def _git_output(argv, **kwargs): + argv = [GIT]+argv + logging.info('Running %r', argv) + return subprocess.check_output(argv, **kwargs) + + +def parse_args(argv): + """This extracts a subset of the arguments that this bootstrap script cares + about. Currently this consists of: + * an override for the recipe engine in the form of `-O recipe_engin=/path` + * the --package option. + """ + PREFIX = 'recipe_engine=' + + p = argparse.ArgumentParser(add_help=False) + p.add_argument('-O', '--project-override', action='append') + p.add_argument('--package', type=os.path.abspath) + args, _ = p.parse_known_args(argv) + for override in args.project_override or (): + if override.startswith(PREFIX): + return override[len(PREFIX):], args.package + return None, args.package + + +def checkout_engine(engine_path, repo_root, recipes_cfg_path): + dep, recipes_path = parse(repo_root, recipes_cfg_path) + if dep is None: + # we're running from the engine repo already! + return os.path.join(repo_root, recipes_path) + + url = dep.url + + if not engine_path and url.startswith('file://'): + engine_path = urlparse.urlparse(url).path + + if not engine_path: + revision = dep.revision + subpath = dep.path_override + branch = dep.branch + + # Ensure that we have the recipe engine cloned. + engine = os.path.join(recipes_path, '.recipe_deps', 'recipe_engine') + engine_path = os.path.join(engine, subpath) + + with open(os.devnull, 'w') as NUL: + # Note: this logic mirrors the logic in recipe_engine/fetch.py + _git_check_call(['init', engine], stdout=NUL) + + try: + _git_check_call(['rev-parse', '--verify', '%s^{commit}' % revision], + cwd=engine, stdout=NUL, stderr=NUL) + except subprocess.CalledProcessError: + _git_check_call(['fetch', url, branch], cwd=engine, stdout=NUL, + stderr=NUL) + + try: + _git_check_call(['diff', '--quiet', revision], cwd=engine) + except subprocess.CalledProcessError: + _git_check_call(['reset', '-q', '--hard', revision], cwd=engine) + + return engine_path + + +def main(): + if '--verbose' in sys.argv: + logging.getLogger().setLevel(logging.INFO) + + args = sys.argv[1:] + engine_override, recipes_cfg_path = parse_args(args) + + if recipes_cfg_path: + # calculate repo_root from recipes_cfg_path + repo_root = os.path.dirname( + os.path.dirname( + os.path.dirname(recipes_cfg_path))) + else: + # find repo_root with git and calculate recipes_cfg_path + repo_root = (_git_output( + ['rev-parse', '--show-toplevel'], + cwd=os.path.abspath(os.path.dirname(__file__))).strip()) + repo_root = os.path.abspath(repo_root) + recipes_cfg_path = os.path.join(repo_root, 'infra', 'config', 'recipes.cfg') + args = ['--package', recipes_cfg_path] + args + + engine_path = checkout_engine(engine_override, repo_root, recipes_cfg_path) + + return _subprocess_call([ + sys.executable, '-u', + os.path.join(engine_path, 'recipes.py')] + args) + + +if __name__ == '__main__': + sys.exit(main())
diff --git a/infra/recipes/gn.expected/linux.json b/infra/recipes/gn.expected/linux.json new file mode 100644 index 0000000..6631ad0 --- /dev/null +++ b/infra/recipes/gn.expected/linux.json
@@ -0,0 +1,16 @@ +[ + { + "cmd": [ + "git", + "clone", + "https://gn.googlesource.com/gn", + "[START_DIR]/gn" + ], + "name": "checkout" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file
diff --git a/infra/recipes/gn.expected/mac.json b/infra/recipes/gn.expected/mac.json new file mode 100644 index 0000000..6631ad0 --- /dev/null +++ b/infra/recipes/gn.expected/mac.json
@@ -0,0 +1,16 @@ +[ + { + "cmd": [ + "git", + "clone", + "https://gn.googlesource.com/gn", + "[START_DIR]/gn" + ], + "name": "checkout" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file
diff --git a/infra/recipes/gn.expected/win.json b/infra/recipes/gn.expected/win.json new file mode 100644 index 0000000..c3438b0 --- /dev/null +++ b/infra/recipes/gn.expected/win.json
@@ -0,0 +1,16 @@ +[ + { + "cmd": [ + "git", + "clone", + "https://gn.googlesource.com/gn", + "[START_DIR]\\gn" + ], + "name": "checkout" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file
diff --git a/infra/recipes/gn.py b/infra/recipes/gn.py new file mode 100644 index 0000000..7a4224f --- /dev/null +++ b/infra/recipes/gn.py
@@ -0,0 +1,21 @@ +# Copyright 2018 The Chromium Authors. All rights reserved. +# Use of this source code is governed under the Apache License, Version 2.0 +# that can be found in the LICENSE file. +"""Recipe for building GN.""" + +DEPS = [ + 'recipe_engine/path', + 'recipe_engine/platform', + 'recipe_engine/step', +] + + +def RunSteps(api): + src_dir = api.path['start_dir'].join('gn') + api.step('checkout', + ['git', 'clone', 'https://gn.googlesource.com/gn', src_dir]) + + +def GenTests(api): + for platform in ('linux', 'mac', 'win'): + yield api.test(platform) + api.platform.name(platform)