CQ/CI configuration and recipe
Change-Id: Ib94c9710ed181409c1c8e7300e1b8a9d0dd385c6
Reviewed-on: https://gn-review.googlesource.com/1040
Reviewed-by: Vadim Shtayura <vadimsh@chromium.org>
Reviewed-by: Andrii Shyshkalov <tandrii@google.com>
Reviewed-by: Petr Hosek <phosek@google.com>
diff --git a/.gitignore b/.gitignore
index 9447ff9..8c85b91 100644
--- a/.gitignore
+++ b/.gitignore
@@ -53,6 +53,7 @@
Thumbs.db
# Settings directories for eclipse
/.externalToolBuilders/
+/.recipe_deps/
/.settings/
/.vs/
# Visual Studio Code
diff --git a/.style.yapf b/.style.yapf
new file mode 100644
index 0000000..de0c6a7
--- /dev/null
+++ b/.style.yapf
@@ -0,0 +1,2 @@
+[style]
+based_on_style = chromium
diff --git a/infra/README.recipes.md b/infra/README.recipes.md
new file mode 100644
index 0000000..72fb3b3
--- /dev/null
+++ b/infra/README.recipes.md
@@ -0,0 +1,19 @@
+<!--- AUTOGENERATED BY `./recipes.py test train` -->
+# Package documentation for [gn]()
+## Table of Contents
+
+**[Recipes](#Recipes)**
+ * [gn](#recipes-gn) — Recipe for building GN.
+## Recipes
+
+### *recipes* / [gn](/infra/recipes/gn.py)
+
+[DEPS](/infra/recipes/gn.py#6): [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/step][recipe_engine/recipe_modules/step]
+
+Recipe for building GN.
+
+— **def [RunSteps](/infra/recipes/gn.py#13)(api):**
+
+[recipe_engine/recipe_modules/path]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/10f67fe223a7973a95bc5fff219d1a7a3d95a326/README.recipes.md#recipe_modules-path
+[recipe_engine/recipe_modules/platform]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/10f67fe223a7973a95bc5fff219d1a7a3d95a326/README.recipes.md#recipe_modules-platform
+[recipe_engine/recipe_modules/step]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/10f67fe223a7973a95bc5fff219d1a7a3d95a326/README.recipes.md#recipe_modules-step
diff --git a/infra/config/cq.cfg b/infra/config/cq.cfg
new file mode 100644
index 0000000..bb3a274
--- /dev/null
+++ b/infra/config/cq.cfg
@@ -0,0 +1,27 @@
+# See http://luci-config.appspot.com/schemas/projects/refs:cq.cfg for the
+# documentation of this file format.
+
+version: 1
+cq_name: "gn"
+git_repo_url: "https://gn.googlesource.com/gn"
+gerrit {}
+verifiers {
+ try_job {
+ buckets {
+ name: "luci.gn.try",
+ builders {
+ name: "linux"
+ }
+ builders {
+ name: "mac"
+ }
+ builders {
+ name: "win"
+ }
+ }
+ }
+ gerrit_cq_ability {
+ committer_list: "project-gn-committers"
+ dry_run_access_list: "project-gn-tryjob-access"
+ }
+}
diff --git a/infra/config/recipes.cfg b/infra/config/recipes.cfg
new file mode 100644
index 0000000..1e725fe
--- /dev/null
+++ b/infra/config/recipes.cfg
@@ -0,0 +1,12 @@
+{
+ "api_version": 2,
+ "deps": {
+ "recipe_engine": {
+ "branch": "master",
+ "revision": "10f67fe223a7973a95bc5fff219d1a7a3d95a326",
+ "url": "https://chromium.googlesource.com/infra/luci/recipes-py"
+ }
+ },
+ "project_id": "gn",
+ "recipes_path": "infra"
+}
diff --git a/infra/config/refs.cfg b/infra/config/refs.cfg
new file mode 100644
index 0000000..465672f
--- /dev/null
+++ b/infra/config/refs.cfg
@@ -0,0 +1,8 @@
+# See RefsCfg message in
+# http://luci-config.appspot.com/schemas/projects:refs.cfg for the
+# documentation of this file format.
+
+refs {
+ name: "refs/heads/master"
+ config_path: "infra/config"
+}
diff --git a/infra/recipes.py b/infra/recipes.py
new file mode 100755
index 0000000..fe6589d
--- /dev/null
+++ b/infra/recipes.py
@@ -0,0 +1,216 @@
+#!/usr/bin/env python
+
+# Copyright 2017 The LUCI Authors. All rights reserved.
+# Use of this source code is governed under the Apache License, Version 2.0
+# that can be found in the LICENSE file.
+
+"""Bootstrap script to clone and forward to the recipe engine tool.
+
+*******************
+** DO NOT MODIFY **
+*******************
+
+This is a copy of https://chromium.googlesource.com/infra/luci/recipes-py/+/master/doc/recipes.py.
+To fix bugs, fix in the googlesource repo then run the autoroller.
+"""
+
+import argparse
+import json
+import logging
+import os
+import random
+import subprocess
+import sys
+import time
+import urlparse
+
+from collections import namedtuple
+
+from cStringIO import StringIO
+
+# The dependency entry for the recipe_engine in the client repo's recipes.cfg
+#
+# url (str) - the url to the engine repo we want to use.
+# revision (str) - the git revision for the engine to get.
+# path_override (str) - the subdirectory in the engine repo we should use to
+# find it's recipes.py entrypoint. This is here for completeness, but will
+# essentially always be empty. It would be used if the recipes-py repo was
+# merged as a subdirectory of some other repo and you depended on that
+# subdirectory.
+# branch (str) - the branch to fetch for the engine as an absolute ref (e.g.
+# refs/heads/master)
+# repo_type ("GIT"|"GITILES") - An ignored enum which will be removed soon.
+EngineDep = namedtuple('EngineDep',
+ 'url revision path_override branch repo_type')
+
+
+class MalformedRecipesCfg(Exception):
+ def __init__(self, msg, path):
+ super(MalformedRecipesCfg, self).__init__('malformed recipes.cfg: %s: %r'
+ % (msg, path))
+
+
+def parse(repo_root, recipes_cfg_path):
+ """Parse is a lightweight a recipes.cfg file parser.
+
+ Args:
+ repo_root (str) - native path to the root of the repo we're trying to run
+ recipes for.
+ recipes_cfg_path (str) - native path to the recipes.cfg file to process.
+
+ Returns (as tuple):
+ engine_dep (EngineDep|None): The recipe_engine dependency, or None, if the
+ current repo IS the recipe_engine.
+ recipes_path (str) - native path to where the recipes live inside of the
+ current repo (i.e. the folder containing `recipes/` and/or
+ `recipe_modules`)
+ """
+ with open(recipes_cfg_path, 'rU') as fh:
+ pb = json.load(fh)
+
+ try:
+ if pb['api_version'] != 2:
+ raise MalformedRecipesCfg('unknown version %d' % pb['api_version'],
+ recipes_cfg_path)
+
+ # If we're running ./doc/recipes.py from the recipe_engine repo itself, then
+ # return None to signal that there's no EngineDep.
+ if pb['project_id'] == 'recipe_engine':
+ return None, pb.get('recipes_path', '')
+
+ engine = pb['deps']['recipe_engine']
+
+ if 'url' not in engine:
+ raise MalformedRecipesCfg(
+ 'Required field "url" in dependency "recipe_engine" not found',
+ recipes_cfg_path)
+
+ engine.setdefault('revision', '')
+ engine.setdefault('path_override', '')
+ engine.setdefault('branch', 'refs/heads/master')
+ recipes_path = pb.get('recipes_path', '')
+
+ # TODO(iannucci): only support absolute refs
+ if not engine['branch'].startswith('refs/'):
+ engine['branch'] = 'refs/heads/' + engine['branch']
+
+ engine.setdefault('repo_type', 'GIT')
+ if engine['repo_type'] not in ('GIT', 'GITILES'):
+ raise MalformedRecipesCfg(
+ 'Unsupported "repo_type" value in dependency "recipe_engine"',
+ recipes_cfg_path)
+
+ recipes_path = os.path.join(
+ repo_root, recipes_path.replace('/', os.path.sep))
+ return EngineDep(**engine), recipes_path
+ except KeyError as ex:
+ raise MalformedRecipesCfg(ex.message, recipes_cfg_path)
+
+
+GIT = 'git.bat' if sys.platform.startswith(('win', 'cygwin')) else 'git'
+
+
+def _subprocess_call(argv, **kwargs):
+ logging.info('Running %r', argv)
+ return subprocess.call(argv, **kwargs)
+
+
+def _git_check_call(argv, **kwargs):
+ argv = [GIT]+argv
+ logging.info('Running %r', argv)
+ subprocess.check_call(argv, **kwargs)
+
+
+def _git_output(argv, **kwargs):
+ argv = [GIT]+argv
+ logging.info('Running %r', argv)
+ return subprocess.check_output(argv, **kwargs)
+
+
+def parse_args(argv):
+ """This extracts a subset of the arguments that this bootstrap script cares
+ about. Currently this consists of:
+ * an override for the recipe engine in the form of `-O recipe_engin=/path`
+ * the --package option.
+ """
+ PREFIX = 'recipe_engine='
+
+ p = argparse.ArgumentParser(add_help=False)
+ p.add_argument('-O', '--project-override', action='append')
+ p.add_argument('--package', type=os.path.abspath)
+ args, _ = p.parse_known_args(argv)
+ for override in args.project_override or ():
+ if override.startswith(PREFIX):
+ return override[len(PREFIX):], args.package
+ return None, args.package
+
+
+def checkout_engine(engine_path, repo_root, recipes_cfg_path):
+ dep, recipes_path = parse(repo_root, recipes_cfg_path)
+ if dep is None:
+ # we're running from the engine repo already!
+ return os.path.join(repo_root, recipes_path)
+
+ url = dep.url
+
+ if not engine_path and url.startswith('file://'):
+ engine_path = urlparse.urlparse(url).path
+
+ if not engine_path:
+ revision = dep.revision
+ subpath = dep.path_override
+ branch = dep.branch
+
+ # Ensure that we have the recipe engine cloned.
+ engine = os.path.join(recipes_path, '.recipe_deps', 'recipe_engine')
+ engine_path = os.path.join(engine, subpath)
+
+ with open(os.devnull, 'w') as NUL:
+ # Note: this logic mirrors the logic in recipe_engine/fetch.py
+ _git_check_call(['init', engine], stdout=NUL)
+
+ try:
+ _git_check_call(['rev-parse', '--verify', '%s^{commit}' % revision],
+ cwd=engine, stdout=NUL, stderr=NUL)
+ except subprocess.CalledProcessError:
+ _git_check_call(['fetch', url, branch], cwd=engine, stdout=NUL,
+ stderr=NUL)
+
+ try:
+ _git_check_call(['diff', '--quiet', revision], cwd=engine)
+ except subprocess.CalledProcessError:
+ _git_check_call(['reset', '-q', '--hard', revision], cwd=engine)
+
+ return engine_path
+
+
+def main():
+ if '--verbose' in sys.argv:
+ logging.getLogger().setLevel(logging.INFO)
+
+ args = sys.argv[1:]
+ engine_override, recipes_cfg_path = parse_args(args)
+
+ if recipes_cfg_path:
+ # calculate repo_root from recipes_cfg_path
+ repo_root = os.path.dirname(
+ os.path.dirname(
+ os.path.dirname(recipes_cfg_path)))
+ else:
+ # find repo_root with git and calculate recipes_cfg_path
+ repo_root = (_git_output(
+ ['rev-parse', '--show-toplevel'],
+ cwd=os.path.abspath(os.path.dirname(__file__))).strip())
+ repo_root = os.path.abspath(repo_root)
+ recipes_cfg_path = os.path.join(repo_root, 'infra', 'config', 'recipes.cfg')
+ args = ['--package', recipes_cfg_path] + args
+
+ engine_path = checkout_engine(engine_override, repo_root, recipes_cfg_path)
+
+ return _subprocess_call([
+ sys.executable, '-u',
+ os.path.join(engine_path, 'recipes.py')] + args)
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/infra/recipes/gn.expected/linux.json b/infra/recipes/gn.expected/linux.json
new file mode 100644
index 0000000..6631ad0
--- /dev/null
+++ b/infra/recipes/gn.expected/linux.json
@@ -0,0 +1,16 @@
+[
+ {
+ "cmd": [
+ "git",
+ "clone",
+ "https://gn.googlesource.com/gn",
+ "[START_DIR]/gn"
+ ],
+ "name": "checkout"
+ },
+ {
+ "name": "$result",
+ "recipe_result": null,
+ "status_code": 0
+ }
+]
\ No newline at end of file
diff --git a/infra/recipes/gn.expected/mac.json b/infra/recipes/gn.expected/mac.json
new file mode 100644
index 0000000..6631ad0
--- /dev/null
+++ b/infra/recipes/gn.expected/mac.json
@@ -0,0 +1,16 @@
+[
+ {
+ "cmd": [
+ "git",
+ "clone",
+ "https://gn.googlesource.com/gn",
+ "[START_DIR]/gn"
+ ],
+ "name": "checkout"
+ },
+ {
+ "name": "$result",
+ "recipe_result": null,
+ "status_code": 0
+ }
+]
\ No newline at end of file
diff --git a/infra/recipes/gn.expected/win.json b/infra/recipes/gn.expected/win.json
new file mode 100644
index 0000000..c3438b0
--- /dev/null
+++ b/infra/recipes/gn.expected/win.json
@@ -0,0 +1,16 @@
+[
+ {
+ "cmd": [
+ "git",
+ "clone",
+ "https://gn.googlesource.com/gn",
+ "[START_DIR]\\gn"
+ ],
+ "name": "checkout"
+ },
+ {
+ "name": "$result",
+ "recipe_result": null,
+ "status_code": 0
+ }
+]
\ No newline at end of file
diff --git a/infra/recipes/gn.py b/infra/recipes/gn.py
new file mode 100644
index 0000000..7a4224f
--- /dev/null
+++ b/infra/recipes/gn.py
@@ -0,0 +1,21 @@
+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed under the Apache License, Version 2.0
+# that can be found in the LICENSE file.
+"""Recipe for building GN."""
+
+DEPS = [
+ 'recipe_engine/path',
+ 'recipe_engine/platform',
+ 'recipe_engine/step',
+]
+
+
+def RunSteps(api):
+ src_dir = api.path['start_dir'].join('gn')
+ api.step('checkout',
+ ['git', 'clone', 'https://gn.googlesource.com/gn', src_dir])
+
+
+def GenTests(api):
+ for platform in ('linux', 'mac', 'win'):
+ yield api.test(platform) + api.platform.name(platform)
