Add a GN auto-roller script.
This CL adds a script in //tools/gn/bin/roll_gn.py that, given no
arguments, will post three CLs and three tryjobs in order to:
* build a new GN binary at tip-of-tree
* update the buildtools repo to the newly built binaries
* and then update DEPS to point to the leatest buildtools
The script must be run on Linux by someone with commit access to both
src/ and buildtools/.
One can also run the steps independently for testing purposes.
It can only be run in a clean chromium checkout; it should error out
in most cases if something bad happens, but the error checking isn't
yet foolproof.
R=brettw@chromium.org, scottmg@chromium.org, thakis@chromium.org
Review URL: https://codereview.chromium.org/1230293003
Cr-Original-Commit-Position: refs/heads/master@{#339191}
Cr-Mirrored-From: https://chromium.googlesource.com/chromium/src
Cr-Mirrored-Commit: 97ec9ac8703dc10eaef5e60984bb3ecdfad344bd
diff --git a/tools/gn/bin/roll_gn.py b/tools/gn/bin/roll_gn.py
new file mode 100644
index 0000000..6691ecc
--- /dev/null
+++ b/tools/gn/bin/roll_gn.py
@@ -0,0 +1,384 @@
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""An auto-roller for GN binaries into Chromium.
+
+This script is used to update the GN binaries that a Chromium
+checkout uses. In order to update the binaries, one must follow
+four steps in order:
+
+1. Trigger try jobs to build a new GN binary at tip-of-tree and upload
+ the newly-built binaries into the right Google CloudStorage bucket.
+2. Wait for the try jobs to complete.
+3. Update the buildtools repo with the .sha1 hashes of the newly built
+ binaries.
+4. Update Chromium's DEPS file to the new version of the buildtools repo.
+
+The script has four commands that correspond to the four steps above:
+'build', 'wait', 'roll_buildtools', and 'roll_deps'.
+
+The script has a fifth command, 'roll', that runs the four in order.
+
+If given no arguments, the script will run the 'roll' command.
+
+It can only be run on linux in a clean Chromium checkout; it should
+error out in most cases if something bad happens, but the error checking
+isn't yet foolproof.
+
+"""
+
+from __future__ import print_function
+
+import argparse
+import json
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import time
+import urllib2
+
+depot_tools_path = None
+for p in os.environ['PATH'].split(os.pathsep):
+ if (p.rstrip(os.sep).endswith('depot_tools') and
+ os.path.isfile(os.path.join(p, 'gclient.py'))):
+ depot_tools_path = p
+
+assert depot_tools_path
+if not depot_tools_path in sys.path:
+ sys.path.insert(0, depot_tools_path)
+
+third_party_path = os.path.join(depot_tools_path, 'third_party')
+if not third_party_path in sys.path:
+ sys.path.insert(0, third_party_path)
+
+import upload
+
+
+CHROMIUM_REPO = 'https://chromium.googlesource.com/chromium/src.git'
+
+CODE_REVIEW_SERVER = 'https://codereview.chromium.org'
+
+
+class GNRoller(object):
+ def __init__(self):
+ self.chromium_src_dir = None
+ self.buildtools_dir = None
+ self.old_gn_commitish = None
+ self.new_gn_commitish = None
+ self.old_gn_version = None
+ self.new_gn_version = None
+ self.reviewer = 'dpranke@chromium.org'
+ if os.getenv('USER') == 'dpranke':
+ self.reviewer = 'brettw@chromium.org'
+
+ def Roll(self):
+ parser = argparse.ArgumentParser()
+ parser.usage = __doc__
+ parser.add_argument('command', nargs='?', default='roll',
+ help='build|roll|roll_buildtools|roll_deps|wait'
+ ' (%(default)s is the default)')
+
+ args = parser.parse_args()
+ command = args.command
+ ret = self.SetUp()
+ if not ret and command in ('roll', 'build'):
+ ret = self.TriggerBuild()
+ if not ret and command in ('roll', 'wait'):
+ ret = self.WaitForBuildToFinish()
+ if not ret and command in ('roll', 'roll_buildtools'):
+ ret = self.RollBuildtools()
+ if not ret and command in ('roll', 'roll_deps'):
+ ret = self.RollDEPS()
+
+ return ret
+
+ def SetUp(self):
+ if sys.platform != 'linux2':
+ print('roll_gn is only tested and working on Linux for now.')
+ return 1
+
+ ret, out, _ = self.Call('git config --get remote.origin.url')
+ origin = out.strip()
+ if ret or origin != CHROMIUM_REPO:
+ print('Not in a Chromium repo? git config --get remote.origin.url '
+ 'returned %d: %s' % (ret, origin))
+ return 1
+
+ ret, _, _ = self.Call('git diff -q')
+ if ret:
+ print("Checkout is dirty, exiting")
+ return 1
+
+ _, out, _ = self.Call('git rev-parse --show-toplevel', cwd=os.getcwd())
+ self.chromium_src_dir = out.strip()
+ self.buildtools_dir = os.path.join(self.chromium_src_dir, 'buildtools')
+
+ self.new_gn_commitish, self.new_gn_version = self.GetNewVersions()
+
+ _, out, _ = self.Call('gn --version')
+ self.old_gn_version = out.strip()
+
+ _, out, _ = self.Call('git crrev-parse %s' % self.old_gn_version)
+ self.old_gn_commitish = out.strip()
+ return 0
+
+ def GetNewVersions(self):
+ _, out, _ = self.Call('git log -1 --grep Cr-Commit-Position')
+ commit_msg = out.splitlines()
+ first_line = commit_msg[0]
+ new_gn_commitish = first_line.split()[1]
+
+ last_line = commit_msg[-1]
+ new_gn_version = re.sub('.*master@{#(\d+)}', '\\1', last_line)
+
+ return new_gn_commitish, new_gn_version
+
+ def TriggerBuild(self):
+ ret, _, _ = self.Call('git new-branch build_gn_%s' % self.new_gn_version)
+ if ret:
+ print('Failed to create a new branch for build_gn_%s' %
+ self.new_gn_version)
+ return 1
+
+ self.MakeDummyDepsChange()
+
+ ret, out, err = self.Call('git commit -a -m "Build gn at %s"' %
+ self.new_gn_version)
+ if ret:
+ print('git commit failed: %s' % out + err)
+ return 1
+
+ print('Uploading CL to build GN at {#%s} - %s' %
+ (self.new_gn_version, self.new_gn_commitish))
+ ret, out, err = self.Call('git cl upload -f')
+ if ret:
+ print('git-cl upload failed: %s' % out + err)
+ return 1
+
+ print('Starting try jobs')
+ self.Call('git-cl try -b linux_chromium_gn_upload '
+ '-b mac_chromium_gn_upload '
+ '-b win8_chromium_gn_upload -r %s' % self.new_gn_commitish)
+
+ return 0
+
+ def MakeDummyDepsChange(self):
+ with open('DEPS') as fp:
+ deps_content = fp.read()
+ new_deps = deps_content.replace("'buildtools_revision':",
+ "'buildtools_revision': ")
+
+ with open('DEPS', 'w') as fp:
+ fp.write(new_deps)
+
+ def WaitForBuildToFinish(self):
+ print('Checking build')
+ results = self.CheckBuild()
+ while any(r['state'] == 'pending' for r in results.values()):
+ print()
+ print('Sleeping for 30 seconds')
+ time.sleep(30)
+ print('Checking build')
+ results = self.CheckBuild()
+ return 0 if all(r['state'] == 'success' for r in results.values()) else 1
+
+ def CheckBuild(self):
+ _, out, _ = self.Call('git-cl issue')
+
+ issue = int(out.split()[2])
+
+ _, out, _ = self.Call('git config user.email')
+ email = ''
+ rpc_server = upload.GetRpcServer(CODE_REVIEW_SERVER, email)
+ try:
+ props = json.loads(rpc_server.Send('/api/%d' % issue))
+ except Exception as _e:
+ raise
+
+ patchset = int(props['patchsets'][-1])
+
+ try:
+ patchset_data = json.loads(rpc_server.Send('/api/%d/%d' %
+ (issue, patchset)))
+ except Exception as _e:
+ raise
+
+ TRY_JOB_RESULT_STATES = ('success', 'warnings', 'failure', 'skipped',
+ 'exception', 'retry', 'pending')
+ try_job_results = patchset_data['try_job_results']
+ if not try_job_results:
+ print('No try jobs found on most recent patchset')
+ return 1
+
+ results = {}
+ for job in try_job_results:
+ builder = job['builder']
+ if builder == 'linux_chromium_gn_upload':
+ platform = 'linux64'
+ elif builder == 'mac_chromium_gn_upload':
+ platform = 'mac'
+ elif builder == 'win8_chromium_gn_upload':
+ platform = 'win'
+ else:
+ print('Unexpected builder: %s')
+ continue
+
+ state = TRY_JOB_RESULT_STATES[int(job['result'])]
+ url_str = ' %s' % job['url']
+ build = url_str.split('/')[-1]
+
+ sha1 = '-'
+ results.setdefault(platform, {'build': -1, 'sha1': '', 'url': url_str})
+
+ if state == 'success':
+ jsurl = url_str.replace('/builders/', '/json/builders/')
+ fp = urllib2.urlopen(jsurl)
+ js = json.loads(fp.read())
+ fp.close()
+ for step in js['steps']:
+ if step['name'] == 'gn sha1':
+ sha1 = step['text'][1]
+
+ if results[platform]['build'] < build:
+ results[platform]['build'] = build
+ results[platform]['sha1'] = sha1
+ results[platform]['state'] = state
+ results[platform]['url'] = url_str
+
+ for platform, r in results.items():
+ print(platform)
+ print(' sha1: %s' % r['sha1'])
+ print(' state: %s' % r['state'])
+ print(' build: %s' % r['build'])
+ print(' url: %s' % r['url'])
+ print()
+
+ return results
+
+ def RollBuildtools(self):
+ results = self.CheckBuild()
+ if not all(r['state'] == 'success' for r in results.values()):
+ print("Roll isn't done or didn't succeed, exiting:")
+ return 1
+
+ desc = self.GetBuildtoolsDesc()
+
+ self.Call('git new-branch roll_buildtools_gn_%s' % self.new_gn_version,
+ cwd=self.buildtools_dir)
+
+ for platform in results:
+ fname = 'gn.exe.sha1' if platform == 'win' else 'gn.sha1'
+ path = os.path.join(self.buildtools_dir, platform, fname)
+ with open(path, 'w') as fp:
+ fp.write('%s\n' % results[platform]['sha1'])
+
+ desc_file = tempfile.NamedTemporaryFile(delete=False)
+ try:
+ desc_file.write(desc)
+ desc_file.close()
+ self.Call('git commit -a -F %s' % desc_file.name,
+ cwd=self.buildtools_dir)
+ self.Call('git-cl upload -f --send-mail',
+ cwd=self.buildtools_dir)
+ finally:
+ os.remove(desc_file.name)
+
+ self.Call('git cl push', cwd=self.buildtools_dir)
+
+ # Fetch the revision we just committed so that RollDEPS will find it.
+ self.Call('git cl fetch', cwd=self.buildtools_dir)
+
+ return 0
+
+ def RollDEPS(self):
+ _, out, _ = self.Call('git rev-parse origin/master',
+ cwd=self.buildtools_dir)
+ new_buildtools_commitish = out.strip()
+
+ new_deps_lines = []
+ old_buildtools_commitish = ''
+ with open(os.path.join(self.chromium_src_dir, 'DEPS')) as fp:
+ for l in fp.readlines():
+ m = re.match(".*'buildtools_revision':.*'(.+)',", l)
+ if m:
+ old_buildtools_commitish = m.group(1)
+ new_deps_lines.append(" 'buildtools_revision': '%s'," %
+ new_buildtools_commitish)
+ else:
+ new_deps_lines.append(l)
+
+ if not old_buildtools_commitish:
+ print('Could not update DEPS properly, exiting')
+ return 1
+
+ with open('DEPS', 'w') as fp:
+ fp.write(''.join(new_deps_lines) + '\n')
+
+ desc = self.GetDEPSRollDesc(old_buildtools_commitish,
+ new_buildtools_commitish)
+ desc_file = tempfile.NamedTemporaryFile(delete=False)
+ try:
+ desc_file.write(desc)
+ desc_file.close()
+ self.Call('git commit -a -F %s' % desc_file.name)
+ self.Call('git-cl upload -f --send-mail --commit-queue')
+ finally:
+ os.remove(desc_file.name)
+ return 0
+
+ def GetBuildtoolsDesc(self):
+ gn_changes = self.GetGNChanges()
+ return (
+ 'Roll gn %s..%s (r%s:%s)\n'
+ '\n'
+ '%s'
+ '\n'
+ 'TBR=%s\n' % (
+ self.old_gn_commitish,
+ self.new_gn_commitish,
+ self.old_gn_version,
+ self.new_gn_version,
+ gn_changes,
+ self.reviewer,
+ ))
+
+ def GetDEPSRollDesc(self, old_buildtools_commitish, new_buildtools_commitish):
+ gn_changes = self.GetGNChanges()
+
+ return (
+ 'Roll DEPS %s..%s\n'
+ '\n'
+ ' in order to roll GN %s..%s (r%s:%s)\n'
+ '\n'
+ '%s'
+ '\n'
+ 'TBR=%s\n' % (
+ old_buildtools_commitish,
+ new_buildtools_commitish,
+ self.old_gn_commitish,
+ self.new_gn_commitish,
+ self.old_gn_version,
+ self.new_gn_version,
+ gn_changes,
+ self.reviewer,
+ ))
+
+ def GetGNChanges(self):
+ _, out, _ = self.Call(
+ "git log --pretty=' %h %s' " +
+ "%s..%s tools/gn" % (self.old_gn_commitish, self.new_gn_commitish))
+ return out
+
+ def Call(self, cmd, cwd=None):
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True,
+ cwd=(cwd or self.chromium_src_dir))
+ out, err = proc.communicate()
+ return proc.returncode, out, err
+
+
+if __name__ == '__main__':
+ roller = GNRoller()
+ sys.exit(roller.Roll())
