build/win/merge_pgc_files.py

#!/usr/bin/env python
# Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Merge the PGC files generated during the profiling step to the PGD database.

This is required to workaround a flakyness in pgomgr.exe where it can run out
of address space while trying to merge all the PGC files at the same time.
"""

import glob
import json
import optparse
import os
import subprocess
import sys


script_dir = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, os.path.join(script_dir, os.pardir))

import vs_toolchain


# Number of PGC files that should be merged in each iteration, merging all
# the files one by one is really slow but merging more than 10 at a time doesn't
# really seem to impact the total time (when merging 180 files).
#
# Number of pgc merged per iteration  |  Time (in min)
# 1                                   |  27.2
# 10                                  |  12.8
# 20                                  |  12.0
# 30                                  |  11.5
# 40                                  |  11.4
# 50                                  |  11.5
# 60                                  |  11.6
# 70                                  |  11.6
# 80                                  |  11.7
#
# TODO(sebmarchand): Measure the memory usage of pgomgr.exe to see how it get
#     affected by the number of pgc files.
_BATCH_SIZE_DEFAULT = 10


def find_pgomgr(chrome_checkout_dir):
  """Find pgomgr.exe."""
  win_toolchain_json_file = os.path.join(chrome_checkout_dir, 'build',
      'win_toolchain.json')
  if not os.path.exists(win_toolchain_json_file):
    raise Exception('The toolchain JSON file is missing.')
  with open(win_toolchain_json_file) as temp_f:
    toolchain_data = json.load(temp_f)
  if not os.path.isdir(toolchain_data['path']):
    raise Exception('The toolchain JSON file is invalid.')

  # Always use the x64 version of pgomgr (the x86 one doesn't work on the bot's
  # environment).
  pgomgr_dir = None
  if toolchain_data['version'] == '2017':
    vc_tools_root = vs_toolchain.FindVCToolsRoot()
    pgomgr_dir = os.path.join(vc_tools_root, 'HostX64', 'x64')

  pgomgr_path = os.path.join(pgomgr_dir, 'pgomgr.exe')
  if not os.path.exists(pgomgr_path):
    raise Exception('pgomgr.exe is missing from %s.' % pgomgr_dir)

  return pgomgr_path


def merge_pgc_files(pgomgr_path, files, pgd_path):
  """Merge all the pgc_files in |files| to |pgd_path|."""
  merge_command = [
      pgomgr_path,
      '/merge'
  ]
  merge_command.extend(files)
  merge_command.append(pgd_path)
  proc = subprocess.Popen(merge_command, stdout=subprocess.PIPE)
  stdout, _ = proc.communicate()
  print stdout
  return proc.returncode


def main():
  parser = optparse.OptionParser(usage='%prog [options]')
  parser.add_option('--checkout-dir', help='The Chrome checkout directory.')
  parser.add_option('--target-cpu', help='[DEPRECATED] The target\'s bitness.')
  parser.add_option('--build-dir', help='Chrome build directory.')
  parser.add_option('--binary-name', help='The binary for which the PGC files '
                    'should be merged, without extension.')
  parser.add_option('--files-per-iter', help='The number of PGC files to merge '
                    'in each iteration, default to %d.' % _BATCH_SIZE_DEFAULT,
                    type='int', default=_BATCH_SIZE_DEFAULT)
  options, _ = parser.parse_args()

  if not options.checkout_dir:
    parser.error('--checkout-dir is required')
  if not options.build_dir:
    parser.error('--build-dir is required')
  if not options.binary_name:
    parser.error('--binary-name is required')

  # Starts by finding pgomgr.exe.
  pgomgr_path = find_pgomgr(options.checkout_dir)

  pgc_files = glob.glob(os.path.join(options.build_dir,
                                     '%s*.pgc' % options.binary_name))
  pgd_file = os.path.join(options.build_dir, '%s.pgd' % options.binary_name)

  def _split_in_chunks(items, chunk_size):
    """Split |items| in chunks of size |chunk_size|.

    Source: http://stackoverflow.com/a/312464
    """
    for i in xrange(0, len(items), chunk_size):
      yield items[i:i + chunk_size]
  for chunk in _split_in_chunks(pgc_files, options.files_per_iter):
    files_to_merge = []
    for pgc_file in chunk:
      files_to_merge.append(
          os.path.join(options.build_dir, os.path.basename(pgc_file)))
    ret = merge_pgc_files(pgomgr_path, files_to_merge, pgd_file)
    # pgomgr.exe sometimes fails to merge too many files at the same time (it
    # usually complains that a stream is missing, but if you try to merge this
    # file individually it works), try to merge all the PGCs from this batch one
    # at a time instead. Don't fail the build if we can't merge a file.
    # TODO(sebmarchand): Report this to Microsoft, check if this is still
    # happening with VS2017.
    if ret != 0:
      print ('Error while trying to merge several PGC files at the same time, '
             'trying to merge them one by one.')
      for pgc_file in chunk:
        ret = merge_pgc_files(
            pgomgr_path,
            [os.path.join(options.build_dir, os.path.basename(pgc_file))],
            pgd_file
        )
        if ret != 0:
          print 'Error while trying to merge %s, continuing.' % pgc_file


if __name__ == '__main__':
  sys.exit(main())