cq/scripts/helpers/gn_utils.py

"""Compilation of utility functions for working with GN."""

from __future__ import absolute_import
import collections
import os
import re

from helpers import branch_utils
import six

# TODO(mbjorge|maasen): generate these based on the base path of the repo
# Default location for GN files
GN_SHA1 = 'chromium/src/buildtools/linux64/gn.sha1'
GN_BINARY = 'chromium/src/buildtools/linux64/gn'

# DEPRECATED: Do not reference this variable in new code. Prefer to use a
# path lookup for the chromium/src project instead.
GN_CHROMIUM_BASE_DIR = 'chromium/src'

GENERATE_GN_HELPER_SCRIPT = 'scripts/helpers/generate_gn_helper.sh'

GN_EXTENSIONS = ('.gn', '.gni')

GN_DONE_MSG = r'Done. Wrote \d+ targets from \d+ files'
GN_HEADER_OK_MSG = r'Header dependency check OK'
GN_ERROR_MSG = (
    r'ERROR at //(?P<filename>[^:]+):(?P<line>\d+):\d+: (?P<error>.+?)\n')
GN_ERROR_DIVIDER_MSG = r'___________________'
GN_FORMAT_ERROR_MSG = r':(?P<linenum>\d+):(?P<column>\d+): '


class GnUtilsError(Exception):
  """Gn Utils specific exception."""


def gn_check_error_to_comments(gn_check_output):
  """Convert a gn check error message to comments for a gerrit review.

  Args:
    gn_check_output: output from the `gn check` command

  Returns:
    Comment to be added to a review message.
    The comment is a dictionary with filenames as keys.
    Each filename maps to an array of line-numbered comments for that file.
  """
  assert isinstance(gn_check_output, six.string_types)

  comments = collections.defaultdict(list)

  # Check that GN check did not succeed
  if re.search(GN_DONE_MSG, gn_check_output) or re.search(
      GN_HEADER_OK_MSG, gn_check_output):
    return comments

  error_msg = re.compile(GN_ERROR_MSG)
  error_divider_msg = re.compile(GN_ERROR_DIVIDER_MSG)

  filename, linenum, message = '', '', []
  for line in gn_check_output.splitlines(True):
    # Look for the start of a new error
    match = error_msg.match(line)
    if match:
      filename = match.group('filename')
      linenum = match.group('line')
      message = ['ERROR: {}\n'.format(match.group('error'))]
      continue

    # Look for the end of the current error
    match = error_divider_msg.match(line)
    if match:
      comment = {'line': linenum, 'message': ''.join(message)}
      comments[filename].append(comment)
      filename, linenum, message = '', '', []
      continue

    # Add the contents of the error to the current comment
    message.append(line)

  # The last error message does not have a trailing divider
  if message:
    comment = {'line': linenum, 'message': ''.join(message)}
    comments[filename].append(comment)

  return comments


def gn_format_error_to_comments(filename, error_msg):
  """Convert a gn format error message to comments for a gerrit review.

  Args:
    filename: name of the file that caused the error
    error_msg: Output message from gn format. Example:
               "ERROR at /path/to/bad/file.gn:10:12: Invalid bar."

  Returns:
    Comment to be added to a review message.
    Example:
    {'path/to/bad/file.gn': [{'line': 10, 'message': 'Invalid bar.'}]}
  """
  assert error_msg.startswith('ERROR at ')
  assert filename in error_msg

  lines = error_msg.splitlines(True)
  match = re.search(GN_FORMAT_ERROR_MSG, lines[0])
  linenum = match.group('linenum')
  message = error_msg[match.end():]

  return {filename: [{'line': linenum, 'message': message}]}


def filepath_to_gn_path(filename):
  """convert chromium/src/a/b/BUILD.gn to //a/b:*."""
  if not filename.endswith('BUILD.gn'):
    raise ValueError('Invalid path %s. Must be a BUILD.gn file.', filename)
  if not filename.startswith(GN_CHROMIUM_BASE_DIR):
    raise ValueError('Invalid path %s. Must start with %s',
                     filename, GN_CHROMIUM_BASE_DIR)
  target_path = os.path.relpath(os.path.dirname(filename), GN_CHROMIUM_BASE_DIR)
  if target_path == '.':
    target_path = ''
  return  '//{}:*'.format(target_path)


def gn_label_to_target_name(gn_label):
  """Convert '//foo/bar:baz' to 'baz'."""
  if not gn_label:
    raise ValueError('Input label must not be empty.')
  if gn_label[-1] in [':', '/']:
    raise ValueError('A GN label must not end with {!r}: {}'.format(
        gn_label[-1], gn_label))

  if ':' in gn_label:
    return gn_label.split(':')[-1]
  if '/' in gn_label:
    return gn_label.split('/')[-1]

  return gn_label


def get_gn_binary(executor):
  """Returns the path to a gn binary.

  This attempts to download the latest version of GN. If the download fails,
  then an older version will be used. If GN cannot be downloaded and no
  older version can be found, this throws a GnUtilsError.

  Args:
    executor: Executes the download of GN via executor.exec_subproces

  Returns:
    A relative path to the GN binary.

  Raises:
    GnUtilsError: if GN cannot be found or downloaded.
  """
  ret, _, _ = _download_gn(GN_SHA1, executor)

  # If the download succeeded, trust that the binary is in the correct place
  # and return. If the download fails, an old copy could still exist and be
  # used.
  if ret == 0 or _verify_gn_exists():
    return GN_BINARY
  raise GnUtilsError('GN binary could not be found or downloaded.')


def _verify_gn_exists():
  """Returns boolean if gn exists."""
  return os.path.exists(GN_BINARY)


def _download_gn(sha1_path, executor):
  """Downloads the latest GN from google storage.

  Args:
    sha1_path: Path to gn.sha1 used to download GN.
               Using gn_utils.GN_SHA1 will result in GN
               being placed at gn_utils.GN_BINARY
    executor: Executes the download via executor.exec_subprocess

  Returns:
    (return_code, stdout, stderr) from the download process

  """
  assert isinstance(sha1_path, str)
  assert hasattr(executor, 'exec_subprocess')
  command = ['download_from_google_storage',
             '--no_resume',
             '--platform=linux*',
             '--no_auth',
             '--bucket', 'chromium-gn',
             '-s', sha1_path]
  return executor.exec_subprocess(command)