cq/scripts/slave/step/gn_check_step.py

"""Build step class for running `gn check`."""
import logging
import os

from helpers import git_utils
from helpers import gn_utils
from helpers import py_env_utils
from slave import base_step

_GN_ROOT_DIR_PROJECT = 'chromium/src'
_GN_CHECK_OUT_DIR = 'out_gn_check'
_MESH_TAG = '-mesh-eng'

# Bare bones GN configs that allow for basic smoke tests to run during
# presubmit. All the full builds also do GN checks as part of the build,
# so the full GN configs will also be tested.
_DESKTOP_VIDEO_GN_ARGS = ['is_chromecast=true',
                          'chromecast_branding="internal"']
_DESKTOP_AUDIO_GN_ARGS = ['is_chromecast=true',
                          'chromecast_branding="internal"',
                          'is_cast_audio_only=true']
_GN_CONFIGS = [_DESKTOP_VIDEO_GN_ARGS,
               _DESKTOP_AUDIO_GN_ARGS,]


class GnCheckStepBase(base_step.BaseStep):
  """Base step for running GnCheck.

  The key part here being shared is the method for parsing gn check output
  and making it into a comment.
  """

  def __init__(self, name:str='gn check', **kwargs):
    super().__init__(name=name, **kwargs)

  def run(self):
    raise NotImplementedError

  def gn_check_to_review(self, stdout):
    """Converts gn_check |stdout| into comments and adds them to the review."""
    comments = gn_utils.gn_check_error_to_comments(stdout)

    # GN makes comments relative to the root GN directory,
    # but we want the file comments to be relative to |self.directory|
    directory_prefix = self.directory
    gn_root_dir = self.get_project_path(_GN_ROOT_DIR_PROJECT)
    if directory_prefix.startswith(gn_root_dir):
      directory_prefix = directory_prefix[len(gn_root_dir + os.path.sep):]

    # GN can generate comments for files not directly included in the CL, so
    # split these comments out to the review message
    files_in_cl = git_utils.changed_files(self, directory=self.directory)
    file_comments, review_comments = {}, {}
    for filename, comment_list in comments.items():
      full_filename = filename
      if directory_prefix and filename.startswith(directory_prefix):
        filename = filename[len(directory_prefix + os.path.sep):]

      if filename in files_in_cl:
        file_comments[filename] = comment_list
      else:
        review_comments[full_filename] = comment_list

      # To limit comment filesize, only add the first comment. In theory, seeing
      # one GN check issue will inspire people to run gn check locally to clean
      # up the other issues.
      logging.info('All GN check comments: %s', comments)
      break

    # Generate the review message
    review_message = [
        'Run `gn check out_chromecast_$PRODUCT/release` or ',
        '`lunch $PRODUCT && make build.ninja ENABLE_GN_CHECK=1`',
    ]
    for filename, comments in review_comments.items():
      review_message.append(filename)
      for comment in comments:
        review_message.append('LINE: ' + comment['line'])
        review_message.append(comment['message'])
        review_message.append('----------')
      review_message.append('')

    self.add_review({
        'message': '\n'.join(review_message),
        'comments': file_comments
    })


class GnCheckStep(GnCheckStepBase):
  """Build step class for running gn check via generate_gn helper."""

  def __init__(self, name: str='gn check', **kwargs):
    """Creates a GnCheckStep instance.

    Args:
      **kwargs: Any additional args to pass to BaseStep.
    """
    super().__init__(name=name, **kwargs)

  def run(self):
    """Runs gn gen --check and reports any errors.

    Sets a build property "review" with the output if there are errors.

    Returns:
      True iff gn check reported no errors.
    """
    out_dir = os.path.join(self.get_tmp_dir(), _GN_CHECK_OUT_DIR)
    root_dir = self.get_project_path(_GN_ROOT_DIR_PROJECT)
    all_checks_passed = True
    for gn_config in _GN_CONFIGS:
      command = [gn_utils.GN_BINARY,
                 'gen', out_dir,
                 '--root={}'.format(root_dir),
                 '--args={}'.format(' '.join(gn_config)),
                 '--check']
      returncode, stdout, _ = self.exec_subprocess(command, quiet=True)

      if returncode != 0:
        all_checks_passed = False
        self.gn_check_to_review(stdout)

    return all_checks_passed


class GnCheckForOta(GnCheckStepBase):
  """Step for running gn check on OTA builds."""

  def __init__(self, build_name, **kwargs):
    """Creats a GnCheckForOta instance.

    Args:
      build_name: Name of build (e.g. pepperoni-eng, tvdefault-eng)
      **kwargs: Any additional args to pass to BaseStep.
    """
    GnCheckStepBase.__init__(self, **kwargs)
    self._enable_gn_check = gn_utils.is_gn_check_enabled(self.manifest_branch)
    self._product = build_name.split('-')[0]
    self._is_mesh_build = build_name.endswith(_MESH_TAG)
    self._build_name = build_name.replace(_MESH_TAG, '-eng')

  def rerun_if_necessary(self, returncode, stdout, stderr, command, env):
    """Do some cleanup and rerun |command| if it failed previously.

    In the majority of runs, this function will just be a no-op.

    Occasionally, running `make` will produce a corrupted
    AllocationTestHarness.P file. This bad file will stay around until it is
    cleaned out, causing builds to fail unnecessarily. This checks for this case
    and fixes it. (b/31939016)

    Args:
      returncode: Returncode from previous run of |command|
      stdout: Stdout from previous run of |command|
      stderr: Stderr from previous run of |command|
      command: Command to potentially re-run (as a list of strings)
      env: Extra environment variables to use

    Returns:
      (returncode, stdout, stderr) from |command|
    """
    if returncode != 0 and 'AllocationTestHarness.P' in stderr:
      logging.warning('Bad AllocationTestHarness.P file was detected'
                      ' (b/31939016). Cleaning and then re-trying the previous'
                      ' command.')
      clean_cmd = ['make', 'clean']
      self.exec_subprocess(clean_cmd)

      returncode, stdout, stderr = self.exec_subprocess(command, env=env)

    return returncode, stdout, stderr

  def run(self):
    """"Builds the build.ninja target, passing --check to `gn gen`.

    Returns:
      True iff there were no gn check errors.
    """
    # `make PRODUCT-$product-build.ninja` is essentally equivalent to
    # `lunch $product-eng && make build.ninja`. The build.ninja target
    # then invokes `gn gen --check`.
    command = [
        'make', '-j{}'.format(self.get_num_jobs()),
        'PRODUCT-' + self._product + '-build.ninja',
        'ENABLE_GN_CHECK=1',
        'DISABLE_AUTO_INSTALLCLEAN=true',
    ]

    if self._is_mesh_build:
      command += ['USE_GWIFI=1']

    command += self.build_accelerator.make_flags
    env = self.build_accelerator.environment_variables

    returncode, stdout, stderr = self.exec_subprocess(command, env=env)

    returncode, stdout, _ = self.rerun_if_necessary(returncode, stdout, stderr,
                                                    command, env)

    if returncode == 0:
      logging.info('gn check reported no issues.')
      return True

    self.gn_check_to_review(stdout)
    return False