cq/scripts/helpers/rbe_utils.py

""" Utility functions for working with re-client tooling.

Functions for starting and stopping reproxy.
"""

from __future__ import absolute_import
import os

RBE_SERVICE='remotebuildexecution.googleapis.com:443'
DEFAULT_INSTANCE = 'projects/nest-rbe-prod/instances/us-central'
RBE_REPROXY_ADDR = 'unix:///tmp/reproxy.sock'

NEW_RBE_REL_PATH = 'chromium/src/buildtools/reclient'
OLD_RBE_REL_PATH = 'chromium/src/buildtools/rbe'


def _rbe_path():
  """Path to RBE binaries relative to chromium/src

  The path was moved from rbe -> reclient, but some old branches still use
  the old path, so test the new path and fall back to the old path if needed.
  """
  if os.path.exists(NEW_RBE_REL_PATH):
    return NEW_RBE_REL_PATH
  return OLD_RBE_REL_PATH

def _depscan_path():
  return os.path.join(_rbe_path(), 'dependency_scanner_go_plugin.so')

def _rbe_start_command():
  """Command to start the remote execution proxy (reproxy)."""
  return [os.path.join(_rbe_path(), 'bootstrap'),
          '-re_proxy=' + os.path.join(_rbe_path(), 'reproxy')]

def _rbe_stop_command():
  """Command to stop the remote execution proxy (reproxy, via bootstrap)."""
  return [os.path.join(_rbe_path(), 'bootstrap'),
          '--shutdown']

def is_installed():
  """Returns True if the rbe bootstrap client is available."""
  return os.path.exists(os.path.join(_rbe_path(), 'bootstrap'))


def start_reproxy(executor):
  """Runs 'reproxy <args>' to start the reproxy and connect to the rbe backend.

  This is run in a non-blocking fashion as the reproxy should be running for
  the duration of the receipe.

  Args:
    executor: Executes the subprocess.

  Returns:
    (returncode, stdout, stderr)
  """
  return executor.exec_subprocess(_rbe_start_command(),
                                  env=_get_rbe_env(executor))


def stop_reproxy(executor):
  """Runs 'bootstrap --shutdown' which stops the reproxy process.

  Args:
    executor: Executes the subprocess.

  Returns:
    (returncode, stdout, stderr)
  """
  return executor.exec_subprocess(_rbe_stop_command(),
                                  env=_get_rbe_env(executor))


def _get_rbe_env(executor):
  """Returns the dict of environment vars to pass to bootstrap/reproxy.

  Args:
    tmp_dir: Temporary directory to place logs.

  Returns:
    dict of environment variables.
  """
  # Set flags to reproxy via env variables.
  tmp_dir = executor.get_tmp_dir()
  log_dir = executor.get_gcs_dir()
  env_values = {
      'RBE_use_gce_credentials': 'true',
      'RBE_service': RBE_SERVICE,
      'RBE_instance': executor.get_property('rbe_instance', DEFAULT_INSTANCE),
      'RBE_server_address': RBE_REPROXY_ADDR,
      'RBE_clean_include_paths': 'true',
      'RBE_log_dir': log_dir,
      # The files generated by the proxy are large and not very useful.
      'RBE_proxy_log_dir': tmp_dir,
      'RBE_output_dir': log_dir,
      # Set the clang plugins to ignore during deps scanning.
      'RBE_clang_depscan_ignored_plugins': 
        'find-bad-constructs,blink-gc-plugin',
  }
  # The dependency_scanner_go_plugin.so is removed in newer versions of RBE
  # But older branches still need it. Include only if needed.
  if os.path.exists(_depscan_path()):
    env_values['RBE_cpp_dependency_scanner_plugin'] = _depscan_path()
  return env_values