cq/scripts/slave/step/code_coverage_step.py - manifest_repos/continuous-tests - Git at Google

 """Build step class for generating code coverage reports."""
 import fileinput
 import logging
 import os
 import pathlib
 import shutil
 import time
 from slave import base_step
 from slave.step import cast_shell_step
 from cq.scripts.helpers import coverage_utils

 COVERAGE_SCRIPT_PATH = 'tools/code_coverage/coverage.py'
 COVERAGE_REPORT_PATH = 'code_coverage_html'
 COVERAGE_LCOV_PATH = f'{COVERAGE_REPORT_PATH}/linux/coverage.lcov'
 STEP_DATA_TEST_LIST = 'step_data_test_list'

 # This is a list of test suites to filter out of the coverage
 # run.  Tests are added to this list of they crash, or otherwise disrupt the
 # generation of code coverage data.
 TESTS_TO_FILTER_OUT = [
     'bindings_browsertests',  # b/178190494
     'cast_shell_browsertests',  # b/178190494
     'cast_shell_internal_browsertests',  # b/178190494
 ]

 # TODO(b/232528731): Can potentially try to find where this is defined
 #                    dynamically to avoid hardcoding it. If possible,
 #                    use a configuration variable instead.
 PREFIX_FILTER = '/b/f/w'


 class RunCoverageCommandStep(cast_shell_step.NonOtaBuildBaseStep):
   """Build step class generating code coverage data."""

   def __init__(self, branding, build_type, product, build_flavor,
                sanitizer=None, build_args_product=None, junit=False,
                ignore_filename_regex=None, report_format=None,
                build_out_dir=None, test_env=None, **kwargs):
     """Creates a RunCoverageCommandStep instance.

     Args:
       branding: Build branding (chromium or chrome).
       build_type: Type of system to build for ('x86', 'clang', or 'arm').
       product: product to build for (ie. 'chromecast', 'audio').
       build_flavor: Flavor of build ('Debug', 'Eng', 'Release').
       sanitizer: Clang sanitizer to run with ('asan', 'msan', 'tsan', 'ubsan').
       build_args_product: The name of one of the
         chromecast/internal/build/args/product files to use for the GN args.
       junit: If true, run junit tests, otherwise run gtests (Default False).
       ignore_filename_regex: Skip source code files with file paths that match
         the given regular expression.
       report_format: Output format of the "llvm-cov show/export" command
         ('html', 'text', 'lcov').
       build_out_dir: Build output directory.
       test_env: Dictionary of env variables to set while executing tests.
       **kwargs: Any additional args to pass to BaseStep.
     """
     cast_shell_step.NonOtaBuildBaseStep.__init__(
         self, 'generate coverage data', branding, build_type,
         product, build_flavor, sanitizer=sanitizer, timeout_secs=0,
         build_args_product=build_args_product, **kwargs)

     self._junit = junit
     self._ignore_filename_regex = ignore_filename_regex
     self._format = report_format
     self._build_out_dir = build_out_dir
     self._test_env = test_env

   def _get_build_out_dir(self, use_flavor=False):
     if self._build_out_dir:
       return self._build_out_dir

     out_dir = ['out', self.build_type_dir, self.build_product_dir, 'gn']
     if self._sanitizer:
       out_dir.append(self._sanitizer)

     if use_flavor:
       return os.path.join('_'.join(out_dir), self.build_flavor_dir)

     return '_'.join(out_dir)

   def _generate_coverage_args(self, test_list):
     """Generates unit test related arguments for the coverage script.

        The coverage script takes in a list of unit test binaries as
        an argument, and a corresponding test command for that binary.
        This function reads from a list of test commands and constructs
        a list of binaries and the command for that binary to be parsed
        by coverage.

     Args:
       test_list: A list where each element represents a test command.

     Returns:
       A list of binaries and a list of commands arguments.
     """
     test_binary_list = []
     test_command_list = []

     for test_binary_command in test_list:
       test_command = test_binary_command.strip().split(' ')
       test_name = test_command[0]
       if test_name in TESTS_TO_FILTER_OUT:
         # Skip filtered tests
         continue
       test_binary_list.append(test_name)
       test_command_list.append('--command')
       test_command_list.append(
           self._construct_formatted_test_command(test_command))

     return test_binary_list, test_command_list

   def _construct_formatted_test_command(self, test_command):
     """Formats a test command to be parsed by the coverage script.

     Args:
       test_command: A list representation of a test command.

     Returns:
       String argument representation of test command for the coverage script.
     """
     test_path = os.path.join(self._get_build_out_dir(use_flavor=True),
                              test_command[0])
     return '{} {}'.format(test_path, ' '.join(test_command[1:]))

   def _get_test_list(self):
     return self.get_step_data(STEP_DATA_TEST_LIST)

   def get_lcov_output_file_paths(self):
     return [pathlib.Path(self.get_gcs_dir()) / COVERAGE_LCOV_PATH]

   def run(self):
     """Runs the chromium coverage script and generates a report.

     Returns:
       True iff there were no errors.
     """
     # JUnit tests are not supported.
     if self._junit:
       return True
     else:
       test_list = self._get_test_list()

     test_binary_list, test_command_list = self._generate_coverage_args(
         test_list)

     coverage_script_path = os.path.join(self.get_project_path('chromium/src'),
                                         COVERAGE_SCRIPT_PATH)
     vpython3_path = os.path.join(
         self.get_project_path('chromium/tools/depot_tools'), 'vpython3')
     report_path = os.path.join(self.get_gcs_dir(), COVERAGE_REPORT_PATH)

     command = [vpython3_path, coverage_script_path]
     command += test_binary_list
     command += [
       '-o', report_path, '-b', self._get_build_out_dir(use_flavor=True), '-v']
     if self._format:
       command += ['--format', self._format]
     if self._ignore_filename_regex:
       command += ['--ignore-filename-regex', self._ignore_filename_regex]
     command += test_command_list

     # This is necessary because generating the HTML for code coverage can
     # take 3+ hours, in which there is no stdout printed at all. Since a
     # build is killed if there is no stdout for 2 hours, this block outputs
     # every 1 minute to keep the build alive.
     # TODO(kevinmk): Work out a way to have the coverage command output
     # something meaningful during the HTML generation step.
     subprocess, log_fd, log_file_path = self.exec_non_blocking_subprocess(
         command, env=self._test_env)

     logging.info('Writing to FD: %d and path %s', log_fd, log_file_path)

     while subprocess.poll() is None:
       logging.info('[%s] Waiting for completion...',
                    time.strftime('%Y-%m-%d %H:%M:%S %Z'))
       try:
         out_files = os.listdir(report_path)
         logging.info('Generated Files: %s', str(out_files))
       except FileNotFoundError:
         pass  # Ignore the error if the folder hasn't been created yet.
       time.sleep(60)

     try:
       shutil.copyfile(log_file_path, os.path.join(self.get_gcs_dir(),
                                                   'coverage.py.log'))
       with open(log_file_path, 'r') as f:
         print(f.read())
     except IOError:
       logging.warning('Log file does not exist.')

     return True


 class CollectLcovCoverageStep(base_step.BaseStep):
   """Collects and alters the LCOV files for uploading as artifacts."""

   def __init__(self, lcov_files, eureka_root, **kwargs):
     super().__init__('Collect LCOV Coverage', eureka_root=eureka_root, **kwargs)
     self._lcov_files = [pathlib.Path(f) for f in lcov_files]
     self._workspace_root = pathlib.Path(eureka_root)

   def run(self):
     gcs_dir = pathlib.Path(self.get_gcs_dir())
     artifact_directory = gcs_dir / 'coverage'
     artifact_directory.mkdir(exist_ok=True)
     lcov_output_property = []
     for lcov_file in self._lcov_files:
       output_filename = _transform_file_path(lcov_file)
       output_path = artifact_directory / output_filename
       logging.info('Copying %s to %s', lcov_file, output_path)
       coverage_utils.rewrite_file_paths_in_lcov(
           lcov_file, output_path, self._workspace_root)
       lcov_output_property.append(str(output_path.relative_to(gcs_dir)))
     if lcov_output_property:
       self.set_build_property('coverage_lcov_files', lcov_output_property)
     return True


 def _transform_file_path(file_path):
   """Constructs a unique filename from path."""
   return '_'.join(file_path.resolve().relative_to('/').parts)
	"""Build step class for generating code coverage reports."""
	import fileinput
	import logging
	import os
	import pathlib
	import shutil
	import time
	from slave import base_step
	from slave.step import cast_shell_step
	from cq.scripts.helpers import coverage_utils

	COVERAGE_SCRIPT_PATH = 'tools/code_coverage/coverage.py'
	COVERAGE_REPORT_PATH = 'code_coverage_html'
	COVERAGE_LCOV_PATH = f'{COVERAGE_REPORT_PATH}/linux/coverage.lcov'
	STEP_DATA_TEST_LIST = 'step_data_test_list'

	# This is a list of test suites to filter out of the coverage
	# run. Tests are added to this list of they crash, or otherwise disrupt the
	# generation of code coverage data.
	TESTS_TO_FILTER_OUT = [
	'bindings_browsertests', # b/178190494
	'cast_shell_browsertests', # b/178190494
	'cast_shell_internal_browsertests', # b/178190494
	]

	# TODO(b/232528731): Can potentially try to find where this is defined
	# dynamically to avoid hardcoding it. If possible,
	# use a configuration variable instead.
	PREFIX_FILTER = '/b/f/w'


	class RunCoverageCommandStep(cast_shell_step.NonOtaBuildBaseStep):
	"""Build step class generating code coverage data."""

	def __init__(self, branding, build_type, product, build_flavor,
	sanitizer=None, build_args_product=None, junit=False,
	ignore_filename_regex=None, report_format=None,
	build_out_dir=None, test_env=None, **kwargs):
	"""Creates a RunCoverageCommandStep instance.

	Args:
	branding: Build branding (chromium or chrome).
	build_type: Type of system to build for ('x86', 'clang', or 'arm').
	product: product to build for (ie. 'chromecast', 'audio').
	build_flavor: Flavor of build ('Debug', 'Eng', 'Release').
	sanitizer: Clang sanitizer to run with ('asan', 'msan', 'tsan', 'ubsan').
	build_args_product: The name of one of the
	chromecast/internal/build/args/product files to use for the GN args.
	junit: If true, run junit tests, otherwise run gtests (Default False).
	ignore_filename_regex: Skip source code files with file paths that match
	the given regular expression.
	report_format: Output format of the "llvm-cov show/export" command
	('html', 'text', 'lcov').
	build_out_dir: Build output directory.
	test_env: Dictionary of env variables to set while executing tests.
	**kwargs: Any additional args to pass to BaseStep.
	"""
	cast_shell_step.NonOtaBuildBaseStep.__init__(
	self, 'generate coverage data', branding, build_type,
	product, build_flavor, sanitizer=sanitizer, timeout_secs=0,
	build_args_product=build_args_product, **kwargs)

	self._junit = junit
	self._ignore_filename_regex = ignore_filename_regex
	self._format = report_format
	self._build_out_dir = build_out_dir
	self._test_env = test_env

	def _get_build_out_dir(self, use_flavor=False):
	if self._build_out_dir:
	return self._build_out_dir

	out_dir = ['out', self.build_type_dir, self.build_product_dir, 'gn']
	if self._sanitizer:
	out_dir.append(self._sanitizer)

	if use_flavor:
	return os.path.join('_'.join(out_dir), self.build_flavor_dir)

	return '_'.join(out_dir)

	def _generate_coverage_args(self, test_list):
	"""Generates unit test related arguments for the coverage script.

	The coverage script takes in a list of unit test binaries as
	an argument, and a corresponding test command for that binary.
	This function reads from a list of test commands and constructs
	a list of binaries and the command for that binary to be parsed
	by coverage.

	Args:
	test_list: A list where each element represents a test command.

	Returns:
	A list of binaries and a list of commands arguments.
	"""
	test_binary_list = []
	test_command_list = []

	for test_binary_command in test_list:
	test_command = test_binary_command.strip().split(' ')
	test_name = test_command[0]
	if test_name in TESTS_TO_FILTER_OUT:
	# Skip filtered tests
	continue
	test_binary_list.append(test_name)
	test_command_list.append('--command')
	test_command_list.append(
	self._construct_formatted_test_command(test_command))

	return test_binary_list, test_command_list

	def _construct_formatted_test_command(self, test_command):
	"""Formats a test command to be parsed by the coverage script.

	Args:
	test_command: A list representation of a test command.

	Returns:
	String argument representation of test command for the coverage script.
	"""
	test_path = os.path.join(self._get_build_out_dir(use_flavor=True),
	test_command[0])
	return '{} {}'.format(test_path, ' '.join(test_command[1:]))

	def _get_test_list(self):
	return self.get_step_data(STEP_DATA_TEST_LIST)

	def get_lcov_output_file_paths(self):
	return [pathlib.Path(self.get_gcs_dir()) / COVERAGE_LCOV_PATH]

	def run(self):
	"""Runs the chromium coverage script and generates a report.

	Returns:
	True iff there were no errors.
	"""
	# JUnit tests are not supported.
	if self._junit:
	return True
	else:
	test_list = self._get_test_list()

	test_binary_list, test_command_list = self._generate_coverage_args(
	test_list)

	coverage_script_path = os.path.join(self.get_project_path('chromium/src'),
	COVERAGE_SCRIPT_PATH)
	vpython3_path = os.path.join(
	self.get_project_path('chromium/tools/depot_tools'), 'vpython3')
	report_path = os.path.join(self.get_gcs_dir(), COVERAGE_REPORT_PATH)

	command = [vpython3_path, coverage_script_path]
	command += test_binary_list
	command += [
	'-o', report_path, '-b', self._get_build_out_dir(use_flavor=True), '-v']
	if self._format:
	command += ['--format', self._format]
	if self._ignore_filename_regex:
	command += ['--ignore-filename-regex', self._ignore_filename_regex]
	command += test_command_list

	# This is necessary because generating the HTML for code coverage can
	# take 3+ hours, in which there is no stdout printed at all. Since a
	# build is killed if there is no stdout for 2 hours, this block outputs
	# every 1 minute to keep the build alive.
	# TODO(kevinmk): Work out a way to have the coverage command output
	# something meaningful during the HTML generation step.
	subprocess, log_fd, log_file_path = self.exec_non_blocking_subprocess(
	command, env=self._test_env)

	logging.info('Writing to FD: %d and path %s', log_fd, log_file_path)

	while subprocess.poll() is None:
	logging.info('[%s] Waiting for completion...',
	time.strftime('%Y-%m-%d %H:%M:%S %Z'))
	try:
	out_files = os.listdir(report_path)
	logging.info('Generated Files: %s', str(out_files))
	except FileNotFoundError:
	pass # Ignore the error if the folder hasn't been created yet.
	time.sleep(60)

	try:
	shutil.copyfile(log_file_path, os.path.join(self.get_gcs_dir(),
	'coverage.py.log'))
	with open(log_file_path, 'r') as f:
	print(f.read())
	except IOError:
	logging.warning('Log file does not exist.')

	return True


	class CollectLcovCoverageStep(base_step.BaseStep):
	"""Collects and alters the LCOV files for uploading as artifacts."""

	def __init__(self, lcov_files, eureka_root, **kwargs):
	super().__init__('Collect LCOV Coverage', eureka_root=eureka_root, **kwargs)
	self._lcov_files = [pathlib.Path(f) for f in lcov_files]
	self._workspace_root = pathlib.Path(eureka_root)

	def run(self):
	gcs_dir = pathlib.Path(self.get_gcs_dir())
	artifact_directory = gcs_dir / 'coverage'
	artifact_directory.mkdir(exist_ok=True)
	lcov_output_property = []
	for lcov_file in self._lcov_files:
	output_filename = _transform_file_path(lcov_file)
	output_path = artifact_directory / output_filename
	logging.info('Copying %s to %s', lcov_file, output_path)
	coverage_utils.rewrite_file_paths_in_lcov(
	lcov_file, output_path, self._workspace_root)
	lcov_output_property.append(str(output_path.relative_to(gcs_dir)))
	if lcov_output_property:
	self.set_build_property('coverage_lcov_files', lcov_output_property)
	return True


	def _transform_file_path(file_path):
	"""Constructs a unique filename from path."""
	return '_'.join(file_path.resolve().relative_to('/').parts)