| """Build step classes for building and running unit tests.""" |
| import logging |
| import os |
| import re |
| |
| import six |
| |
| from helpers import branch_utils |
| from helpers import git_utils |
| from helpers import gn_utils |
| from helpers import ninja_utils |
| from slave import base_step |
| from slave.step import cast_shell_step |
| from slave.step import sanitizer_unittest_step |
| from slave.step import shell_step |
| |
| _BUILD_CHROMECAST_SCRIPT = 'build/build_chromecast.py' |
| |
| RUN_TEST_LIST_PATH = 'tests/run_test_list.txt' |
| RUN_JUNIT_TEST_LIST_PATH = 'junit/run_junit_test_list.txt' |
| |
| STEP_DATA_TEST_LIST = 'step_data_test_list' |
| STEP_DATA_JUNIT_TEST_LIST = 'step_data_junit_test_list' |
| |
| ANDROID_TEST_RUNNER_PROJECT = 'test' |
| ANDROID_TEST_RUNNER_PATH = 'unit_test/atv_unit_test_runner.py' |
| TRANSFORM_XML_TO_JSON_PATH = 'unit_test/transform_xml_results_to_json.py' |
| |
| UNIT_TEST_REPORT_PATH = 'unit_test_results' |
| |
| # This is the maximum number of modified files a single change can have |
| # before we assume that we need to run all unit tests. This value exists |
| # because it takes time to calculate which tests need to be run for a set |
| # of files and for very large changes, like chromium merges, it's just better |
| # to assume we need to run all tests. |
| MAX_TARGETS_FOR_ALL_TESTS = 1000 |
| |
| # There are some files that GN does not know about that could affect the |
| # unit tests. If any of these files are touched, then build and run all the |
| # unit tests in order to be safe. |
| # A map of <project-name> -> <list of file regexes to match>. |
| FORCE_RUN_ALL_TESTS_WHITELIST = { |
| 'chromecast/internal': [r'build/.*', r'vendor/google3/prebuilts.json'], |
| 'vendor/amlogic': [r'google3_prebuilts.json'], |
| 'vendor/marvell': [r'google3_prebuilts.json'], |
| } |
| |
| |
| class GnExecutionError(RuntimeError): |
| """Exception raised when a gn command returns a non-zero return code.""" |
| |
| |
| class BuildAffectedUnitTestsStep(cast_shell_step.NonOtaBuildBaseStep): |
| """Build step class for building unit tests.""" |
| |
| def __init__(self, |
| sanitizer=None, |
| code_coverage=False, |
| absolute_coverage=False, |
| build_args_product=None, |
| build_args_official=None, |
| junit=False, |
| **kwargs): |
| """Instantiate the class. |
| |
| Args: |
| sanitizer: Clang saniziter to run with ('asan', 'msan', 'tsan', 'ubsan'). |
| code_coverage: True if code coverage is enabled for this build. |
| absolute_coverage: True if you want to run coverage for all unit tests. |
| build_args_product: The name of one of the |
| chromecast/internal/build/args/product files to use for the GN args. |
| build_args_official: Whether the build is an official build. MUST be set |
| for any build that will ship to users. Disables debugging; do not set |
| for development builds. |
| junit: If True, build junit tests. Otherwise, build gtests (Default False) |
| **kwargs: Any additional args to pass to BaseStep. |
| """ |
| if junit: |
| step_name = 'build affected junit tests' |
| else: |
| step_name = 'build affected gtest unit tests' |
| cast_shell_step.NonOtaBuildBaseStep.__init__( |
| self, |
| step_name, |
| build_args_product=build_args_product, |
| build_args_official=build_args_official, |
| **kwargs) |
| self._code_coverage = code_coverage |
| self._absolute_coverage = absolute_coverage |
| self._junit = junit |
| self._run_only_affected_tests = ( |
| self.get_property('build_system') != 'catabuilder' and |
| branch_utils.is_branch_equal_to_or_later_than(self.manifest_branch, |
| '1.27')) |
| self._changed_files = None |
| |
| def BuildUnitTests(self): |
| archive_dir = self.get_gcs_dir() |
| |
| build_script = os.path.join( |
| self.get_project_path('chromecast/internal'), _BUILD_CHROMECAST_SCRIPT) |
| command = [ |
| 'python3', build_script, '--archive_dir', archive_dir, |
| '--build_args_product', self._build_args_product, '--build_number', |
| self.build_number, '--chrome_root', |
| self.get_project_path('chromium/src') |
| ] |
| command += self.build_accelerator.build_chromecast_flags |
| |
| # TODO(b/73231972) remove unused flags once migrated to new builds |
| if self._build_args_official: |
| command += ['--official'] |
| |
| command += ['--tests_only'] |
| |
| if self.get_property('build_system') == 'catabuilder': |
| command += ['--enable_gn_check'] |
| |
| if self._junit: |
| command += ['--use_junit'] |
| |
| if self._code_coverage: |
| command += ['--enable_code_coverage'] |
| |
| if not self._must_run_all_tests(): |
| # Run `gn gen` to make sure the out folder exists before `gn refs` is run. |
| returncode, _, _ = self.exec_subprocess(command + ['--gn_gen_only']) |
| if returncode != 0: |
| return returncode |
| |
| # Determine which tests are affected and only build those tests. |
| affected_tests = self._get_affected_tests() |
| if affected_tests: |
| logging.info('Building %s tests affected by this change.', |
| len(affected_tests)) |
| affected_tests_arg = ','.join(sorted(affected_tests)) |
| command += ['--whitelisted_tests', affected_tests_arg] |
| else: |
| logging.info( |
| 'Affected test calculation returned no results, will build all ' |
| 'tests instead.') |
| |
| returncode, stdout, _ = self.exec_subprocess(command) |
| |
| if returncode != 0: |
| comments = ninja_utils.ninja_failure_to_comments(stdout, max_errors=3) |
| for comment in comments: |
| self.add_review({'message': comment}) |
| else: |
| returncode = self.ArchiveUnitTests() |
| |
| return returncode |
| |
| def _get_unit_test_out_dir(self): |
| """Generate build-specific out dir name. |
| |
| The out_dir needs to match the name of the build_args_product. |
| |
| E.g. out/atv_arm_eng for Android eng builds. |
| |
| Returns: |
| Path to UnittestStep out dir |
| """ |
| return os.path.join(['out', self._build_args_product]) |
| |
| def ArchiveUnitTests(self, dest_name='test_deps.zip'): # pylint: disable=method-hidden |
| """Saving build_tests_android.zip to gcs_dir. |
| |
| Args: |
| dest_name: desired archive name after upload. |
| |
| Returns: |
| 0 if succeed, returncode > 0 if failed |
| """ |
| # Skip archive step if no build_tests_android.zip expected |
| # build_tests_android.zip been generated by GetTestDepsTargets in |
| # chromecast/internal/build/build_chromecast.py |
| is_android_or_atv = any(product in self._build_args_product |
| for product in ['cube', 'soba', 'udon', 'atv']) |
| if not is_android_or_atv or self._junit: |
| return 0 |
| out_dir = self._get_unit_test_out_dir() |
| |
| archive_path = 'chromium/src/{}/{}/tests/build_tests_android.zip'.format( |
| out_dir) |
| |
| # Archive only if build_tests_android.zip was generated |
| if not os.path.isfile(archive_path): |
| return 0 |
| |
| archive_dir = os.path.join(self.get_gcs_dir(), 'artifacts/tests') |
| command = ['mkdir', '-p', archive_dir] |
| returncode, _, _ = self.exec_subprocess(command) |
| if not returncode: |
| command = ['cp', archive_path, os.path.join(archive_dir, dest_name)] |
| returncode, _, _ = self.exec_subprocess(command) |
| |
| return returncode |
| |
| def _get_changed_files(self): |
| if self._changed_files is None: |
| if self.patch_project: |
| all_patches = [{ |
| 'project': self.patch_project, |
| 'branch': self.get_property('patch_branch')}] + self.depends_on_list |
| else: |
| all_patches = self.depends_on_list |
| self._changed_files = git_utils.get_changed_files_in_all_patches( |
| self, all_patches, self.get_project_path_lookup_table()) |
| return self._changed_files |
| |
| def _must_run_all_tests(self): |
| """Returns if all the unittests must be ran based on the changed files.""" |
| |
| if self._absolute_coverage: |
| return True |
| |
| if not self._run_only_affected_tests: |
| return True |
| |
| for project, patterns in FORCE_RUN_ALL_TESTS_WHITELIST.items(): |
| for pattern in patterns: |
| full_pattern = os.path.join(self.get_project_path(project), pattern) |
| if any(re.search(full_pattern, f) for f in self._get_changed_files()): |
| return True |
| |
| # .gni files can get imported anywhere and cause deps, configs, etc. |
| # to change. It would be too burdensome to try to find the affected |
| # targets in this case, so we need to build/run all the tests. |
| if any(f.endswith('.gni') for f in self._get_changed_files()): |
| return True |
| |
| return False |
| |
| def _get_affected_tests(self): |
| """Return a list of the testing binaries affected by the changed files. |
| |
| Uses gn to determine which test binaries are affected. |
| |
| Returns: |
| A set of testing binaries affected by the changed files. |
| |
| Raises: |
| GnUtilsError: If the gn binary cannot be found. |
| GnExecutionError: If the gn command fails. |
| """ |
| |
| if not self._get_changed_files(): |
| return set() |
| |
| # Non-gn files can be passed to gn refs normally. If a CL modifies a |
| # BUILD.gn file, then instead pass a GN label that matches against |
| # all targets in that BUILD.gn file, otherwise GN will effectively |
| # ignore the file. (e.g. //foo/bar/BUILD.gn -> //foo/bar:*). |
| gn_refs_targets = [] |
| for f in self._get_changed_files(): |
| # Only care about the BUILD.gn files in chromium/src. Some projects |
| # outside chromium/src also contain BUILD.gn files, |
| # e.g. <eurekaroot>/external/gwifi/libchrome-cros, which should |
| # NOT be considered to affect unit tests |
| if f.endswith('BUILD.gn') and f.startswith('chromium/src'): |
| gn_refs_targets.append(gn_utils.filepath_to_gn_path(f)) |
| else: |
| gn_refs_targets.append(f) |
| |
| gn_binary = gn_utils.get_gn_binary(self) |
| |
| is_android_or_atv = any(product in self._build_args_product |
| for product in ['cube', 'soba', 'udon', 'atv']) |
| if is_android_or_atv: |
| return self._get_android_affected_tests(gn_binary) |
| |
| return self._get_linux_affected_tests(gn_binary, gn_refs_targets) |
| |
| def _get_linux_affected_tests(self, gn_binary, targets): |
| # If there's just way too many targets then just use all unit tests |
| # otherwise we could be waiting a very long time calculating that we |
| # do, in fact, need to run all the unit tests |
| if len(targets) > MAX_TARGETS_FOR_ALL_TESTS: |
| command = [ |
| gn_binary, 'refs', |
| '--root={}'.format(self.get_project_path('chromium/src')), |
| self._outdir, '--testonly=true', '--as=output', '--type=executable', |
| '--all', '-q', '*' |
| ] |
| else: |
| # The non-android test templates generate normal test executables, so |
| # look for those. |
| targets_str = '\n'.join(targets + ['']) |
| tmp_file_path = self.write_temp_file(targets_str) |
| |
| command = [ |
| gn_binary, 'refs', |
| '--root={}'.format(self.get_project_path('chromium/src')), |
| self._outdir, '--testonly=true', '--as=output', '--type=executable', |
| '--all', '-q', '@' + tmp_file_path |
| ] |
| |
| returncode, stdout, _ = self.exec_subprocess(command) |
| |
| if returncode != 0: |
| raise GnExecutionError( |
| 'Could not determine the tests affected by this change.') |
| |
| return set(test.strip() for test in stdout.splitlines() if test.strip()) |
| |
| def _get_android_affected_tests(self, gn_binary): |
| |
| # Determining affected tests for Android builds has been non-deterministic. |
| # (b/145934413, b/132997084, etc.) allowing breaking changes through |
| # presubmit. Until that is fixed, always run all test suites. |
| command = [ |
| gn_binary, 'refs', |
| '--root={}'.format(self.get_project_path('chromium/src')), self._outdir, |
| '--testonly=true', '--as=label', |
| '--type={}'.format('group' if self._junit else 'action'), '--all', '-q', |
| '*' |
| ] |
| |
| returncode, stdout, _ = self.exec_subprocess(command) |
| |
| if returncode != 0: |
| raise GnExecutionError( |
| 'Could not determine the tests affected by this change.') |
| |
| # Convert each label returned in |stdout| to a GN target name. Some of the |
| # labels are legitimate test targets, but many are intermediate targets |
| # which are internal details of GN templates. Do some basic filtering to |
| # cut the list down. The rest will be filtered by the cast_test_list. |
| return set( |
| gn_utils.gn_label_to_target_name(label.strip()) |
| for label in stdout.splitlines() |
| if label.strip() and '__' not in label) |
| |
| def run(self): |
| """Builds unittests. |
| |
| Returns: |
| True iff there were no errors. |
| """ |
| returncode = self.BuildUnitTests() |
| |
| if returncode != 0: |
| return False |
| |
| run_test_list = ( |
| RUN_JUNIT_TEST_LIST_PATH if self._junit else RUN_TEST_LIST_PATH) |
| step_data_key = ( |
| STEP_DATA_JUNIT_TEST_LIST if self._junit else STEP_DATA_TEST_LIST) |
| |
| # Load the tests to be run. |
| test_list = self.exec_read_file(os.path.join(self._outdir, |
| run_test_list)).splitlines() |
| |
| # Load the tests affected by this set of patches. |
| try: |
| affected_tests = self._get_affected_tests() |
| except gn_utils.GnUtilsError as e: |
| self.add_review({'message': str(e)}) |
| return False |
| except RuntimeError as e: |
| self.add_review({'message': str(e)}) |
| return False |
| |
| if not self._must_run_all_tests() and affected_tests: |
| # Filter the test list for tests that are affected by the changed files |
| # and save this in step data. |
| test_list = [ |
| test_cmd for test_cmd in test_list |
| if test_cmd.split()[0] in affected_tests |
| ] |
| self.add_step_data(step_data_key, test_list) |
| |
| return True |
| |
| |
| class RunAffectedUnitTestsStep(cast_shell_step.NonOtaBuildBaseStep): |
| """Build step class for running x86 unit tests, if any.""" |
| |
| def __init__(self, |
| report_url, |
| recipe_steps, |
| sanitizer=None, |
| build_args_product=None, |
| build_args_official=None, |
| junit=False, |
| enable_network_service=False, |
| **kwargs): |
| """Instantiate the class. |
| |
| Args: |
| report_url: The URL the code coverage report can be accessed at. |
| recipe_steps: Reference to the list of steps for this recipe. |
| sanitizer: Clang saniziter to run with ('asan', 'msan', 'tsan', 'ubsan'). |
| build_args_product: The name of one of the |
| chromecast/internal/build/args/product files to use for the GN args. |
| junit: If true, run junit tests, otherwise run gtests (Default False). |
| enable_network_service: If true, include --enable-features=NetworkService |
| **kwargs: Any additional args to pass to BaseStep. |
| """ |
| if junit: |
| step_name = 'run affected junit tests' |
| else: |
| step_name = 'run affected gtest unit tests' |
| cast_shell_step.NonOtaBuildBaseStep.__init__( |
| self, |
| step_name, |
| build_args_product=build_args_product, |
| build_args_official=build_args_official, |
| **kwargs) |
| |
| assert isinstance(report_url, six.string_types) |
| assert isinstance(recipe_steps, list) |
| assert build_args_product |
| |
| self._report_url = report_url |
| self._recipe_steps = recipe_steps |
| self._sanitizer = sanitizer |
| self._junit = junit |
| self._enable_network_service = enable_network_service |
| self._step_kwargs = kwargs |
| |
| # Index to insert new steps at. This is to make sure that different steps that |
| # are included after the RunAffectedUnitTestsStep actually run after it, |
| # rather than the new steps being appended after them. |
| self._insert_index = None |
| |
| def get_report_dir(self): |
| """Returns a unit test result directory.""" |
| report_dir = os.path.join(self.get_gcs_dir(), UNIT_TEST_REPORT_PATH) |
| if not os.path.isdir(report_dir): |
| os.mkdir(report_dir) |
| return report_dir |
| |
| def get_gtest_xml_log_uri(self, test_name): |
| """Returns a unit test xml log uri for the given gtest.""" |
| return '{}/test_detail_{}.xml'.format(self.get_report_dir(), test_name) |
| |
| def get_gtest_step(self, test_name, test_command): |
| """Returns a step that runs the unit test executable.""" |
| if self._sanitizer: |
| return sanitizer_unittest_step.SanitizerUnitTestShellStep( |
| name=test_name, |
| command=test_command, |
| directory=self._outdir, |
| sanitizer=self._sanitizer, |
| **self._step_kwargs) |
| |
| return shell_step.ShellStep( |
| name=test_name, |
| command=test_command, |
| directory=self._outdir, |
| **self._step_kwargs) |
| |
| def get_junit_step(self, test_name): |
| """Returns a step that runs a junit test.""" |
| return self.get_android_runner_step(test_name, 'junit') |
| |
| def get_android_runner_step(self, test_name, test_type): |
| """Returns a step that runs an android test.""" |
| test_runner = self._get_test_runner() |
| pythonpath = self._get_python_path() |
| # Single device, should be single test. |
| wrapper_command = [ |
| 'python3', |
| test_runner, |
| '--debug', # Dumps test output to stdout. |
| '--type', |
| test_type, |
| '--test_result_dir', |
| self.get_gcs_dir(), |
| '--unit_test_binary_dir', |
| self._outdir, |
| '--unittest_name', |
| test_name |
| ] |
| return shell_step.ShellStep( |
| name=test_name, |
| command=wrapper_command, |
| directory='', |
| env={'PYTHONPATH': pythonpath}, |
| **self._step_kwargs) |
| |
| def get_json_step(self, test_name): |
| """Returns a step that generates test_result.json from existing xml.""" |
| report_dir = self.get_report_dir() |
| test_project_dir = self.get_project_path(ANDROID_TEST_RUNNER_PROJECT) |
| xml_to_json = os.path.join(test_project_dir, TRANSFORM_XML_TO_JSON_PATH) |
| wrapper_command = [ |
| 'python3', xml_to_json, '--xml_dir_path', report_dir, '--branch_name', |
| self.manifest_branch, '--build_number', self.build_number, |
| '--test_name', test_name |
| ] |
| wrapper_command += [ |
| '--build_args_product', |
| self._build_args_product, |
| ] |
| return shell_step.ShellStep( |
| name='summary json for {}'.format(test_name), |
| command=wrapper_command, |
| directory='', |
| env={'PYTHONPATH': test_project_dir}, |
| **self._step_kwargs) |
| |
| def _add_step_to_recipe_steps(self, step): |
| if self._insert_index is None: |
| self._insert_index = self._recipe_steps.index(self) + 1 |
| |
| self._recipe_steps.insert(self._insert_index, step) |
| self._insert_index += 1 |
| |
| def add_gtest_tests(self): |
| test_list = self.get_step_data(STEP_DATA_TEST_LIST) |
| |
| for test_exec_text in test_list: |
| test_command = test_exec_text.split(' ') |
| test_name = test_command[0] |
| test_command[0] = './' + test_name |
| if (self._enable_network_service and |
| branch_utils.is_branch_equal_to_or_later_than(self.manifest_branch, |
| '1.41')): |
| test_command.append( |
| '--enable-features=NetworkService,NetworkServiceInProcess') |
| # TODO(b/130182642): Re-enable these tests once Chromium supports |
| # network service in unit tests. |
| disabled_tests = (':CastDevToolsManagerDelegateTest.*' |
| ':CastMediaBlockerTest.*' |
| ':CastSessionIdMapTest.*') |
| for i, part in enumerate(test_command): |
| if part.startswith('--gtest_filter=-'): |
| test_command[i] = part + disabled_tests |
| break |
| if i == len(test_command) - 1: |
| test_command.append('--gtest_filter=-' + disabled_tests) |
| |
| test_command.append('--gtest_output=xml:{}'.format( |
| self.get_gtest_xml_log_uri(test_name))) |
| self._add_step_to_recipe_steps( |
| self.get_gtest_step(test_name, test_command)) |
| |
| self._add_step_to_recipe_steps(self.get_json_step('unit_test')) |
| |
| def add_junit_tests(self): |
| test_list = self.get_step_data(STEP_DATA_JUNIT_TEST_LIST) |
| for test_exec_text in test_list: |
| test_name = test_exec_text.split(' ')[0] |
| self._add_step_to_recipe_steps(self.get_junit_step(test_name)) |
| |
| def _get_test_runner(self): |
| test_project_dir = self.get_project_path(ANDROID_TEST_RUNNER_PROJECT) |
| return os.path.join(test_project_dir, ANDROID_TEST_RUNNER_PATH) |
| |
| def _get_python_path(self): |
| test_project_dir = self.get_project_path(ANDROID_TEST_RUNNER_PROJECT) |
| return ':'.join( |
| [test_project_dir, |
| self.get_project_path('continuous-tests')]) |
| |
| def run(self): |
| """Adds a separate step for each Chromium test case to be executed. |
| |
| Returns: |
| True, it shouldn't be possible for this step to fail. |
| """ |
| |
| original_step_count = len(self._recipe_steps) |
| if self._junit: |
| self.add_junit_tests() |
| else: |
| self.add_gtest_tests() |
| new_steps_count = len(self._recipe_steps) - original_step_count |
| if new_steps_count: |
| logging.info('Running %s tests affecting this change.', new_steps_count) |
| else: |
| logging.info('No affected tests were detected for this change. None ' |
| 'will be run.') |
| |
| return True |
