| from __future__ import absolute_import |
| import os |
| import re |
| import collections |
| |
| from slave import base_step |
| from slave.step import code_coverage_step |
| |
| BUILD_VARIANT = 'eng' |
| |
| TARGET_PRODUCT = { |
| 'gq': 'gq_x86_64', |
| 'newman': 'newman_x86_64', |
| 'nq': 'nq_x86_64', |
| 'nq_exp': 'nq_exp_x86_64', |
| 'spencer': 'spencer_x86_64', |
| 'venus': 'venus_x86_64', |
| } |
| |
| BUILD_ITEM = { |
| 'gq': ['curl', 'nldaemon', 'nldaemon-cli', 'chromium-iot'], |
| 'newman': 'reconnect-tests', |
| 'nq': ['curl', 'nldaemon', 'nldaemon-cli', 'chromium-iot'], |
| 'nq_exp': ['curl', 'nldaemon', 'nldaemon-cli', 'chromium-iot', |
| 'libcast_camera_hal', 'libamvenc_api_c328', 'libdma_api_c328', |
| 'libcutils'], |
| 'spencer': ['curl', 'nldaemon', 'nldaemon-cli', 'chromium-iot'], |
| 'venus': ['curl', 'nldaemon', 'nldaemon-cli', 'chromium-iot'], |
| 'iot-tests': 'chromium-iot-tests' |
| } |
| |
| _MAKE_ERROR_PATTERN = ( |
| r'^(?P<filename>[^:]+):(?P<linenum>\d+):(?P<column>\d+): ' |
| 'error: (?P<err_msg>.*)$', |
| r'^(?P<filename>[^\-]+)-(?P<linenum>\d+): error: (?P<err_msg>.*)$', |
| ) |
| |
| IOT_TESTS_LIST = 'tests/iot/run_test_list.txt' |
| |
| OUT_DIR = 'out/target/product/newman_x86_64/system/bin' |
| NCCM_OUT_DIR = 'chromium/src/out/Release/' |
| NCCM_BUILD_ITEM = os.path.join("chromecast/internal/iot_services", |
| "nestcam_context_manager:nccm_unittests") |
| |
| PHOTON_OUT_DIR = 'chromium/src/out/Release/' |
| PHOTON_BUILD_ITEMS = [ os.path.join("chromecast/internal/iot_services", |
| "photon_hal:photon_hal_group"), |
| os.path.join("chromecast/internal/iot_services", |
| "photon_hal:photon_hal_tests") ] |
| |
| SIM_SYSTEM = [ |
| "out/target/product/{}/system/bin/", |
| "out/target/product/{}/system/etc/", |
| "out/target/product/{}/system/lib/", |
| "out/target/product/{}/symbols/system/chrome/bin/", |
| "out/target/product/{}/symbols/system/chrome/lib/", |
| ] |
| SIM_LIB = [ |
| "chromium/src/build/linux/debian_sid_amd64-sysroot/usr/lib/x86_64-linux-gnu/", |
| os.path.join("chromium/src/build/linux/debian_sid_amd64-sysroot/lib", |
| "x86_64-linux-gnu/libcom_err.so.2.1"), |
| os.path.join("chromium/src/build/linux/debian_sid_amd64-sysroot/lib", |
| "x86_64-linux-gnu/libkeyutils.so.1.9"), |
| os.path.join("chromium/src/third_party/android_ndk/toolchains/llvm/prebuilt", |
| "linux-x86_64/lib64/libc++.so.1"), |
| ] |
| SIM_CONFIG = "virtual-home/cirque-park/build_eureka/{}/" |
| MOJO_EVENTS = "virtual-home/cirque-park/build_eureka/mojo_events/" |
| OPENWEAVE = "virtual-home/cirque-park/openweave/" |
| SAMPLE_VIDEO = "vendor/nest/reconnect/data/test/proxy_video.h264" |
| SAMPLE_AUDIO = "vendor/nest/reconnect/data/test/proxy_audio_mono.opus" |
| SAMPLE_JPEG = "vendor/nest/reconnect/data/test/{}_sample.jpeg" |
| |
| CODE_COVERAGE_GN_ARGS = 'use_clang_coverage=true is_component_build=false' |
| |
| def make_error_to_comments(stderr, directory): |
| """ |
| parse stderr with regex error pattern and return comments dict |
| Args: |
| stderr (str): logs from stderr |
| Return: |
| (dict) comments: { |
| "filename": |
| { "linenum": <int>, |
| "message": <str> error message, |
| } |
| } |
| """ |
| comments = collections.defaultdict(list) |
| |
| for line in stderr.splitlines(): |
| match = [m for m in [ |
| re.match(err_pat, line) for err_pat in _MAKE_ERROR_PATTERN |
| ] if m is not None] |
| if match: |
| match = match[0] |
| filename = match.group('filename') |
| if directory and filename.startswith(directory): |
| filename = filename[len(directory) + len(os.path.sep):] |
| linenum = int(match.group('linenum')) |
| comments[filename].append({'line': linenum, 'message': line}) |
| return comments |
| |
| |
| class MakeCleanSim(base_step.BaseStep): |
| |
| def __init__(self, **kwargs): |
| self.product = kwargs['product'] |
| base_step.BaseStep.__init__(self, |
| name='make clean x86_64 {}'.format( |
| self.product.upper()), |
| halt_on_failure=True, |
| **kwargs) |
| |
| def run(self): |
| |
| command = [ |
| 'make', |
| 'TARGET_PRODUCT={}'.format(TARGET_PRODUCT[self.product]), |
| 'TARGET_BUILD_VARIANT={}'.format(BUILD_VARIANT), |
| 'clean' |
| ] |
| returncode, _, stderr = self.exec_subprocess(command) |
| if returncode != 0: |
| comment = make_error_to_comments(stderr, '') |
| self.add_review({ |
| 'comments': comment |
| }) |
| return False |
| |
| return True |
| |
| |
| class BuildSimulatorStep(base_step.BaseStep): |
| |
| def __init__(self, code_coverage=False, **kwargs): |
| self.product = kwargs['product'] |
| self._code_coverage = code_coverage |
| base_step.BaseStep.__init__(self, |
| name='build x86_64 {}'.format( |
| self.product.upper()), |
| halt_on_failure=True, |
| **kwargs) |
| |
| def run(self): |
| command = [ |
| 'make', |
| '-j{}'.format(self.get_num_jobs()), |
| 'TARGET_PRODUCT={}'.format(TARGET_PRODUCT[self.product]), |
| 'TARGET_BUILD_VARIANT={}'.format(BUILD_VARIANT) |
| ] |
| |
| if self._code_coverage: |
| command.append( |
| 'COMMAND_LINE_GN_ARGS=\'{}\''.format(CODE_COVERAGE_GN_ARGS)) |
| |
| command += self.build_accelerator.make_flags |
| command += BUILD_ITEM[self.product] |
| returncode, _, stderr = self.exec_subprocess(command) |
| if returncode != 0: |
| comment = make_error_to_comments(stderr, '') |
| self.add_review({ |
| 'comments': comment |
| }) |
| return False |
| return True |
| |
| |
| class CreateSimTarStep(base_step.BaseStep): |
| |
| def __init__(self, **kwargs): |
| self.product = kwargs['product'] |
| base_step.BaseStep.__init__(self, |
| name='tar-ing sim artifacts', |
| halt_on_failure=True, |
| **kwargs) |
| |
| def run(self): |
| tar_filename = '{}.tar.gz'.format(TARGET_PRODUCT[self.product]) |
| command = [ |
| 'tar', |
| 'czf', |
| os.path.join(self.get_gcs_dir(), tar_filename), |
| SIM_CONFIG.format("{}_x86".format(self.product)), |
| MOJO_EVENTS, |
| OPENWEAVE, |
| SAMPLE_VIDEO, |
| SAMPLE_AUDIO, |
| SAMPLE_JPEG.format(self.product) |
| ] |
| command += [path.format(TARGET_PRODUCT[self.product] + "-" + BUILD_VARIANT) for path in SIM_SYSTEM] |
| command += SIM_LIB |
| returncode, _, stderr = self.exec_subprocess(command) |
| if returncode != 0: |
| comment = make_error_to_comments(stderr, '') |
| self.add_review({ |
| 'comments': comment |
| }) |
| else: |
| self.set_build_property('ota_files', [tar_filename]) |
| |
| return True |
| |
| |
| class BuildChromiumIotTestsStep(base_step.BaseStep): |
| |
| def __init__(self, code_coverage=False, **kwargs): |
| self.product = kwargs['product'] |
| self.out_dir = kwargs['out_dir'] |
| self._code_coverage = code_coverage |
| base_step.BaseStep.__init__(self, |
| name='build chromium-iot-tests', |
| halt_on_failure=True, |
| **kwargs) |
| |
| def run(self): |
| |
| command = [ |
| 'make', |
| '-j{}'.format(self.get_num_jobs()), |
| 'TARGET_PRODUCT={}'.format(TARGET_PRODUCT[self.product]), |
| 'TARGET_BUILD_VARIANT={}'.format(BUILD_VARIANT), |
| '{}'.format(BUILD_ITEM['iot-tests']) |
| ] |
| |
| if self._code_coverage: |
| command.append( |
| 'COMMAND_LINE_GN_ARGS=\'{}\''.format(CODE_COVERAGE_GN_ARGS)) |
| |
| command += self.build_accelerator.make_flags |
| |
| returncode, _, stderr = self.exec_subprocess(command) |
| if returncode != 0: |
| comment = make_error_to_comments(stderr, '') |
| self.add_review({ |
| 'comments': comment |
| }) |
| return False |
| |
| test_list = self.exec_read_file( |
| os.path.join(self.out_dir, IOT_TESTS_LIST)).splitlines() |
| self.add_step_data(code_coverage_step.STEP_DATA_TEST_LIST, test_list) |
| return True |
| |
| |
| class RunChromiumIotTestsStep(base_step.BaseStep): |
| |
| def __init__(self, **kwargs): |
| self.cwd = kwargs['cwd'] |
| self.product = kwargs['product'] |
| base_step.BaseStep.__init__(self, |
| name='run chromium-iot-tests', |
| halt_on_failure=True, |
| **kwargs) |
| def get_commands(self): |
| commands = [] |
| with open(os.path.join(self.cwd, IOT_TESTS_LIST), 'r') as f: |
| for line in f: |
| commands.append(["./{}".format(line.rstrip())]) |
| return commands |
| |
| def run(self): |
| commands = self.get_commands() |
| target_product = TARGET_PRODUCT[self.product] |
| env = { |
| 'LD_LIBRARY_PATH': ":".join( |
| ['../../../../prebuilt/toolchain/x86_64/usr/lib64', |
| os.path.join('../../../../out/target/product', |
| f'{target_product}-{BUILD_VARIANT}/system/lib') |
| ]) |
| } |
| for command in commands: |
| returncode, _, _ = self.exec_subprocess(command, cwd=self.cwd, env=env) |
| # return early if any unittest fails |
| if returncode != 0: |
| return False |
| return True |
| |
| |
| class BuildReconnectProxyTestStep(base_step.BaseStep): |
| |
| def __init__(self, **kwargs): |
| self.product = kwargs['product'] |
| base_step.BaseStep.__init__(self, |
| name='build reconnect-tests', |
| halt_on_failure=True, |
| **kwargs) |
| |
| def run(self): |
| |
| command = [ |
| 'make', |
| '-j{}'.format(self.get_num_jobs()), |
| 'TARGET_PRODUCT={}'.format(TARGET_PRODUCT[self.product]), |
| 'TARGET_BUILD_VARIANT={}'.format(BUILD_VARIANT), |
| '{}'.format(BUILD_ITEM[self.product]) |
| ] |
| returncode, _, stderr = self.exec_subprocess(command) |
| if returncode != 0: |
| comment = make_error_to_comments(stderr, '') |
| self.add_review({ |
| 'comments': comment |
| }) |
| return False |
| return True |
| |
| |
| class RunReconnectTestsStep(base_step.BaseStep): |
| |
| def __init__(self, **kwargs): |
| base_step.BaseStep.__init__(self, |
| name='run reconnect-tests', |
| halt_on_failure=True, |
| **kwargs) |
| |
| def run(self): |
| |
| command = [ |
| './{}'.format(os.path.join(OUT_DIR, 'reconnect-tests')) |
| ] |
| returncode, _, _ = self.exec_subprocess(command) |
| return returncode == 0 |
| |
| |
| class BuildNccmUnitTestStep(base_step.BaseStep): |
| |
| def __init__(self, **kwargs): |
| self.out_dir = kwargs.get('out_dir', NCCM_OUT_DIR) |
| base_step.BaseStep.__init__(self, |
| name='build nccm_unittests', |
| halt_on_failure=True, |
| **kwargs) |
| |
| def run(self): |
| |
| command = [ |
| 'autoninja', |
| '-C', |
| self.out_dir, |
| NCCM_BUILD_ITEM |
| ] |
| returncode, _, stderr = self.exec_subprocess(command) |
| if returncode != 0: |
| comment = make_error_to_comments(stderr, '') |
| self.add_review({ |
| 'comments': comment |
| }) |
| return False |
| |
| self.add_step_data( |
| code_coverage_step.STEP_DATA_TEST_LIST, ['nccm_unittests']) |
| return True |
| |
| |
| class RunNccmUnitTestStep(base_step.BaseStep): |
| |
| def __init__(self, **kwargs): |
| self.out_dir = kwargs['out_dir'] |
| base_step.BaseStep.__init__(self, |
| name='run nccm_unittests', |
| halt_on_failure=True, |
| **kwargs) |
| |
| def run(self): |
| |
| command = [ |
| ('./{}').format(os.path.join(self.out_dir, 'nccm_unittests'))] |
| returncode, _, _ = self.exec_subprocess(command) |
| return returncode == 0 |
| |
| |
| class BuildAndRunNldaemonUnitTestStep(base_step.BaseStep): |
| |
| def __init__(self, **kwargs): |
| self.product = kwargs['product'] |
| super().__init__( |
| name='run nldaemon_unittests', |
| halt_on_failure=True, |
| **kwargs, |
| ) |
| |
| def run(self): |
| if self.product != 'gq': |
| # gq_x86_64 is experimental as of 2020-Dec-17 |
| # Not blocking others if the tests fails. |
| return True |
| |
| # We need to do a clean build so the coverage data can be generated correctly. |
| command = [ |
| 'make', |
| 'TARGET_PRODUCT={}'.format(TARGET_PRODUCT[self.product]), |
| 'TARGET_BUILD_VARIANT={}'.format(BUILD_VARIANT), |
| 'clean' |
| ] |
| returncode, _, stderr = self.exec_subprocess(command) |
| if returncode != 0: |
| comment = make_error_to_comments(stderr, '') |
| self.add_review({ |
| 'comments': comment |
| }) |
| return False |
| |
| env = { |
| 'LD_LIBRARY_PATH': ":".join( |
| [os.path.join('chromium/src/third_party/android_ndk', |
| 'toolchains/llvm/prebuilt/linux-x86_64/lib64'), |
| os.path.join('out/target/product', |
| '{}/system/lib'.format(TARGET_PRODUCT[self.product])) |
| ]) |
| } |
| command = [ |
| 'make', |
| '-j{}'.format(self.get_num_jobs()), |
| 'ENABLE_CODE_COVERAGE=1', |
| 'TARGET_PRODUCT={}'.format(TARGET_PRODUCT[self.product]), |
| 'TARGET_BUILD_VARIANT={}'.format(BUILD_VARIANT), |
| 'nldaemon_unittests_run' |
| ] |
| returncode, _, stderr = self.exec_subprocess(command, env=env) |
| if returncode != 0: |
| comment = make_error_to_comments(stderr, '') |
| self.add_review({ |
| 'comments': comment |
| }) |
| return False |
| return True |
| |
| class BuildPhotonStep(base_step.BaseStep): |
| |
| def __init__(self, **kwargs): |
| self.out_dir = kwargs.get('out_dir', PHOTON_OUT_DIR) |
| base_step.BaseStep.__init__(self, |
| name='build photon', |
| halt_on_failure=True, |
| **kwargs) |
| |
| def run(self): |
| command = [ |
| 'autoninja', |
| '-C', |
| self.out_dir |
| ] |
| command += PHOTON_BUILD_ITEMS |
| returncode, _, stderr = self.exec_subprocess(command) |
| if returncode != 0: |
| comment = make_error_to_comments(stderr, '') |
| self.add_review({ |
| 'comments': comment |
| }) |
| return False |
| return True |
| |
| class RunPhotonUnitTestStep(base_step.BaseStep): |
| |
| def __init__(self, **kwargs): |
| self.out_dir = kwargs['out_dir'] |
| base_step.BaseStep.__init__(self, |
| name='run photon_hal_tests', |
| halt_on_failure=True, |
| **kwargs) |
| |
| def run(self): |
| command = [ |
| ('./{}').format(os.path.join(self.out_dir, 'photon_hal_tests'))] |
| returncode, _, _ = self.exec_subprocess(command) |
| return returncode == 0 |
