| """Recipe to build and test fuchsia-cast.""" |
| import copy |
| import os |
| import re |
| import tempfile |
| |
| from helpers import branch_utils |
| from slave import base_recipe |
| from slave.step import cipd_upload_step |
| from slave.step import fuchsia_cipd_upload_step |
| from slave.step import fuchsia_emulator_step |
| from slave.step import fuchsia_unittest_step |
| from slave.step import gn_step |
| from slave.step import landmine_step |
| from slave.step import make_clean_step |
| from slave.step import ninja_step |
| from slave.step import shell_step |
| from slave.test import test_generator |
| |
| |
| BUILD_CONFIGS = { |
| 'arm64_fuchsia': { |
| 'architecture': 'arm64', |
| 'build_args_product': 'fuchsia_arm64_user', |
| 'build_args_flavor': 'Release', |
| 'build_target': 'chromecast/internal/fuchsia:all', |
| 'test_target': 'chromecast/internal/fuchsia:chromecast_fuchsia_tests', |
| 'run_gtests': False, |
| 'run_catatester': True, |
| 'build_args': [] |
| }, |
| 'arm64_fuchsia_eng': { |
| 'architecture': 'arm64', |
| 'build_args_product': 'fuchsia_arm64_eng', |
| 'build_args_flavor': 'Eng', |
| 'build_target': 'chromecast/internal/fuchsia:all', |
| 'test_target': 'chromecast/internal/fuchsia:chromecast_fuchsia_tests', |
| 'run_gtests': False, |
| 'build_args': [] |
| }, |
| 'arm64_fuchsia_eng_optimized': { |
| 'architecture': 'arm64', |
| 'build_args_product': 'fuchsia_arm64_eng_optimized', |
| 'build_args_flavor': 'EngOptimized', |
| 'build_target': 'chromecast/internal/fuchsia:all', |
| 'test_target': 'chromecast/internal/fuchsia:chromecast_fuchsia_tests', |
| 'run_gtests': False, |
| 'build_args': [] |
| }, |
| 'x64_fuchsia': { |
| 'architecture': 'amd64', |
| 'build_args_product': 'fuchsia_x64_user', |
| 'build_args_flavor': 'Release', |
| 'build_target': 'chromecast/internal/fuchsia:all', |
| 'test_target': 'chromecast/internal/fuchsia:chromecast_fuchsia_tests', |
| 'run_gtests': True, |
| 'build_args': ['dcheck_always_on=true'] |
| }, |
| 'x64_fuchsia_eng': { |
| 'architecture': 'amd64', |
| 'build_args_product': 'fuchsia_x64_eng', |
| 'build_args_flavor': 'Eng', |
| 'build_target': 'chromecast/internal/fuchsia:all', |
| 'test_target': 'chromecast/internal/fuchsia:chromecast_fuchsia_tests', |
| 'run_gtests': True, |
| 'build_args': [] |
| }, |
| 'x64_fuchsia_eng_optimized': { |
| 'architecture': 'amd64', |
| 'build_args_product': 'fuchsia_x64_eng_optimized', |
| 'build_args_flavor': 'EngOptimized', |
| 'build_target': 'chromecast/internal/fuchsia:all', |
| 'test_target': 'chromecast/internal/fuchsia:chromecast_fuchsia_tests', |
| 'run_gtests': True, |
| 'build_args': [] |
| }, |
| } |
| |
| SHORTLIVED_BRANCH_PATTERN = ( |
| r'(?P<major>\d+)\.(?P<minor>\d+)_shortlived_(?P<build>\d+)') |
| |
| CIPD_UPLOADS_FILE_NAME = 'cipd_uploads' |
| |
| |
| def GetValidBuildNames(): |
| return list(BUILD_CONFIGS.keys()) |
| |
| |
| def CreateRecipe(build_name: str, **kwargs): |
| """Builds a recipe object.""" |
| config = BUILD_CONFIGS[build_name] |
| return FuchsiaRecipe(config, **kwargs) |
| |
| |
| class FuchsiaRecipe(base_recipe.BaseRecipe): |
| """Recipe to lint a repository.""" |
| |
| def __init__(self, config, **kwargs): |
| base_recipe.BaseRecipe.__init__( |
| self, enable_build_accelerator=True, **kwargs) |
| self._build_config = config |
| self._fuchsia_workdir = None |
| self._properties = kwargs.get('properties', {}) |
| |
| def get_fuchsia_bucket(self, chromium_root): |
| """Get the current Fuchsia bucket to get artifacts from. |
| |
| Usually, this will pull artifacts from either the public or private Fuchsia |
| SDK buckets. For go/feureka-dd builds, it will pull from a temporary bucket. |
| |
| Args: |
| chromium_root: Path to chromium source directory. |
| |
| Returns: |
| GCP bucket. For example: 'fuchsia-sdk' or 'fuchsia'. |
| """ |
| bucket_path = os.path.join(chromium_root, 'build/fuchsia/sdk-bucket.txt') |
| bucket = 'fuchsia' |
| # Only go/feureka-dd writes to the bucket file |
| with open(bucket_path) as f: |
| # Trim any whitespace on each line |
| bucket = f.readline().rstrip() |
| return bucket or 'fuchsia' |
| |
| def get_num_jobs(self): # pylint:disable=method-hidden |
| """Count of jobs to use.""" |
| setup_steps = self.get_setup_steps() |
| return setup_steps[0].get_num_jobs(multiplier=10) |
| |
| def create_temp_directory(self): |
| return tempfile.TemporaryDirectory().name |
| |
| def get_steps(self): |
| # Use one of the setup steps to get chromium root |
| cwd = os.getcwd() |
| setup_steps = self.get_setup_steps() |
| chromium_root = setup_steps[0].get_project_path('chromium/src') |
| gcs_dir = setup_steps[0].get_gcs_dir() |
| jobs = self.get_num_jobs() |
| architecture = self._build_config.get('architecture') |
| build_args_product = self._build_config.get('build_args_product') |
| build_args_flavor = self._build_config.get('build_args_flavor') |
| build_args = copy.deepcopy(self._build_config.get('build_args')) |
| if self.build_system == 'catabuilder': |
| build_args.append('is_official_build=true') |
| build_target = self._build_config.get('build_target') |
| test_target = self._build_config.get('test_target') |
| run_gtests = self._build_config.get('run_gtests') |
| run_catatester = self._build_config.get('run_catatester', False) |
| post_submit_upload = self.build_system == 'catabuilder' |
| self._fuchsia_workdir = self.create_temp_directory() |
| |
| # Extend buildargs with incremental build id |
| incremental_build_prop = ('cast_build_incremental="{}"'.format( |
| self.build_number)) |
| build_args.append(incremental_build_prop) |
| out_dir = os.path.join(chromium_root, 'out_{}'.format(build_args_product), |
| build_args_flavor) |
| steps = [] |
| if self.build_system == 'catabuilder': |
| steps.append(make_clean_step.MakeCleanStep(**self._step_kwargs)) |
| else: |
| steps += [ |
| landmine_step.LandmineStep(**self._step_kwargs), |
| ] |
| # Generate gn gen step |
| gn_gen_step = gn_step.GnGenStep( |
| build_args_product=build_args_product, |
| build_args=build_args, |
| out_dir=out_dir, |
| halt_on_failure=True, |
| cwd=cwd, |
| **self._step_kwargs) |
| steps.append(gn_gen_step) |
| # Generate gn args step |
| gn_args_step = gn_step.GnArgsStep( |
| out_dir=out_dir, cwd=cwd, halt_on_failure=True, **self._step_kwargs) |
| steps.append(gn_args_step) |
| # Generate ninja build step |
| ninja_build_step = ninja_step.NinjaStep( |
| out_dir=out_dir, |
| target=build_target, |
| cwd=cwd, |
| jobs=jobs, |
| halt_on_failure=True, |
| **self._step_kwargs) |
| steps.append(ninja_build_step) |
| # Generate ninja build tests step |
| max_timeout_secs = 4 * 60 * 60 |
| ninja_build_tests_step = ninja_step.NinjaStep( |
| out_dir=out_dir, |
| target=test_target, |
| halt_on_failure=True, |
| timeout_secs=max_timeout_secs, |
| cwd=cwd, |
| jobs=jobs, |
| **self._step_kwargs) |
| steps.append(ninja_build_tests_step) |
| if run_gtests: |
| bucket = self.get_fuchsia_bucket(chromium_root) |
| |
| # Start FEMU |
| femu_start_step = fuchsia_emulator_step.FuchsiaEmulatorStartStep( |
| bucket=bucket, |
| chromium_root=chromium_root, |
| out_dir=out_dir, |
| fuchsia_workdir=self._fuchsia_workdir, |
| **self._step_kwargs) |
| steps.append(femu_start_step) |
| |
| step = fuchsia_unittest_step.FuchsiaUnitTestStep( |
| out_dir=out_dir, |
| recipe_steps=steps, |
| cwd=cwd, |
| **self._step_kwargs) |
| steps.append(step) |
| |
| # Upload the artifacts to CIPD |
| refs = {self._manifest_branch} |
| |
| tags = { |
| 'version': self.build_number, |
| } |
| |
| far_pkg_vars = { |
| 'targetarch': architecture |
| } |
| |
| cipd_uploads_file = os.path.join(out_dir, 'gen', CIPD_UPLOADS_FILE_NAME) |
| cipd_yaml_dir = chromium_root |
| steps.append(fuchsia_cipd_upload_step.FuchsiaCIPDUploadStep( |
| steps, cipd_uploads_file, cipd_yaml_dir, |
| refs=refs, |
| tags=tags, |
| pkg_vars=far_pkg_vars, |
| **self._step_kwargs)) |
| |
| # Package gunit test dependencies and upload to gcs |
| # only if recipe is running from Catabuilder. |
| if post_submit_upload and run_catatester: |
| max_timeout_secs = 30 * 60 |
| ninja_test_deps_archive_step = ninja_step.NinjaStep( |
| out_dir=out_dir, |
| target='chromecast_test_deps_archive', |
| halt_on_failure=True, |
| timeout_secs=max_timeout_secs, |
| cwd=cwd, |
| jobs=jobs, |
| **self._step_kwargs) |
| steps.append(ninja_test_deps_archive_step) |
| |
| src_path = os.path.join(out_dir, 'test_deps.tar.gz') |
| dst_path = os.path.join(gcs_dir, 'artifacts', 'tests') |
| copy_deps_steps = [ |
| shell_step.ShellStep( |
| name='Create test deps destination folder', |
| command=[ |
| 'mkdir', |
| '-p', |
| dst_path, |
| ], |
| directory=os.getcwd(), |
| **self._step_kwargs), |
| shell_step.ShellStep( |
| name='Move test_deps.tar.gz to upload folder', |
| command=['mv', src_path, dst_path], |
| directory=os.getcwd(), |
| **self._step_kwargs) |
| ] |
| steps.extend(copy_deps_steps) |
| return steps |
| |
| def get_teardown_steps(self): |
| """Returns a list of all steps that should be executed as part of teardown. |
| |
| Teardown steps are always called, even if previous steps fail. |
| |
| Returns: |
| List of slave.base_step.BaseStep commands to be executed. |
| """ |
| return [ |
| fuchsia_emulator_step.FuchsiaEmulatorTeardownStep(**self._step_kwargs) |
| ] + super().get_teardown_steps() |
