cq/scripts/slave/recipe/clusterfuzz_builder.py

"""Recipe to build and upload fuzz testing targets to clusterfuzz GCS Bucket"""
import os

from helpers import branch_utils
from helpers import gcs_utils
from slave import base_recipe
from slave.step import gcs_upload_step
from slave.step import gn_step
from slave.step import make_clean_step
from slave.step import ninja_step
from slave.step import shell_step

BUILD_CONFIGS = {
    'tcp_socket_impl_fuzzer': {
        'archive_name': 'tcp_socket_impl_fuzzer',
        'build_args_product': 'video_x64_eng',
        'build_target': ('chromecast/internal/smart_home/actions/'
                         'platform/cast:tcp_socket_impl_fuzzer'),
        'run_gtests': False,
        'build_args': [
            'use_libfuzzer=true',
            'is_asan=true',
            'is_component_build=false',
        ],
        'gcs_bucket': 'catabuilder-clusterfuzz-results',
        'gcs_path': 'tcp_socket_impl_fuzzer',
    },
    'udp_socket_impl_fuzzer': {
        'archive_name': 'udp_socket_impl_fuzzer',
        'build_args_product': 'video_x64_eng',
        'build_target': ('chromecast/internal/smart_home/actions/'
                         'platform/cast:udp_socket_impl_fuzzer'),
        'run_gtests': False,
        'build_args': [
            'use_libfuzzer=true',
            'is_asan=true',
            'is_component_build=false',
        ],
        'gcs_bucket': 'catabuilder-clusterfuzz-results',
        'gcs_path': 'udp_socket_impl_fuzzer',
    },
}


def GetValidBuildNames():
  return list(BUILD_CONFIGS.keys())


def CreateRecipe(build_name: str, **kwargs):
  """Builds a recipe object."""
  config = BUILD_CONFIGS[build_name]
  return FuzzRecipe(config, **kwargs)


class FuzzRecipe(base_recipe.BaseRecipe):
  """Recipe to build and upload fuzz testing targets to clusterfuzz GCS"""

  def __init__(self, config, **kwargs):
    base_recipe.BaseRecipe.__init__(
        self, enable_build_accelerator=True, **kwargs)
    self._build_config = config

  def get_num_jobs(self):  # pylint:disable=method-hidden
    """Count of jobs to use."""
    setup_steps = self.get_setup_steps()
    return setup_steps[0].get_num_jobs()

  def get_steps(self):
    cwd = os.getcwd()
    gcs_bucket = self._build_config.get('gcs_bucket')
    gcs_path = self._build_config.get('gcs_path')
    jobs = self.get_num_jobs()
    build_args = self._build_config.get('build_args')
    build_args_product = self._build_config.get('build_args_product')
    build_target = self._build_config.get('build_target')
    post_submit_upload = self.build_system == 'catabuilder'
    archive_name = self._build_config.get('archive_name')
    fuzz_archive = '{}.zip'.format(self.build_number)

    # Going to build target to out directory
    out_dir = os.path.join('out', archive_name, self.build_number)
    steps = []
    if self.build_system == 'catabuilder':
      steps.append(make_clean_step.MakeCleanStep(**self._step_kwargs))

    # Generate gn gen step
    gn_gen_step = gn_step.GnGenStep(
        build_args_product=build_args_product,
        build_args=build_args,
        out_dir=out_dir,
        halt_on_failure=True,
        cwd=cwd,
        **self._step_kwargs)
    steps.append(gn_gen_step)

    # Generate gn args step
    gn_args_step = gn_step.GnArgsStep(
        out_dir=out_dir, cwd=cwd, halt_on_failure=True, **self._step_kwargs)
    steps.append(gn_args_step)

    # Generate ninja build step
    ninja_build_step = ninja_step.NinjaStep(
        out_dir=out_dir,
        target=build_target,
        cwd=cwd,
        jobs=jobs,
        halt_on_failure=True,
        **self._step_kwargs)
    steps.append(ninja_build_step)

    # Zip build artifacts excluding obj file
    zip_output_step = shell_step.ShellStep(
        name='Zip build artifacts',
        command=['zip', '-rq', fuzz_archive,
                  '.', '-x', "'*obj*'"],
        directory=os.getcwd()+'/'+out_dir,
        **self._step_kwargs)

    steps.append(zip_output_step)

    # Upload Fuzz Testing build output to gcs
    # only if recipe is running from Catabuilder.
    if post_submit_upload:
      # Write and upload targets.list file
      contents = '\n'.join(
          BUILD_CONFIGS[config]['gcs_path'] for config in BUILD_CONFIGS)
      targets_list_file = self._executor.write_temp_file(contents)

      gcs_step = gcs_upload_step.GcsUploadStep(
          gcs_bucket,
          'targets.list',
          name='Clusterfuzz GCS upload for targets.list',
          src_path=targets_list_file,
          **self._step_kwargs)

      gcs_step = gcs_upload_step.GcsUploadStep(
          gcs_bucket,
          gcs_path,
          name='Clusterfuzz GCS upload for target: ' + archive_name,
          src_path=os.getcwd()+'/'+out_dir+'/'+fuzz_archive,
          **self._step_kwargs)
      steps.append(gcs_step)

    return steps