cq/scripts/slave/recipe/eureka_kythe_builder.py

"""Recipe to create and publish KZIPs for Kythe.

See go/eureka-cs-indexing for more context.
"""
import copy
import os

from slave import base_recipe
from slave.step import gcs_upload_step
from slave.step import gn_step
from slave.step import kythe_step
from slave.step import make_clean_step
from slave.step import ninja_step
from slave.test import test_generator


# WARNING: the build_args_product and build_args_flavor are used to construct
# the naming of the out folder. As per the vnames.json file, Kythe expects this
# exact naming in order to properly index and serve generated files.
BUILD_CONFIGS = {
    'fuchsia_eureka_kythe_prod': {
        'build_args_product': 'fuchsia_arm64_user',
        'build_args_flavor': 'Release',
        'build_args': ['is_official_build=true'],
        'build_target': 'cast_agent_exe',
        'gcs_folder': 'prod'
    },
    'fuchsia_eureka_kythe_staging': {
        'build_args_product': 'fuchsia_arm64_user',
        'build_args_flavor': 'Release',
        'build_args': ['is_official_build=true'],
        'build_target': 'cast_agent_exe',
        'gcs_folder': 'staging'
    },
}

GCS_UPLOAD_BUCKET = 'eureka-kythe'

# Expect that merged KZIPs are at least 1 GB
MIN_EXPECTED_KZIP_SIZE = 1 * 1024 * 1024 * 1024


def GetValidBuildNames():
  return list(BUILD_CONFIGS.keys())


def CreateRecipe(build_name: str, **kwargs):
  config = BUILD_CONFIGS[build_name]
  return EurekaKytheRecipe(config, **kwargs)


class EurekaKytheRecipe(base_recipe.BaseRecipe):

  def __init__(self, config, **kwargs):
    base_recipe.BaseRecipe.__init__(
        self, enable_build_accelerator=True, **kwargs)
    self._build_config = config

  def get_clean_steps(self):
    return [make_clean_step.MakeCleanStep(**self._step_kwargs)]

  # Needed so that we can properly mock expectations in test.
  def get_num_jobs(self):  # pylint:disable=method-hidden
    """Count of jobs to use."""
    setup_steps = self.get_setup_steps()
    return setup_steps[0].get_num_jobs(multiplier=10)

  def get_steps(self):
    steps = []

    cwd = os.getcwd()
    continuous_tests_root = os.path.abspath(
        self.get_setup_steps()[0].get_project_path('continuous-tests'))
    chromium_root = self.get_setup_steps()[0].get_project_path('chromium/src')
    build_args = copy.deepcopy(self._build_config.get('build_args'))
    build_args_product = self._build_config.get('build_args_product')
    build_args_flavor = self._build_config.get('build_args_flavor')
    build_target = self._build_config.get('build_target')
    gcs_folder = self._build_config.get('gcs_folder')
    out_dir = os.path.join(chromium_root, 'out_{}'.format(build_args_product),
                           build_args_flavor)

    # STEP: Generate gn gen for build
    gn_gen_step = gn_step.GnGenStep(
        build_args_product=build_args_product,
        build_args=build_args,
        out_dir=out_dir,
        halt_on_failure=True,
        cwd=cwd,
        **self._step_kwargs)
    steps.append(gn_gen_step)

    # STEP: List gn args
    gn_args_step = gn_step.GnArgsStep(
        out_dir=out_dir, cwd=cwd, halt_on_failure=True, **self._step_kwargs)
    steps.append(gn_args_step)

    # STEP: Generate ninja build
    num_jobs = self.get_num_jobs()
    ninja_build_step = ninja_step.NinjaStep(
        out_dir=out_dir,
        target=build_target,
        cwd=cwd,
        jobs=num_jobs,
        halt_on_failure=True,
        **self._step_kwargs)
    steps.append(ninja_build_step)

    # STEP: Generate compilation database
    gn_gen_step = gn_step.GnGenCompilationDBStep(
        out_dir=out_dir,
        cwd=cwd,
        build_args_product=build_args_product,
        target_name=build_target,
        halt_on_failure=True,
        **self._step_kwargs)
    compdb_file = gn_gen_step.get_compdb_file_location()
    steps.append(gn_gen_step)

    # STEP: Generate KZIPs
    kzip_gen_step = kythe_step.GenerateKzipStep(
        cwd=cwd,
        tests_root=continuous_tests_root,
        chromium_root=chromium_root,
        out_dir=out_dir,
        compdb_file=compdb_file,
        min_expected_kzip_bytes=MIN_EXPECTED_KZIP_SIZE,
        halt_on_failures=True,
        **self._step_kwargs)
    kzip_file = kzip_gen_step.get_kythe_output_file()
    steps.append(kzip_gen_step)

    # STEP: Upload to GCS
    upload_step = gcs_upload_step.GcsUploadStep(
        gcs_bucket=GCS_UPLOAD_BUCKET,
        gcs_path=gcs_folder,
        src_path=kzip_file,
        **self._step_kwargs)
    steps.append(upload_step)

    return steps