| """Script to identify the latest build number for testing. |
| |
| This script is called before each test to identify the latest build number and |
| path to the latest build zip file. |
| |
| By default, we find the common set of two build flavors (eng and user) and then |
| search for a good build hint from the greatest build number, once such hint is |
| found, the build number will be returned. |
| |
| Two types of build hints are currently in-place: |
| 1. (post-unit-test) good_to_test hint is created after unit test completion |
| (regardless test pass or fail). It is used by regular test. |
| 2. (pre-unit-test) test_deps.zip hint is created when OTA and test binaries are |
| successfully produced. It is used by unit test. |
| |
| Typical usages: |
| 1. Get good build for regular test: |
| get_latest_build.py --output_mode="number" --branch=1.8 --product=anchovy |
| 2. Get a good build for unit test: |
| get_latest_build.py --output_mode="number" --branch=1.8 --product=anchovy |
| --flavor=eng --build_triangulation_list=eng --use_unit_test_hint |
| 3. Get a good build for unit test and claim it (no one will get this build) |
| get_latest_build.py --output_mode="number" --branch=1.8 --product=anchovy |
| --flavor=eng --build_triangulation_list=eng --use_unit_test_hint |
| --claim_unit_test_hint |
| |
| One the build is clained, the hint file name will change and will not be |
| discovered by another build searcher. |
| """ |
| |
| import datetime |
| import optparse |
| import os |
| import random |
| import re |
| import subprocess |
| import sys |
| import tempfile |
| import time |
| |
| __author__ = 'billiani@google.com (Leonardo Billiani)' |
| |
| # Commands to list the build zip files: |
| # Eg. path: gs://gtv-te/internal/gtv-3.0/cosmo-tests/32908/cosmo-tests-32908.zip |
| GS_LIST_BUILDS_CMD = 'gsutil ls gs://%s/%s/%s-%s/%s/%s' |
| # gs://gtv-eureka/internal/1.28f/biggie-eng/100872/factory/biggie-fct-100872.zip |
| GS_LIST_FCT_BUILDS_CMD = 'gsutil ls gs://%s/%s/%s-%s/%s/factory/%s' |
| OTA_FILE_PATTERN = '%s-%s-%s.zip' |
| OTA_FILE_PATTERN_WITH_BOARD = '%s-%s-%s-%s.zip' |
| GS_COPY_BUILD_CMD = 'gsutil cp %s %s' |
| |
| # Path to marker file |
| GS_LIST_FILE = 'gsutil ls gs://%s/%s/%s-%s/%s/%s' |
| |
| # Rename marker file |
| GS_MV_MARKER_CMD = 'gsutil mv %s %s' |
| |
| # Regex to extract build number from a build zip path: |
| # Eg. 32 from gs://gtv-te/internal/gtv-3.0/cosmo-tests/32/cosmo-tests-32.zip |
| BUILD_NUMBER_RE = re.compile(r'\/(\d+)\/') |
| |
| # Supported "flavors" and "output type" of build: |
| BUILD_FLAVORS = { |
| 'darcy': 'arm', |
| 'eng': 'eng', |
| 'eng-clang': 'eng-clang', |
| 'fugu': 'x86', |
| 'imx7d_pico': 'arm', |
| 'molly': 'arm', |
| 'tests': 'tests', |
| 'user': 'user'} |
| |
| BUILD_OUTPUT_TYPES = { |
| 'fct': 'fct', |
| 'ota': 'ota', |
| 'symbols': 'symbols', |
| 'tests': 'tests'} |
| |
| # Chromecast Product Cloud Storage Bucket |
| PRODUCT_GCS_BUCKET = 'gtv-eureka' |
| |
| # Chromecast Unit Test Binary Cloud Storage Bucket |
| UNIT_TEST_GCS_BUCKET = 'chromecast-unit-test' |
| |
| # Root repositories for different products |
| PRODUCT_ROOT_REPOSITORY = { |
| 'anchovy': '%s/internal' % PRODUCT_GCS_BUCKET, |
| 'androidtv': '%s/internal' % PRODUCT_GCS_BUCKET, |
| 'androidthings': '%s/internal' % PRODUCT_GCS_BUCKET, |
| 'assistantdefault': '%s/internal' % PRODUCT_GCS_BUCKET, |
| 'audiodefault': '%s/cast_audio_reference' % PRODUCT_GCS_BUCKET, |
| 'biggie': '%s/internal' % PRODUCT_GCS_BUCKET, |
| 'bubbletea': '%s/cast_tv_mstar' % PRODUCT_GCS_BUCKET, |
| 'chorizo': '%s/internal' % PRODUCT_GCS_BUCKET, |
| 'dragonfly': '%s/cast_tv' % PRODUCT_GCS_BUCKET, |
| 'estelle': '%s/internal' % PRODUCT_GCS_BUCKET, |
| 'firefly': '%s/cast_tv' % PRODUCT_GCS_BUCKET, |
| 'guabao': '%s/cast_tv_mstar' % PRODUCT_GCS_BUCKET, |
| 'Mars_US': '%s/cast_tv_mediatek' % PRODUCT_GCS_BUCKET, |
| 'mozart': '%s/cast_audio_new' % PRODUCT_GCS_BUCKET, |
| 'mushroom': '%s/internal' % PRODUCT_GCS_BUCKET, |
| 'N00007': '%s/cast_tv_novatek' % PRODUCT_GCS_BUCKET, |
| 'np8340': '%s/cast_audio_new' % PRODUCT_GCS_BUCKET, |
| 'pepperoni': '%s/internal' % PRODUCT_GCS_BUCKET, |
| 'pineapple': '%s/internal' % PRODUCT_GCS_BUCKET, |
| 'redwood': '%s/cast_tv_marvell' % PRODUCT_GCS_BUCKET, |
| 'salami': '%s/internal' % PRODUCT_GCS_BUCKET, |
| 'steak': '%s/internal' % PRODUCT_GCS_BUCKET, |
| 'sullivan': '%s/cast_audio_new' % PRODUCT_GCS_BUCKET, |
| 'suncake': '%s/cast_tv_mstar' % PRODUCT_GCS_BUCKET, |
| 'taroball': '%s/cast_tv_mstar' % PRODUCT_GCS_BUCKET, |
| 'wutang': '%s/cast_audio_new' % PRODUCT_GCS_BUCKET} |
| UNIT_TEST_ROOT_REPOSITORY = { |
| 'anchovy': '%s/internal' % UNIT_TEST_GCS_BUCKET, |
| 'assistantdefault': '%s/internal' % UNIT_TEST_GCS_BUCKET, |
| 'audiodefault': '%s/cast_audio_reference' % UNIT_TEST_GCS_BUCKET, |
| 'biggie': '%s/internal' % UNIT_TEST_GCS_BUCKET, |
| 'bubbletea': '%s/cast_tv_mstar' % UNIT_TEST_GCS_BUCKET, |
| 'chorizo': '%s/internal' % UNIT_TEST_GCS_BUCKET, |
| 'default': '%s/internal' % UNIT_TEST_GCS_BUCKET, |
| 'dragonfly': '%s/cast_tv' % UNIT_TEST_GCS_BUCKET, |
| 'estelle': '%s/internal' % UNIT_TEST_GCS_BUCKET, |
| 'firefly': '%s/cast_tv' % UNIT_TEST_GCS_BUCKET, |
| 'guabao': '%s/cast_tv_mstar' % UNIT_TEST_GCS_BUCKET, |
| 'Mars_US': '%s/cast_tv_mediatek' % UNIT_TEST_GCS_BUCKET, |
| 'mozart': '%s/cast_audio_new' % UNIT_TEST_GCS_BUCKET, |
| 'mushroom': '%s/internal' % UNIT_TEST_GCS_BUCKET, |
| 'N00007': '%s/cast_tv_novatek' % UNIT_TEST_GCS_BUCKET, |
| 'np8340': '%s/cast_audio_new' % UNIT_TEST_GCS_BUCKET, |
| 'pepperoni': '%s/internal' % UNIT_TEST_GCS_BUCKET, |
| 'pineapple': '%s/internal' % UNIT_TEST_GCS_BUCKET, |
| 'redwood': '%s/cast_tv2' % UNIT_TEST_GCS_BUCKET, |
| 'salami': '%s/internal' % UNIT_TEST_GCS_BUCKET, |
| 'steak': '%s/internal' % UNIT_TEST_GCS_BUCKET, |
| 'sullivan': '%s/cast_audio_new' % UNIT_TEST_GCS_BUCKET, |
| 'suncake': '%s/cast_tv_mstar' % UNIT_TEST_GCS_BUCKET, |
| 'taroball': '%s/cast_tv_mstar' % UNIT_TEST_GCS_BUCKET, |
| 'wutang': '%s/cast_audio_new' % UNIT_TEST_GCS_BUCKET} |
| |
| STD_ROOT_REPOSITORY = 'gtv-te' |
| |
| # Run modes for this script: |
| OUTPUT_MODES = {'path': 'path', 'number': 'number'} |
| |
| # Supported tablet models: |
| TABLETS = ['xoomtablet', 'nexus10tablet'] |
| |
| # Supported Android products: |
| ANDROID_PRODUCTS = ['androidtv', 'androidthings', 'soba'] |
| |
| # Build flavor list is used to triangulate a good build. For example, by |
| # default, we get the greatest common build number from both eng and user |
| # build. |
| DEFAULT_TRIANGULATION_LIST = ( |
| ' '.join([BUILD_FLAVORS['eng'], BUILD_FLAVORS['user']])) |
| |
| # List of repositories that only build eng flavor |
| ENG_ONLY_REPOSITORY_LIST = ['chromium_tot'] |
| |
| # A good build marker file is generated when a build can finish unit test. |
| GOOD_BUILD_MARKER = 'good_to_test' |
| |
| # Directory with all the test identifiers |
| TEST_IDENTIFIER_DIR = 'run_identifiers' |
| |
| # Run identifiers path |
| IDS_REPOSITORY = 'fragglerock/%s' % TEST_IDENTIFIER_DIR |
| |
| # Unit test marker is the zip file which is used for unit test slaves. |
| UNIT_TEST_MARKER_DIR = 'gs://%s/%s/%s-%s/%s/' |
| UNIT_TEST_MARKER = 'test_deps.tar.gz' |
| CLAIMED_UNIT_TEST_MARKER = 'test_deps_claimed.tar.gz' |
| |
| BAD_BUILD_NUMBER = '0' |
| RANDOM_BACKOFF_SECS = 60 |
| DEFAULT_MAX_BUILD_LIST_LEN = 50 |
| |
| # Cast OTA file name override |
| # Key=hardware-type, Value=OTA-file-name |
| CAST_OTA_OVERRIDE_DICT = { |
| 'bubbletea': 'usb.bin', |
| 'guabao': 'usb.bin', |
| 'Mars_US': 'upgrade_loader_no_perm_data.pkg', |
| 'mozart': 'mozart-b2-ota.zip', |
| 'N00007': 'update.img', |
| 'np8340': '8506_linux_demo_dbg.bin', |
| 'sullivan': '8580_linux_demo_dbg.bin', |
| 'suncake': 'usb.bin', |
| 'taroball': 'usb.bin', |
| 'wutang': '8507_linux_demo_dbg.bin'} |
| |
| # Assumption: all products should have eng builds |
| # Overrides the triangulation list for products without user builds. |
| TRIANGULATION_OVERRIDE_DICT = { |
| 'bubbletea': 'eng', |
| 'dragonfly': 'eng', |
| 'guabao': 'eng', |
| 'Mars_US': 'eng', |
| 'mozart': 'eng', |
| 'N00007': 'eng', |
| 'np8340': 'eng', |
| 'sullivan': 'eng', |
| 'suncake': 'eng', |
| 'taroball': 'eng', |
| 'wutang': 'eng'} |
| |
| # Options for the build selection policy. |
| ALWAYS_USE_LATEST = 'always_use_latest' |
| BACKTRACK_THEN_LATEST = 'backtrack_then_latest' |
| ONLY_BACKTRACK = 'only_backtrack' |
| SECOND_TO_LATEST = 'second_to_latest' |
| BUILD_SELECTION_POLICIES = [ |
| ALWAYS_USE_LATEST, |
| BACKTRACK_THEN_LATEST, |
| ONLY_BACKTRACK, |
| SECOND_TO_LATEST |
| ] |
| DEFAULT_BUILD_SELECTION_POLICY = ONLY_BACKTRACK |
| |
| |
| def ParseFlags(): |
| """Parses command line flags.""" |
| parser = optparse.OptionParser() |
| parser.add_option('--branch', |
| default='master', |
| help='branch to get latest build number') |
| parser.add_option('--product', |
| help='product to get latest build number') |
| parser.add_option('--build_output_type', |
| default=BUILD_OUTPUT_TYPES['ota'], |
| type='choice', |
| choices=list(BUILD_OUTPUT_TYPES.keys()), |
| help=str(list(BUILD_OUTPUT_TYPES.keys()))) |
| parser.add_option('--build_number', |
| help='build number') |
| parser.add_option('--build_output_path', |
| default='', |
| help=('Complete path to where the build file is copied')) |
| parser.add_option('--output_mode', |
| type='choice', |
| choices=list(OUTPUT_MODES.keys()), |
| help=str(list(OUTPUT_MODES.keys()))) |
| parser.add_option('--flavor', |
| default=BUILD_FLAVORS['eng'], |
| type='choice', |
| choices=list(BUILD_FLAVORS.keys()), |
| help=str(list(BUILD_FLAVORS.keys()))) |
| parser.add_option('--build_triangulation_list', |
| default=DEFAULT_TRIANGULATION_LIST, |
| help='list of flavors used to determine latest build') |
| parser.add_option('--use_unit_test_hint', |
| action='store_true', |
| default=False, |
| help='Use unit test zip file as marker for a good build') |
| parser.add_option('--claim_unit_test_hint', |
| action='store_true', |
| default=False, |
| help='Change the unit test marker file name if True') |
| parser.add_option('--board_type', |
| default='none', |
| help='Chromecast device board type, e.g. lexx-b3') |
| parser.add_option('--repo_alias', |
| default='', |
| help='Repository alias, e.g. internal or partner') |
| parser.add_option('--test_identifier', |
| default='', |
| help=('Test identifier from buildbot. If specified, ' |
| 'it gets the latest non tested build.')) |
| parser.add_option('--use_gn', |
| default=1, |
| type=int, |
| help=('Positive integers for using GN; otherwise, no GN.')) |
| parser.add_option('--build_selection_policy', |
| type='choice', |
| choices=[ALWAYS_USE_LATEST, |
| BACKTRACK_THEN_LATEST, |
| ONLY_BACKTRACK, |
| SECOND_TO_LATEST], |
| default=ONLY_BACKTRACK, |
| help=('Defines the policy to select a build depending ' |
| 'on if it\'s been tested or not. ' |
| '"%s" chooses the latest build, whether it\'s ' |
| 'been tested or not. ' |
| '"%s" chooses that latest non-tested build ' |
| 'or the latest available if all have been tested. ' |
| '"%s" chooses that latest non-tested build ' |
| 'and exits with failure if all have been tested. ' |
| '"%s" chooses the second to latest good build. ' |
| 'Default: %%default' |
| % (ALWAYS_USE_LATEST, |
| BACKTRACK_THEN_LATEST, |
| ONLY_BACKTRACK, |
| SECOND_TO_LATEST))) |
| parser.add_option('--max_build_selection_range', |
| default=DEFAULT_MAX_BUILD_LIST_LEN, |
| type=int, |
| help=('Maximum number of builds to be used in build ' |
| 'selection process. This flag is used in conjunction ' |
| 'with build_selection_policy')) |
| parser.add_option('-v', |
| action='store_true', |
| default=False, |
| help='Run in unit test mode') |
| (flags, _) = parser.parse_args() |
| if flags.v: |
| return None |
| if not flags.output_mode: |
| parser.print_help() |
| sys.exit() |
| if flags.build_number and flags.output_mode == 'build': |
| parser.print_help() |
| sys.exit() |
| return flags |
| |
| |
| test_exit = subprocess.call('gsutil ls &> /dev/null', shell=True) |
| if test_exit: |
| print('gsutil not available') |
| sys.exit() |
| |
| |
| class AndroidTvBuildHelper(object): |
| """Class to handle Android TV builds.""" |
| # Cloud Storage location (this is pre-defined) |
| GS_PREFIX = 'gs://%s' % UNIT_TEST_ROOT_REPOSITORY['default'] |
| # Product Cast Shell APK location |
| GS_APK_PREFIX = 'gs://%s' % PRODUCT_ROOT_REPOSITORY['androidtv'] |
| # Cast Shell APK file pattern |
| CAST_SHELL_APK = 'CastShellInternal-*-%s.apk' |
| # List command |
| GS_LIST_CMD = 'gsutil ls' |
| |
| # Supported flavors: x86 for Fugu and arm for Molly. |
| SUPPORTED_FLAVOR = ['x86', 'arm'] |
| # Supported scope: internal or public |
| SUPPORTED_SCOPE = ['internal', 'public'] |
| |
| def __init__(self, product='androidtv', use_gn=1): |
| """Constructor.""" |
| self._product = product |
| self._gn_dir_suffix = self._GetGnDirSuffix(use_gn) |
| |
| def _GetGnDirSuffix(self, use_gn): |
| """Returns gn directory suffix.""" |
| if use_gn <= 0: |
| return '' |
| return '_gn' |
| |
| def _GetGsDirPattern(self): |
| """Returns GCS directory pattern.""" |
| return '%s/%s/content_shell_'+ self._product + '_%s_%s%s/*/%s' |
| |
| def GetLatestBuildNumber(self, |
| branch, |
| flavor, |
| claim_hint=True, |
| test_identifier='', |
| scope='internal'): |
| """Returns the most recent un-test build number. |
| |
| Args: |
| branch: String of branch name, e.g. unfork_m37. |
| flavor: String for device flavor, e.g. fugu or molly. |
| claim_hint: Optional boolean if a hint needs to be claimed. |
| test_identifier: Optional string for test complete hint. |
| scope: Optional string for scope defined, interna or public. |
| |
| Returns: |
| String: of the most recent build number. |
| """ |
| build_to_return = 0 |
| if claim_hint: |
| gs_prefix = self.GS_PREFIX |
| marker = UNIT_TEST_MARKER |
| else: |
| gs_prefix = self.GS_APK_PREFIX |
| marker = GOOD_BUILD_MARKER |
| search_dir = self._GetGsDirPattern() % (gs_prefix, |
| branch, |
| BUILD_FLAVORS[flavor], |
| scope, |
| self._gn_dir_suffix, |
| marker) |
| gs_list_cmd = ' '.join([self.GS_LIST_CMD, search_dir]) |
| output = _Execute(gs_list_cmd) |
| get_build_num_regex = re.compile(search_dir.replace('*', r'(\d+)')) |
| search_dir_list = sorted( |
| output.split(), reverse=True)[:DEFAULT_MAX_BUILD_LIST_LEN] |
| for line in search_dir_list: # pylint: disable=too-many-nested-blocks |
| match = get_build_num_regex.search(line) |
| if match: |
| if match.group(1) > build_to_return: |
| if not test_identifier: |
| build_to_return = match.group(1) |
| else: |
| # Check if the test was completed already |
| if test_identifier and not claim_hint: |
| test_name = test_identifier.split('#')[0] |
| is_tested_cmd = GS_LIST_FILE % ( |
| IDS_REPOSITORY, |
| branch, |
| flavor, |
| 'eng', |
| match.group(1), |
| os.path.join(TEST_IDENTIFIER_DIR, test_name + '*')) |
| if not _Execute(is_tested_cmd): |
| build_to_return = match.group(1) |
| |
| return str(build_to_return) |
| |
| def ClaimBuildNumber(self, |
| build_number, |
| branch, |
| flavor='fugu', |
| scope='internal'): |
| """Renames the test zip file. |
| |
| Args: |
| build_number: String containing a build number. |
| branch: String of branch name, e.g. unfork_m37. |
| flavor: String for device flavor, e.g. arm or x86. |
| scope: String for scope defined, interna or public. |
| """ |
| path_pattern = self._GetGsDirPattern().replace('*', build_number) |
| original_test_file = ( |
| path_pattern % (self.GS_PREFIX, |
| branch, |
| BUILD_FLAVORS[flavor], |
| scope, |
| self._gn_dir_suffix, |
| UNIT_TEST_MARKER)) |
| claimed_test_file = ( |
| path_pattern % (self.GS_PREFIX, |
| branch, |
| BUILD_FLAVORS[flavor], |
| scope, |
| self._gn_dir_suffix, |
| CLAIMED_UNIT_TEST_MARKER)) |
| _Execute(GS_MV_MARKER_CMD % (original_test_file, claimed_test_file)) |
| |
| def GetBuildPath(self, |
| build_number, |
| branch, |
| flavor='fugu', |
| scope='internal', |
| claim_hint=True): |
| """Renames the test zip file. |
| |
| Args: |
| build_number: String containing a build number. |
| branch: String of branch name, e.g. unfork_m37. |
| flavor: Optional string for device flavor, e.g. fugu or molly. |
| scope: Optional string for scope defined, interna or public. |
| claim_hint: Optional boolean if build hint needs to be claimed. |
| |
| Returns: |
| String: of path to the claimed test zip file. |
| """ |
| if claim_hint: |
| gs_prefix = self.GS_PREFIX |
| marker = CLAIMED_UNIT_TEST_MARKER |
| else: |
| gs_prefix = self.GS_APK_PREFIX |
| marker = self.CAST_SHELL_APK % BUILD_FLAVORS[flavor] |
| path_pattern = self._GetGsDirPattern() % (gs_prefix, |
| branch, |
| BUILD_FLAVORS[flavor], |
| scope, |
| self._gn_dir_suffix, |
| marker) |
| |
| return path_pattern.replace('*', build_number) |
| |
| |
| def _Execute(command, exit_on_stderr=False): |
| """Executes given command and returns stdout string.""" |
| proc = subprocess.Popen( |
| command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
| stdoutdata, stderrdata = proc.communicate() |
| if exit_on_stderr: |
| error_message_list = ( |
| ['CommandException', |
| 'InvalidUriError', |
| 'GSResponseError']) |
| for error_message in error_message_list: |
| if error_message in stderrdata: |
| raise IOError('%s: %s' % (command, stderrdata)) |
| return stdoutdata |
| |
| |
| def GetRepoGcsRootPath(product, repo_alias=''): |
| """Returns Cloud Storage root path for a given product.""" |
| # If repository alias is provided, use it for the GCS root |
| # Otherwise, look it up from the pre-defined product list |
| if repo_alias: |
| root_path = '%s/%s' % (PRODUCT_GCS_BUCKET, repo_alias) |
| else: |
| root_path = PRODUCT_ROOT_REPOSITORY.get(product, STD_ROOT_REPOSITORY) |
| |
| return root_path |
| |
| |
| def GetUnitTestGcsRootPath(product, repo_alias=''): |
| """Returns Unit Test Cloud Storage root path for a given product.""" |
| # If repository alias is provided, use it for the GCS root |
| # Otherwise, look it up from the pre-defined product list |
| if repo_alias: |
| root_path = '%s/%s' % (UNIT_TEST_GCS_BUCKET, repo_alias) |
| else: |
| root_path = UNIT_TEST_ROOT_REPOSITORY.get(product, |
| UNIT_TEST_ROOT_REPOSITORY['default']) |
| |
| return root_path |
| |
| |
| def GetListBuildsCommand(**kwargs): |
| """Returns command to list builds matching the given parameters. |
| |
| Args: |
| **kwargs: Key/Value dictionary containing values for: |
| branch, product, build_number, flavor, build_output_type. |
| |
| Returns: |
| String containing command to list the builds. |
| """ |
| if not kwargs.get('build_number'): |
| kwargs['build_number'] = '[0-9]*' |
| |
| kwargs['root_path'] = GetRepoGcsRootPath( |
| kwargs['product'], |
| kwargs.get('repo_alias')) |
| ota_file_name = '' |
| if kwargs and kwargs.get('board_type') != 'none': |
| ota_file_name = OTA_FILE_PATTERN_WITH_BOARD % ( |
| kwargs['product'], |
| kwargs['build_output_type'], |
| kwargs.get('board_type'), |
| kwargs['build_number']) |
| else: |
| ota_file_name = OTA_FILE_PATTERN % ( |
| kwargs['product'], |
| kwargs['build_output_type'], |
| kwargs['build_number']) |
| # Override the OTA file name for specific devices (which don't use OTA zip). |
| if kwargs.get('product') in CAST_OTA_OVERRIDE_DICT.keys(): |
| ota_file_name = CAST_OTA_OVERRIDE_DICT[kwargs['product']] |
| |
| list_cmd_fmt = GS_LIST_BUILDS_CMD |
| # Override the command when OTA output type is fct for factory build. |
| if kwargs and kwargs.get('build_output_type') == 'fct': |
| list_cmd_fmt = GS_LIST_FCT_BUILDS_CMD |
| list_cmd = list_cmd_fmt % ( |
| kwargs['root_path'], |
| kwargs['branch'], |
| kwargs['product'], |
| kwargs['flavor'], |
| kwargs['build_number'], |
| ota_file_name) |
| |
| return list_cmd |
| |
| |
| def GetBuildsListByFlavor(repo_alias, branch, product, flavor, board_type): |
| """Returns list of eng build paths matching the specified parameters.""" |
| return _Execute( |
| GetListBuildsCommand( |
| repo_alias=repo_alias, |
| branch=branch, |
| product=product, |
| flavor=flavor, |
| build_output_type=BUILD_OUTPUT_TYPES['ota'], |
| board_type=board_type)).split() |
| |
| |
| def ToBuildNumber(build_path): |
| """Returns a build number for the build zip at build_path. |
| |
| If a build number can not be parsed, 0 is returned. |
| |
| Args: |
| build_path: String containing build file path. |
| |
| Returns: |
| String containing the build number. |
| """ |
| match = BUILD_NUMBER_RE.search(build_path) |
| if match: |
| return match.group(1) |
| return '0' |
| |
| |
| def _IsBuildMarked(command_pattern, build_num, marker_file_name): |
| """Executes the command and returns if the marker is in stdout.""" |
| command = command_pattern % (build_num, marker_file_name) |
| stdout = _Execute(command) |
| return marker_file_name in stdout |
| |
| |
| def _IsBuildTested(has_been_tested_cmd, build_num): |
| """Executes the command and returns if the test ID is in stdout.""" |
| if not has_been_tested_cmd: |
| return False |
| command = has_been_tested_cmd % (build_num) |
| stdout = _Execute(command) |
| # Only match the first part of the test_identifier |
| test_name = FLAGS.test_identifier.split('#')[0] |
| return test_name in stdout |
| |
| |
| def GetBuildNumberWithMarker( |
| build_number_list, |
| command_pattern, |
| marker_file_name=GOOD_BUILD_MARKER, |
| has_been_tested_cmd='', |
| build_selection_policy=DEFAULT_BUILD_SELECTION_POLICY, |
| max_build_selection_range=DEFAULT_MAX_BUILD_LIST_LEN): |
| """Iterates all the builds and returns the greatest number. |
| |
| Examine the build number list from high to low, if a marker file is found, |
| return that build number. |
| |
| Args: |
| build_number_list: List containing builds in integers. |
| command_pattern: String containing command with build number to replace. |
| marker_file_name: Optional string containing the marker file name. |
| has_been_tested_cmd: String with the command to verify the test did not run. |
| build_selection_policy: The policy used to select the build. |
| max_build_selection_range: The maximum number of builds to be used for |
| build selection process. |
| |
| Returns: |
| String: build number found or zero if not found. |
| """ |
| is_good = lambda n: _IsBuildMarked(command_pattern, n, marker_file_name) |
| is_tested = lambda n: _IsBuildTested(has_been_tested_cmd, n) |
| if isinstance(build_number_list, list): |
| build_nums = sorted( |
| build_number_list, reverse=True)[:max_build_selection_range] |
| first_good = None |
| for build_num in build_nums: |
| if not is_good(build_num): |
| continue |
| tested = is_tested(build_num) |
| if first_good: |
| if not tested and build_selection_policy == SECOND_TO_LATEST: |
| return str(build_num) |
| else: |
| first_good = build_num |
| if build_selection_policy == ALWAYS_USE_LATEST: |
| return str(build_num) |
| if not tested and build_selection_policy != SECOND_TO_LATEST: |
| return str(build_num) |
| if build_selection_policy == BACKTRACK_THEN_LATEST and first_good: |
| return str(first_good) |
| return str(BAD_BUILD_NUMBER) |
| |
| |
| def ClaimBuildNumber(repo_alias, branch, product, flavor, build_number): |
| """Returns the good build number after claimed; or zero if fails to claim. |
| |
| Claim the build by changing the zip file name |
| However, if the claim fails, we will return 0 to indicate an error. |
| |
| Args: |
| repo_alias: String containing repository alias (e.g. internal/partner). |
| branch: String containing branch name (e.g. unfork_m37). |
| product: String containing product name (e.g. anchovy). |
| flavor: String containing one particular build flavor (e.g. eng). |
| build_number: String containing the build number (e.g. 19219). |
| |
| Returns: |
| String: build_number value if claim is successful; otherwise, zero. |
| """ |
| good_build_num = build_number |
| unit_test_root_repo = GetUnitTestGcsRootPath(product, repo_alias) |
| hint_path_pattern = UNIT_TEST_MARKER_DIR % ( |
| unit_test_root_repo, |
| branch, |
| product, |
| flavor, |
| good_build_num) + '%s' |
| original_hint_path = hint_path_pattern % UNIT_TEST_MARKER |
| claimed_hint_path = hint_path_pattern % CLAIMED_UNIT_TEST_MARKER |
| mv_cmd = GS_MV_MARKER_CMD % (original_hint_path, claimed_hint_path) |
| try: |
| _Execute(mv_cmd, exit_on_stderr=True) |
| except IOError: |
| good_build_num = '0' |
| return good_build_num |
| |
| |
| def GetLatestBuildNumber(repo_alias, branch, product, |
| triangulation_list=DEFAULT_TRIANGULATION_LIST, |
| flavor=BUILD_FLAVORS['eng'], board_type='', |
| build_selection_policy=DEFAULT_BUILD_SELECTION_POLICY, |
| max_build_selection_range=DEFAULT_MAX_BUILD_LIST_LEN): |
| """Prints latest successful build number for given branch and product. |
| |
| By default, take pre-cautions by cross examining two flavors (eng and user) |
| to determine a common latest build. However, it allows to use one flavor. |
| |
| Args: |
| repo_alias: String containing repository alias (e.g. internal/partner). |
| branch: String containing branch name (e.g. unfork_m37). |
| product: String containing product name (e.g. anchovy). |
| triangulation_list: Optional string containing flavor names separated |
| by spaces. |
| flavor: Optional string containing one particular build flavor (e.g. eng). |
| board_type: Optional string for device board type. |
| build_selection_policy: The policy used to select the build. |
| max_build_selection_range: The maximum number of builds to be used for |
| build selection process. |
| |
| Returns: |
| String: build number value or 0 if no build found. |
| """ |
| common_build_numbers = [] |
| |
| if product in TRIANGULATION_OVERRIDE_DICT: # pylint: disable=consider-using-get |
| triangulation_list = TRIANGULATION_OVERRIDE_DICT[product] |
| |
| # Special treat eng only repositories |
| if repo_alias in ENG_ONLY_REPOSITORY_LIST: |
| triangulation_list = BUILD_FLAVORS['eng'] |
| |
| for internal_flavor in triangulation_list.split(): |
| build_numbers = list(map( |
| ToBuildNumber, |
| GetBuildsListByFlavor( |
| repo_alias, branch, product, internal_flavor, board_type))) |
| if common_build_numbers: |
| common_build_numbers = ( |
| list(set(common_build_numbers) & set(build_numbers))) |
| else: |
| common_build_numbers = build_numbers |
| int_build_numbers = list(map(int, common_build_numbers)) |
| try: |
| # A good build should have good_to_test file in bigstore |
| if product in PRODUCT_ROOT_REPOSITORY: |
| cloud_storage = GetRepoGcsRootPath(product, repo_alias) |
| marker_file_name = GOOD_BUILD_MARKER |
| if FLAGS.use_unit_test_hint: |
| # Wait for random seconds (0-20) to avoid file operation collision |
| time.sleep(random.randint(0, RANDOM_BACKOFF_SECS)) |
| cloud_storage = GetUnitTestGcsRootPath(product, repo_alias) |
| marker_file_name = UNIT_TEST_MARKER |
| cmd_pattern = GS_LIST_FILE % (cloud_storage, |
| branch, |
| product, |
| flavor, |
| '%s', |
| '%s') |
| has_been_tested_cmd = '' |
| |
| # Only get latest non tested build |
| if FLAGS.test_identifier: |
| # Only match the first part of the test_identifier |
| test_name = FLAGS.test_identifier.split('#')[0] |
| has_been_tested_cmd = GS_LIST_FILE % (IDS_REPOSITORY, |
| branch, |
| product, |
| flavor, |
| '%s', |
| os.path.join( |
| TEST_IDENTIFIER_DIR, |
| test_name + '*')) |
| |
| good_build_num = GetBuildNumberWithMarker( |
| int_build_numbers, |
| cmd_pattern, |
| marker_file_name, |
| has_been_tested_cmd, |
| build_selection_policy=build_selection_policy, |
| max_build_selection_range=max_build_selection_range) |
| |
| if FLAGS.claim_unit_test_hint and good_build_num != BAD_BUILD_NUMBER: |
| ClaimBuildNumber( |
| repo_alias, branch, product, flavor, good_build_num) |
| return good_build_num |
| return str(int_build_numbers[-1]) |
| except TypeError: |
| return BAD_BUILD_NUMBER |
| |
| |
| def GetBuildPath(repo_alias, branch, product, build_number, build_output_type, |
| flavor, board_type): |
| """Prints the build path for build number in branch and product specified.""" |
| build_paths = _Execute( |
| GetListBuildsCommand( |
| repo_alias=repo_alias, |
| branch=branch, product=product, build_number=build_number, |
| flavor=flavor, build_output_type=build_output_type, |
| board_type=board_type)).split() |
| return build_paths[-1] |
| |
| |
| def GetOutputPath(build_path, output_path): |
| """Returns the output path where the build file is copied. |
| |
| Args: |
| build_path: String containing the bigstore build path (Eg. gs://gtv-te/...). |
| output_path: String optionally containing location to copy the build to. If |
| empty, a temp directory path is used. |
| |
| Returns: |
| String containing the correct path where the build should be copied. |
| |
| Raises: |
| IOError: when the path is empty or does not exist |
| """ |
| if output_path: |
| if os.path.isdir(output_path): |
| raise IOError('Path (%s) must include file name.' % output_path) |
| return output_path |
| temp_dir = tempfile.mkdtemp() |
| return os.path.join(temp_dir, os.path.basename(build_path)) |
| |
| |
| def CopyToDir(build_path, output_path, cp_cmd_template=GS_COPY_BUILD_CMD): |
| """Copies the given bigstore build_path to the given output_path. |
| |
| Args: |
| build_path: String containing the bigstore build path (Eg. gs://gtv-te/...). |
| output_path: String containing the location to copy the build to. |
| cp_cmd_template: Formatting string to interpolate to build the cp command |
| |
| Returns: |
| String containing the location to copy the build to. |
| """ |
| _Execute(cp_cmd_template % (build_path, output_path)) |
| return output_path |
| |
| |
| def main(): |
| # For tablets, there's not really a valid buildnumber that's useful |
| # so instead we use a datetime string to specify unique runs |
| if FLAGS.product in TABLETS: |
| print(str(datetime.datetime.now().strftime('%d%B%Y%I%M%p'))) |
| |
| # TODO(kevinfei): we have too many products and each does not fall into the |
| # same way to retrieve build. Re-organize this script for the readability |
| # and long-term maintainability. |
| # Android TV is a prodcut which needs unit test coverage. Once a build is |
| # available, test hint (test_deps.zip) will be available in the Cloud |
| # Storage (e.g. gs://chromecast-unit-test/internal/content_shell_androidtv_ |
| # x86_internal_gn/19912/test_deps.zip. |
| # To claim this build, rename the file name to test_deps_claimed.zip. |
| # When the functional test (e.g. Cast Compliance) starts, the marker file |
| # will be storage in the Cloud Storage (different bucket though) and the |
| # test will check for the existence of that file to determine if that build |
| # should be tested. |
| elif FLAGS.product in ANDROID_PRODUCTS: |
| atv_helper = AndroidTvBuildHelper(FLAGS.product, FLAGS.use_gn) |
| if FLAGS.output_mode == 'path': |
| claimed_file_path = atv_helper.GetBuildPath( |
| FLAGS.build_number, |
| FLAGS.branch, |
| FLAGS.flavor, |
| claim_hint=FLAGS.claim_unit_test_hint) |
| print(CopyToDir(claimed_file_path, FLAGS.build_output_path)) |
| else: |
| latest_build = ( |
| atv_helper.GetLatestBuildNumber( |
| FLAGS.branch, |
| FLAGS.flavor, |
| claim_hint=FLAGS.claim_unit_test_hint, |
| test_identifier=FLAGS.test_identifier)) |
| if FLAGS.claim_unit_test_hint and latest_build != BAD_BUILD_NUMBER: |
| try: |
| atv_helper.ClaimBuildNumber(latest_build, |
| FLAGS.branch, |
| FLAGS.flavor) |
| except IOError: |
| latest_build = BAD_BUILD_NUMBER |
| if latest_build == BAD_BUILD_NUMBER: |
| exit(1) |
| else: |
| print(latest_build) |
| |
| else: |
| if FLAGS.output_mode == 'path': |
| build_path = GetBuildPath(FLAGS.repo_alias, FLAGS.branch, FLAGS.product, |
| FLAGS.build_number, FLAGS.build_output_type, |
| FLAGS.flavor, FLAGS.board_type) |
| output_path = GetOutputPath(build_path, FLAGS.build_output_path) |
| print(CopyToDir(build_path, output_path)) |
| else: |
| latest_build = GetLatestBuildNumber( |
| FLAGS.repo_alias, |
| FLAGS.branch, |
| FLAGS.product, |
| triangulation_list=FLAGS.build_triangulation_list, |
| flavor=FLAGS.flavor, |
| board_type=FLAGS.board_type, |
| build_selection_policy=FLAGS.build_selection_policy, |
| max_build_selection_range=FLAGS.max_build_selection_range) |
| if latest_build == BAD_BUILD_NUMBER: |
| exit(1) |
| else: |
| print(latest_build) |
| |
| |
| if __name__ == '__main__': |
| FLAGS = ParseFlags() |
| main() |
