| # Copyright 2017 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Triggers and processes results from flag try jobs. |
| |
| For more information, see: http://bit.ly/flag-try-jobs |
| """ |
| |
| import argparse |
| import sys |
| |
| from blinkpy.common.host import Host |
| from blinkpy.common.net.git_cl import GitCL |
| from blinkpy.common.path_finder import PathFinder |
| from blinkpy.web_tests.models.test_configuration import TestConfiguration |
| from blinkpy.web_tests.models.typ_types import Expectation, TestExpectations, ResultType |
| |
| # TODO(skobes): use blinkpy/config/builders.json instead of hardcoding these. |
| BUILDER_CONFIGS = { |
| 'linux-rel': TestConfiguration('Linux', '', 'release'), |
| 'mac-rel': TestConfiguration('Mac', '', 'release'), |
| 'win7-rel': TestConfiguration('Win', '', 'release') |
| } |
| BUILDER_BUCKETS = { |
| 'linux-rel': 'luci.chromium.try', |
| 'mac-rel': 'luci.chromium.try', |
| 'win7-rel': 'luci.chromium.try', |
| } |
| FLAG_FILE = 'additional-driver-flag.setting' |
| |
| |
| class TryFlag(object): |
| def __init__(self, argv, host, git_cl): |
| self._args = parse_args(argv) |
| self._host = host |
| self._git_cl = git_cl |
| self._expectations = [] |
| self._filesystem = self._host.filesystem |
| self._path_finder = PathFinder(self._filesystem) |
| self._git = self._host.git() |
| |
| def _force_flag_for_test_runner(self): |
| flag = self._args.flag |
| path = self._path_finder.path_from_web_tests(FLAG_FILE) |
| self._filesystem.write_text_file(path, flag + '\n') |
| self._git.add_list([path]) |
| self._git.commit_locally_with_message( |
| 'Flag try job: force %s for run_web_tests.py.' % flag) |
| |
| def _flag_expectations_path(self): |
| return self._path_finder.path_from_web_tests( |
| 'FlagExpectations', self._args.flag.lstrip('-')) |
| |
| def _clear_expectations(self): |
| path = self._flag_expectations_path() |
| self._filesystem.write_text_file(path, '') |
| self._git.add_list([path]) |
| self._git.commit_locally_with_message( |
| 'Flag try job: clear expectations for %s.' % self._args.flag) |
| |
| def _tests_in_flag_expectations(self): |
| path = self._flag_expectations_path() |
| content = self._filesystem.read_text_file(path) |
| test_expectations = TestExpectations() |
| test_expectations.parse_tagged_list(content) |
| return { |
| test_name |
| for test_name in test_expectations.individual_exps.keys() |
| } |
| |
| def trigger(self): |
| self._force_flag_for_test_runner() |
| if self._args.regenerate: |
| self._clear_expectations() |
| self._git_cl.run([ |
| 'upload', '--bypass-hooks', '-f', '-m', |
| 'Flag try job for %s.' % self._args.flag |
| ]) |
| for builder in sorted(BUILDER_BUCKETS): |
| bucket = BUILDER_BUCKETS[builder] |
| self._git_cl.trigger_try_jobs([builder], bucket) |
| |
| def _create_expectation_line(self, result, test_configuration): |
| expected_results = set( |
| [res for res in result.actual_results().split()]) |
| tag = test_configuration.version |
| reason = '' |
| if self._args.bug: |
| reason = 'crbug.com/' + self._args.bug |
| return Expectation( |
| test=result.test_name(), |
| results=expected_results, |
| tags=set([tag]), |
| reason=reason) |
| |
| def _process_result(self, build, result): |
| if not result.did_run_as_expected(): |
| self._expectations.append( |
| self._create_expectation_line( |
| result, BUILDER_CONFIGS[build.builder_name])) |
| |
| def update(self): |
| self._host.print_('Fetching results...') |
| # TODO: Get jobs from the _tryflag branch. Current branch for now. |
| jobs = self._git_cl.latest_try_jobs( |
| builder_names=BUILDER_CONFIGS.keys()) |
| results_fetcher = self._host.results_fetcher |
| for build in sorted(jobs): |
| self._host.print_('-- %s: %s/results.html' % |
| (BUILDER_CONFIGS[build.builder_name].version, |
| results_fetcher.results_url( |
| build.builder_name, build.build_number))) |
| results = results_fetcher.fetch_results(build, True) |
| results.for_each_test( |
| lambda result, b=build: self._process_result(b, result)) |
| |
| # TODO: Write to flag expectations file. For now, stdout. :) |
| unexpected_failures = [] |
| unexpected_passes = [] |
| tests_in_flag_expectations = self._tests_in_flag_expectations() |
| for exp in self._expectations: |
| if ResultType.Pass not in exp.results: |
| unexpected_failures.append(exp) |
| elif exp.test in tests_in_flag_expectations: |
| unexpected_passes.append(exp) |
| unexpected_passes = sorted(unexpected_passes, key=lambda e: e.test) |
| unexpected_failures = sorted(unexpected_failures, key=lambda e: e.test) |
| self._print_all(unexpected_passes, 'unexpected passes') |
| self._print_all(unexpected_failures, 'unexpected failures') |
| |
| def _print_all(self, exps, description): |
| self._host.print_('\n### %s %s:\n' % (len(exps), description)) |
| for exp in exps: |
| self._host.print_(exp.to_string()) |
| |
| def run(self): |
| action = self._args.action |
| if action == 'trigger': |
| self.trigger() |
| elif action == 'update': |
| self.update() |
| else: |
| print >> self._host.stderr, 'specify "trigger" or "update"' |
| return 1 |
| return 0 |
| |
| |
| def parse_args(argv): |
| parser = argparse.ArgumentParser( |
| description=__doc__, |
| formatter_class=argparse.RawDescriptionHelpFormatter) |
| parser.add_argument('action', help='"trigger" or "update"') |
| parser.add_argument('--bug', help='crbug number for expectation lines') |
| parser.add_argument( |
| '--flag', |
| required=True, |
| help='flag to force-enable in run_web_tests.py') |
| parser.add_argument( |
| '--regenerate', |
| action='store_true', |
| help='clear the flag expectations before triggering') |
| return parser.parse_args(argv) |
| |
| |
| def main(): |
| host = Host() |
| return TryFlag(sys.argv[1:], host, GitCL(host)).run() |
