| # Copyright 2016 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Updates TestExpectations based on results in builder bots. |
| |
| Scans the TestExpectations file and uses results from actual builder bots runs |
| to remove tests that are marked as flaky or failing but don't fail in the |
| specified way. |
| |
| --type=flake updates only lines that incude a 'Pass' expectation plus at |
| least one other expectation. |
| |
| --type=fail updates lines that include only 'Failure', 'Timeout', or |
| 'Crash' expectations. |
| |
| E.g. If a test has this expectation: |
| bug(test) fast/test.html [ Failure Pass ] |
| |
| And all the runs on builders have passed the line will be removed. |
| |
| Additionally, the runs don't all have to be Passing to remove the line; |
| as long as the non-Passing results are of a type not specified in the |
| expectation this line will be removed. For example, if this is the |
| expectation: |
| |
| bug(test) fast/test.html [ Crash Pass ] |
| |
| But the results on the builders show only Passes and Timeouts, the line |
| will be removed since there's no Crash results. |
| """ |
| |
| import argparse |
| import logging |
| import webbrowser |
| |
| from blinkpy.tool.commands.flaky_tests import FlakyTests |
| from blinkpy.web_tests.models.test_expectations import TestExpectations, SPECIAL_PREFIXES |
| from blinkpy.web_tests.models.typ_types import ResultType |
| |
| _log = logging.getLogger(__name__) |
| |
| CHROMIUM_BUG_PREFIX = 'crbug.com/' |
| |
| |
| def main(host, bot_test_expectations_factory, argv): |
| parser = argparse.ArgumentParser( |
| epilog=__doc__, formatter_class=argparse.RawTextHelpFormatter) |
| parser.add_argument( |
| '--type', |
| choices=['flake', 'fail', 'all'], |
| default='all', |
| help='type of expectations to update (default: %(default)s)') |
| parser.add_argument( |
| '--verbose', |
| '-v', |
| action='store_true', |
| default=False, |
| help='enable more verbose logging') |
| parser.add_argument( |
| '--remove-missing', |
| action='store_true', |
| default=False, |
| help='also remove lines if there were no results, ' |
| 'e.g. Android-only expectations for tests ' |
| 'that are not in SmokeTests') |
| # TODO(crbug.com/1077883): Including cq results might introduce false |
| # negatives. An in-review change that caused a test with fail test |
| # expectation but no longer fails, to fail again, will result in |
| # the test expectation not being removed even if the change |
| # does not ultimately land. |
| parser.add_argument( |
| '--include-cq-results', |
| action='store_true', |
| default=False, |
| help='include results from cq.') |
| parser.add_argument( |
| '--show-results', |
| '-s', |
| action='store_true', |
| default=False, |
| help='Open results dashboard for all removed lines') |
| args = parser.parse_args(argv) |
| |
| logging.basicConfig( |
| level=logging.DEBUG if args.verbose else logging.INFO, |
| format='%(levelname)s: %(message)s') |
| |
| port = host.port_factory.get() |
| expectations_file = port.path_to_generic_test_expectations_file() |
| if not host.filesystem.isfile(expectations_file): |
| _log.warning("Didn't find generic expectations file at: " + |
| expectations_file) |
| return 1 |
| |
| remover = ExpectationsRemover(host, port, bot_test_expectations_factory, |
| webbrowser, args.type, args.remove_missing, |
| args.include_cq_results) |
| |
| test_expectations = remover.get_updated_test_expectations() |
| if args.show_results: |
| remover.show_removed_results() |
| |
| host.filesystem.write_text_file(expectations_file, test_expectations) |
| remover.print_suggested_commit_description() |
| return 0 |
| |
| |
| class ExpectationsRemover(object): |
| def __init__(self, |
| host, |
| port, |
| bot_test_expectations_factory, |
| browser, |
| type_flag='all', |
| remove_missing=False, |
| include_cq_results=False): |
| self._host = host |
| self._port = port |
| self._expectations_factory = bot_test_expectations_factory |
| self._browser = browser |
| self._expectations_to_remove_list = None |
| self._type = type_flag |
| self._bug_numbers = set() |
| self._remove_missing = remove_missing |
| self._include_cq_results = include_cq_results |
| self._builder_results_by_path = self._get_builder_results_by_path() |
| self._removed_test_names = set() |
| self._version_to_os = {} |
| |
| def _can_delete_line(self, expectation): |
| """Returns whether a given line in the expectations can be removed. |
| |
| Uses results from builder bots to determine if a given line is stale and |
| can safely be removed from the TestExpectations file. (i.e. remove if |
| the bots show that it's not flaky.) There are also some rules about when |
| not to remove lines (e.g. never remove lines with Rebaseline |
| expectations, don't remove non-flaky expectations, etc.) |
| |
| Args: |
| test_expectation_line (TestExpectationLine): A line in the test |
| expectation file to test for possible removal. |
| |
| Returns: |
| True if the line can be removed, False otherwise. |
| """ |
| expected_results = expectation.results |
| |
| if not expected_results: |
| return False |
| |
| # Don't check lines that have expectations like Skip or Slow. |
| if ResultType.Skip in expected_results or expectation.is_slow_test: |
| return False |
| |
| # Don't check consistent passes. |
| if len(expected_results) == 1 and ResultType.Pass in expected_results: |
| return False |
| |
| # Don't check flakes in fail mode. |
| if self._type == 'fail' and ResultType.Pass in expected_results: |
| return False |
| |
| # Don't check failures in flake mode. |
| if self._type == 'flake' and ResultType.Pass not in expected_results: |
| return False |
| |
| # Initialize OS version to OS dictionary. |
| if not self._version_to_os: |
| for os, os_versions in \ |
| self._port.configuration_specifier_macros().items(): |
| for version in os_versions: |
| self._version_to_os[version.lower()] = os.lower() |
| |
| # The line can be deleted if none of the expectations appear in the |
| # actual results or only a PASS expectation appears in the actual |
| # results. |
| builders_checked = [] |
| builders = [] |
| for config in self._port.all_test_configurations(): |
| if set(expectation.tags).issubset( |
| set([ |
| self._version_to_os[config.version.lower()], |
| config.version.lower(), |
| config.build_type.lower() |
| ])): |
| try_server_configs = [False] |
| if self._include_cq_results: |
| try_server_configs.append(True) |
| for is_try_server in try_server_configs: |
| builder_name = self._host.builders.builder_name_for_specifiers( |
| config.version, config.build_type, is_try_server) |
| if not builder_name: |
| _log.debug('No builder with config %s for %s', |
| config, 'CQ' if is_try_server else 'CI') |
| # For many configurations, there is no matching builder in |
| # blinkpy/common/config/builders.json. We ignore these |
| # configurations and make decisions based only on configurations |
| # with actual builders. |
| continue |
| builders.append(builder_name) |
| |
| for builder_name in builders: |
| |
| builders_checked.append(builder_name) |
| |
| if builder_name not in self._builder_results_by_path.keys(): |
| _log.error('Failed to find results for builder "%s"', |
| builder_name) |
| return False |
| |
| results_by_path = self._builder_results_by_path[builder_name] |
| |
| # No results means the tests were all skipped, or all results are passing. |
| if expectation.test not in results_by_path.keys(): |
| if self._remove_missing: |
| continue |
| return False |
| |
| results_for_single_test = set(results_by_path[expectation.test]) |
| expectations_met = expected_results & results_for_single_test |
| if (expectations_met != set([ResultType.Pass]) |
| and expectations_met != set([])): |
| return False |
| if builders_checked: |
| _log.debug('Checked builders:\n %s', |
| '\n '.join(builders_checked)) |
| else: |
| _log.warning('No matching builders for line, deleting line.') |
| return True |
| |
| def _get_builder_results_by_path(self): |
| """Returns a dictionary of results for each builder. |
| |
| Returns a dictionary where each key is a builder and value is a dictionary containing |
| the distinct results for each test. E.g. |
| |
| { |
| 'WebKit Linux Precise': { |
| 'test1.html': ['PASS', 'IMAGE'], |
| 'test2.html': ['PASS'], |
| }, |
| 'WebKit Mac10.10': { |
| 'test1.html': ['PASS', 'IMAGE'], |
| 'test2.html': ['PASS', 'TEXT'], |
| } |
| } |
| """ |
| builder_results_by_path = {} |
| builders = [] |
| if self._include_cq_results: |
| builders = self._host.builders.all_builder_names() |
| else: |
| builders = self._host.builders.all_continuous_builder_names() |
| |
| for builder_name in builders: |
| expectations_for_builder = ( |
| self._expectations_factory.expectations_for_builder(builder_name) |
| ) |
| |
| if not expectations_for_builder: |
| # This is not fatal since we may not need to check these |
| # results. If we do need these results we'll log an error later |
| # when trying to check against them. |
| _log.warning( |
| 'Downloaded results are missing results for builder "%s"', |
| builder_name) |
| continue |
| |
| builder_results_by_path[builder_name] = ( |
| expectations_for_builder.all_results_by_path()) |
| return builder_results_by_path |
| |
| def get_updated_test_expectations(self): |
| """Filters out passing lines from TestExpectations file. |
| |
| Reads the current TestExpectations file and, using results from the |
| build bots, removes lines that are passing. That is, removes lines that |
| were not needed to keep the bots green. |
| |
| Returns: |
| A TestExpectations object with the passing lines filtered out. |
| """ |
| generic_exp_path = self._port.path_to_generic_test_expectations_file() |
| raw_test_expectations = self._host.filesystem.read_text_file( |
| generic_exp_path) |
| expectations_dict = {generic_exp_path: raw_test_expectations} |
| test_expectations = TestExpectations( |
| port=self._port, expectations_dict=expectations_dict) |
| removed_exps = [] |
| lines = [] |
| |
| for exp in test_expectations.get_updated_lines(generic_exp_path): |
| # only get expectations objects for non glob patterns |
| if not exp.test or exp.is_glob: |
| continue |
| |
| if self._can_delete_line(exp): |
| reason = exp.reason or '' |
| self._bug_numbers.update([ |
| reason[len(CHROMIUM_BUG_PREFIX):] |
| for reason in reason.split() |
| if reason.startswith(CHROMIUM_BUG_PREFIX) |
| ]) |
| self._removed_test_names.add(exp.test) |
| removed_exps.append(exp) |
| _log.info('Deleting line "%s"' % exp.to_string().strip()) |
| |
| if removed_exps: |
| test_expectations.remove_expectations(generic_exp_path, |
| removed_exps) |
| |
| return '\n'.join([ |
| e.to_string() |
| for e in test_expectations.get_updated_lines(generic_exp_path) |
| ]) |
| |
| def show_removed_results(self): |
| """Opens a browser showing the removed lines in the results dashboard. |
| |
| Opens the results dashboard in the browser, showing all the tests for |
| lines removed from the TestExpectations file, allowing the user to |
| manually confirm the results. |
| """ |
| url = self._flakiness_dashboard_url() |
| _log.info('Opening results dashboard: ' + url) |
| self._browser.open(url) |
| |
| def print_suggested_commit_description(self): |
| """Prints the body of a suggested CL description after removing some lines.""" |
| |
| expectation_type = '' |
| if self._type != 'all': |
| expectation_type = self._type + ' ' |
| dashboard_url = self._flakiness_dashboard_url() |
| bugs = ', '.join(sorted(self._bug_numbers)) |
| message = ( |
| 'Remove %sTestExpectations which are not failing in the specified way.\n\n' |
| 'This change was made by the update_expectations.py script.\n\n' |
| 'Recent test results history:\n%s\n\n' |
| 'Bug: %s') % (expectation_type, dashboard_url, bugs) |
| _log.info('Suggested commit description:\n' + message) |
| |
| def _flakiness_dashboard_url(self): |
| removed_test_names = ','.join(sorted(self._removed_test_names)) |
| return FlakyTests.FLAKINESS_DASHBOARD_URL % removed_test_names |
