| # Copyright (C) 2010 Google Inc. All rights reserved. |
| # |
| # Redistribution and use in source and binary forms, with or without |
| # modification, are permitted provided that the following conditions are |
| # met: |
| # |
| # * Redistributions of source code must retain the above copyright |
| # notice, this list of conditions and the following disclaimer. |
| # * Redistributions in binary form must reproduce the above |
| # copyright notice, this list of conditions and the following disclaimer |
| # in the documentation and/or other materials provided with the |
| # distribution. |
| # * Neither the Google name nor the names of its |
| # contributors may be used to endorse or promote products derived from |
| # this software without specific prior written permission. |
| # |
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| import base64 |
| import json |
| import time |
| |
| from blinkpy.common import exit_codes |
| from blinkpy.common.system.crash_logs import CrashLogs |
| from blinkpy.web_tests.models.test_configuration import TestConfiguration |
| from blinkpy.web_tests.port.base import Port, VirtualTestSuite |
| from blinkpy.web_tests.port.driver import DeviceFailure, Driver, DriverOutput |
| from blinkpy.w3c.wpt_manifest import BASE_MANIFEST_NAME |
| |
| # Here we use a non-standard location for the web tests, to ensure that |
| # this works. The path contains a '.' in the name because we've seen bugs |
| # related to this before. |
| WEB_TEST_DIR = '/test.checkout/wtests' |
| PERF_TEST_DIR = '/test.checkout/PerformanceTests' |
| |
| |
| # This sets basic expectations for a test. Each individual expectation |
| # can be overridden by a keyword argument in TestList.add(). |
| class TestInstance(object): |
| def __init__(self, name): |
| self.name = name |
| self.base = name[(name.rfind('/') + 1):name.rfind('.')] |
| self.crash = False |
| self.web_process_crash = False |
| self.exception = False |
| self.keyboard = False |
| self.error = '' |
| self.timeout = False |
| self.device_failure = False |
| self.leak = False |
| |
| # The values of each field are treated as raw byte strings. They |
| # will be converted to unicode strings where appropriate using |
| # FileSystem.read_text_file(). |
| self.actual_text = self.base + '-txt' |
| self.actual_checksum = self.base + '-checksum' |
| |
| # We add the '\x8a' for the image file to prevent the value from |
| # being treated as UTF-8 (the character is invalid) |
| self.actual_image = self.base + '\x8a' + '-png' + 'tEXtchecksum\x00' + self.actual_checksum |
| |
| self.expected_text = self.actual_text |
| self.expected_image = self.actual_image |
| |
| self.actual_audio = None |
| self.expected_audio = None |
| |
| |
| # This is an in-memory list of tests, what we want them to produce, and |
| # what we want to claim are the expected results. |
| class TestList(object): |
| def __init__(self): |
| self.tests = {} |
| |
| def add(self, name, **kwargs): |
| test = TestInstance(name) |
| for key, value in kwargs.items(): |
| test.__dict__[key] = value |
| self.tests[name] = test |
| |
| def add_reference(self, |
| name, |
| actual_checksum='checksum', |
| actual_image='FAIL'): |
| self.add( |
| name, |
| actual_checksum=actual_checksum, |
| actual_image=actual_image, |
| actual_text=None, |
| expected_text=None, |
| expected_image=None) |
| |
| def add_reftest(self, |
| name, |
| reference_name, |
| same_image=True, |
| actual_text=None, |
| expected_text=None, |
| crash=False, |
| error=''): |
| self.add( |
| name, |
| actual_checksum='checksum', |
| actual_image='FAIL', |
| expected_image=None, |
| actual_text=actual_text, |
| expected_text=expected_text, |
| crash=crash, |
| error=error) |
| if same_image: |
| self.add_reference(reference_name) |
| else: |
| self.add_reference( |
| reference_name, actual_checksum='diff', actual_image='DIFF') |
| |
| def keys(self): |
| return self.tests.keys() |
| |
| def __contains__(self, item): |
| return item in self.tests |
| |
| def __getitem__(self, item): |
| return self.tests[item] |
| |
| |
| # |
| # These numbers may need to be updated whenever we add or delete tests. This includes virtual tests. |
| # |
| TOTAL_TESTS = 176 |
| TOTAL_WONTFIX = 3 |
| TOTAL_SKIPS = 26 + TOTAL_WONTFIX |
| TOTAL_CRASHES = 78 |
| |
| UNEXPECTED_PASSES = 2 |
| UNEXPECTED_NON_VIRTUAL_FAILURES = 34 |
| UNEXPECTED_FAILURES = 67 |
| |
| |
| def unit_test_list(): |
| tests = TestList() |
| tests.add('failures/expected/crash.html', crash=True) |
| tests.add('failures/expected/exception.html', exception=True) |
| tests.add('failures/expected/device_failure.html', device_failure=True) |
| tests.add('failures/expected/timeout.html', timeout=True) |
| tests.add('failures/expected/leak.html', leak=True) |
| tests.add( |
| 'failures/expected/image.html', |
| actual_image='image_fail-pngtEXtchecksum\x00checksum_fail', |
| expected_image='image-pngtEXtchecksum\x00checksum-png') |
| tests.add( |
| 'failures/expected/image_checksum.html', |
| actual_checksum='image_checksum_fail-checksum', |
| actual_image='image_checksum_fail-png') |
| tests.add( |
| 'failures/expected/audio.html', |
| actual_audio=base64.b64encode('audio_fail-wav'), |
| expected_audio='audio-wav', |
| actual_text=None, |
| expected_text=None, |
| actual_image=None, |
| expected_image=None, |
| actual_checksum=None) |
| tests.add( |
| 'failures/unexpected/image-mismatch.html', |
| actual_image='image_fail-pngtEXtchecksum\x00checksum_fail', |
| expected_image='image-pngtEXtchecksum\x00checksum-png') |
| tests.add( |
| 'failures/unexpected/no-image-generated.html', |
| expected_image='image-pngtEXtchecksum\x00checksum-png', |
| actual_image=None, |
| actual_checksum=None) |
| tests.add( |
| 'failures/unexpected/no-image-baseline.html', |
| actual_image='image_fail-pngtEXtchecksum\x00checksum_fail', |
| expected_image=None) |
| tests.add( |
| 'failures/unexpected/audio-mismatch.html', |
| actual_audio=base64.b64encode('audio_fail-wav'), |
| expected_audio='audio-wav', |
| actual_text=None, |
| expected_text=None, |
| actual_image=None, |
| expected_image=None, |
| actual_checksum=None) |
| tests.add( |
| 'failures/unexpected/no-audio-baseline.html', |
| actual_audio=base64.b64encode('audio_fail-wav'), |
| actual_text=None, |
| expected_text=None, |
| actual_image=None, |
| expected_image=None, |
| actual_checksum=None) |
| tests.add( |
| 'failures/unexpected/no-audio-generated.html', |
| expected_audio=base64.b64encode('audio_fail-wav'), |
| actual_text=None, |
| expected_text=None, |
| actual_image=None, |
| expected_image=None, |
| actual_checksum=None) |
| tests.add( |
| 'failures/unexpected/text-mismatch-overlay.html', |
| actual_text='"invalidations": [\nfail', |
| expected_text='"invalidations": [\npass') |
| tests.add( |
| 'failures/unexpected/no-text-baseline.html', |
| actual_text='"invalidations": [\nfail', |
| expected_text=None) |
| tests.add( |
| 'failures/unexpected/no-text-generated.html', |
| actual_text=None, |
| expected_text='"invalidations": [\npass') |
| tests.add('failures/expected/keyboard.html', keyboard=True) |
| tests.add( |
| 'failures/expected/newlines_leading.html', |
| expected_text='\nfoo\n', |
| actual_text='foo\n') |
| tests.add( |
| 'failures/expected/newlines_trailing.html', |
| expected_text='foo\n\n', |
| actual_text='foo\n') |
| tests.add( |
| 'failures/expected/newlines_with_excess_CR.html', |
| expected_text='foo\r\r\r\n', |
| actual_text='foo\n') |
| tests.add('failures/expected/text.html', actual_text='text_fail-png') |
| tests.add('failures/expected/crash_then_text.html') |
| tests.add('failures/expected/skip_text.html', actual_text='text diff') |
| tests.add('failures/flaky/text.html') |
| tests.add('failures/unexpected/*/text.html', actual_text='text_fail-png') |
| tests.add('failures/unexpected/missing_text.html', expected_text=None) |
| tests.add( |
| 'failures/unexpected/missing_check.html', |
| expected_image='missing-check-png') |
| tests.add('failures/unexpected/missing_image.html', expected_image=None) |
| tests.add( |
| 'failures/unexpected/missing_render_tree_dump.html', |
| actual_text="""layer at (0,0) size 800x600 |
| RenderView at (0,0) size 800x600 |
| layer at (0,0) size 800x34 |
| RenderBlock {HTML} at (0,0) size 800x34 |
| RenderBody {BODY} at (8,8) size 784x18 |
| RenderText {#text} at (0,0) size 133x18 |
| text run at (0,0) width 133: "This is an image test!" |
| """, |
| expected_text=None) |
| tests.add('failures/unexpected/crash.html', crash=True) |
| tests.add('failures/unexpected/crash-with-sample.html', crash=True) |
| tests.add('failures/unexpected/crash-with-delayed-log.html', crash=True) |
| tests.add( |
| 'failures/unexpected/crash-with-stderr.html', |
| crash=True, |
| error='mock-std-error-output') |
| tests.add( |
| 'failures/unexpected/web-process-crash-with-stderr.html', |
| web_process_crash=True, |
| error='mock-std-error-output') |
| tests.add('failures/unexpected/pass.html') |
| tests.add( |
| 'failures/unexpected/text-checksum.html', |
| actual_text='text-checksum_fail-txt', |
| actual_checksum='text-checksum_fail-checksum') |
| tests.add( |
| 'failures/unexpected/text-image-checksum.html', |
| actual_text='text-image-checksum_fail-txt', |
| actual_image= |
| 'text-image-checksum_fail-pngtEXtchecksum\x00checksum_fail', |
| actual_checksum='text-image-checksum_fail-checksum') |
| tests.add( |
| 'failures/unexpected/checksum-with-matching-image.html', |
| actual_checksum='text-image-checksum_fail-checksum') |
| tests.add( |
| 'failures/unexpected/image-only.html', |
| expected_text=None, |
| actual_text=None, |
| actual_image='image-only_fail-pngtEXtchecksum\x00checksum_fail', |
| actual_checksum='image-only_fail-checksum') |
| tests.add('failures/unexpected/skip_pass.html') |
| tests.add('failures/unexpected/text.html', actual_text='text_fail-txt') |
| tests.add('failures/unexpected/text_then_crash.html') |
| tests.add('failures/unexpected/timeout.html', timeout=True) |
| tests.add('failures/unexpected/leak.html', leak=True) |
| tests.add('http/tests/passes/text.html') |
| tests.add('http/tests/passes/image.html') |
| tests.add('http/tests/ssl/text.html') |
| tests.add('passes/args.html') |
| tests.add('passes/error.html', error='stuff going to stderr') |
| tests.add('passes/image.html', actual_text=None, expected_text=None) |
| tests.add( |
| 'passes/audio.html', |
| actual_audio=base64.b64encode('audio-wav'), |
| expected_audio='audio-wav', |
| actual_text=None, |
| expected_text=None, |
| actual_image=None, |
| expected_image=None, |
| actual_checksum=None, |
| expected_checksum=None) |
| tests.add('passes/platform_image.html') |
| tests.add('passes/slow.html') |
| tests.add( |
| 'passes/checksum_in_image.html', |
| expected_image='tEXtchecksum\x00checksum_in_image-checksum') |
| tests.add('passes/skipped/skip.html') |
| tests.add( |
| 'failures/unexpected/testharness.html', |
| actual_text= |
| 'This is a testharness.js-based test.\nFAIL: bah\nHarness: the test ran to completion.' |
| ) |
| |
| # Note that here the checksums don't match/ but the images do, so this test passes "unexpectedly". |
| # See https://bugs.webkit.org/show_bug.cgi?id=69444 . |
| tests.add( |
| 'failures/unexpected/checksum.html', |
| actual_checksum='checksum_fail-checksum') |
| |
| # Text output files contain "\r\n" on Windows. This may be |
| # helpfully filtered to "\r\r\n" by our Python/Cygwin tooling. |
| tests.add( |
| 'passes/text.html', |
| expected_text='\nfoo\n\n', |
| actual_text='\nfoo\r\n\r\r\n') |
| |
| # For reftests. |
| tests.add_reftest('passes/reftest.html', 'passes/reftest-expected.html') |
| # This adds a different virtual reference to ensure that that also works. |
| tests.add_reference('virtual/virtual_passes/passes/reftest-expected.html') |
| |
| tests.add_reftest( |
| 'passes/reftest-with-text.html', |
| 'passes/reftest-with-text-expected.html', |
| actual_text='reftest', |
| expected_text='reftest') |
| tests.add_reftest( |
| 'passes/mismatch.html', |
| 'passes/mismatch-expected-mismatch.html', |
| same_image=False) |
| tests.add_reftest('passes/svgreftest.svg', |
| 'passes/svgreftest-expected.svg') |
| tests.add_reftest('passes/xhtreftest.xht', |
| 'passes/xhtreftest-expected.html') |
| tests.add_reftest( |
| 'passes/phpreftest.php', |
| 'passes/phpreftest-expected-mismatch.svg', |
| same_image=False) |
| tests.add_reftest( |
| 'failures/expected/reftest.html', |
| 'failures/expected/reftest-expected.html', |
| same_image=False) |
| tests.add_reftest( |
| 'failures/unexpected/reftest-with-matching-text.html', |
| 'failures/unexpected/reftest-with-matching-text-expected.html', |
| same_image=False, |
| actual_text='reftest', |
| expected_text='reftest') |
| tests.add_reftest( |
| 'failures/unexpected/reftest-with-mismatching-text.html', |
| 'failures/unexpected/reftest-with-mismatching-text-expected.html', |
| actual_text='reftest', |
| expected_text='reftest-different') |
| tests.add_reftest('failures/expected/mismatch.html', |
| 'failures/expected/mismatch-expected-mismatch.html') |
| tests.add_reftest( |
| 'failures/unexpected/crash-reftest.html', |
| 'failures/unexpected/crash-reftest-expected.html', |
| crash=True) |
| tests.add_reftest( |
| 'failures/unexpected/reftest.html', |
| 'failures/unexpected/reftest-expected.html', |
| same_image=False) |
| tests.add_reftest( |
| 'failures/unexpected/reftest-mismatch-with-text-mismatch-with-stderr.html', |
| 'failures/unexpected/reftest-mismatch-with-text-mismatch-with-stderr-expected.html', |
| same_image=False, |
| actual_text='actual', |
| expected_text='expected', |
| error='oops') |
| tests.add_reftest('failures/unexpected/mismatch.html', |
| 'failures/unexpected/mismatch-expected-mismatch.html') |
| tests.add( |
| 'failures/unexpected/reftest-nopixel.html', |
| actual_checksum=None, |
| actual_image=None, |
| expected_image=None) |
| tests.add( |
| 'failures/unexpected/reftest-nopixel-expected.html', |
| actual_checksum=None, |
| actual_image=None) |
| |
| tests.add('websocket/tests/passes/text.html') |
| |
| # For testing that we don't run tests under platform/. Note that these don't contribute to TOTAL_TESTS. |
| tests.add('platform/test-mac-10.10/http/test.html') |
| tests.add('platform/test-win-win7/http/test.html') |
| |
| # For testing if perf tests are running in a locked shard. |
| tests.add('perf/foo/test.html') |
| tests.add('perf/foo/test-ref.html') |
| |
| # For testing that virtual test suites don't expand names containing themselves |
| # See webkit.org/b/97925 and base_unittest.PortTest.test_tests(). |
| tests.add('passes/test-virtual-passes.html') |
| tests.add('passes/virtual_passes/test-virtual-passes.html') |
| |
| tests.add('passes_two/test-virtual-passes.html') |
| |
| tests.add( |
| 'passes/testharness.html', |
| actual_text='This is a testharness.js-based test.\nPASS: bah\n' |
| 'Harness: the test ran to completion.', |
| expected_text=None, |
| actual_checksum=None, |
| actual_image=None, |
| expected_checksum=None, |
| expected_image=None) |
| tests.add( |
| 'failures/unexpected/testharness.html', |
| actual_text='This is a testharness.js-based test.\nFAIL: bah\n' |
| 'Harness: the test ran to completion.', |
| expected_text=None, |
| actual_checksum=None, |
| actual_image=None, |
| expected_checksum=None, |
| expected_image=None) |
| |
| tests.add('virtual/virtual_empty_bases/physical1.html') |
| tests.add('virtual/virtual_empty_bases/dir/physical2.html') |
| |
| return tests |
| |
| |
| # Here we synthesize an in-memory filesystem from the test list |
| # in order to fully control the test output and to demonstrate that |
| # we don't need a real filesystem to run the tests. |
| def add_unit_tests_to_mock_filesystem(filesystem): |
| # Add the test_expectations file. |
| filesystem.maybe_make_directory(WEB_TEST_DIR) |
| if not filesystem.exists(WEB_TEST_DIR + '/TestExpectations'): |
| filesystem.write_text_file( |
| WEB_TEST_DIR + '/TestExpectations', """ |
| # results: [ Pass Failure Crash Timeout Skip ] |
| failures/expected/audio.html [ Failure ] |
| failures/expected/crash.html [ Crash ] |
| failures/expected/crash_then_text.html [ Failure ] |
| failures/expected/device_failure.html [ Crash ] |
| failures/expected/exception.html [ Crash ] |
| failures/expected/image.html [ Failure ] |
| failures/expected/image_checksum.html [ Failure ] |
| failures/expected/keyboard.html [ Crash ] |
| failures/expected/leak.html [ Failure ] |
| failures/expected/mismatch.html [ Failure ] |
| failures/expected/newlines_leading.html [ Failure ] |
| failures/expected/newlines_trailing.html [ Failure ] |
| failures/expected/newlines_with_excess_CR.html [ Failure ] |
| failures/expected/reftest.html [ Failure ] |
| failures/expected/skip_text.html [ Skip ] |
| failures/expected/text.html [ Failure ] |
| failures/expected/timeout.html [ Timeout ] |
| failures/unexpected/pass.html [ Failure ] |
| failures/unexpected/skip_pass.html [ Skip ] |
| crbug.com/123 passes/skipped/skip.html [ Skip ] |
| passes/text.html [ Pass ] |
| virtual/skipped/failures/expected* [ Skip ] |
| """) |
| |
| if not filesystem.exists(WEB_TEST_DIR + '/NeverFixTests'): |
| filesystem.write_text_file( |
| WEB_TEST_DIR + '/NeverFixTests', """ |
| # results: [ Pass Failure Crash Timeout Skip ] |
| failures/expected/keyboard.html [ Skip ] |
| failures/expected/exception.html [ Skip ] |
| failures/expected/device_failure.html [ Skip ] |
| virtual/virtual_failures/failures/expected/keyboard.html [ Skip ] |
| virtual/virtual_failures/failures/expected/exception.html [ Skip ] |
| virtual/virtual_failures/failures/expected/device_failure.html [ Skip ] |
| """) |
| |
| if not filesystem.exists(WEB_TEST_DIR + '/SlowTests'): |
| filesystem.write_text_file( |
| WEB_TEST_DIR + '/SlowTests', """ |
| # results: [ Slow ] |
| passes/slow.html [ Slow ] |
| """) |
| |
| # FIXME: This test was only being ignored because of missing a leading '/'. |
| # Fixing the typo causes several tests to assert, so disabling the test entirely. |
| # Add in a file should be ignored by port.find_test_files(). |
| #files[WEB_TEST_DIR + '/userscripts/resources/iframe.html'] = 'iframe' |
| |
| def add_file(test, suffix, contents): |
| dirname = filesystem.join(WEB_TEST_DIR, |
| test.name[0:test.name.rfind('/')]) |
| base = test.base |
| filesystem.maybe_make_directory(dirname) |
| filesystem.write_binary_file( |
| filesystem.join(dirname, base + suffix), contents) |
| |
| # Add each test and the expected output, if any. |
| test_list = unit_test_list() |
| for test in test_list.tests.values(): |
| add_file(test, test.name[test.name.rfind('.'):], '') |
| if test.expected_audio: |
| add_file(test, '-expected.wav', test.expected_audio) |
| if test.expected_text: |
| add_file(test, '-expected.txt', test.expected_text) |
| if test.expected_image: |
| add_file(test, '-expected.png', test.expected_image) |
| |
| filesystem.write_text_file( |
| filesystem.join(WEB_TEST_DIR, 'virtual', 'virtual_passes', 'passes', |
| 'args-expected.txt'), 'args-txt --virtual-arg') |
| |
| filesystem.maybe_make_directory( |
| filesystem.join(WEB_TEST_DIR, 'external', 'wpt')) |
| filesystem.write_text_file( |
| filesystem.join(WEB_TEST_DIR, 'external', BASE_MANIFEST_NAME), |
| '{"manifest": "base"}') |
| |
| # Clear the list of written files so that we can watch what happens during testing. |
| filesystem.clear_written_files() |
| |
| |
| def add_manifest_to_mock_filesystem(port): |
| # Disable manifest update otherwise they'll be overwritten. |
| port.set_option_default('manifest_update', False) |
| filesystem = port.host.filesystem |
| filesystem.write_text_file( |
| WEB_TEST_DIR + '/external/wpt/MANIFEST.json', |
| json.dumps({ |
| 'items': { |
| 'testharness': { |
| 'dom': { |
| 'ranges': { |
| 'Range-attributes.html': ['acbdef123', [None, {}]], |
| 'Range-attributes-slow.html': |
| ['abcdef123', [None, { |
| 'timeout': 'long' |
| }]], |
| }, |
| }, |
| 'console': { |
| 'console-is-a-namespace.any.js': [ |
| 'abcdef1234', |
| ['console/console-is-a-namespace.any.html', {}], |
| [ |
| 'console/console-is-a-namespace.any.worker.html', |
| { |
| 'timeout': 'long' |
| } |
| ], |
| ], |
| }, |
| 'html': { |
| 'parse.html': [ |
| 'abcdef123', |
| ['html/parse.html?run_type=uri', {}], |
| [ |
| 'html/parse.html?run_type=write', { |
| 'timeout': 'long' |
| } |
| ], |
| ], |
| }, |
| }, |
| 'manual': {}, |
| 'reftest': { |
| 'html': { |
| 'dom': { |
| 'elements': { |
| 'global-attributes': { |
| 'dir_auto-EN-L.html': [ |
| 'abcdef123', |
| [ |
| None, |
| [[ |
| '/html/dom/elements/global-attributes/dir_auto-EN-L-ref.html', |
| '==' |
| ]], { |
| 'timeout': 'long' |
| } |
| ], |
| ] |
| } |
| } |
| } |
| } |
| }, |
| 'crashtest': { |
| 'portals': { |
| 'portals-no-frame-crash.html': |
| ['abcdef123', [None, {}]], |
| }, |
| }, |
| } |
| })) |
| filesystem.write_text_file( |
| WEB_TEST_DIR + '/external/wpt/dom/ranges/Range-attributes.html', '') |
| filesystem.write_text_file( |
| WEB_TEST_DIR + '/external/wpt/dom/ranges/Range-attributes-slow.html', |
| '') |
| filesystem.write_text_file( |
| WEB_TEST_DIR + '/external/wpt/console/console-is-a-namespace.any.js', |
| '') |
| filesystem.write_text_file( |
| WEB_TEST_DIR + '/external/wpt/common/blank.html', 'foo') |
| |
| filesystem.write_text_file( |
| WEB_TEST_DIR + '/wpt_internal/MANIFEST.json', |
| json.dumps({ |
| 'items': { |
| 'testharness': { |
| 'dom': { |
| 'bar.html': ['abcdef123', [None, {}]] |
| } |
| } |
| } |
| })) |
| filesystem.write_text_file(WEB_TEST_DIR + '/wpt_internal/dom/bar.html', |
| 'baz') |
| |
| |
| class TestPort(Port): |
| port_name = 'test' |
| default_port_name = 'test-mac-mac10.10' |
| |
| FALLBACK_PATHS = { |
| 'win7': ['test-win-win7', 'test-win-win10'], |
| 'win10': ['test-win-win10'], |
| 'mac10.10': ['test-mac-mac10.10', 'test-mac-mac10.11'], |
| 'mac10.11': ['test-mac-mac10.11'], |
| 'trusty': ['test-linux-trusty', 'test-win-win10'], |
| 'precise': |
| ['test-linux-precise', 'test-linux-trusty', 'test-win-win10'], |
| } |
| |
| @classmethod |
| def determine_full_port_name(cls, host, options, port_name): |
| if port_name == 'test': |
| return TestPort.default_port_name |
| return port_name |
| |
| def __init__(self, host, port_name=None, **kwargs): |
| Port.__init__(self, host, port_name or TestPort.default_port_name, |
| **kwargs) |
| self._tests = unit_test_list() |
| self._flakes = set() |
| |
| # FIXME: crbug.com/279494. This needs to be in the "real web tests |
| # dir" in a mock filesystem, rather than outside of the checkout, so |
| # that tests that want to write to a TestExpectations file can share |
| # this between "test" ports and "real" ports. This is the result of |
| # rebaseline_unittest.py having tests that refer to "real" port names |
| # and real builders instead of fake builders that point back to the |
| # test ports. rebaseline_unittest.py needs to not mix both "real" ports |
| # and "test" ports |
| |
| self._generic_expectations_path = WEB_TEST_DIR + '/TestExpectations' |
| self._results_directory = None |
| |
| self._operating_system = 'mac' |
| if self._name.startswith('test-win'): |
| self._operating_system = 'win' |
| elif self._name.startswith('test-linux'): |
| self._operating_system = 'linux' |
| |
| self.port_name = self._operating_system |
| |
| version_map = { |
| 'test-win-win7': 'win7', |
| 'test-win-win10': 'win10', |
| 'test-mac-mac10.10': 'mac10.10', |
| 'test-mac-mac10.11': 'mac10.11', |
| 'test-linux-precise': 'precise', |
| 'test-linux-trusty': 'trusty', |
| } |
| self._version = version_map[self._name] |
| |
| if self._operating_system == 'linux': |
| self._architecture = 'x86_64' |
| |
| self.all_systems = (('mac10.10', 'x86'), ('mac10.11', 'x86'), |
| ('win7', 'x86'), ('win10', 'x86'), |
| ('precise', 'x86_64'), ('trusty', 'x86_64')) |
| |
| self.all_build_types = ('debug', 'release') |
| |
| # To avoid surprises when introducing new macros, these are |
| # intentionally fixed in time. |
| self.configuration_specifier_macros_dict = { |
| 'mac': ['mac10.10', 'mac10.11'], |
| 'win': ['win7', 'win10'], |
| 'linux': ['precise', 'trusty'] |
| } |
| |
| def look_for_new_samples(self, crashed_processes, start_time): |
| del start_time |
| sample_files = {} |
| for cp in crashed_processes: |
| if cp[0].endswith('crash-with-sample.html'): |
| sample_file = cp[0].replace('.html', '_sample.txt') |
| self._filesystem.maybe_make_directory( |
| self._filesystem.dirname(sample_file)) |
| self._filesystem.write_binary_file(sample_file, |
| 'crash sample file') |
| sample_files[cp[0]] = sample_file |
| return sample_files |
| |
| def _flag_specific_expectations_path(self): |
| flags = [f[2:] for f in self._specified_additional_driver_flags()] |
| if not flags: |
| return None |
| return self._filesystem.join(self.web_tests_dir(), 'FlagExpectations', |
| flags[0]) |
| |
| def look_for_new_crash_logs(self, crashed_processes, start_time): |
| del start_time |
| crash_logs = {} |
| for cp in crashed_processes: |
| if cp[0].endswith('-with-delayed-log.html'): |
| crash_logs[cp[0]] = ('delayed crash log', '/tmp') |
| return crash_logs |
| |
| def _path_to_driver(self, target=None): |
| # This routine shouldn't normally be called, but it is called by |
| # the mock_drt Driver. We return something, but make sure it's useless. |
| return 'MOCK _path_to_driver' |
| |
| def default_child_processes(self): |
| return 1 |
| |
| def check_build(self, needs_http, printer): |
| return exit_codes.OK_EXIT_STATUS |
| |
| def check_sys_deps(self): |
| return exit_codes.OK_EXIT_STATUS |
| |
| def default_configuration(self): |
| return 'Release' |
| |
| def diff_image(self, |
| expected_contents, |
| actual_contents, |
| max_channel_diff=None, |
| max_pixels_diff=None): |
| diffed = actual_contents != expected_contents |
| if not actual_contents and not expected_contents: |
| return (None, None) |
| if not actual_contents or not expected_contents: |
| return (True, None) |
| if diffed: |
| return ('< %s\n---\n> %s\n' % (expected_contents, actual_contents), |
| None) |
| return (None, None) |
| |
| def web_tests_dir(self): |
| return WEB_TEST_DIR |
| |
| def _perf_tests_dir(self): |
| return PERF_TEST_DIR |
| |
| def name(self): |
| return self._name |
| |
| def operating_system(self): |
| return self._operating_system |
| |
| def default_results_directory(self): |
| return '/tmp' |
| |
| def setup_test_run(self): |
| pass |
| |
| def _driver_class(self): |
| return TestDriver |
| |
| def start_http_server(self, additional_dirs, number_of_drivers): |
| pass |
| |
| def start_websocket_server(self): |
| pass |
| |
| def acquire_http_lock(self): |
| pass |
| |
| def stop_http_server(self): |
| pass |
| |
| def stop_websocket_server(self): |
| pass |
| |
| def release_http_lock(self): |
| pass |
| |
| def path_to_apache(self): |
| return '/usr/sbin/httpd' |
| |
| def path_to_apache_config_file(self): |
| return self._filesystem.join(self.apache_config_directory(), |
| 'httpd.conf') |
| |
| def path_to_generic_test_expectations_file(self): |
| return self._generic_expectations_path |
| |
| def all_test_configurations(self): |
| """Returns a sequence of the TestConfigurations the port supports.""" |
| # By default, we assume we want to test every graphics type in |
| # every configuration on every system. |
| test_configurations = [] |
| for version, architecture in self.all_systems: |
| for build_type in self.all_build_types: |
| test_configurations.append( |
| TestConfiguration( |
| version=version, |
| architecture=architecture, |
| build_type=build_type)) |
| return test_configurations |
| |
| def configuration_specifier_macros(self): |
| return self.configuration_specifier_macros_dict |
| |
| def virtual_test_suites(self): |
| return [ |
| VirtualTestSuite(prefix='virtual_passes', |
| bases=['passes', 'passes_two'], |
| args=['--virtual-arg']), |
| VirtualTestSuite(prefix='skipped', |
| bases=['failures/expected'], |
| args=['--virtual-arg-skipped']), |
| VirtualTestSuite( |
| prefix='virtual_failures', |
| bases=['failures/expected', 'failures/unexpected'], |
| args=['--virtual-arg-failures']), |
| VirtualTestSuite(prefix='virtual_wpt', |
| bases=['external/wpt'], |
| args=['--virtual-arg-wpt']), |
| VirtualTestSuite(prefix='virtual_wpt_dom', |
| bases=['external/wpt/dom', 'wpt_internal/dom'], |
| args=['--virtual-arg-wpt-dom']), |
| VirtualTestSuite(prefix='virtual_empty_bases', |
| bases=[], |
| args=['--virtual-arg-empty-bases']), |
| VirtualTestSuite(prefix='mixed_wpt', |
| bases=['http', 'external/wpt/dom'], |
| args=['--virtual-arg']), |
| ] |
| |
| |
| class TestDriver(Driver): |
| """Test/Dummy implementation of the driver interface.""" |
| next_pid = 1 |
| |
| # pylint: disable=protected-access |
| |
| def __init__(self, *args, **kwargs): |
| super(TestDriver, self).__init__(*args, **kwargs) |
| self.started = False |
| self.pid = 0 |
| |
| def cmd_line(self, per_test_args): |
| return [self._port._path_to_driver()] + \ |
| self._port.get_option('additional_driver_flag', []) + per_test_args |
| |
| def run_test(self, driver_input): |
| if not self.started: |
| self.started = True |
| self.pid = TestDriver.next_pid |
| TestDriver.next_pid += 1 |
| |
| start_time = time.time() |
| test_name = driver_input.test_name |
| test_args = driver_input.args or [] |
| test = self._port._tests[test_name] |
| if test.keyboard: |
| raise KeyboardInterrupt |
| if test.exception: |
| raise ValueError('exception from ' + test_name) |
| if test.device_failure: |
| raise DeviceFailure('device failure in ' + test_name) |
| |
| audio = None |
| actual_text = test.actual_text |
| crash = test.crash |
| web_process_crash = test.web_process_crash |
| leak = test.leak |
| |
| if 'flaky/text.html' in test_name and not test_name in self._port._flakes: |
| self._port._flakes.add(test_name) |
| actual_text = 'flaky text failure' |
| |
| if 'crash_then_text.html' in test_name: |
| if test_name in self._port._flakes: |
| actual_text = 'text failure' |
| else: |
| self._port._flakes.add(test_name) |
| crashed_process_name = self._port.driver_name() |
| crashed_pid = 1 |
| crash = True |
| |
| if 'text_then_crash.html' in test_name: |
| if test_name in self._port._flakes: |
| crashed_process_name = self._port.driver_name() |
| crashed_pid = 1 |
| crash = True |
| else: |
| self._port._flakes.add(test_name) |
| actual_text = 'text failure' |
| |
| if actual_text and test_args and test_name == 'passes/args.html': |
| actual_text = actual_text + ' ' + ' '.join(test_args) |
| |
| if test.actual_audio: |
| audio = base64.b64decode(test.actual_audio) |
| crashed_process_name = None |
| crashed_pid = None |
| |
| leak_log = '' |
| if leak: |
| leak_log = 'leak detected' |
| |
| crash_log = '' |
| if crash: |
| crashed_process_name = self._port.driver_name() |
| crashed_pid = 1 |
| crash_log = 'crash log' |
| elif web_process_crash: |
| crashed_process_name = 'WebProcess' |
| crashed_pid = 2 |
| crash_log = 'web process crash log' |
| |
| if crashed_process_name: |
| crash_logs = CrashLogs(self._port.host) |
| crash_log = crash_logs.find_newest_log(crashed_process_name, |
| None) or crash_log |
| |
| if 'crash-reftest.html' in test_name: |
| crashed_process_name = self._port.driver_name() |
| crashed_pid = 3 |
| crash = True |
| crash_log = 'reftest crash log' |
| if test.actual_checksum == driver_input.image_hash: |
| image = None |
| else: |
| image = test.actual_image |
| return DriverOutput( |
| actual_text, |
| image, |
| test.actual_checksum, |
| audio, |
| crash=(crash or web_process_crash), |
| crashed_process_name=crashed_process_name, |
| crashed_pid=crashed_pid, |
| crash_log=crash_log, |
| test_time=time.time() - start_time, |
| timeout=test.timeout, |
| error=test.error, |
| pid=self.pid, |
| leak=test.leak, |
| leak_log=leak_log) |
| |
| def stop(self, timeout_secs=0.0): |
| self.started = False |
