Add web test expectation removal builders module
Adds blinkpy.web_tests.stale_expectation_removal.builders and its
associated unittests.
As a side-effect, also updates some of the shared and GPU code to
accept an iterable for a certain field instead of a single
string.
Bug: 1222827
Change-Id: Ib06549f77b69b2a0400dafddd087121810dc1887
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3069392
Auto-Submit: Brian Sheedy <[email protected]>
Commit-Queue: Bruce Dawson <[email protected]>
Reviewed-by: Bruce Dawson <[email protected]>
Reviewed-by: Xianzhu Wang <[email protected]>
Cr-Commit-Position: refs/heads/master@{#908898}
diff --git a/BUILD.gn b/BUILD.gn
index 291e3be..3bb7c25 100644
--- a/BUILD.gn
+++ b/BUILD.gn
@@ -1325,6 +1325,7 @@
"//build/gn_helpers.py",
"//build/config/gclient_args.gni",
"//components/crash/content/tools/generate_breakpad_symbols.py",
+ "//testing/unexpected_passes_common/",
"//third_party/blink/renderer/bindings/scripts/",
"//third_party/blink/renderer/build/scripts/",
"//third_party/blink/tools/",
diff --git a/content/test/gpu/unexpected_passes/gpu_builders.py b/content/test/gpu/unexpected_passes/gpu_builders.py
index 89d3c23..214472c 100644
--- a/content/test/gpu/unexpected_passes/gpu_builders.py
+++ b/content/test/gpu/unexpected_passes/gpu_builders.py
@@ -81,7 +81,7 @@
self._fake_ci_builders = {}
for try_builder, ci_builder_list in fake_try_builders.items():
for ci in ci_builder_list:
- self._fake_ci_builders[ci] = try_builder
+ self._fake_ci_builders.setdefault(ci, set()).add(try_builder)
return self._fake_ci_builders
diff --git a/content/test/gpu/unexpected_passes/gpu_builders_unittest.py b/content/test/gpu/unexpected_passes/gpu_builders_unittest.py
index 1b81ba7..5aaeaa2f 100755
--- a/content/test/gpu/unexpected_passes/gpu_builders_unittest.py
+++ b/content/test/gpu/unexpected_passes/gpu_builders_unittest.py
@@ -66,3 +66,7 @@
if 'telemetry_gpu_integration_test' in isolate and 'android' in isolate:
return
self.fail('Did not find any Android-specific isolate names')
+
+
+if __name__ == '__main__':
+ unittest.main(verbosity=2)
diff --git a/testing/unexpected_passes_common/__init__.py b/testing/unexpected_passes_common/__init__.py
index fa6e05e2..8a659964 100644
--- a/testing/unexpected_passes_common/__init__.py
+++ b/testing/unexpected_passes_common/__init__.py
@@ -6,7 +6,8 @@
import sys
CHROMIUM_SRC_DIR = os.path.join(os.path.dirname(__file__), '..', '..')
-sys.path.append(os.path.join(CHROMIUM_SRC_DIR, 'content', 'test', 'gpu'))
+TYP_PATH = os.path.join(CHROMIUM_SRC_DIR, 'third_party', 'catapult',
+ 'third_party', 'typ')
-from gpu_tests import path_util
-path_util.SetupTypPath()
\ No newline at end of file
+if TYP_PATH not in sys.path:
+ sys.path.append(TYP_PATH)
diff --git a/testing/unexpected_passes_common/builders.py b/testing/unexpected_passes_common/builders.py
index 77bc43c..a15cc63 100644
--- a/testing/unexpected_passes_common/builders.py
+++ b/testing/unexpected_passes_common/builders.py
@@ -144,7 +144,7 @@
fake_builders = self.GetFakeCiBuilders()
if ci_builder in fake_builders:
- mirrored_builders.add(fake_builders[ci_builder])
+ mirrored_builders |= fake_builders[ci_builder]
logging.debug('%s is a fake CI builder mirrored by %s', ci_builder,
fake_builders[ci_builder])
return mirrored_builders, True
@@ -213,9 +213,9 @@
"""Gets a mapping of fake CI builders to their mirrored trybots.
Returns:
- A dict of string -> string. Each key is a CI builder that doesn't actually
- exist and each value is a try builder that mirrors the CI builder but does
- exist.
+ A dict of string -> set(string). Each key is a CI builder that doesn't
+ actually exist and each value is a set of try builders that mirror the CI
+ builder but do exist.
"""
raise NotImplementedError()
diff --git a/testing/unexpected_passes_common/builders_unittest.py b/testing/unexpected_passes_common/builders_unittest.py
index 91f3924..f0b106d 100755
--- a/testing/unexpected_passes_common/builders_unittest.py
+++ b/testing/unexpected_passes_common/builders_unittest.py
@@ -172,7 +172,7 @@
def testFakeCiBuilder(self):
"""Tests that a fake CI builder gets properly mapped."""
- self._fake_ci_mock.return_value = {'foo_ci': 'foo_try'}
+ self._fake_ci_mock.return_value = {'foo_ci': {'foo_try'}}
try_builder, found_mirror = (
self._builders_instance._GetMirroredBuildersForCiBuilder('foo_ci'))
self.assertTrue(found_mirror)
diff --git a/third_party/blink/tools/blinkpy/web_tests/stale_expectation_removal/__init__.py b/third_party/blink/tools/blinkpy/web_tests/stale_expectation_removal/__init__.py
index e69de29..faf03ce 100644
--- a/third_party/blink/tools/blinkpy/web_tests/stale_expectation_removal/__init__.py
+++ b/third_party/blink/tools/blinkpy/web_tests/stale_expectation_removal/__init__.py
@@ -0,0 +1,14 @@
+# Copyright 2021 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import sys
+
+CHROMIUM_SRC_DIR = os.path.realpath(
+ os.path.join(os.path.dirname(__file__), '..', '..', '..', '..', '..',
+ '..'))
+TESTING_DIR = os.path.join(CHROMIUM_SRC_DIR, 'testing')
+
+if TESTING_DIR not in sys.path:
+ sys.path.append(TESTING_DIR)
diff --git a/third_party/blink/tools/blinkpy/web_tests/stale_expectation_removal/builders.py b/third_party/blink/tools/blinkpy/web_tests/stale_expectation_removal/builders.py
new file mode 100644
index 0000000..5502514
--- /dev/null
+++ b/third_party/blink/tools/blinkpy/web_tests/stale_expectation_removal/builders.py
@@ -0,0 +1,65 @@
+# Copyright 2021 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Web test-specific impl of the unexpected passes' builders module."""
+
+from unexpected_passes_common import builders
+
+
+class WebTestBuilders(builders.Builders):
+ def _BuilderRunsTestOfInterest(self, test_map, _):
+ tests = test_map.get('isolated_scripts', [])
+ for t in tests:
+ if t.get('isolate_name') in self.GetIsolateNames():
+ return True
+ return False
+
+ def GetIsolateNames(self):
+ return {
+ 'blink_web_tests',
+ 'webgpu_blink_web_tests',
+ }
+
+ def GetFakeCiBuilders(self):
+ return {
+ # chromium.fyi
+ 'linux-blink-rel-dummy': {
+ 'linux-blink-rel',
+ 'v8_linux_blink_rel',
+ },
+ 'mac10.12-blink-rel-dummy': {
+ 'mac10.12-blink-rel',
+ },
+ 'mac10.13-blink-rel-dummy': {
+ 'mac10.13-blink-rel',
+ },
+ 'mac10.14-blink-rel-dummy': {
+ 'mac10.14-blink-rel',
+ },
+ 'mac10.15-blink-rel-dummy': {
+ 'mac10.15-blink-rel',
+ },
+ 'mac11.0-blink-rel-dummy': {
+ 'mac11.0-blink-rel',
+ },
+ 'WebKit Linux composite_after_paint Dummy Builder': {
+ 'linux_layout_tests_composite_after_paint',
+ },
+ 'WebKit Linux layout_ng_disabled Builder': {
+ 'linux_layout_tests_layout_ng_disabled',
+ },
+ 'win7-blink-rel-dummy': {
+ 'win7-blink-rel',
+ },
+ 'win10-blink-rel-dummy': {
+ 'win10-blink-rel',
+ },
+ 'win10.20h2-blink-rel-dummy': {
+ 'win10.20h2-blink-rel',
+ },
+ }
+
+ def GetNonChromiumBuilders(self):
+ return {
+ 'DevTools Linux (chromium)',
+ }
diff --git a/third_party/blink/tools/blinkpy/web_tests/stale_expectation_removal/builders_unittest.py b/third_party/blink/tools/blinkpy/web_tests/stale_expectation_removal/builders_unittest.py
new file mode 100755
index 0000000..a1ff905
--- /dev/null
+++ b/third_party/blink/tools/blinkpy/web_tests/stale_expectation_removal/builders_unittest.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env vpython3
+# Copyright 2021 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import unittest
+
+from blinkpy.web_tests.stale_expectation_removal import builders
+
+
+class BuilderRunsTestOfInterestUnittest(unittest.TestCase):
+ def setUp(self):
+ self.instance = builders.WebTestBuilders()
+
+ def testMatch(self):
+ """Tests that a match can be successfully found."""
+ test_map = {
+ 'isolated_scripts': [
+ {
+ 'isolate_name': 'blink_web_tests',
+ },
+ ],
+ }
+ self.assertTrue(
+ self.instance._BuilderRunsTestOfInterest(test_map, None))
+
+ test_map = {
+ 'isolated_scripts': [
+ {
+ 'isolate_name': 'webgpu_blink_web_tests',
+ },
+ ],
+ }
+ self.assertTrue(
+ self.instance._BuilderRunsTestOfInterest(test_map, None))
+
+ def testNoMatch(self):
+ test_map = {
+ 'isolated_scripts': [
+ {
+ 'isolate_name': 'foo_web_tests',
+ },
+ ],
+ }
+ self.assertFalse(
+ self.instance._BuilderRunsTestOfInterest(test_map, None))
+
+
+if __name__ == '__main__':
+ unittest.main(verbosity=2)
diff --git a/third_party/blink/tools/blinkpy/web_tests/stale_expectation_removal/remove_stale_expectations.py b/third_party/blink/tools/blinkpy/web_tests/stale_expectation_removal/remove_stale_expectations.py
index ffa6fd7..fdea36a 100644
--- a/third_party/blink/tools/blinkpy/web_tests/stale_expectation_removal/remove_stale_expectations.py
+++ b/third_party/blink/tools/blinkpy/web_tests/stale_expectation_removal/remove_stale_expectations.py
@@ -8,6 +8,9 @@
assert sys.version_info[0] == 3
+from blinkpy.web_tests.stale_expectation_removal import builders
+from unexpected_passes_common import builders as common_builders
+
def ParseArgs():
parser = argparse.ArgumentParser(description=(
@@ -78,6 +81,8 @@
raise RuntimeError(
'Script is still under active development and not currently functional'
)
+ builders_instance = builders.WebTestBuilders()
+ common_builders.RegisterInstance(builders_instance)
return 0
