[rb] Generate per-browser test targets

Author: p0deje

.github/workflows/ci-ruby.yml

@@ -56,7 +56,11 @@ jobs:
       cache-key: rb-unit-test-${{ matrix.ruby-version }}
       os: ${{ matrix.os }}
       ruby-version: ${{ matrix.ruby-version }}
-      run: bazel test //rb/spec/unit/...
+      run: >
+        bazel test
+        --build_tests_only
+        --test_size_filters small
+        //rb/spec/...
 
   integration-tests-local:
     name: Local Tests
@@ -90,10 +94,12 @@ jobs:
       os: ${{ matrix.os }}
       run: >
         bazel test
-        --define browser=${{ matrix.browser }}
+        --build_tests_only
         --flaky_test_attempts 3
         --local_test_jobs 1
-        //rb/spec/integration/...
+        --test_size_filters large
+        --test_tag_filters ${{ matrix.browser }}
+        //rb/spec/...
 
   integration-tests-remote:
     name: Remote Tests
@@ -119,8 +125,9 @@ jobs:
       java-version: 11
       run: >
         bazel test
-        --define browser=${{ matrix.browser }}
-        --define remote=true
+        --build_tests_only
         --flaky_test_attempts 3
         --local_test_jobs 1
-        //rb/spec/integration/...
+        --test_size_filters large
+        --test_tag_filters ${{ matrix.browser }}-remote
+        //rb/spec/...

.skipped-tests

@@ -31,6 +31,7 @@
 -//py:test-chrome-test/selenium/webdriver/chrome/chrome_service_tests.py
 -//py:test-chrome-test/selenium/webdriver/chrome/proxy_tests.py
 -//py:unit-test/unit/selenium/webdriver/common/cdp_module_fallback_tests.py
--//rb/spec/integration/selenium/webdriver/chrome:service
--//rb/spec/integration/selenium/webdriver/edge:service
--//rb/spec/integration/selenium/webdriver/firefox:service
+-//rb/spec/integration/selenium/webdriver/chrome:service-chrome
+-//rb/spec/integration/selenium/webdriver/edge:service-edge
+-//rb/spec/integration/selenium/webdriver/firefox:service-firefox
+-//rb/spec/integration/selenium/webdriver/firefox:service-firefox-beta

README.md

@@ -334,37 +334,45 @@ bazel test //py:all
 
 Test targets:
 
-| Command                                                                              | Description                                    |
-|--------------------------------------------------------------------------------------|------------------------------------------------|
-| `bazel test //rb/...`                                                                | Run unit, integration tests (Chrome) and lint  |
-| `bazel test //rb:lint`                                                               | Run RuboCop linter                             |
-| `bazel test //rb/spec/...`                                                           | Run unit and integration tests (Chrome)        |
-| `bazel test --test_size_filters large //rb/...`                                      | Run integration tests using (Chrome)           |
-| `bazel test //rb/spec/integration/...`                                               | Run integration tests using (Chrome)           |
-| `bazel test //rb/spec/integration/... --define browser=firefox`                      | Run integration tests using (Firefox)          |
-| `bazel test //rb/spec/integration/... --define remote=true`                          | Run integration tests using (Chrome and Grid)  |
-| `bazel test //rb/spec/integration/... --define browser=firefox --define remote=true` | Run integration tests using (Firefox and Grid) |
-| `bazel test --test_size_filters small //rb/...`                                      | Run unit tests                                 |
-| `bazel test //rb/spec/unit/...`                                                      | Run unit tests                                 |
-
-Ruby test modules have the same name as the spec file with `_spec.rb` removed, so you can run them individually:
-
-| Test file                                                      | Test target                                              |
-|----------------------------------------------------------------|----------------------------------------------------------|
-| `rb/spec/integration/selenium/webdriver/chrome/driver_spec.rb` | `//rb/spec/integration/selenium/webdriver/chrome:driver` |
-| `rb/spec/unit/selenium/webdriver/proxy_spec.rb`                | `//rb/spec/unit/selenium/webdriver:proxy`                |
+| Command                                                                          | Description                                        |
+| -------------------------------------------------------------------------------- | -------------------------------------------------- |
+| `bazel test //rb/...`                                                            | Run unit, all integration tests and lint           |
+| `bazel test //rb:lint`                                                           | Run RuboCop linter                                 |
+| `bazel test //rb/spec/...`                                                       | Run unit and integration tests for all browsers    |
+| `bazel test //rb/spec/... --test_size_filters small`                             | Run unit tests                                     |
+| `bazel test //rb/spec/unit/...`                                                  | Run unit tests                                     |
+| `bazel test //rb/spec/... --test_size_filters large`                             | Run integration tests for all browsers             |
+| `bazel test //rb/spec/integration/...`                                           | Run integration tests for all browsers             |
+| `bazel test //rb/spec/integration/... --test_tag_filters firefox`                | Run integration tests for local Firefox only       |
+| `bazel test //rb/spec/integration/... --test_tag_filters firefox-remote`         | Run integration tests for remote Firefox only      |
+| `bazel test //rb/spec/integration/... --test_tag_filters firefox,firefox-remote` | Run integration tests for local and remote Firefox |
+
+Ruby test targets have the same name as the spec file with `_spec.rb` removed, so you can run them individually.
+Integration tests targets also have a browser and remote suffix to control which browser to pick and whether to use Grid.
+
+| Test file                                               | Test target                                                      |
+| ------------------------------------------------------- | ---------------------------------------------------------------- |
+| `rb/spec/unit/selenium/webdriver/proxy_spec.rb`         | `//rb/spec/unit/selenium/webdriver:proxy`                        |
+| `rb/spec/integration/selenium/webdriver/driver_spec.rb` | `//rb/spec/integration/selenium/webdriver:driver-chrome`         |
+| `rb/spec/integration/selenium/webdriver/driver_spec.rb` | `//rb/spec/integration/selenium/webdriver:driver-chrome-remote`  |
+| `rb/spec/integration/selenium/webdriver/driver_spec.rb` | `//rb/spec/integration/selenium/webdriver:driver-firefox`        |
+| `rb/spec/integration/selenium/webdriver/driver_spec.rb` | `//rb/spec/integration/selenium/webdriver:driver-firefox-remote` |
 
 Supported browsers:
 
 * `chrome`
 * `edge`
 * `firefox`
+* `firefox-beta`
 * `ie`
-* `safari` (cannot be run in parallel - use `--local_test_jobs 1`)
-* `safari-preview` (cannot be run in parallel - use `--local_test_jobs 1`)
+* `safari`
+* `safari-preview`
 
 Useful command line options:
 
+* `--pin_browsers` - use browsers and drivers downloaded by Bazel
+* `--headless` - run browsers in headless mode (supported be Chrome, Edge and Firefox)
+
 In addition to the [Common Options Examples](#common-options-examples), here are some additional Ruby specific ones:
 * `--test_arg "-tfocus"` - test only [focused specs](https://relishapp.com/rspec/rspec-core/v/3-12/docs/filtering/inclusion-filters)
 * `--test_arg "-eTimeouts"` - test only specs which name include "Timeouts"
@@ -380,8 +388,11 @@ Supported environment variables for use with `--test_env`:
 - `HEADLESS` - for chrome, edge and firefox; runs tests in headless mode
 - `DISABLE_BUILD_CHECK` - for chrome and edge; whether to ignore driver and browser version mismatches (allows testing Canary builds)
 - `CHROME_BINARY` - path to test specific Chrome browser
+- `CHROMEDRIVER_BINARY` - path to test specific ChromeDriver
 - `EDGE_BINARY` - path to test specific Edge browser
+- `MSEDGEDRIVER_BINARY` - path to test specific msedgedriver
 - `FIREFOX_BINARY` - path to test specific Firefox browser
+- `GECKODRIVER_BINARY` - path to test specific GeckoDriver
 
 To run with a specific version of Ruby you can change the version in `rb/.ruby-version` or from command line:
 

common/browsers.bzl

@@ -1,5 +1,8 @@
 COMMON_TAGS = [
     "browser-test",
+    # We have to use no-sandbox at the moment because Firefox crashes
+    # when run under sandbox: https://bugzilla.mozilla.org/show_bug.cgi?id=1382498.
+    # For Chromium-based browser, we can just pass `--no-sandbox` flag.
     "no-sandbox",
     "requires-network",
 ]

common/extensions/BUILD.bazel

@@ -11,7 +11,7 @@ filegroup(
         "//java/test/org/openqa/selenium/firefox:__pkg__",
         "//javascript/node/selenium-webdriver:__pkg__",
         "//py:__pkg__",
-        "//rb:__pkg__",
+        "//rb/spec:__subpackages__",
     ],
 )
 

rb/spec/BUILD.bazel

@@ -8,14 +8,14 @@ rb_library(
 )
 
 # List of dependencies can be gathered by running:
-# bazel query 'kind("rb_.* rule", //rb/spec/...) except //rb/spec:spec' | xargs -I{} echo '"{}",'
+# bazel query 'kind("rb_.* rule", //rb/spec/...) except attr(tags, "browser-test", //rb/spec/...) except //rb/spec:spec' | xargs -I{} echo '"{}",'
 
 rb_library(
     name = "spec",
     testonly = True,
     visibility = ["//rb:__pkg__"],
     deps = [
-        ":rspec_matchers",
+        "//rb/spec:rspec_matchers",
         "//rb/spec/integration/selenium/webdriver:action_builder",
         "//rb/spec/integration/selenium/webdriver:bidi",
         "//rb/spec/integration/selenium/webdriver:devtools",
@@ -52,6 +52,7 @@ rb_library(
         "//rb/spec/integration/selenium/webdriver/remote:driver",
         "//rb/spec/integration/selenium/webdriver/remote:element",
         "//rb/spec/integration/selenium/webdriver/safari:driver",
+        "//rb/spec/unit/selenium:devtools",
         "//rb/spec/unit/selenium:server",
         "//rb/spec/unit/selenium/devtools:cdp_client_generator",
         "//rb/spec/unit/selenium/webdriver:file_reaper",
@@ -67,6 +68,7 @@ rb_library(
         "//rb/spec/unit/selenium/webdriver/chrome:service",
         "//rb/spec/unit/selenium/webdriver/common:action_builder",
         "//rb/spec/unit/selenium/webdriver/common:credentials",
+        "//rb/spec/unit/selenium/webdriver/common:driver_finder",
         "//rb/spec/unit/selenium/webdriver/common:logger",
         "//rb/spec/unit/selenium/webdriver/common:selenium_manager",
         "//rb/spec/unit/selenium/webdriver/common:service",

rb/spec/integration/selenium/webdriver/BUILD.bazel

@@ -12,10 +12,7 @@ rb_library(
         "//rb:manager-linux",
         "//rb:manager-macos",
         "//rb:manager-windows",
-    ] + select({
-        "//rb/spec/integration:remote": ["//java/src/org/openqa/selenium/grid:selenium_server_deploy.jar"],
-        "//conditions:default": [],
-    }),
+    ],
     visibility = ["//rb/spec:__subpackages__"],
     deps = [
         "//rb/lib:selenium-webdriver",
@@ -27,14 +24,7 @@ rb_library(
         "//rb/lib/selenium/webdriver:remote",
         "//rb/spec:rspec_matchers",
         "@bundle",
-    ] + select({
-        "//rb/spec/integration:edge": ["//rb/lib/selenium/webdriver:edge"],
-        "//rb/spec/integration:firefox": ["//rb/lib/selenium/webdriver:firefox"],
-        "//rb/spec/integration:ie": ["//rb/lib/selenium/webdriver:ie"],
-        "//rb/spec/integration:safari": ["//rb/lib/selenium/webdriver:safari"],
-        "//rb/spec/integration:safari-preview": ["//rb/lib/selenium/webdriver:safari"],
-        "//conditions:default": ["//rb/lib/selenium/webdriver:chrome"],
-    }),
+    ],
 )
 
 [
@@ -79,25 +69,9 @@ rb_integration_test(
 rb_integration_test(
     name = "driver",
     srcs = ["driver_spec.rb"],
-    deps = [
-        "//rb/lib/selenium/webdriver:chrome",
-        "//rb/lib/selenium/webdriver:edge",
-        "//rb/lib/selenium/webdriver:firefox",
-        "//rb/lib/selenium/webdriver:ie",
-        "//rb/lib/selenium/webdriver:remote",
-        "//rb/lib/selenium/webdriver:safari",
-    ],
 )
 
 rb_integration_test(
     name = "element",
     srcs = ["element_spec.rb"],
-    deps = [
-        "//rb/lib/selenium/webdriver:chrome",
-        "//rb/lib/selenium/webdriver:edge",
-        "//rb/lib/selenium/webdriver:firefox",
-        "//rb/lib/selenium/webdriver:ie",
-        "//rb/lib/selenium/webdriver:remote",
-        "//rb/lib/selenium/webdriver:safari",
-    ],
 )

rb/spec/integration/selenium/webdriver/chrome/BUILD.bazel

@@ -4,7 +4,7 @@ load("//rb/spec:tests.bzl", "rb_integration_test")
     rb_integration_test(
         name = file[:-8],
         srcs = [file],
-        deps = ["//rb/lib/selenium/webdriver:chrome"],
+        browsers = ["chrome"],  # No need to run in other browsers.
     )
     for file in glob(["*_spec.rb"])
 ]

rb/spec/integration/selenium/webdriver/edge/BUILD.bazel

@@ -4,7 +4,7 @@ load("//rb/spec:tests.bzl", "rb_integration_test")
     rb_integration_test(
         name = file[:-8],
         srcs = [file],
-        deps = ["//rb/lib/selenium/webdriver:edge"],
+        browsers = ["edge"],  # No need to run in other browsers.
     )
     for file in glob(["*_spec.rb"])
 ]

rb/spec/integration/selenium/webdriver/firefox/BUILD.bazel

@@ -4,7 +4,12 @@ load("//rb/spec:tests.bzl", "rb_integration_test")
     rb_integration_test(
         name = file[:-8],
         srcs = [file],
-        deps = ["//rb/lib/selenium/webdriver:firefox"],
+        # No need to run in other browsers.
+        browsers = [
+            "firefox",
+            "firefox-beta",
+        ],
+        data = ["//common/extensions"],
     )
     for file in glob(["*_spec.rb"])
 ]