intermediate commit

google/cloud/storage/_helpers.py

@@ -30,20 +30,28 @@
 from google.cloud.storage.retry import DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED
 
 
-STORAGE_EMULATOR_ENV_VAR = "STORAGE_EMULATOR_HOST"
+STORAGE_EMULATOR_ENV_VAR = "STORAGE_EMULATOR_HOST"  # Despite name, includes scheme.
 """Environment variable defining host for Storage emulator."""
 
-_API_ENDPOINT_OVERRIDE_ENV_VAR = "API_ENDPOINT_OVERRIDE"
+_API_ENDPOINT_OVERRIDE_ENV_VAR = "API_ENDPOINT_OVERRIDE"  # Includes scheme
 """This is an experimental configuration variable. Use api_endpoint instead."""
 
 _API_VERSION_OVERRIDE_ENV_VAR = "API_VERSION_OVERRIDE"
 """This is an experimental configuration variable used for internal testing."""
 
+_DEFAULT_UNIVERSE_DOMAIN = "googleapis.com"
+
+_STORAGE_HOST_TEMPLATE = "storage.{universe_domain}"
+
+_TRUE_DEFAULT_STORAGE_HOST = _STORAGE_HOST_TEMPLATE.format(universe_domain=_DEFAULT_UNIVERSE_DOMAIN)
+
 _DEFAULT_STORAGE_HOST = os.getenv(
-    _API_ENDPOINT_OVERRIDE_ENV_VAR, "https://storage.googleapis.com"
+    _API_ENDPOINT_OVERRIDE_ENV_VAR, _TRUE_DEFAULT_STORAGE_HOST
 )
 """Default storage host for JSON API."""
 
+_DEFAULT_SCHEME = "https://"
+
 _API_VERSION = os.getenv(_API_VERSION_OVERRIDE_ENV_VAR, "v1")
 """API version of the default storage host"""
 
@@ -72,8 +80,23 @@
 )
 
 
-def _get_storage_host():
-    return os.environ.get(STORAGE_EMULATOR_ENV_VAR, _DEFAULT_STORAGE_HOST)
+def _get_storage_emulator_override():
+    return os.environ.get(STORAGE_EMULATOR_ENV_VAR, None)
+
+
+def _get_api_endpoint_override():
+    """This is an experimental configuration variable. Use api_endpoint instead."""
+    if _DEFAULT_STORAGE_HOST != _TRUE_DEFAULT_STORAGE_HOST:
+        return DEFAULT_SCHEME + _DEFAULT_STORAGE_HOST
+    return None
+
+
+def _get_default_storage_base_url():
+    return _DEFAULT_SCHEME + _DEFAULT_STORAGE_HOST
+
+
+def _use_client_cert():
+    return os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE") == "true"
 
 
 def _get_environ_project():

google/cloud/storage/_http.py

@@ -21,8 +21,14 @@
 
 
 class Connection(_http.JSONConnection):
-    """A connection to Google Cloud Storage via the JSON REST API. Mutual TLS feature will be
-    enabled if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "true".
+    """A connection to Google Cloud Storage via the JSON REST API.
+
+    Mutual TLS will be enabled if the "GOOGLE_API_USE_CLIENT_CERTIFICATE"
+    environment variable is set to the exact string "true" (case-sensitive).
+
+    Mutual TLS is not compatible with any API endpoint or universe domain
+    override at this time. If such settings are enabled along with
+    "GOOGLE_API_USE_CLIENT_CERTIFICATE", a ValueError will be raised.
 
     :type client: :class:`~google.cloud.storage.client.Client`
     :param client: The client that owns the current connection.
@@ -31,10 +37,11 @@ class Connection(_http.JSONConnection):
     :param client_info: (Optional) instance used to generate user agent.
 
     :type api_endpoint: str
-    :param api_endpoint: (Optional) api endpoint to use.
+    :param api_endpoint: (Optional) api endpoint to use. This may be a universe
+        domain if one was specified.
     """
 
-    DEFAULT_API_ENDPOINT = _helpers._DEFAULT_STORAGE_HOST
+    DEFAULT_API_ENDPOINT = _helpers._get_default_storage_base_url()
     DEFAULT_API_MTLS_ENDPOINT = "https://storage.mtls.googleapis.com"
 
     def __init__(self, client, client_info=None, api_endpoint=None):

google/cloud/storage/blob.py

@@ -57,10 +57,10 @@
 from google.cloud.storage._helpers import _raise_if_more_than_one_set
 from google.cloud.storage._helpers import _api_core_retry_to_resumable_media_retry
 from google.cloud.storage._helpers import _get_default_headers
+from google.cloud.storage._helpers import _get_default_storage_base_url
 from google.cloud.storage._signing import generate_signed_url_v2
 from google.cloud.storage._signing import generate_signed_url_v4
 from google.cloud.storage._helpers import _NUM_RETRIES_MESSAGE
-from google.cloud.storage._helpers import _DEFAULT_STORAGE_HOST
 from google.cloud.storage._helpers import _API_VERSION
 from google.cloud.storage.acl import ACL
 from google.cloud.storage.acl import ObjectACL
@@ -80,7 +80,7 @@
 from google.cloud.storage.fileio import BlobWriter
 
 
-_API_ACCESS_ENDPOINT = _DEFAULT_STORAGE_HOST
+_API_ACCESS_ENDPOINT = _get_default_storage_base_url()
 _DEFAULT_CONTENT_TYPE = "application/octet-stream"
 _DOWNLOAD_URL_TEMPLATE = "{hostname}/download/storage/{api_version}{path}?alt=media"
 _BASE_UPLOAD_TEMPLATE = (

google/cloud/storage/client.py

@@ -31,9 +31,12 @@
 from google.cloud.exceptions import NotFound
 
 from google.cloud.storage._helpers import _get_environ_project
-from google.cloud.storage._helpers import _get_storage_host
-from google.cloud.storage._helpers import _DEFAULT_STORAGE_HOST
+from google.cloud.storage._helpers import _get_storage_emulator_override
+from google.cloud.storage._helpers import _get_api_endpoint_override
+from google.cloud.storage._helpers import _STORAGE_HOST_TEMPLATE
 from google.cloud.storage._helpers import _bucket_bound_hostname_url
+from google.cloud.storage._helpers import _DEFAULT_UNIVERSE_DOMAIN
+from google.cloud.storage._helpers import _DEFAULT_SCHEME
 
 from google.cloud.storage._http import Connection
 from google.cloud.storage._signing import (
@@ -87,7 +90,7 @@ class Client(ClientWithProject):
 
     :type client_options: :class:`~google.api_core.client_options.ClientOptions` or :class:`dict`
     :param client_options: (Optional) Client options used to set user options on the client.
-        API Endpoint should be set through client_options.
+        A non-default universe domain or api endpoint should be set through client_options.
 
     :type use_auth_w_custom_endpoint: bool
     :param use_auth_w_custom_endpoint:
@@ -135,32 +138,78 @@ def __init__(
         self._initial_client_options = client_options
         self._extra_headers = extra_headers
 
-        kw_args = {"client_info": client_info}
-
-        # `api_endpoint` should be only set by the user via `client_options`,
-        # or if the _get_storage_host() returns a non-default value (_is_emulator_set).
-        # `api_endpoint` plays an important role for mTLS, if it is not set,
-        # then mTLS logic will be applied to decide which endpoint will be used.
-        storage_host = _get_storage_host()
-        _is_emulator_set = storage_host != _DEFAULT_STORAGE_HOST
-        kw_args["api_endpoint"] = storage_host if _is_emulator_set else None
+        connection_kw_args = {"client_info": client_info}
 
         if client_options:
             if isinstance(client_options, dict):
                 client_options = google.api_core.client_options.from_dict(
                     client_options
                 )
-            if client_options.api_endpoint:
-                api_endpoint = client_options.api_endpoint
-                kw_args["api_endpoint"] = api_endpoint
+
+        if client_options and client_options.universe_domain:
+            self._universe_domain = client_options.universe_domain
+        else:
+            self._universe_domain = None
+
+        storage_emulator_override = _get_storage_emulator_override()
+        api_endpoint_override = _get_api_endpoint_override()
+
+        # Determine the api endpoint. The rules are as follows:
+
+        # 1. If the `api_endpoint` is set in `client_options`, use that as the
+        #    endpoint.
+        if client_options and client_options.api_endpoint:
+            api_endpoint = client_options.api_endpoint
+
+        # 2. Elif the "STORAGE_EMULATOR_HOST" env var is set, then use that as the
+        #    endpoint.
+        elif storage_emulator_override:
+            api_endpoint = storage_emulator_override
+
+        # 3. Elif the "API_ENDPOINT_OVERRIDE" env var is set, then use that as the
+        #    endpoint.
+        elif api_endpoint_override:
+            api_endpoint = api_endpoint_override
+
+        # 4. Elif the `universe_domain` is set in `client_options`,
+        #    create the endpoint using that as the default.
+        #
+        #    Mutual TLS is not compatible with a non-default universe domain
+        #    at this time. If such settings are enabled along with the
+        #    "GOOGLE_API_USE_CLIENT_CERTIFICATE" env variable, a ValueError will
+        #    be raised.
+
+        elif self._universe_domain:
+            # The final decision of whether to use mTLS takes place in
+            # google-auth-library-python. We peek at the environment variable
+            # here only to issue an exception in case of a conflict.
+            if helpers._use_client_cert():
+                raise ValueError(
+                    "The \"GOOGLE_API_USE_CLIENT_CERTIFICATE\" env variable is "
+                    "set to \"true\" and a non-default universe domain is "
+                    "configured. mTLS is not supported in any universe other than"
+                    "googleapis.com.")
+            api_endpoint = _DEFAULT_SCHEME + _STORAGE_HOST_TEMPLATE.format(
+                universe_domain=self._universe_domain
+            )
+
+        # 5. Else, use the default, which is to use the default
+        #    universe domain of "googleapis.com" and create the endpoint
+        #    "storage.googleapis.com" from that.
+        else:
+            api_endpoint = None
+
+        connection_kw_args["api_endpoint"] = api_endpoint
+
+        self._is_emulator_set = True if storage_emulator_override else False
 
         # If a custom endpoint is set, the client checks for credentials
         # or finds the default credentials based on the current environment.
         # Authentication may be bypassed under certain conditions:
         # (1) STORAGE_EMULATOR_HOST is set (for backwards compatibility), OR
         # (2) use_auth_w_custom_endpoint is set to False.
-        if kw_args["api_endpoint"] is not None:
-            if _is_emulator_set or not use_auth_w_custom_endpoint:
+        if connection_kw_args["api_endpoint"] is not None:
+            if self._is_emulator_set or not use_auth_w_custom_endpoint:
                 if credentials is None:
                     credentials = AnonymousCredentials()
                 if project is None:
@@ -176,11 +225,22 @@ def __init__(
             _http=_http,
         )
 
+        # Validate that the universe domain of the credentials matches the
+        # universe domain of the client.
+        if self._credentials.universe_domain != self.universe_domain:
+            raise ValueError(
+                "The configured universe domain ({client_ud}) does not match "
+                "the universe domain found in the credentials ({cred_ud}). If "
+                "you haven't configured the universe domain explicitly, "
+                "`googleapis.com` is the default.".format(
+                    client_ud=self.universe_domain,
+                    cred_ud=self._credentials.universe_domain))
+
         if no_project:
             self.project = None
 
         # Pass extra_headers to Connection
-        connection = Connection(self, **kw_args)
+        connection = Connection(self, **connection_kw_args)
         connection.extra_headers = extra_headers
         self._connection = connection
         self._batch_stack = _LocalStack()
@@ -201,6 +261,14 @@ def create_anonymous_client(cls):
         client.project = None
         return client
 
+    @property
+    def universe_domain(self):
+        return self._universe_domain or _DEFAULT_UNIVERSE_DOMAIN
+
+    @property
+    def api_endpoint(self):
+        return self._connection.API_BASE_URL
+
     @property
     def _connection(self):
         """Get connection or batch on the client.
@@ -922,8 +990,7 @@ def create_bucket(
             project = self.project
 
         # Use no project if STORAGE_EMULATOR_HOST is set
-        _is_emulator_set = _get_storage_host() != _DEFAULT_STORAGE_HOST
-        if _is_emulator_set:
+        if self._is_emulator_set:
             if project is None:
                 project = _get_environ_project()
             if project is None:
@@ -1338,8 +1405,7 @@ def list_buckets(
             project = self.project
 
         # Use no project if STORAGE_EMULATOR_HOST is set
-        _is_emulator_set = _get_storage_host() != _DEFAULT_STORAGE_HOST
-        if _is_emulator_set:
+        if self._is_emulator_set:
             if project is None:
                 project = _get_environ_project()
             if project is None:

setup.py

@@ -28,7 +28,7 @@
 # 'Development Status :: 5 - Production/Stable'
 release_status = "Development Status :: 5 - Production/Stable"
 dependencies = [
-    "google-auth >= 2.23.3, < 3.0dev",
+    "google-auth >= 2.26.1, < 3.0dev",
     "google-api-core >= 1.31.5, <3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0",
     "google-cloud-core >= 2.3.0, < 3.0dev",
     "google-resumable-media >= 2.6.0",

tests/unit/test__helpers.py

@@ -22,30 +22,30 @@
 GCCL_INVOCATION_TEST_CONST = "gccl-invocation-id/test-invocation-123"
 
 
-class Test__get_storage_host(unittest.TestCase):
-    @staticmethod
-    def _call_fut():
-        from google.cloud.storage._helpers import _get_storage_host
+# class Test__get_storage_host(unittest.TestCase):
+#     @staticmethod
+#     def _call_fut():
+#         from google.cloud.storage._helpers import _get_storage_host
 
-        return _get_storage_host()
+#         return _get_storage_host()
 
-    def test_wo_env_var(self):
-        from google.cloud.storage._helpers import _DEFAULT_STORAGE_HOST
+#     def test_wo_env_var(self):
+#         from google.cloud.storage._helpers import _DEFAULT_STORAGE_HOST
 
-        with mock.patch("os.environ", {}):
-            host = self._call_fut()
+#         with mock.patch("os.environ", {}):
+#             host = self._call_fut()
 
-        self.assertEqual(host, _DEFAULT_STORAGE_HOST)
+#         self.assertEqual(host, _DEFAULT_STORAGE_HOST)
 
-    def test_w_env_var(self):
-        from google.cloud.storage._helpers import STORAGE_EMULATOR_ENV_VAR
+#     def test_w_env_var(self):
+#         from google.cloud.storage._helpers import STORAGE_EMULATOR_ENV_VAR
 
-        HOST = "https://api.example.com"
+#         HOST = "https://api.example.com"
 
-        with mock.patch("os.environ", {STORAGE_EMULATOR_ENV_VAR: HOST}):
-            host = self._call_fut()
+#         with mock.patch("os.environ", {STORAGE_EMULATOR_ENV_VAR: HOST}):
+#             host = self._call_fut()
 
-        self.assertEqual(host, HOST)
+#         self.assertEqual(host, HOST)
 
 
 class Test__get_environ_project(unittest.TestCase):

tests/unit/test__http.py

@@ -89,7 +89,7 @@ def test_metadata_op_has_client_custom_headers(self):
         response._content = data
         http.is_mtls = False
         http.request.return_value = response
-        credentials = mock.Mock(spec=google.auth.credentials.Credentials)
+        credentials = mock.Mock(spec=google.auth.credentials.Credentials, universe_domain=_helpers._DEFAULT_UNIVERSE_DOMAIN)
         client = Client(
             project="project",
             credentials=credentials,

tests/unit/test_batch.py

@@ -20,11 +20,12 @@
 import mock
 import requests
 
+from google.cloud.storage._helpers import _DEFAULT_UNIVERSE_DOMAIN
 
 def _make_credentials():
     import google.auth.credentials
 
-    return mock.Mock(spec=google.auth.credentials.Credentials)
+    return mock.Mock(spec=google.auth.credentials.Credentials, universe_domain=_DEFAULT_UNIVERSE_DOMAIN)
 
 
 def _make_response(status=http.client.OK, content=b"", headers={}):

tests/unit/test_blob.py

@@ -29,6 +29,7 @@
 
 from google.cloud.storage import _helpers
 from google.cloud.storage._helpers import _get_default_headers
+from google.cloud.storage._helpers import _DEFAULT_UNIVERSE_DOMAIN
 from google.cloud.storage.retry import (
     DEFAULT_RETRY,
     DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED,
@@ -5905,7 +5906,7 @@ def test_downloads_w_client_custom_headers(self):
             "x-goog-custom-audit-foo": "bar",
             "x-goog-custom-audit-user": "baz",
         }
-        credentials = mock.Mock(spec=google.auth.credentials.Credentials)
+        credentials = mock.Mock(spec=google.auth.credentials.Credentials, universe_domain=_DEFAULT_UNIVERSE_DOMAIN)
         client = Client(
             project="project", credentials=credentials, extra_headers=custom_headers
         )