From 1de05f6cb1cea27cbfa5bc39dc428755464de130 Mon Sep 17 00:00:00 2001 From: arithmetic1728 <58957152+arithmetic1728@users.noreply.github.com> Date: Thu, 11 Feb 2021 13:08:39 -0800 Subject: [PATCH 1/9] test: update system test for mtls (#518) --- noxfile.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 942525ca9..df36d237e 100644 --- a/noxfile.py +++ b/noxfile.py @@ -113,7 +113,11 @@ def system(session): session.install( "mock", "pytest", "psutil", "google-cloud-testutils", "-c", constraints_path ) - session.install("google-cloud-storage", "-c", constraints_path) + if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "") == "true": + # mTLS test requires pyopenssl and latest google-cloud-storage + session.install("google-cloud-storage", "pyopenssl") + else: + session.install("google-cloud-storage", "-c", constraints_path) session.install("-e", ".[all]", "-c", constraints_path) session.install("ipython", "-c", constraints_path) From 9902789791d0237dbda4695c781b8a056dbeddc0 Mon Sep 17 00:00:00 2001 From: shollyman Date: Tue, 16 Feb 2021 06:49:16 -0800 Subject: [PATCH 2/9] chore: add PARQUET to DestinationFormat enum (#521) --- google/cloud/bigquery/enums.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/google/cloud/bigquery/enums.py b/google/cloud/bigquery/enums.py index 2268808fd..db463afdc 100644 --- a/google/cloud/bigquery/enums.py +++ b/google/cloud/bigquery/enums.py @@ -72,6 +72,9 @@ class DestinationFormat(object): AVRO = "AVRO" """Specifies Avro format.""" + PARQUET = "PARQUET" + """Specifies Parquet format.""" + class Encoding(object): """The character encoding of the data. The default is :attr:`UTF_8`. From eedf93b6636c5ff1bd810c6038cfeaea8ccb64d8 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 16 Feb 2021 08:58:02 -0600 Subject: [PATCH 3/9] docs: clarify `%%bigquery`` magics and fix broken link (#508) --- docs/usage/index.rst | 4 ++-- google/cloud/bigquery/magics/magics.py | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/usage/index.rst b/docs/usage/index.rst index ff4c9d7f1..1d3cc9f64 100644 --- a/docs/usage/index.rst +++ b/docs/usage/index.rst @@ -29,7 +29,7 @@ Integrations with Other Libraries pandas -See also, the :mod:`google.cloud.bigquery.magics` module for integrations -with Jupyter. +See also, the :mod:`google.cloud.bigquery.magics.magics` module for +integrations with Jupyter. diff --git a/google/cloud/bigquery/magics/magics.py b/google/cloud/bigquery/magics/magics.py index 8f343ddcc..6ae7cae12 100644 --- a/google/cloud/bigquery/magics/magics.py +++ b/google/cloud/bigquery/magics/magics.py @@ -14,6 +14,15 @@ """IPython Magics +To use these magics, you must first register them. Run the ``%load_ext`` magic +in a Jupyter notebook cell. + +.. code:: + + %load_ext google.cloud.bigquery + +This makes the ``%%bigquery`` magic available. + .. function:: %%bigquery IPython cell magic to run a query and display the result as a DataFrame From 3e9430faff7f071600acef295cb5feefe767b954 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 16 Feb 2021 09:25:22 -0600 Subject: [PATCH 4/9] chore: remove redundant view code samples (#437) --- docs/snippets.py | 126 ----------------------------------------------- 1 file changed, 126 deletions(-) diff --git a/docs/snippets.py b/docs/snippets.py index 8c106e63d..3f9b9a88c 100644 --- a/docs/snippets.py +++ b/docs/snippets.py @@ -478,132 +478,6 @@ def test_update_table_cmek(client, to_delete): # [END bigquery_update_table_cmek] -@pytest.mark.skip( - reason=( - "update_table() is flaky " - "https://github.com/GoogleCloudPlatform/google-cloud-python/issues/5589" - ) -) -def test_manage_views(client, to_delete): - project = client.project - source_dataset_id = "source_dataset_{}".format(_millis()) - source_dataset_ref = bigquery.DatasetReference(project, source_dataset_id) - source_dataset = bigquery.Dataset(source_dataset_ref) - source_dataset = client.create_dataset(source_dataset) - to_delete.append(source_dataset) - - job_config = bigquery.LoadJobConfig() - job_config.schema = [ - bigquery.SchemaField("name", "STRING"), - bigquery.SchemaField("post_abbr", "STRING"), - ] - job_config.skip_leading_rows = 1 - uri = "gs://cloud-samples-data/bigquery/us-states/us-states.csv" - source_table_id = "us_states" - load_job = client.load_table_from_uri( - uri, source_dataset.table(source_table_id), job_config=job_config - ) - load_job.result() - - shared_dataset_id = "shared_dataset_{}".format(_millis()) - shared_dataset_ref = bigquery.DatasetReference(project, shared_dataset_id) - shared_dataset = bigquery.Dataset(shared_dataset_ref) - shared_dataset = client.create_dataset(shared_dataset) - to_delete.append(shared_dataset) - - # [START bigquery_create_view] - # from google.cloud import bigquery - # client = bigquery.Client() - # project = 'my-project' - # source_dataset_id = 'my_source_dataset' - # source_table_id = 'us_states' - # shared_dataset_ref = bigquery.DatasetReference(project, 'my_shared_dataset') - - # This example shows how to create a shared view of a source table of - # US States. The source table contains all 50 states, while the view will - # contain only states with names starting with 'W'. - view_ref = shared_dataset_ref.table("my_shared_view") - view = bigquery.Table(view_ref) - sql_template = 'SELECT name, post_abbr FROM `{}.{}.{}` WHERE name LIKE "W%"' - view.view_query = sql_template.format(project, source_dataset_id, source_table_id) - view = client.create_table(view) # API request - - print("Successfully created view at {}".format(view.full_table_id)) - # [END bigquery_create_view] - - # [START bigquery_update_view_query] - # from google.cloud import bigquery - # client = bigquery.Client() - # project = 'my-project' - # source_dataset_id = 'my_source_dataset' - # source_table_id = 'us_states' - # shared_dataset_ref = bigquery.DatasetReference(project, 'my_shared_dataset') - - # This example shows how to update a shared view of a source table of - # US States. The view's query will be updated to contain only states with - # names starting with 'M'. - view_ref = shared_dataset_ref.table("my_shared_view") - view = bigquery.Table(view_ref) - sql_template = 'SELECT name, post_abbr FROM `{}.{}.{}` WHERE name LIKE "M%"' - view.view_query = sql_template.format(project, source_dataset_id, source_table_id) - view = client.update_table(view, ["view_query"]) # API request - # [END bigquery_update_view_query] - - # [START bigquery_get_view] - # from google.cloud import bigquery - # client = bigquery.Client() - # shared_dataset_id = 'my_shared_dataset' - project = client.project - shared_dataset_ref = bigquery.DatasetReference(project, shared_dataset_id) - view_ref = shared_dataset_ref.table("my_shared_view") - view = client.get_table(view_ref) # API Request - - # Display view properties - print("View at {}".format(view.full_table_id)) - print("View Query:\n{}".format(view.view_query)) - # [END bigquery_get_view] - assert view.view_query is not None - - analyst_group_email = "example-analyst-group@google.com" - # [START bigquery_grant_view_access] - # from google.cloud import bigquery - # client = bigquery.Client() - - # Assign access controls to the dataset containing the view - # shared_dataset_id = 'my_shared_dataset' - # analyst_group_email = 'data_analysts@example.com' - project = client.project - shared_dataset_ref = bigquery.DatasetReference(project, shared_dataset_id) - shared_dataset = client.get_dataset(shared_dataset_ref) # API request - access_entries = shared_dataset.access_entries - access_entries.append( - bigquery.AccessEntry("READER", "groupByEmail", analyst_group_email) - ) - shared_dataset.access_entries = access_entries - shared_dataset = client.update_dataset( - shared_dataset, ["access_entries"] - ) # API request - - # Authorize the view to access the source dataset - # project = 'my-project' - # source_dataset_id = 'my_source_dataset' - project = client.project - source_dataset_ref = bigquery.DatasetReference(project, source_dataset_id) - source_dataset = client.get_dataset(source_dataset_ref) # API request - view_reference = { - "projectId": project, - "datasetId": shared_dataset_id, - "tableId": "my_shared_view", - } - access_entries = source_dataset.access_entries - access_entries.append(bigquery.AccessEntry(None, "view", view_reference)) - source_dataset.access_entries = access_entries - source_dataset = client.update_dataset( - source_dataset, ["access_entries"] - ) # API request - # [END bigquery_grant_view_access] - - def test_load_table_add_column(client, to_delete): dataset_id = "load_table_add_column_{}".format(_millis()) project = client.project From edd3328fffa3040b2cd3a3c668c90a0e43e4c94c Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Wed, 17 Feb 2021 17:52:56 +0100 Subject: [PATCH 5/9] feat: add determinism level for javascript UDFs (#522) * feat: add determinism level for javascript UDFs * Add enum-like class for routine determinism level --- docs/reference.rst | 1 + google/cloud/bigquery/__init__.py | 2 + google/cloud/bigquery/enums.py | 17 ++++++++ google/cloud/bigquery/routine/__init__.py | 29 +++++++++++++ .../cloud/bigquery/{ => routine}/routine.py | 12 ++++++ tests/system/test_client.py | 1 + tests/unit/routine/test_routine.py | 41 ++++++++++++++++++- 7 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 google/cloud/bigquery/routine/__init__.py rename google/cloud/bigquery/{ => routine}/routine.py (97%) diff --git a/docs/reference.rst b/docs/reference.rst index 3643831cb..6b802e2a5 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -110,6 +110,7 @@ Routine .. autosummary:: :toctree: generated + routine.DeterminismLevel routine.Routine routine.RoutineArgument routine.RoutineReference diff --git a/google/cloud/bigquery/__init__.py b/google/cloud/bigquery/__init__.py index 41f987228..29d375b03 100644 --- a/google/cloud/bigquery/__init__.py +++ b/google/cloud/bigquery/__init__.py @@ -70,6 +70,7 @@ from google.cloud.bigquery.query import StructQueryParameter from google.cloud.bigquery.query import UDFResource from google.cloud.bigquery.retry import DEFAULT_RETRY +from google.cloud.bigquery.routine import DeterminismLevel from google.cloud.bigquery.routine import Routine from google.cloud.bigquery.routine import RoutineArgument from google.cloud.bigquery.routine import RoutineReference @@ -134,6 +135,7 @@ "Compression", "CreateDisposition", "DestinationFormat", + "DeterminismLevel", "ExternalSourceFormat", "Encoding", "QueryPriority", diff --git a/google/cloud/bigquery/enums.py b/google/cloud/bigquery/enums.py index db463afdc..e353b3132 100644 --- a/google/cloud/bigquery/enums.py +++ b/google/cloud/bigquery/enums.py @@ -234,3 +234,20 @@ class WriteDisposition(object): WRITE_EMPTY = "WRITE_EMPTY" """If the table already exists and contains data, a 'duplicate' error is returned in the job result.""" + + +class DeterminismLevel: + """Specifies determinism level for JavaScript user-defined functions (UDFs). + + https://cloud.google.com/bigquery/docs/reference/rest/v2/routines#DeterminismLevel + """ + + DETERMINISM_LEVEL_UNSPECIFIED = "DETERMINISM_LEVEL_UNSPECIFIED" + """The determinism of the UDF is unspecified.""" + + DETERMINISTIC = "DETERMINISTIC" + """The UDF is deterministic, meaning that 2 function calls with the same inputs + always produce the same result, even across 2 query runs.""" + + NOT_DETERMINISTIC = "NOT_DETERMINISTIC" + """The UDF is not deterministic.""" diff --git a/google/cloud/bigquery/routine/__init__.py b/google/cloud/bigquery/routine/__init__.py new file mode 100644 index 000000000..d1c79b05e --- /dev/null +++ b/google/cloud/bigquery/routine/__init__.py @@ -0,0 +1,29 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""User-Defined Routines.""" + + +from google.cloud.bigquery.enums import DeterminismLevel +from google.cloud.bigquery.routine.routine import Routine +from google.cloud.bigquery.routine.routine import RoutineArgument +from google.cloud.bigquery.routine.routine import RoutineReference + + +__all__ = ( + "DeterminismLevel", + "Routine", + "RoutineArgument", + "RoutineReference", +) diff --git a/google/cloud/bigquery/routine.py b/google/cloud/bigquery/routine/routine.py similarity index 97% rename from google/cloud/bigquery/routine.py rename to google/cloud/bigquery/routine/routine.py index f26f20886..103799e8f 100644 --- a/google/cloud/bigquery/routine.py +++ b/google/cloud/bigquery/routine/routine.py @@ -50,6 +50,7 @@ class Routine(object): "return_type": "returnType", "type_": "routineType", "description": "description", + "determinism_level": "determinismLevel", } def __init__(self, routine_ref, **kwargs): @@ -253,6 +254,17 @@ def description(self): def description(self, value): self._properties[self._PROPERTY_TO_API_FIELD["description"]] = value + @property + def determinism_level(self): + """Optional[str]: (experimental) The determinism level of the JavaScript UDF + if defined. + """ + return self._properties.get(self._PROPERTY_TO_API_FIELD["determinism_level"]) + + @determinism_level.setter + def determinism_level(self, value): + self._properties[self._PROPERTY_TO_API_FIELD["determinism_level"]] = value + @classmethod def from_api_repr(cls, resource): """Factory: construct a routine given its API representation. diff --git a/tests/system/test_client.py b/tests/system/test_client.py index 85c044bad..60c3b3fa8 100644 --- a/tests/system/test_client.py +++ b/tests/system/test_client.py @@ -2682,6 +2682,7 @@ def test_create_routine(self): ) ] routine.body = "return maxValue(arr)" + routine.determinism_level = bigquery.DeterminismLevel.DETERMINISTIC query_string = "SELECT `{}`([-100.0, 3.14, 100.0, 42.0]) as max_value;".format( str(routine.reference) ) diff --git a/tests/unit/routine/test_routine.py b/tests/unit/routine/test_routine.py index b02ace1db..0a59e7c5f 100644 --- a/tests/unit/routine/test_routine.py +++ b/tests/unit/routine/test_routine.py @@ -18,6 +18,7 @@ import pytest import google.cloud._helpers +from google.cloud import bigquery from google.cloud import bigquery_v2 @@ -73,6 +74,7 @@ def test_ctor_w_properties(target_class): ) type_ = "SCALAR_FUNCTION" description = "A routine description." + determinism_level = bigquery.DeterminismLevel.NOT_DETERMINISTIC actual_routine = target_class( routine_id, @@ -82,6 +84,7 @@ def test_ctor_w_properties(target_class): return_type=return_type, type_=type_, description=description, + determinism_level=determinism_level, ) ref = RoutineReference.from_string(routine_id) @@ -92,6 +95,9 @@ def test_ctor_w_properties(target_class): assert actual_routine.return_type == return_type assert actual_routine.type_ == type_ assert actual_routine.description == description + assert ( + actual_routine.determinism_level == bigquery.DeterminismLevel.NOT_DETERMINISTIC + ) def test_from_api_repr(target_class): @@ -120,6 +126,7 @@ def test_from_api_repr(target_class): "routineType": "SCALAR_FUNCTION", "someNewField": "someValue", "description": "A routine description.", + "determinismLevel": bigquery.DeterminismLevel.DETERMINISTIC, } actual_routine = target_class.from_api_repr(resource) @@ -152,6 +159,7 @@ def test_from_api_repr(target_class): assert actual_routine.type_ == "SCALAR_FUNCTION" assert actual_routine._properties["someNewField"] == "someValue" assert actual_routine.description == "A routine description." + assert actual_routine.determinism_level == "DETERMINISTIC" def test_from_api_repr_w_minimal_resource(target_class): @@ -177,6 +185,7 @@ def test_from_api_repr_w_minimal_resource(target_class): assert actual_routine.return_type is None assert actual_routine.type_ is None assert actual_routine.description is None + assert actual_routine.determinism_level is None def test_from_api_repr_w_unknown_fields(target_class): @@ -208,6 +217,7 @@ def test_from_api_repr_w_unknown_fields(target_class): "returnType": {"typeKind": "INT64"}, "routineType": "SCALAR_FUNCTION", "description": "A routine description.", + "determinismLevel": bigquery.DeterminismLevel.DETERMINISM_LEVEL_UNSPECIFIED, }, ["arguments"], {"arguments": [{"name": "x", "dataType": {"typeKind": "INT64"}}]}, @@ -220,6 +230,7 @@ def test_from_api_repr_w_unknown_fields(target_class): "returnType": {"typeKind": "INT64"}, "routineType": "SCALAR_FUNCTION", "description": "A routine description.", + "determinismLevel": bigquery.DeterminismLevel.DETERMINISM_LEVEL_UNSPECIFIED, }, ["body"], {"definitionBody": "x * 3"}, @@ -232,6 +243,7 @@ def test_from_api_repr_w_unknown_fields(target_class): "returnType": {"typeKind": "INT64"}, "routineType": "SCALAR_FUNCTION", "description": "A routine description.", + "determinismLevel": bigquery.DeterminismLevel.DETERMINISM_LEVEL_UNSPECIFIED, }, ["language"], {"language": "SQL"}, @@ -244,6 +256,7 @@ def test_from_api_repr_w_unknown_fields(target_class): "returnType": {"typeKind": "INT64"}, "routineType": "SCALAR_FUNCTION", "description": "A routine description.", + "determinismLevel": bigquery.DeterminismLevel.DETERMINISM_LEVEL_UNSPECIFIED, }, ["return_type"], {"returnType": {"typeKind": "INT64"}}, @@ -256,6 +269,7 @@ def test_from_api_repr_w_unknown_fields(target_class): "returnType": {"typeKind": "INT64"}, "routineType": "SCALAR_FUNCTION", "description": "A routine description.", + "determinismLevel": bigquery.DeterminismLevel.DETERMINISM_LEVEL_UNSPECIFIED, }, ["type_"], {"routineType": "SCALAR_FUNCTION"}, @@ -268,13 +282,37 @@ def test_from_api_repr_w_unknown_fields(target_class): "returnType": {"typeKind": "INT64"}, "routineType": "SCALAR_FUNCTION", "description": "A routine description.", + "determinismLevel": bigquery.DeterminismLevel.DETERMINISM_LEVEL_UNSPECIFIED, }, ["description"], {"description": "A routine description."}, ), + ( + { + "arguments": [{"name": "x", "dataType": {"typeKind": "INT64"}}], + "definitionBody": "x * 3", + "language": "SQL", + "returnType": {"typeKind": "INT64"}, + "routineType": "SCALAR_FUNCTION", + "description": "A routine description.", + "determinismLevel": bigquery.DeterminismLevel.DETERMINISM_LEVEL_UNSPECIFIED, + }, + ["determinism_level"], + { + "determinismLevel": bigquery.DeterminismLevel.DETERMINISM_LEVEL_UNSPECIFIED + }, + ), ( {}, - ["arguments", "language", "body", "type_", "return_type", "description"], + [ + "arguments", + "language", + "body", + "type_", + "return_type", + "description", + "determinism_level", + ], { "arguments": None, "definitionBody": None, @@ -282,6 +320,7 @@ def test_from_api_repr_w_unknown_fields(target_class): "returnType": None, "routineType": None, "description": None, + "determinismLevel": None, }, ), ( From 4ffb4e067abdaa54dad6eff49a7fbdb0fa358637 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Wed, 17 Feb 2021 18:30:58 +0100 Subject: [PATCH 6/9] feat: expose reservation usage stats on jobs (#524) * feat: expose reservation usage stats on jobs * Add ReservationUsage to job types in docs * Remove redundant space in docstring. --- docs/reference.rst | 1 + google/cloud/bigquery/job/__init__.py | 2 ++ google/cloud/bigquery/job/base.py | 27 +++++++++++++++++++++++++++ tests/unit/job/test_base.py | 24 ++++++++++++++++++++++++ 4 files changed, 54 insertions(+) diff --git a/docs/reference.rst b/docs/reference.rst index 6b802e2a5..52d916f96 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -62,6 +62,7 @@ Job-Related Types job.QueryPlanEntry job.QueryPlanEntryStep job.QueryPriority + job.ReservationUsage job.SourceFormat job.WriteDisposition job.SchemaUpdateOption diff --git a/google/cloud/bigquery/job/__init__.py b/google/cloud/bigquery/job/__init__.py index 26ecf8d3c..4945841d9 100644 --- a/google/cloud/bigquery/job/__init__.py +++ b/google/cloud/bigquery/job/__init__.py @@ -19,6 +19,7 @@ from google.cloud.bigquery.job.base import _DONE_STATE from google.cloud.bigquery.job.base import _JobConfig from google.cloud.bigquery.job.base import _JobReference +from google.cloud.bigquery.job.base import ReservationUsage from google.cloud.bigquery.job.base import ScriptStatistics from google.cloud.bigquery.job.base import ScriptStackFrame from google.cloud.bigquery.job.base import UnknownJob @@ -51,6 +52,7 @@ "_DONE_STATE", "_JobConfig", "_JobReference", + "ReservationUsage", "ScriptStatistics", "ScriptStackFrame", "UnknownJob", diff --git a/google/cloud/bigquery/job/base.py b/google/cloud/bigquery/job/base.py index 5ba01aa67..d8f5d6528 100644 --- a/google/cloud/bigquery/job/base.py +++ b/google/cloud/bigquery/job/base.py @@ -14,6 +14,7 @@ """Base classes and helpers for job classes.""" +from collections import namedtuple import copy import http import threading @@ -73,6 +74,16 @@ def _error_result_to_exception(error_result): ) +ReservationUsage = namedtuple("ReservationUsage", "name slot_ms") +ReservationUsage.__doc__ = "Job resource usage for a reservation." +ReservationUsage.name.__doc__ = ( + 'Reservation name or "unreserved" for on-demand resources usage.' +) +ReservationUsage.slot_ms.__doc__ = ( + "Total slot milliseconds used by the reservation for a particular job." +) + + class _JobReference(object): """A reference to a job. @@ -305,6 +316,22 @@ def _job_statistics(self): statistics = self._properties.get("statistics", {}) return statistics.get(self._JOB_TYPE, {}) + @property + def reservation_usage(self): + """Job resource usage breakdown by reservation. + + Returns: + List[google.cloud.bigquery.job.ReservationUsage]: + Reservation usage stats. Can be empty if not set from the server. + """ + usage_stats_raw = _helpers._get_sub_prop( + self._properties, ["statistics", "reservationUsage"], default=() + ) + return [ + ReservationUsage(name=usage["name"], slot_ms=int(usage["slotMs"])) + for usage in usage_stats_raw + ] + @property def error_result(self): """Error information about the job as a whole. diff --git a/tests/unit/job/test_base.py b/tests/unit/job/test_base.py index 44bbc2c77..bbeffba50 100644 --- a/tests/unit/job/test_base.py +++ b/tests/unit/job/test_base.py @@ -319,6 +319,30 @@ def test_ended(self): stats["endTime"] = millis self.assertEqual(job.ended, now) + def test_reservation_usage_no_stats(self): + client = _make_client(project=self.PROJECT) + job = self._make_one(self.JOB_ID, client) + job._properties["statistics"] = {} + self.assertEqual(job.reservation_usage, []) + + def test_reservation_usage_stats_exist(self): + from google.cloud.bigquery.job import ReservationUsage + + client = _make_client(project=self.PROJECT) + job = self._make_one(self.JOB_ID, client) + job._properties["statistics"] = { + "reservationUsage": [ + {"name": "slot_foo", "slotMs": "42"}, + {"name": "slot_bar", "slotMs": "123"}, + ], + } + + expected = [ + ReservationUsage(name="slot_foo", slot_ms=42), + ReservationUsage(name="slot_bar", slot_ms=123), + ] + self.assertEqual(job.reservation_usage, expected) + def test__job_statistics(self): statistics = {"foo": "bar"} client = _make_client(project=self.PROJECT) From 01e851d00fc17a780375580776753d78f6d74174 Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Wed, 17 Feb 2021 12:06:05 -0800 Subject: [PATCH 7/9] docs: update python contributing guide (#514) This PR was generated using Autosynth. :rainbow: Synth log will be available here: https://source.cloud.google.com/results/invocations/9d4e6069-5c18-4f79-97fb-99ebae377691/targets - [ ] To automatically regenerate this PR, check this box. Source-Link: https://github.com/googleapis/synthtool/commit/4679e7e415221f03ff2a71e3ffad75b9ec41d87e PiperOrigin-RevId: 344443035 Source-Link: https://github.com/googleapis/googleapis/commit/df4fd38d040c5c8a0869936205bca13fb64b2cff --- CONTRIBUTING.rst | 22 ++++++++++++++++++---- synth.metadata | 7 ++++--- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 15bcd2e28..a0e330e44 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -70,9 +70,14 @@ We use `nox `__ to instrument our tests. - To test your changes, run unit tests with ``nox``:: $ nox -s unit-2.7 - $ nox -s unit-3.7 + $ nox -s unit-3.8 $ ... +- Args to pytest can be passed through the nox command separated by a `--`. For + example, to run a single test:: + + $ nox -s unit-3.8 -- -k + .. note:: The unit tests and system tests are described in the @@ -93,8 +98,12 @@ On Debian/Ubuntu:: ************ Coding Style ************ +- We use the automatic code formatter ``black``. You can run it using + the nox session ``blacken``. This will eliminate many lint errors. Run via:: + + $ nox -s blacken -- PEP8 compliance, with exceptions defined in the linter configuration. +- PEP8 compliance is required, with exceptions defined in the linter configuration. If you have ``nox`` installed, you can test that you have not introduced any non-compliant code via:: @@ -133,13 +142,18 @@ Running System Tests - To run system tests, you can execute:: - $ nox -s system-3.7 + # Run all system tests + $ nox -s system-3.8 $ nox -s system-2.7 + # Run a single system test + $ nox -s system-3.8 -- -k + + .. note:: System tests are only configured to run under Python 2.7 and - Python 3.7. For expediency, we do not run them in older versions + Python 3.8. For expediency, we do not run them in older versions of Python 3. This alone will not run the tests. You'll need to change some local diff --git a/synth.metadata b/synth.metadata index 1c5fecaf8..f91ffab69 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,7 +4,7 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/python-bigquery.git", - "sha": "1823cadee3acf95c516d0479400e4175349ea199" + "sha": "1c6681aba872c00afb16a904a2ba9bae8e9618d3" } }, { @@ -19,14 +19,14 @@ "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "33366574ffb9e11737b3547eb6f020ecae0536e8" + "sha": "4679e7e415221f03ff2a71e3ffad75b9ec41d87e" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "33366574ffb9e11737b3547eb6f020ecae0536e8" + "sha": "4679e7e415221f03ff2a71e3ffad75b9ec41d87e" } } ], @@ -92,6 +92,7 @@ "CONTRIBUTING.rst", "LICENSE", "MANIFEST.in", + "bigquery-v2-py.tar.gz", "docs/_static/custom.css", "docs/_templates/layout.html", "docs/bigquery_v2/services.rst", From 4c9947d7578ee9d03d36b464dd4b627545bfb09e Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Thu, 18 Feb 2021 08:18:03 -0800 Subject: [PATCH 8/9] chore: update automation naming, smaller generated code fixes (#505) This PR was generated using Autosynth. :rainbow: Synth log will be available here: https://source.cloud.google.com/results/invocations/9d4e6069-5c18-4f79-97fb-99ebae377691/targets - [ ] To automatically regenerate this PR, check this box. PiperOrigin-RevId: 350246057 Source-Link: https://github.com/googleapis/googleapis/commit/520682435235d9c503983a360a2090025aa47cd1 PiperOrigin-RevId: 347055288 Source-Link: https://github.com/googleapis/googleapis/commit/dd372aa22ded7a8ba6f0e03a80e06358a3fa0907 --- .coveragerc | 34 +-- .gitignore | 4 +- .kokoro/build.sh | 10 + docs/bigquery_v2/services.rst | 6 - docs/bigquery_v2/types.rst | 1 + google/cloud/bigquery_v2/types/__init__.py | 1 - .../bigquery_v2/types/encryption_config.py | 2 +- google/cloud/bigquery_v2/types/model.py | 216 +++++++++--------- .../cloud/bigquery_v2/types/standard_sql.py | 10 +- synth.metadata | 12 +- synth.py | 2 + 11 files changed, 144 insertions(+), 154 deletions(-) delete mode 100644 docs/bigquery_v2/services.rst diff --git a/.coveragerc b/.coveragerc index 0d8e6297d..23861a8eb 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,38 +1,18 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Generated by synthtool. DO NOT EDIT! [run] branch = True -omit = - google/cloud/__init__.py [report] fail_under = 100 show_missing = True +omit = + google/cloud/bigquery/__init__.py exclude_lines = # Re-enable the standard pragma pragma: NO COVER # Ignore debug-only repr def __repr__ - # Ignore abstract methods - raise NotImplementedError -omit = - */gapic/*.py - */proto/*.py - */core/*.py - */site-packages/*.py - google/cloud/__init__.py + # Ignore pkg_resources exceptions. + # This is added at the module level as a safeguard for if someone + # generates the code and tries to run it without pip installing. This + # makes it virtually impossible to test properly. + except pkg_resources.DistributionNotFound diff --git a/.gitignore b/.gitignore index b9daa52f1..b4243ced7 100644 --- a/.gitignore +++ b/.gitignore @@ -50,8 +50,10 @@ docs.metadata # Virtual environment env/ + +# Test logs coverage.xml -sponge_log.xml +*sponge_log.xml # System test environment variables. system_tests/local_test_setup diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 058f363e1..302cc1e1a 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -40,6 +40,16 @@ python3 -m pip uninstall --yes --quiet nox-automation python3 -m pip install --upgrade --quiet nox python3 -m nox --version +# If this is a continuous build, send the test log to the FlakyBot. +# See https://github.com/googleapis/repo-automation-bots/tree/master/packages/flakybot. +if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]]; then + cleanup() { + chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot + $KOKORO_GFILE_DIR/linux_amd64/flakybot + } + trap cleanup EXIT HUP +fi + # If NOX_SESSION is set, it only runs the specified session, # otherwise run all the sessions. if [[ -n "${NOX_SESSION:-}" ]]; then diff --git a/docs/bigquery_v2/services.rst b/docs/bigquery_v2/services.rst deleted file mode 100644 index 65fbb438c..000000000 --- a/docs/bigquery_v2/services.rst +++ /dev/null @@ -1,6 +0,0 @@ -Services for Google Cloud Bigquery v2 API -========================================= - -.. automodule:: google.cloud.bigquery_v2.services.model_service - :members: - :inherited-members: diff --git a/docs/bigquery_v2/types.rst b/docs/bigquery_v2/types.rst index 41b906514..c36a83e0b 100644 --- a/docs/bigquery_v2/types.rst +++ b/docs/bigquery_v2/types.rst @@ -3,4 +3,5 @@ Types for Google Cloud Bigquery v2 API .. automodule:: google.cloud.bigquery_v2.types :members: + :undoc-members: :show-inheritance: diff --git a/google/cloud/bigquery_v2/types/__init__.py b/google/cloud/bigquery_v2/types/__init__.py index 1e354641a..00dc837c9 100644 --- a/google/cloud/bigquery_v2/types/__init__.py +++ b/google/cloud/bigquery_v2/types/__init__.py @@ -32,7 +32,6 @@ ListModelsResponse, ) - __all__ = ( "EncryptionConfiguration", "ModelReference", diff --git a/google/cloud/bigquery_v2/types/encryption_config.py b/google/cloud/bigquery_v2/types/encryption_config.py index 6fb90f340..2d801bde3 100644 --- a/google/cloud/bigquery_v2/types/encryption_config.py +++ b/google/cloud/bigquery_v2/types/encryption_config.py @@ -30,7 +30,7 @@ class EncryptionConfiguration(proto.Message): r""" Attributes: - kms_key_name (~.wrappers.StringValue): + kms_key_name (google.protobuf.wrappers_pb2.StringValue): Optional. Describes the Cloud KMS encryption key that will be used to protect destination BigQuery table. The BigQuery Service Account diff --git a/google/cloud/bigquery_v2/types/model.py b/google/cloud/bigquery_v2/types/model.py index c3530dec2..8ae158b64 100644 --- a/google/cloud/bigquery_v2/types/model.py +++ b/google/cloud/bigquery_v2/types/model.py @@ -45,7 +45,7 @@ class Model(proto.Message): Attributes: etag (str): Output only. A hash of this resource. - model_reference (~.gcb_model_reference.ModelReference): + model_reference (google.cloud.bigquery_v2.types.ModelReference): Required. Unique identifier for this model. creation_time (int): Output only. The time when this model was @@ -58,7 +58,7 @@ class Model(proto.Message): model. friendly_name (str): Optional. A descriptive name for this model. - labels (Sequence[~.gcb_model.Model.LabelsEntry]): + labels (Sequence[google.cloud.bigquery_v2.types.Model.LabelsEntry]): The labels associated with this model. You can use these to organize and group your models. Label keys and values can be no longer than 63 @@ -81,22 +81,22 @@ class Model(proto.Message): Output only. The geographic location where the model resides. This value is inherited from the dataset. - encryption_configuration (~.encryption_config.EncryptionConfiguration): + encryption_configuration (google.cloud.bigquery_v2.types.EncryptionConfiguration): Custom encryption configuration (e.g., Cloud KMS keys). This shows the encryption configuration of the model data while stored in BigQuery storage. This field can be used with PatchModel to update encryption key for an already encrypted model. - model_type (~.gcb_model.Model.ModelType): + model_type (google.cloud.bigquery_v2.types.Model.ModelType): Output only. Type of the model resource. - training_runs (Sequence[~.gcb_model.Model.TrainingRun]): + training_runs (Sequence[google.cloud.bigquery_v2.types.Model.TrainingRun]): Output only. Information for all training runs in increasing order of start_time. - feature_columns (Sequence[~.standard_sql.StandardSqlField]): + feature_columns (Sequence[google.cloud.bigquery_v2.types.StandardSqlField]): Output only. Input feature columns that were used to train this model. - label_columns (Sequence[~.standard_sql.StandardSqlField]): + label_columns (Sequence[google.cloud.bigquery_v2.types.StandardSqlField]): Output only. Label columns that were used to train this model. The output of the model will have a `predicted_` prefix to these columns. @@ -280,15 +280,15 @@ class RegressionMetrics(proto.Message): matrix factorization models. Attributes: - mean_absolute_error (~.wrappers.DoubleValue): + mean_absolute_error (google.protobuf.wrappers_pb2.DoubleValue): Mean absolute error. - mean_squared_error (~.wrappers.DoubleValue): + mean_squared_error (google.protobuf.wrappers_pb2.DoubleValue): Mean squared error. - mean_squared_log_error (~.wrappers.DoubleValue): + mean_squared_log_error (google.protobuf.wrappers_pb2.DoubleValue): Mean squared log error. - median_absolute_error (~.wrappers.DoubleValue): + median_absolute_error (google.protobuf.wrappers_pb2.DoubleValue): Median absolute error. - r_squared (~.wrappers.DoubleValue): + r_squared (google.protobuf.wrappers_pb2.DoubleValue): R^2 score. """ @@ -319,33 +319,33 @@ class AggregateClassificationMetrics(proto.Message): by counting the total number of correctly predicted rows. Attributes: - precision (~.wrappers.DoubleValue): + precision (google.protobuf.wrappers_pb2.DoubleValue): Precision is the fraction of actual positive predictions that had positive actual labels. For multiclass this is a macro-averaged metric treating each class as a binary classifier. - recall (~.wrappers.DoubleValue): + recall (google.protobuf.wrappers_pb2.DoubleValue): Recall is the fraction of actual positive labels that were given a positive prediction. For multiclass this is a macro-averaged metric. - accuracy (~.wrappers.DoubleValue): + accuracy (google.protobuf.wrappers_pb2.DoubleValue): Accuracy is the fraction of predictions given the correct label. For multiclass this is a micro-averaged metric. - threshold (~.wrappers.DoubleValue): + threshold (google.protobuf.wrappers_pb2.DoubleValue): Threshold at which the metrics are computed. For binary classification models this is the positive class threshold. For multi-class classfication models this is the confidence threshold. - f1_score (~.wrappers.DoubleValue): + f1_score (google.protobuf.wrappers_pb2.DoubleValue): The F1 score is an average of recall and precision. For multiclass this is a macro- averaged metric. - log_loss (~.wrappers.DoubleValue): + log_loss (google.protobuf.wrappers_pb2.DoubleValue): Logarithmic Loss. For multiclass this is a macro-averaged metric. - roc_auc (~.wrappers.DoubleValue): + roc_auc (google.protobuf.wrappers_pb2.DoubleValue): Area Under a ROC Curve. For multiclass this is a macro-averaged metric. """ @@ -369,9 +369,9 @@ class BinaryClassificationMetrics(proto.Message): models. Attributes: - aggregate_classification_metrics (~.gcb_model.Model.AggregateClassificationMetrics): + aggregate_classification_metrics (google.cloud.bigquery_v2.types.Model.AggregateClassificationMetrics): Aggregate classification metrics. - binary_confusion_matrix_list (Sequence[~.gcb_model.Model.BinaryClassificationMetrics.BinaryConfusionMatrix]): + binary_confusion_matrix_list (Sequence[google.cloud.bigquery_v2.types.Model.BinaryClassificationMetrics.BinaryConfusionMatrix]): Binary confusion matrix at multiple thresholds. positive_label (str): @@ -384,27 +384,27 @@ class BinaryConfusionMatrix(proto.Message): r"""Confusion matrix for binary classification models. Attributes: - positive_class_threshold (~.wrappers.DoubleValue): + positive_class_threshold (google.protobuf.wrappers_pb2.DoubleValue): Threshold value used when computing each of the following metric. - true_positives (~.wrappers.Int64Value): + true_positives (google.protobuf.wrappers_pb2.Int64Value): Number of true samples predicted as true. - false_positives (~.wrappers.Int64Value): + false_positives (google.protobuf.wrappers_pb2.Int64Value): Number of false samples predicted as true. - true_negatives (~.wrappers.Int64Value): + true_negatives (google.protobuf.wrappers_pb2.Int64Value): Number of true samples predicted as false. - false_negatives (~.wrappers.Int64Value): + false_negatives (google.protobuf.wrappers_pb2.Int64Value): Number of false samples predicted as false. - precision (~.wrappers.DoubleValue): + precision (google.protobuf.wrappers_pb2.DoubleValue): The fraction of actual positive predictions that had positive actual labels. - recall (~.wrappers.DoubleValue): + recall (google.protobuf.wrappers_pb2.DoubleValue): The fraction of actual positive labels that were given a positive prediction. - f1_score (~.wrappers.DoubleValue): + f1_score (google.protobuf.wrappers_pb2.DoubleValue): The equally weighted average of recall and precision. - accuracy (~.wrappers.DoubleValue): + accuracy (google.protobuf.wrappers_pb2.DoubleValue): The fraction of predictions given the correct label. """ @@ -462,9 +462,9 @@ class MultiClassClassificationMetrics(proto.Message): models. Attributes: - aggregate_classification_metrics (~.gcb_model.Model.AggregateClassificationMetrics): + aggregate_classification_metrics (google.cloud.bigquery_v2.types.Model.AggregateClassificationMetrics): Aggregate classification metrics. - confusion_matrix_list (Sequence[~.gcb_model.Model.MultiClassClassificationMetrics.ConfusionMatrix]): + confusion_matrix_list (Sequence[google.cloud.bigquery_v2.types.Model.MultiClassClassificationMetrics.ConfusionMatrix]): Confusion matrix at different thresholds. """ @@ -472,10 +472,10 @@ class ConfusionMatrix(proto.Message): r"""Confusion matrix for multi-class classification models. Attributes: - confidence_threshold (~.wrappers.DoubleValue): + confidence_threshold (google.protobuf.wrappers_pb2.DoubleValue): Confidence threshold used when computing the entries of the confusion matrix. - rows (Sequence[~.gcb_model.Model.MultiClassClassificationMetrics.ConfusionMatrix.Row]): + rows (Sequence[google.cloud.bigquery_v2.types.Model.MultiClassClassificationMetrics.ConfusionMatrix.Row]): One row per actual label. """ @@ -487,7 +487,7 @@ class Entry(proto.Message): The predicted label. For confidence_threshold > 0, we will also add an entry indicating the number of items under the confidence threshold. - item_count (~.wrappers.Int64Value): + item_count (google.protobuf.wrappers_pb2.Int64Value): Number of items being predicted as this label. """ @@ -504,7 +504,7 @@ class Row(proto.Message): Attributes: actual_label (str): The original label of this row. - entries (Sequence[~.gcb_model.Model.MultiClassClassificationMetrics.ConfusionMatrix.Entry]): + entries (Sequence[google.cloud.bigquery_v2.types.Model.MultiClassClassificationMetrics.ConfusionMatrix.Entry]): Info describing predicted label distribution. """ @@ -540,12 +540,12 @@ class ClusteringMetrics(proto.Message): r"""Evaluation metrics for clustering models. Attributes: - davies_bouldin_index (~.wrappers.DoubleValue): + davies_bouldin_index (google.protobuf.wrappers_pb2.DoubleValue): Davies-Bouldin index. - mean_squared_distance (~.wrappers.DoubleValue): + mean_squared_distance (google.protobuf.wrappers_pb2.DoubleValue): Mean of squared distances between each sample to its cluster centroid. - clusters (Sequence[~.gcb_model.Model.ClusteringMetrics.Cluster]): + clusters (Sequence[google.cloud.bigquery_v2.types.Model.ClusteringMetrics.Cluster]): [Beta] Information for all clusters. """ @@ -555,10 +555,10 @@ class Cluster(proto.Message): Attributes: centroid_id (int): Centroid id. - feature_values (Sequence[~.gcb_model.Model.ClusteringMetrics.Cluster.FeatureValue]): + feature_values (Sequence[google.cloud.bigquery_v2.types.Model.ClusteringMetrics.Cluster.FeatureValue]): Values of highly variant features for this cluster. - count (~.wrappers.Int64Value): + count (google.protobuf.wrappers_pb2.Int64Value): Count of training data rows that were assigned to this cluster. """ @@ -569,10 +569,10 @@ class FeatureValue(proto.Message): Attributes: feature_column (str): The feature column name. - numerical_value (~.wrappers.DoubleValue): + numerical_value (google.protobuf.wrappers_pb2.DoubleValue): The numerical feature value. This is the centroid value for this feature. - categorical_value (~.gcb_model.Model.ClusteringMetrics.Cluster.FeatureValue.CategoricalValue): + categorical_value (google.cloud.bigquery_v2.types.Model.ClusteringMetrics.Cluster.FeatureValue.CategoricalValue): The categorical feature value. """ @@ -580,7 +580,7 @@ class CategoricalValue(proto.Message): r"""Representative value of a categorical feature. Attributes: - category_counts (Sequence[~.gcb_model.Model.ClusteringMetrics.Cluster.FeatureValue.CategoricalValue.CategoryCount]): + category_counts (Sequence[google.cloud.bigquery_v2.types.Model.ClusteringMetrics.Cluster.FeatureValue.CategoricalValue.CategoryCount]): Counts of all categories for the categorical feature. If there are more than ten categories, we return top ten (by count) and return one more CategoryCount with category @@ -594,7 +594,7 @@ class CategoryCount(proto.Message): Attributes: category (str): The name of category. - count (~.wrappers.Int64Value): + count (google.protobuf.wrappers_pb2.Int64Value): The count of training samples matching the category within the cluster. """ @@ -654,23 +654,23 @@ class RankingMetrics(proto.Message): feedback_type=implicit. Attributes: - mean_average_precision (~.wrappers.DoubleValue): + mean_average_precision (google.protobuf.wrappers_pb2.DoubleValue): Calculates a precision per user for all the items by ranking them and then averages all the precisions across all the users. - mean_squared_error (~.wrappers.DoubleValue): + mean_squared_error (google.protobuf.wrappers_pb2.DoubleValue): Similar to the mean squared error computed in regression and explicit recommendation models except instead of computing the rating directly, the output from evaluate is computed against a preference which is 1 or 0 depending on if the rating exists or not. - normalized_discounted_cumulative_gain (~.wrappers.DoubleValue): + normalized_discounted_cumulative_gain (google.protobuf.wrappers_pb2.DoubleValue): A metric to determine the goodness of a ranking calculated from the predicted confidence by comparing it to an ideal rank measured by the original ratings. - average_rank (~.wrappers.DoubleValue): + average_rank (google.protobuf.wrappers_pb2.DoubleValue): Determines the goodness of a ranking by computing the percentile rank from the predicted confidence and dividing it by the original rank. @@ -696,11 +696,11 @@ class ArimaForecastingMetrics(proto.Message): r"""Model evaluation metrics for ARIMA forecasting models. Attributes: - non_seasonal_order (Sequence[~.gcb_model.Model.ArimaOrder]): + non_seasonal_order (Sequence[google.cloud.bigquery_v2.types.Model.ArimaOrder]): Non-seasonal order. - arima_fitting_metrics (Sequence[~.gcb_model.Model.ArimaFittingMetrics]): + arima_fitting_metrics (Sequence[google.cloud.bigquery_v2.types.Model.ArimaFittingMetrics]): Arima model fitting metrics. - seasonal_periods (Sequence[~.gcb_model.Model.SeasonalPeriod.SeasonalPeriodType]): + seasonal_periods (Sequence[google.cloud.bigquery_v2.types.Model.SeasonalPeriod.SeasonalPeriodType]): Seasonal periods. Repeated because multiple periods are supported for one time series. has_drift (Sequence[bool]): @@ -709,7 +709,7 @@ class ArimaForecastingMetrics(proto.Message): time_series_id (Sequence[str]): Id to differentiate different time series for the large-scale case. - arima_single_model_forecasting_metrics (Sequence[~.gcb_model.Model.ArimaForecastingMetrics.ArimaSingleModelForecastingMetrics]): + arima_single_model_forecasting_metrics (Sequence[google.cloud.bigquery_v2.types.Model.ArimaForecastingMetrics.ArimaSingleModelForecastingMetrics]): Repeated as there can be many metric sets (one for each model) in auto-arima and the large-scale case. @@ -720,16 +720,16 @@ class ArimaSingleModelForecastingMetrics(proto.Message): model. Attributes: - non_seasonal_order (~.gcb_model.Model.ArimaOrder): + non_seasonal_order (google.cloud.bigquery_v2.types.Model.ArimaOrder): Non-seasonal order. - arima_fitting_metrics (~.gcb_model.Model.ArimaFittingMetrics): + arima_fitting_metrics (google.cloud.bigquery_v2.types.Model.ArimaFittingMetrics): Arima fitting metrics. has_drift (bool): Is arima model fitted with drift or not. It is always false when d is not 1. time_series_id (str): The id to indicate different time series. - seasonal_periods (Sequence[~.gcb_model.Model.SeasonalPeriod.SeasonalPeriodType]): + seasonal_periods (Sequence[google.cloud.bigquery_v2.types.Model.SeasonalPeriod.SeasonalPeriodType]): Seasonal periods. Repeated because multiple periods are supported for one time series. """ @@ -779,21 +779,21 @@ class EvaluationMetrics(proto.Message): imported models. Attributes: - regression_metrics (~.gcb_model.Model.RegressionMetrics): + regression_metrics (google.cloud.bigquery_v2.types.Model.RegressionMetrics): Populated for regression models and explicit feedback type matrix factorization models. - binary_classification_metrics (~.gcb_model.Model.BinaryClassificationMetrics): + binary_classification_metrics (google.cloud.bigquery_v2.types.Model.BinaryClassificationMetrics): Populated for binary classification/classifier models. - multi_class_classification_metrics (~.gcb_model.Model.MultiClassClassificationMetrics): + multi_class_classification_metrics (google.cloud.bigquery_v2.types.Model.MultiClassClassificationMetrics): Populated for multi-class classification/classifier models. - clustering_metrics (~.gcb_model.Model.ClusteringMetrics): + clustering_metrics (google.cloud.bigquery_v2.types.Model.ClusteringMetrics): Populated for clustering models. - ranking_metrics (~.gcb_model.Model.RankingMetrics): + ranking_metrics (google.cloud.bigquery_v2.types.Model.RankingMetrics): Populated for implicit feedback type matrix factorization models. - arima_forecasting_metrics (~.gcb_model.Model.ArimaForecastingMetrics): + arima_forecasting_metrics (google.cloud.bigquery_v2.types.Model.ArimaForecastingMetrics): Populated for ARIMA models. """ @@ -835,10 +835,10 @@ class DataSplitResult(proto.Message): and evaluation data tables that were used to train the model. Attributes: - training_table (~.table_reference.TableReference): + training_table (google.cloud.bigquery_v2.types.TableReference): Table reference of the training data after split. - evaluation_table (~.table_reference.TableReference): + evaluation_table (google.cloud.bigquery_v2.types.TableReference): Table reference of the evaluation data after split. """ @@ -893,7 +893,7 @@ class GlobalExplanation(proto.Message): features after training. Attributes: - explanations (Sequence[~.gcb_model.Model.GlobalExplanation.Explanation]): + explanations (Sequence[google.cloud.bigquery_v2.types.Model.GlobalExplanation.Explanation]): A list of the top global explanations. Sorted by absolute value of attribution in descending order. @@ -913,7 +913,7 @@ class Explanation(proto.Message): be formatted like .. Overall size of feature name will always be truncated to first 120 characters. - attribution (~.wrappers.DoubleValue): + attribution (google.protobuf.wrappers_pb2.DoubleValue): Attribution of feature. """ @@ -933,22 +933,22 @@ class TrainingRun(proto.Message): r"""Information about a single training query run for the model. Attributes: - training_options (~.gcb_model.Model.TrainingRun.TrainingOptions): + training_options (google.cloud.bigquery_v2.types.Model.TrainingRun.TrainingOptions): Options that were used for this training run, includes user specified and default options that were used. - start_time (~.timestamp.Timestamp): + start_time (google.protobuf.timestamp_pb2.Timestamp): The start time of this training run. - results (Sequence[~.gcb_model.Model.TrainingRun.IterationResult]): + results (Sequence[google.cloud.bigquery_v2.types.Model.TrainingRun.IterationResult]): Output of each iteration run, results.size() <= max_iterations. - evaluation_metrics (~.gcb_model.Model.EvaluationMetrics): + evaluation_metrics (google.cloud.bigquery_v2.types.Model.EvaluationMetrics): The evaluation metrics over training/eval data that were computed at the end of training. - data_split_result (~.gcb_model.Model.DataSplitResult): + data_split_result (google.cloud.bigquery_v2.types.Model.DataSplitResult): Data split result of the training run. Only set when the input data is actually split. - global_explanations (Sequence[~.gcb_model.Model.GlobalExplanation]): + global_explanations (Sequence[google.cloud.bigquery_v2.types.Model.GlobalExplanation]): Global explanations for important features of the model. For multi-class models, there is one entry for each label class. For other models, @@ -962,30 +962,30 @@ class TrainingOptions(proto.Message): max_iterations (int): The maximum number of iterations in training. Used only for iterative training algorithms. - loss_type (~.gcb_model.Model.LossType): + loss_type (google.cloud.bigquery_v2.types.Model.LossType): Type of loss function used during training run. learn_rate (float): Learning rate in training. Used only for iterative training algorithms. - l1_regularization (~.wrappers.DoubleValue): + l1_regularization (google.protobuf.wrappers_pb2.DoubleValue): L1 regularization coefficient. - l2_regularization (~.wrappers.DoubleValue): + l2_regularization (google.protobuf.wrappers_pb2.DoubleValue): L2 regularization coefficient. - min_relative_progress (~.wrappers.DoubleValue): + min_relative_progress (google.protobuf.wrappers_pb2.DoubleValue): When early_stop is true, stops training when accuracy improvement is less than 'min_relative_progress'. Used only for iterative training algorithms. - warm_start (~.wrappers.BoolValue): + warm_start (google.protobuf.wrappers_pb2.BoolValue): Whether to train a model from the last checkpoint. - early_stop (~.wrappers.BoolValue): + early_stop (google.protobuf.wrappers_pb2.BoolValue): Whether to stop early when the loss doesn't improve significantly any more (compared to min_relative_progress). Used only for iterative training algorithms. input_label_columns (Sequence[str]): Name of input label columns in training data. - data_split_method (~.gcb_model.Model.DataSplitMethod): + data_split_method (google.cloud.bigquery_v2.types.Model.DataSplitMethod): The data split type for training and evaluation, e.g. RANDOM. data_split_eval_fraction (float): @@ -1007,13 +1007,13 @@ class TrainingOptions(proto.Message): and the rest are eval data. It respects the order in Orderable data types: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#data-type-properties - learn_rate_strategy (~.gcb_model.Model.LearnRateStrategy): + learn_rate_strategy (google.cloud.bigquery_v2.types.Model.LearnRateStrategy): The strategy to determine learn rate for the current iteration. initial_learn_rate (float): Specifies the initial learning rate for the line search learn rate strategy. - label_class_weights (Sequence[~.gcb_model.Model.TrainingRun.TrainingOptions.LabelClassWeightsEntry]): + label_class_weights (Sequence[google.cloud.bigquery_v2.types.Model.TrainingRun.TrainingOptions.LabelClassWeightsEntry]): Weights associated with each label class, for rebalancing the training data. Only applicable for classification models. @@ -1023,21 +1023,21 @@ class TrainingOptions(proto.Message): item_column (str): Item column specified for matrix factorization models. - distance_type (~.gcb_model.Model.DistanceType): + distance_type (google.cloud.bigquery_v2.types.Model.DistanceType): Distance type for clustering models. num_clusters (int): Number of clusters for clustering models. model_uri (str): [Beta] Google Cloud Storage URI from which the model was imported. Only applicable for imported models. - optimization_strategy (~.gcb_model.Model.OptimizationStrategy): + optimization_strategy (google.cloud.bigquery_v2.types.Model.OptimizationStrategy): Optimization strategy for training linear regression models. hidden_units (Sequence[int]): Hidden units for dnn models. batch_size (int): Batch size for dnn models. - dropout (~.wrappers.DoubleValue): + dropout (google.protobuf.wrappers_pb2.DoubleValue): Dropout probability for dnn models. max_tree_depth (int): Maximum depth of a tree for boosted tree @@ -1046,18 +1046,18 @@ class TrainingOptions(proto.Message): Subsample fraction of the training data to grow tree to prevent overfitting for boosted tree models. - min_split_loss (~.wrappers.DoubleValue): + min_split_loss (google.protobuf.wrappers_pb2.DoubleValue): Minimum split loss for boosted tree models. num_factors (int): Num factors specified for matrix factorization models. - feedback_type (~.gcb_model.Model.FeedbackType): + feedback_type (google.cloud.bigquery_v2.types.Model.FeedbackType): Feedback type that specifies which algorithm to run for matrix factorization. - wals_alpha (~.wrappers.DoubleValue): + wals_alpha (google.protobuf.wrappers_pb2.DoubleValue): Hyperparameter for matrix factoration when implicit feedback type is specified. - kmeans_initialization_method (~.gcb_model.Model.KmeansEnums.KmeansInitializationMethod): + kmeans_initialization_method (google.cloud.bigquery_v2.types.Model.KmeansEnums.KmeansInitializationMethod): The method used to initialize the centroids for kmeans algorithm. kmeans_initialization_column (str): @@ -1071,16 +1071,16 @@ class TrainingOptions(proto.Message): for ARIMA model. auto_arima (bool): Whether to enable auto ARIMA or not. - non_seasonal_order (~.gcb_model.Model.ArimaOrder): + non_seasonal_order (google.cloud.bigquery_v2.types.Model.ArimaOrder): A specification of the non-seasonal part of the ARIMA model: the three components (p, d, q) are the AR order, the degree of differencing, and the MA order. - data_frequency (~.gcb_model.Model.DataFrequency): + data_frequency (google.cloud.bigquery_v2.types.Model.DataFrequency): The data frequency of a time series. include_drift (bool): Include drift when fitting an ARIMA model. - holiday_region (~.gcb_model.Model.HolidayRegion): + holiday_region (google.cloud.bigquery_v2.types.Model.HolidayRegion): The geographical region based on which the holidays are considered in time series modeling. If a valid value is specified, then holiday @@ -1226,23 +1226,23 @@ class IterationResult(proto.Message): r"""Information about a single iteration of the training run. Attributes: - index (~.wrappers.Int32Value): + index (google.protobuf.wrappers_pb2.Int32Value): Index of the iteration, 0 based. - duration_ms (~.wrappers.Int64Value): + duration_ms (google.protobuf.wrappers_pb2.Int64Value): Time taken to run the iteration in milliseconds. - training_loss (~.wrappers.DoubleValue): + training_loss (google.protobuf.wrappers_pb2.DoubleValue): Loss computed on the training data at the end of iteration. - eval_loss (~.wrappers.DoubleValue): + eval_loss (google.protobuf.wrappers_pb2.DoubleValue): Loss computed on the eval data at the end of iteration. learn_rate (float): Learn rate used for this iteration. - cluster_infos (Sequence[~.gcb_model.Model.TrainingRun.IterationResult.ClusterInfo]): + cluster_infos (Sequence[google.cloud.bigquery_v2.types.Model.TrainingRun.IterationResult.ClusterInfo]): Information about top clusters for clustering models. - arima_result (~.gcb_model.Model.TrainingRun.IterationResult.ArimaResult): + arima_result (google.cloud.bigquery_v2.types.Model.TrainingRun.IterationResult.ArimaResult): """ @@ -1252,10 +1252,10 @@ class ClusterInfo(proto.Message): Attributes: centroid_id (int): Centroid id. - cluster_radius (~.wrappers.DoubleValue): + cluster_radius (google.protobuf.wrappers_pb2.DoubleValue): Cluster radius, the average distance from centroid to each point assigned to the cluster. - cluster_size (~.wrappers.Int64Value): + cluster_size (google.protobuf.wrappers_pb2.Int64Value): Cluster size, the total number of points assigned to the cluster. """ @@ -1276,11 +1276,11 @@ class ArimaResult(proto.Message): iteration results. Attributes: - arima_model_info (Sequence[~.gcb_model.Model.TrainingRun.IterationResult.ArimaResult.ArimaModelInfo]): + arima_model_info (Sequence[google.cloud.bigquery_v2.types.Model.TrainingRun.IterationResult.ArimaResult.ArimaModelInfo]): This message is repeated because there are multiple arima models fitted in auto-arima. For non-auto-arima model, its size is one. - seasonal_periods (Sequence[~.gcb_model.Model.SeasonalPeriod.SeasonalPeriodType]): + seasonal_periods (Sequence[google.cloud.bigquery_v2.types.Model.SeasonalPeriod.SeasonalPeriodType]): Seasonal periods. Repeated because multiple periods are supported for one time series. """ @@ -1314,18 +1314,18 @@ class ArimaModelInfo(proto.Message): r"""Arima model information. Attributes: - non_seasonal_order (~.gcb_model.Model.ArimaOrder): + non_seasonal_order (google.cloud.bigquery_v2.types.Model.ArimaOrder): Non-seasonal order. - arima_coefficients (~.gcb_model.Model.TrainingRun.IterationResult.ArimaResult.ArimaCoefficients): + arima_coefficients (google.cloud.bigquery_v2.types.Model.TrainingRun.IterationResult.ArimaResult.ArimaCoefficients): Arima coefficients. - arima_fitting_metrics (~.gcb_model.Model.ArimaFittingMetrics): + arima_fitting_metrics (google.cloud.bigquery_v2.types.Model.ArimaFittingMetrics): Arima fitting metrics. has_drift (bool): Whether Arima model fitted with drift or not. It is always false when d is not 1. time_series_id (str): The id to indicate different time series. - seasonal_periods (Sequence[~.gcb_model.Model.SeasonalPeriod.SeasonalPeriodType]): + seasonal_periods (Sequence[google.cloud.bigquery_v2.types.Model.SeasonalPeriod.SeasonalPeriodType]): Seasonal periods. Repeated because multiple periods are supported for one time series. """ @@ -1482,7 +1482,7 @@ class PatchModelRequest(proto.Message): Required. Dataset ID of the model to patch. model_id (str): Required. Model ID of the model to patch. - model (~.gcb_model.Model): + model (google.cloud.bigquery_v2.types.Model): Required. Patched model. Follows RFC5789 patch semantics. Missing fields are not updated. To clear a field, explicitly @@ -1525,7 +1525,7 @@ class ListModelsRequest(proto.Message): Required. Project ID of the models to list. dataset_id (str): Required. Dataset ID of the models to list. - max_results (~.wrappers.UInt32Value): + max_results (google.protobuf.wrappers_pb2.UInt32Value): The maximum number of results to return in a single response page. Leverage the page tokens to iterate through the entire collection. @@ -1547,7 +1547,7 @@ class ListModelsResponse(proto.Message): r""" Attributes: - models (Sequence[~.gcb_model.Model]): + models (Sequence[google.cloud.bigquery_v2.types.Model]): Models in the requested dataset. Only the following fields are populated: model_reference, model_type, creation_time, last_modified_time and labels. diff --git a/google/cloud/bigquery_v2/types/standard_sql.py b/google/cloud/bigquery_v2/types/standard_sql.py index 80e4632f7..3bc6afedc 100644 --- a/google/cloud/bigquery_v2/types/standard_sql.py +++ b/google/cloud/bigquery_v2/types/standard_sql.py @@ -33,13 +33,13 @@ class StandardSqlDataType(proto.Message): array_element_type="DATE"}} ]}} Attributes: - type_kind (~.standard_sql.StandardSqlDataType.TypeKind): + type_kind (google.cloud.bigquery_v2.types.StandardSqlDataType.TypeKind): Required. The top level type of this field. Can be any standard SQL data type (e.g., "INT64", "DATE", "ARRAY"). - array_element_type (~.standard_sql.StandardSqlDataType): + array_element_type (google.cloud.bigquery_v2.types.StandardSqlDataType): The type of the array's elements, if type_kind = "ARRAY". - struct_type (~.standard_sql.StandardSqlStructType): + struct_type (google.cloud.bigquery_v2.types.StandardSqlStructType): The fields of this struct, in order, if type_kind = "STRUCT". """ @@ -80,7 +80,7 @@ class StandardSqlField(proto.Message): name (str): Optional. The name of this field. Can be absent for struct fields. - type (~.standard_sql.StandardSqlDataType): + type (google.cloud.bigquery_v2.types.StandardSqlDataType): Optional. The type of this parameter. Absent if not explicitly specified (e.g., CREATE FUNCTION statement can omit the return type; in @@ -97,7 +97,7 @@ class StandardSqlStructType(proto.Message): r""" Attributes: - fields (Sequence[~.standard_sql.StandardSqlField]): + fields (Sequence[google.cloud.bigquery_v2.types.StandardSqlField]): """ diff --git a/synth.metadata b/synth.metadata index f91ffab69..dc183a72e 100644 --- a/synth.metadata +++ b/synth.metadata @@ -11,22 +11,22 @@ "git": { "name": "googleapis", "remote": "https://github.com/googleapis/googleapis.git", - "sha": "df4fd38d040c5c8a0869936205bca13fb64b2cff", - "internalRef": "344443035" + "sha": "e13001be33d69042a9505e698f792587a804a5cf", + "internalRef": "358152223" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "4679e7e415221f03ff2a71e3ffad75b9ec41d87e" + "sha": "4dca4132c6d63788c6675e1b1e11e7b9225f8694" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "4679e7e415221f03ff2a71e3ffad75b9ec41d87e" + "sha": "4dca4132c6d63788c6675e1b1e11e7b9225f8694" } } ], @@ -42,6 +42,7 @@ } ], "generatedFiles": [ + ".coveragerc", ".flake8", ".github/CONTRIBUTING.md", ".github/ISSUE_TEMPLATE/bug_report.md", @@ -95,6 +96,7 @@ "bigquery-v2-py.tar.gz", "docs/_static/custom.css", "docs/_templates/layout.html", + "docs/bigquery_v2/model_service.rst", "docs/bigquery_v2/services.rst", "docs/bigquery_v2/types.rst", "docs/conf.py", @@ -127,4 +129,4 @@ "setup.cfg", "testing/.gitignore" ] -} \ No newline at end of file +} diff --git a/synth.py b/synth.py index 341c5832f..3ab271c96 100644 --- a/synth.py +++ b/synth.py @@ -33,6 +33,8 @@ library, excludes=[ "docs/index.rst", + "docs/bigquery_v2/*_service.rst", + "docs/bigquery_v2/services.rst", "README.rst", "noxfile.py", "setup.py", From 1862de798e09b81c9bbbf06b00a438b5f57daf79 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 18 Feb 2021 10:52:56 -0600 Subject: [PATCH 9/9] chore: release 2.9.0 (#526) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 14 ++++++++++++++ google/cloud/bigquery/version.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 768b7b036..51fad831e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,20 @@ [1]: https://pypi.org/project/google-cloud-bigquery/#history +## [2.9.0](https://www.github.com/googleapis/python-bigquery/compare/v2.8.0...v2.9.0) (2021-02-18) + + +### Features + +* add determinism level for javascript UDFs ([#522](https://www.github.com/googleapis/python-bigquery/issues/522)) ([edd3328](https://www.github.com/googleapis/python-bigquery/commit/edd3328fffa3040b2cd3a3c668c90a0e43e4c94c)) +* expose reservation usage stats on jobs ([#524](https://www.github.com/googleapis/python-bigquery/issues/524)) ([4ffb4e0](https://www.github.com/googleapis/python-bigquery/commit/4ffb4e067abdaa54dad6eff49a7fbdb0fa358637)) + + +### Documentation + +* clarify `%%bigquery`` magics and fix broken link ([#508](https://www.github.com/googleapis/python-bigquery/issues/508)) ([eedf93b](https://www.github.com/googleapis/python-bigquery/commit/eedf93b6636c5ff1bd810c6038cfeaea8ccb64d8)) +* update python contributing guide ([#514](https://www.github.com/googleapis/python-bigquery/issues/514)) ([01e851d](https://www.github.com/googleapis/python-bigquery/commit/01e851d00fc17a780375580776753d78f6d74174)) + ## [2.8.0](https://www.github.com/googleapis/python-bigquery/compare/v2.7.0...v2.8.0) (2021-02-08) diff --git a/google/cloud/bigquery/version.py b/google/cloud/bigquery/version.py index 0a9aecb37..b2a8c5535 100644 --- a/google/cloud/bigquery/version.py +++ b/google/cloud/bigquery/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.8.0" +__version__ = "2.9.0"