From 6ef9fb9d5cc49b244be7476c439bccaa8e443bb9 Mon Sep 17 00:00:00 2001 From: shollyman Date: Wed, 18 Jan 2023 09:57:21 -0800 Subject: [PATCH 01/16] testing: remove test_create_table_with_policy (#1470) This is superseded by a version that tests using real tags in test_create_table_with_real_custom_policy Fixes: https://github.com/googleapis/python-bigquery/issues/1468 --- tests/system/test_client.py | 51 ------------------------------------- 1 file changed, 51 deletions(-) diff --git a/tests/system/test_client.py b/tests/system/test_client.py index 575898209..14a9b04d4 100644 --- a/tests/system/test_client.py +++ b/tests/system/test_client.py @@ -335,57 +335,6 @@ def test_create_table(self): self.assertTrue(_table_exists(table)) self.assertEqual(table.table_id, table_id) - def test_create_table_with_policy(self): - from google.cloud.bigquery.schema import PolicyTagList - - dataset = self.temp_dataset(_make_dataset_id("create_table_with_policy")) - table_id = "test_table" - policy_1 = PolicyTagList( - names=[ - "projects/{}/locations/us/taxonomies/1/policyTags/2".format( - Config.CLIENT.project - ), - ] - ) - policy_2 = PolicyTagList( - names=[ - "projects/{}/locations/us/taxonomies/3/policyTags/4".format( - Config.CLIENT.project - ), - ] - ) - - schema = [ - bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), - bigquery.SchemaField( - "secret_int", "INTEGER", mode="REQUIRED", policy_tags=policy_1 - ), - ] - table_arg = Table(dataset.table(table_id), schema=schema) - self.assertFalse(_table_exists(table_arg)) - - table = helpers.retry_403(Config.CLIENT.create_table)(table_arg) - self.to_delete.insert(0, table) - - self.assertTrue(_table_exists(table)) - self.assertEqual(policy_1, table.schema[1].policy_tags) - - # Amend the schema to replace the policy tags - new_schema = table.schema[:] - old_field = table.schema[1] - new_schema[1] = bigquery.SchemaField( - name=old_field.name, - field_type=old_field.field_type, - mode=old_field.mode, - description=old_field.description, - fields=old_field.fields, - policy_tags=policy_2, - ) - - table.schema = new_schema - table2 = Config.CLIENT.update_table(table, ["schema"]) - self.assertEqual(policy_2, table2.schema[1].policy_tags) - def test_create_table_with_real_custom_policy(self): from google.cloud.bigquery.schema import PolicyTagList From bdfe888f44c7114954e3b15f3139ad3a9ade83c2 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Wed, 18 Jan 2023 19:20:16 +0000 Subject: [PATCH 02/16] chore(deps): update dependency packaging to v23 (#1462) * chore(deps): update dependency packaging to v23 * bump google-cloud-bigquery Co-authored-by: Anthonios Partheniou Co-authored-by: aribray <45905583+aribray@users.noreply.github.com> --- samples/geography/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index 0f475e0fd..994d579aa 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -13,7 +13,7 @@ geopandas===0.10.2; python_version == '3.7' geopandas==0.12.2; python_version >= '3.8' google-api-core==2.11.0 google-auth==2.16.0 -google-cloud-bigquery==3.4.1 +google-cloud-bigquery==3.4.2 google-cloud-bigquery-storage==2.18.0 google-cloud-core==2.3.2 google-crc32c==1.5.0 @@ -24,7 +24,7 @@ idna==3.4 libcst==0.4.9 munch==2.5.0 mypy-extensions==0.4.3 -packaging==21.3 +packaging==23.0 pandas===1.3.5; python_version == '3.7' pandas==1.5.2; python_version >= '3.8' proto-plus==1.22.2 From 0dab7d25ace4b63d2984485e7b0c5bb38f20476f Mon Sep 17 00:00:00 2001 From: Mario Torres Jr <105736410+Mattix23@users.noreply.github.com> Date: Wed, 18 Jan 2023 13:53:49 -0600 Subject: [PATCH 03/16] =?UTF-8?q?docs:=20revise=20delete=20label=20table?= =?UTF-8?q?=20code=20sample,=20add=20TODO=20to=20clean=20up=20sni=E2=80=A6?= =?UTF-8?q?=20(#1466)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs: revise delete label table code sample, add TODO to clean up snippets.py * changed name of test function to align with file name Co-authored-by: aribray <45905583+aribray@users.noreply.github.com> Co-authored-by: Anthonios Partheniou --- docs/snippets.py | 2 + samples/snippets/delete_label_table.py | 43 +++++++++++++++++++++ samples/snippets/delete_label_table_test.py | 34 ++++++++++++++++ 3 files changed, 79 insertions(+) create mode 100644 samples/snippets/delete_label_table.py create mode 100644 samples/snippets/delete_label_table_test.py diff --git a/docs/snippets.py b/docs/snippets.py index 85856eb3e..eca2b1353 100644 --- a/docs/snippets.py +++ b/docs/snippets.py @@ -203,6 +203,8 @@ def test_manage_table_labels(client, to_delete): # [END bigquery_get_table_labels] assert table.labels == labels + # TODO(Mattix23): After code sample is updated from cloud.google.com delete this + # [START bigquery_delete_label_table] # from google.cloud import bigquery # client = bigquery.Client() diff --git a/samples/snippets/delete_label_table.py b/samples/snippets/delete_label_table.py new file mode 100644 index 000000000..0e9eaaf8f --- /dev/null +++ b/samples/snippets/delete_label_table.py @@ -0,0 +1,43 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.cloud import bigquery + + +def delete_label_table(table_id: str, label_key: str) -> bigquery.Table: + orig_table_id = table_id + orig_label_key = label_key + # [START bigquery_delete_label_table] + from google.cloud import bigquery + + client = bigquery.Client() + + # TODO(dev): Change table_id to the full name of the table you wish to delete from. + table_id = "your-project.your_dataset.your_table_name" + # TODO(dev): Change label_key to the name of the label you want to remove. + label_key = "color" + # [END bigquery_delete_label_table] + table_id = orig_table_id + label_key = orig_label_key + # [START bigquery_delete_label_table] + table = client.get_table(table_id) # API request + + # To delete a label from a table, set its value to None + table.labels[label_key] = None + + table = client.update_table(table, ["labels"]) # API request + + print(f"Deleted label '{label_key}' from {table_id}.") + # [END bigquery_delete_label_table] + return table diff --git a/samples/snippets/delete_label_table_test.py b/samples/snippets/delete_label_table_test.py new file mode 100644 index 000000000..54acae77f --- /dev/null +++ b/samples/snippets/delete_label_table_test.py @@ -0,0 +1,34 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import typing + +import delete_label_table + +if typing.TYPE_CHECKING: + import pytest + + +def test_delete_label_table( + capsys: "pytest.CaptureFixture[str]", + table_id: str, +) -> None: + + table = delete_label_table.delete_label_table(table_id, "color") + + out, _ = capsys.readouterr() + assert "Deleted" in out + assert "color" in out + assert table_id in out + assert table.labels is None or "color" not in table.labels From f0ace2ac2307ef359511a235f80f5ce9e46264c1 Mon Sep 17 00:00:00 2001 From: aribray <45905583+aribray@users.noreply.github.com> Date: Wed, 18 Jan 2023 16:29:06 -0600 Subject: [PATCH 04/16] docs: adds snippet for creating table with external data config (#1420) * docs: add samples for creating table with external data configuration and creating an external table definition Co-authored-by: Anthonios Partheniou --- docs/usage/tables.rst | 9 +++ ...reate_table_external_data_configuration.py | 66 +++++++++++++++++++ samples/tests/conftest.py | 18 ++++- ...reate_table_external_data_configuration.py | 32 +++++++++ 4 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 samples/create_table_external_data_configuration.py create mode 100644 samples/tests/test_create_table_external_data_configuration.py diff --git a/docs/usage/tables.rst b/docs/usage/tables.rst index d924fe214..105e93637 100644 --- a/docs/usage/tables.rst +++ b/docs/usage/tables.rst @@ -58,6 +58,15 @@ Create an empty table with the :start-after: [START bigquery_create_table] :end-before: [END bigquery_create_table] +Create a table using an external data source with the +:func:`~google.cloud.bigquery.client.Client.create_table` method: + +.. literalinclude:: ../samples/create_table_external_data_configuration.py + :language: python + :dedent: 4 + :start-after: [START bigquery_create_table_external_data_configuration] + :end-before: [END bigquery_create_table_external_data_configuration] + Create a clustered table with the :func:`~google.cloud.bigquery.client.Client.create_table` method: diff --git a/samples/create_table_external_data_configuration.py b/samples/create_table_external_data_configuration.py new file mode 100644 index 000000000..068f91555 --- /dev/null +++ b/samples/create_table_external_data_configuration.py @@ -0,0 +1,66 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def create_table_external_data_configuration( + table_id: str, +) -> None: + """Create a table using an external data source""" + orig_table_id = table_id + # [START bigquery_create_table_external_data_configuration] + # [START bigquery_create_external_table_definition] + from google.cloud import bigquery + + # Construct a BigQuery client object. + client = bigquery.Client() + + # TODO(developer): Set table_id to the ID of the table to create. + table_id = "your-project.your_dataset.your_table_name" + # [END bigquery_create_table_external_data_configuration] + table_id = orig_table_id + # [START bigquery_create_table_external_data_configuration] + + # TODO(developer): Set the external source format of your table. + # Note that the set of allowed values for external data sources is + # different than the set used for loading data (see :class:`~google.cloud.bigquery.job.SourceFormat`). + external_source_format = "AVRO" + + # TODO(developer): Set the source_uris to point to your data in Google Cloud + source_uris = [ + "gs://cloud-samples-data/bigquery/federated-formats-reference-file-schema/a-twitter.avro", + "gs://cloud-samples-data/bigquery/federated-formats-reference-file-schema/b-twitter.avro", + "gs://cloud-samples-data/bigquery/federated-formats-reference-file-schema/c-twitter.avro", + ] + + # Create ExternalConfig object with external source format + external_config = bigquery.ExternalConfig(external_source_format) + # Set source_uris that point to your data in Google Cloud + external_config.source_uris = source_uris + + # TODO(developer) You have the option to set a reference_file_schema_uri, which points to + # a reference file for the table schema + reference_file_schema_uri = "gs://cloud-samples-data/bigquery/federated-formats-reference-file-schema/b-twitter.avro" + + external_config.reference_file_schema_uri = reference_file_schema_uri + # [END bigquery_create_external_table_definition] + + table = bigquery.Table(table_id) + # Set the external data configuration of the table + table.external_data_configuration = external_config + table = client.create_table(table) # Make an API request. + + print( + f"Created table with external source format {table.external_data_configuration.source_format}" + ) + # [END bigquery_create_table_external_data_configuration] diff --git a/samples/tests/conftest.py b/samples/tests/conftest.py index b7a2ad587..99bd2e367 100644 --- a/samples/tests/conftest.py +++ b/samples/tests/conftest.py @@ -13,7 +13,7 @@ # limitations under the License. import datetime -from typing import Iterator +from typing import Iterator, List import uuid import google.auth @@ -47,6 +47,22 @@ def random_table_id(dataset_id: str) -> str: return "{}.{}".format(dataset_id, random_table_id) +@pytest.fixture +def avro_source_uris() -> List[str]: + avro_source_uris = [ + "gs://cloud-samples-data/bigquery/federated-formats-reference-file-schema/a-twitter.avro", + "gs://cloud-samples-data/bigquery/federated-formats-reference-file-schema/b-twitter.avro", + "gs://cloud-samples-data/bigquery/federated-formats-reference-file-schema/c-twitter.avro", + ] + return avro_source_uris + + +@pytest.fixture +def reference_file_schema_uri() -> str: + reference_file_schema_uri = "gs://cloud-samples-data/bigquery/federated-formats-reference-file-schema/b-twitter.avro" + return reference_file_schema_uri + + @pytest.fixture def random_dataset_id(client: bigquery.Client) -> Iterator[str]: now = datetime.datetime.now() diff --git a/samples/tests/test_create_table_external_data_configuration.py b/samples/tests/test_create_table_external_data_configuration.py new file mode 100644 index 000000000..bf4cf17d4 --- /dev/null +++ b/samples/tests/test_create_table_external_data_configuration.py @@ -0,0 +1,32 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import typing + +from .. import create_table_external_data_configuration + +if typing.TYPE_CHECKING: + import pytest + + +def test_create_table_external_data_configuration( + capsys: "pytest.CaptureFixture[str]", + random_table_id: str, +) -> None: + + create_table_external_data_configuration.create_table_external_data_configuration( + random_table_id + ) + out, _ = capsys.readouterr() + assert "Created table with external source format AVRO" in out From 7fe04dbe0e2811a5cfea08a0e1809437410c3ffa Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Mon, 23 Jan 2023 15:47:40 +0000 Subject: [PATCH 05/16] chore(deps): update all dependencies (#1471) Co-authored-by: Anthonios Partheniou --- samples/geography/requirements-test.txt | 2 +- samples/geography/requirements.txt | 2 +- samples/magics/requirements-test.txt | 2 +- samples/magics/requirements.txt | 2 +- samples/snippets/requirements-test.txt | 2 +- samples/snippets/requirements.txt | 4 ++-- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/samples/geography/requirements-test.txt b/samples/geography/requirements-test.txt index 1e6b7c5ea..100e0639c 100644 --- a/samples/geography/requirements-test.txt +++ b/samples/geography/requirements-test.txt @@ -1,2 +1,2 @@ -pytest==7.2.0 +pytest==7.2.1 mock==5.0.1 diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index 994d579aa..c8c063ba8 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -34,7 +34,7 @@ pyasn1-modules==0.2.8 pycparser==2.21 pyparsing==3.0.9 python-dateutil==2.8.2 -pytz==2022.7 +pytz==2022.7.1 PyYAML==6.0 requests==2.28.2 rsa==4.9 diff --git a/samples/magics/requirements-test.txt b/samples/magics/requirements-test.txt index 56aa0f432..e8f3982c7 100644 --- a/samples/magics/requirements-test.txt +++ b/samples/magics/requirements-test.txt @@ -1,3 +1,3 @@ google-cloud-testutils==1.3.3 -pytest==7.2.0 +pytest==7.2.1 mock==5.0.1 diff --git a/samples/magics/requirements.txt b/samples/magics/requirements.txt index 463829c8f..9fd09f557 100644 --- a/samples/magics/requirements.txt +++ b/samples/magics/requirements.txt @@ -11,5 +11,5 @@ matplotlib==3.6.3; python_version >= '3.8' pandas===1.3.5; python_version == '3.7' pandas==1.5.2; python_version >= '3.8' pyarrow==10.0.1 -pytz==2022.7 +pytz==2022.7.1 typing-extensions==4.4.0 diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt index 56aa0f432..e8f3982c7 100644 --- a/samples/snippets/requirements-test.txt +++ b/samples/snippets/requirements-test.txt @@ -1,3 +1,3 @@ google-cloud-testutils==1.3.3 -pytest==7.2.0 +pytest==7.2.1 mock==5.0.1 diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 1ecf3fc45..8a7b97563 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,5 +1,5 @@ db-dtypes==1.0.5 -google-cloud-bigquery==3.4.1 +google-cloud-bigquery==3.4.2 google-cloud-bigquery-storage==2.18.0 google-auth-oauthlib==0.8.0 grpcio==1.51.1 @@ -12,5 +12,5 @@ matplotlib==3.6.3; python_version >= '3.8' pandas===1.3.5; python_version == '3.7' pandas==1.5.2; python_version >= '3.8' pyarrow==10.0.1 -pytz==2022.7 +pytz==2022.7.1 typing-extensions==4.4.0 From fa8f6ec86892c78a44a52884073d159ac6690238 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Mon, 23 Jan 2023 16:31:18 +0000 Subject: [PATCH 06/16] chore(deps): update all dependencies (#1475) --- samples/geography/requirements.txt | 4 ++-- samples/magics/requirements.txt | 2 +- samples/snippets/requirements.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index c8c063ba8..3a28bf2b5 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -17,7 +17,7 @@ google-cloud-bigquery==3.4.2 google-cloud-bigquery-storage==2.18.0 google-cloud-core==2.3.2 google-crc32c==1.5.0 -google-resumable-media==2.4.0 +google-resumable-media==2.4.1 googleapis-common-protos==1.58.0 grpcio==1.51.1 idna==3.4 @@ -26,7 +26,7 @@ munch==2.5.0 mypy-extensions==0.4.3 packaging==23.0 pandas===1.3.5; python_version == '3.7' -pandas==1.5.2; python_version >= '3.8' +pandas==1.5.3; python_version >= '3.8' proto-plus==1.22.2 pyarrow==10.0.1 pyasn1==0.4.8 diff --git a/samples/magics/requirements.txt b/samples/magics/requirements.txt index 9fd09f557..2122f12db 100644 --- a/samples/magics/requirements.txt +++ b/samples/magics/requirements.txt @@ -9,7 +9,7 @@ ipython==8.8.0; python_version >= '3.9' matplotlib===3.5.3; python_version == '3.7' matplotlib==3.6.3; python_version >= '3.8' pandas===1.3.5; python_version == '3.7' -pandas==1.5.2; python_version >= '3.8' +pandas==1.5.3; python_version >= '3.8' pyarrow==10.0.1 pytz==2022.7.1 typing-extensions==4.4.0 diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 8a7b97563..7b0487f06 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -10,7 +10,7 @@ ipython==8.8.0; python_version >= '3.9' matplotlib===3.5.3; python_version == '3.7' matplotlib==3.6.3; python_version >= '3.8' pandas===1.3.5; python_version == '3.7' -pandas==1.5.2; python_version >= '3.8' +pandas==1.5.3; python_version >= '3.8' pyarrow==10.0.1 pytz==2022.7.1 typing-extensions==4.4.0 From a71888a60d1e5e5815ab459fe24368ad5b0d032a Mon Sep 17 00:00:00 2001 From: adhiggs <38925938+adhiggs@users.noreply.github.com> Date: Mon, 23 Jan 2023 16:08:12 -0800 Subject: [PATCH 07/16] docs(samples): table variable fix (#1287) Updated table variable in "# Print row data in tabular format." to point at the correct table_id. Fixes https://togithub.com/googleapis/python-bigquery/issues/1286 --- samples/browse_table_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/browse_table_data.py b/samples/browse_table_data.py index 6a56253bf..74b903aa3 100644 --- a/samples/browse_table_data.py +++ b/samples/browse_table_data.py @@ -47,7 +47,7 @@ def browse_table_data(table_id: str) -> None: print("Downloaded {} rows from table {}".format(len(rows), table_id)) # Print row data in tabular format. - rows_iter = client.list_rows(table, max_results=10) + rows_iter = client.list_rows(table_id, max_results=10) format_string = "{!s:<16} " * len(rows_iter.schema) field_names = [field.name for field in rows_iter.schema] print(format_string.format(*field_names)) # Prints column headers. From e69c81d1ef87a37ccdcca79cc73502acbc2b23ee Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Wed, 25 Jan 2023 18:43:49 +0000 Subject: [PATCH 08/16] chore(deps): update dependency google-cloud-bigquery-storage to v2.18.1 (#1476) Co-authored-by: Anthonios Partheniou --- samples/geography/requirements.txt | 2 +- samples/magics/requirements.txt | 2 +- samples/snippets/requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index 3a28bf2b5..a6a33af5a 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -14,7 +14,7 @@ geopandas==0.12.2; python_version >= '3.8' google-api-core==2.11.0 google-auth==2.16.0 google-cloud-bigquery==3.4.2 -google-cloud-bigquery-storage==2.18.0 +google-cloud-bigquery-storage==2.18.1 google-cloud-core==2.3.2 google-crc32c==1.5.0 google-resumable-media==2.4.1 diff --git a/samples/magics/requirements.txt b/samples/magics/requirements.txt index 2122f12db..ba90dab78 100644 --- a/samples/magics/requirements.txt +++ b/samples/magics/requirements.txt @@ -1,5 +1,5 @@ db-dtypes==1.0.5 -google-cloud-bigquery-storage==2.18.0 +google-cloud-bigquery-storage==2.18.1 google-auth-oauthlib==0.8.0 grpcio==1.51.1 ipywidgets==8.0.4 diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 7b0487f06..b20934886 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,6 +1,6 @@ db-dtypes==1.0.5 google-cloud-bigquery==3.4.2 -google-cloud-bigquery-storage==2.18.0 +google-cloud-bigquery-storage==2.18.1 google-auth-oauthlib==0.8.0 grpcio==1.51.1 ipywidgets==8.0.4 From 58eff2c746aaa436b15e54bba8688a6c72d008cf Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Thu, 26 Jan 2023 17:08:45 +0000 Subject: [PATCH 09/16] chore(deps): update dependency geojson to v3 (#1478) --- samples/geography/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index a6a33af5a..1a332cdaf 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -8,7 +8,7 @@ cligj==0.7.2 dataclasses==0.8; python_version < '3.7' db-dtypes==1.0.5 Fiona==1.8.22 -geojson==2.5.0 +geojson==3.0.0 geopandas===0.10.2; python_version == '3.7' geopandas==0.12.2; python_version >= '3.8' google-api-core==2.11.0 From 9792ae0f1d28c8c9e28c89b0c57b642bb8f1b5ab Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Sat, 28 Jan 2023 11:39:21 +0000 Subject: [PATCH 10/16] chore(deps): update all dependencies (#1479) --- samples/geography/requirements.txt | 2 +- samples/magics/requirements.txt | 4 ++-- samples/snippets/requirements.txt | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index 1a332cdaf..294cb095d 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -28,7 +28,7 @@ packaging==23.0 pandas===1.3.5; python_version == '3.7' pandas==1.5.3; python_version >= '3.8' proto-plus==1.22.2 -pyarrow==10.0.1 +pyarrow==11.0.0 pyasn1==0.4.8 pyasn1-modules==0.2.8 pycparser==2.21 diff --git a/samples/magics/requirements.txt b/samples/magics/requirements.txt index ba90dab78..2446aa5e8 100644 --- a/samples/magics/requirements.txt +++ b/samples/magics/requirements.txt @@ -5,11 +5,11 @@ grpcio==1.51.1 ipywidgets==8.0.4 ipython===7.31.1; python_version == '3.7' ipython===8.0.1; python_version == '3.8' -ipython==8.8.0; python_version >= '3.9' +ipython==8.9.0; python_version >= '3.9' matplotlib===3.5.3; python_version == '3.7' matplotlib==3.6.3; python_version >= '3.8' pandas===1.3.5; python_version == '3.7' pandas==1.5.3; python_version >= '3.8' -pyarrow==10.0.1 +pyarrow==11.0.0 pytz==2022.7.1 typing-extensions==4.4.0 diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index b20934886..a85653d42 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -6,11 +6,11 @@ grpcio==1.51.1 ipywidgets==8.0.4 ipython===7.31.1; python_version == '3.7' ipython===8.0.1; python_version == '3.8' -ipython==8.8.0; python_version >= '3.9' +ipython==8.9.0; python_version >= '3.9' matplotlib===3.5.3; python_version == '3.7' matplotlib==3.6.3; python_version >= '3.8' pandas===1.3.5; python_version == '3.7' pandas==1.5.3; python_version >= '3.8' -pyarrow==10.0.1 +pyarrow==11.0.0 pytz==2022.7.1 typing-extensions==4.4.0 From b63899f8a4608607eb5627f0ea5701d673537317 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Mon, 30 Jan 2023 10:38:10 +0000 Subject: [PATCH 11/16] chore(deps): update dependency shapely to v2.0.1 (#1480) --- samples/geography/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index 294cb095d..d3fee4c2f 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -38,7 +38,7 @@ pytz==2022.7.1 PyYAML==6.0 requests==2.28.2 rsa==4.9 -Shapely==2.0.0 +Shapely==2.0.1 six==1.16.0 typing-extensions==4.4.0 typing-inspect==0.8.0 From 903dcddf7ee7a408dd205f53efc1c1feea9cd0ff Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Mon, 30 Jan 2023 18:34:14 +0000 Subject: [PATCH 12/16] chore(deps): update dependency fiona to v1.9.0 (#1482) --- samples/geography/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index d3fee4c2f..284614d17 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -7,7 +7,7 @@ click-plugins==1.1.1 cligj==0.7.2 dataclasses==0.8; python_version < '3.7' db-dtypes==1.0.5 -Fiona==1.8.22 +Fiona==1.9.0 geojson==3.0.0 geopandas===0.10.2; python_version == '3.7' geopandas==0.12.2; python_version >= '3.8' From f32df1fb74e4aea24cd8a4099040ad2f7436e54d Mon Sep 17 00:00:00 2001 From: Mario Torres Jr <105736410+Mattix23@users.noreply.github.com> Date: Mon, 30 Jan 2023 14:55:16 -0600 Subject: [PATCH 13/16] feat: add __str__ method to DatasetReference (#1477) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add __str__ method to DatasetReference * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot --- google/cloud/bigquery/dataset.py | 3 +++ tests/unit/test_dataset.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index c30204067..0edd29359 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -215,6 +215,9 @@ def __ne__(self, other): def __hash__(self): return hash(self._key()) + def __str__(self): + return f"{self.project}.{self._dataset_id}" + def __repr__(self): return "DatasetReference{}".format(self._key()) diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index 856674daf..5e26a0c03 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -622,6 +622,10 @@ def test___repr__(self): expected = "DatasetReference('project1', 'dataset1')" self.assertEqual(repr(dataset), expected) + def test___str__(self): + dataset = self._make_one("project1", "dataset1") + self.assertEqual(str(dataset), "project1.dataset1") + class TestDataset(unittest.TestCase): from google.cloud.bigquery.dataset import DatasetReference From 24793947f8fc8aaec38d8c78ebbb3a9420992a21 Mon Sep 17 00:00:00 2001 From: Mario Torres Jr <105736410+Mattix23@users.noreply.github.com> Date: Mon, 30 Jan 2023 15:25:23 -0600 Subject: [PATCH 14/16] chore: delete duplicate code from snippets.py (#1481) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: delete duplicate code from snippets.py * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot --- docs/snippets.py | 60 ------------------------------------------------ 1 file changed, 60 deletions(-) diff --git a/docs/snippets.py b/docs/snippets.py index eca2b1353..e1d9ae839 100644 --- a/docs/snippets.py +++ b/docs/snippets.py @@ -164,66 +164,6 @@ def test_create_partitioned_table(client, to_delete): "https://github.com/GoogleCloudPlatform/google-cloud-python/issues/5589" ) ) -def test_manage_table_labels(client, to_delete): - dataset_id = "label_table_dataset_{}".format(_millis()) - table_id = "label_table_{}".format(_millis()) - project = client.project - dataset_ref = bigquery.DatasetReference(project, dataset_id) - dataset = bigquery.Dataset(dataset_ref) - client.create_dataset(dataset) - to_delete.append(dataset) - - table = bigquery.Table(dataset.table(table_id), schema=SCHEMA) - - labels = {"color": "green"} - table.labels = labels - table = client.create_table(table) - - # TODO(Mattix23): After code sample is updated from cloud.google.com delete this - - # [START bigquery_get_table_labels] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_id = 'my_dataset' - # table_id = 'my_table' - - project = client.project - dataset_ref = bigquery.DatasetReference(project, dataset_id) - table_ref = dataset_ref.table(table_id) - table = client.get_table(table_ref) # API Request - - # View table labels - print("Table ID: {}".format(table_id)) - print("Labels:") - if table.labels: - for label, value in table.labels.items(): - print("\t{}: {}".format(label, value)) - else: - print("\tTable has no labels defined.") - # [END bigquery_get_table_labels] - assert table.labels == labels - - # TODO(Mattix23): After code sample is updated from cloud.google.com delete this - - # [START bigquery_delete_label_table] - # from google.cloud import bigquery - # client = bigquery.Client() - # project = client.project - # dataset_ref = bigquery.DatasetReference(project, dataset_id) - # table_ref = dataset_ref.table('my_table') - # table = client.get_table(table_ref) # API request - - # This example table starts with one label - assert table.labels == {"color": "green"} - # To delete a label from a table, set its value to None - table.labels["color"] = None - - table = client.update_table(table, ["labels"]) # API request - - assert table.labels == {} - # [END bigquery_delete_label_table] - - @pytest.mark.skip( reason=( "update_table() is flaky " From bd1da9aa0a40b02b7d5409a0b094d8380e255c91 Mon Sep 17 00:00:00 2001 From: Mario Torres Jr <105736410+Mattix23@users.noreply.github.com> Date: Tue, 31 Jan 2023 12:06:02 -0600 Subject: [PATCH 15/16] feat: add preserveAsciiControlCharacter to LoadJobConfig (#1484) --- google/cloud/bigquery/job/load.py | 13 +++++++++++++ tests/unit/job/test_load_config.py | 14 ++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/google/cloud/bigquery/job/load.py b/google/cloud/bigquery/job/load.py index 5c7f26841..14a7fa30b 100644 --- a/google/cloud/bigquery/job/load.py +++ b/google/cloud/bigquery/job/load.py @@ -311,6 +311,19 @@ def null_marker(self): def null_marker(self, value): self._set_sub_prop("nullMarker", value) + @property + def preserve_ascii_control_characters(self): + """Optional[bool]: Preserves the embedded ASCII control characters when sourceFormat is set to CSV. + + See: + https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.preserve_ascii_control_characters + """ + return self._get_sub_prop("preserveAsciiControlCharacters") + + @preserve_ascii_control_characters.setter + def preserve_ascii_control_characters(self, value): + self._set_sub_prop("preserveAsciiControlCharacters", bool(value)) + @property def projection_fields(self) -> Optional[List[str]]: """Optional[List[str]]: If diff --git a/tests/unit/job/test_load_config.py b/tests/unit/job/test_load_config.py index 5a0c5a83f..7f77fc085 100644 --- a/tests/unit/job/test_load_config.py +++ b/tests/unit/job/test_load_config.py @@ -424,6 +424,20 @@ def test_null_marker_setter(self): config.null_marker = null_marker self.assertEqual(config._properties["load"]["nullMarker"], null_marker) + def test_preserve_ascii_control_characters_missing(self): + config = self._get_target_class()() + self.assertIsNone(config.preserve_ascii_control_characters) + + def test_preserve_ascii_control_characters_hit(self): + config = self._get_target_class()() + config._properties["load"]["preserveAsciiControlCharacters"] = True + self.assertTrue(config.preserve_ascii_control_characters) + + def test_preserve_ascii_control_characters_setter(self): + config = self._get_target_class()() + config.preserve_ascii_control_characters = True + self.assertTrue(config._properties["load"]["preserveAsciiControlCharacters"]) + def test_projection_fields_miss(self): config = self._get_target_class()() self.assertIsNone(config.projection_fields) From 204779c57e706c002ababff72f8e45f1e376b960 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 1 Feb 2023 12:41:48 -0600 Subject: [PATCH 16/16] chore(main): release 3.5.0 (#1472) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 15 +++++++++++++++ google/cloud/bigquery/version.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a5b4d179..454d362f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,21 @@ [1]: https://pypi.org/project/google-cloud-bigquery/#history +## [3.5.0](https://github.com/googleapis/python-bigquery/compare/v3.4.2...v3.5.0) (2023-01-31) + + +### Features + +* Add __str__ method to DatasetReference ([#1477](https://github.com/googleapis/python-bigquery/issues/1477)) ([f32df1f](https://github.com/googleapis/python-bigquery/commit/f32df1fb74e4aea24cd8a4099040ad2f7436e54d)) +* Add preserveAsciiControlCharacter to LoadJobConfig ([#1484](https://github.com/googleapis/python-bigquery/issues/1484)) ([bd1da9a](https://github.com/googleapis/python-bigquery/commit/bd1da9aa0a40b02b7d5409a0b094d8380e255c91)) + + +### Documentation + +* Adds snippet for creating table with external data config ([#1420](https://github.com/googleapis/python-bigquery/issues/1420)) ([f0ace2a](https://github.com/googleapis/python-bigquery/commit/f0ace2ac2307ef359511a235f80f5ce9e46264c1)) +* Revise delete label table code sample, add TODO to clean up sni… ([#1466](https://github.com/googleapis/python-bigquery/issues/1466)) ([0dab7d2](https://github.com/googleapis/python-bigquery/commit/0dab7d25ace4b63d2984485e7b0c5bb38f20476f)) +* **samples:** Table variable fix ([#1287](https://github.com/googleapis/python-bigquery/issues/1287)) ([a71888a](https://github.com/googleapis/python-bigquery/commit/a71888a60d1e5e5815ab459fe24368ad5b0d032a)) + ## [3.4.2](https://github.com/googleapis/python-bigquery/compare/v3.4.1...v3.4.2) (2023-01-13) diff --git a/google/cloud/bigquery/version.py b/google/cloud/bigquery/version.py index d38bb4619..13194aa56 100644 --- a/google/cloud/bigquery/version.py +++ b/google/cloud/bigquery/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "3.4.2" +__version__ = "3.5.0"