diff --git a/CHANGELOG.md b/CHANGELOG.md index d5efd7dd6..869d063e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,32 @@ [1]: https://pypi.org/project/google-cloud-bigquery/#history +## [3.3.6](https://github.com/googleapis/python-bigquery/compare/v3.3.4...v3.3.6) (2022-11-02) + + +### Features + +* Reconfigure tqdm progress bar in %%bigquery magic ([#1355](https://github.com/googleapis/python-bigquery/issues/1355)) ([506f781](https://github.com/googleapis/python-bigquery/commit/506f781c2dd775193336ab9432f32148250ed81d)) + + +### Bug Fixes + +* Corrects test for non-existent attribute ([#1395](https://github.com/googleapis/python-bigquery/issues/1395)) ([a80f436](https://github.com/googleapis/python-bigquery/commit/a80f436f2e75a8fb680316f17a22eecb31a7101d)) +* **deps:** Allow protobuf 3.19.5 ([#1379](https://github.com/googleapis/python-bigquery/issues/1379)) ([3e4a074](https://github.com/googleapis/python-bigquery/commit/3e4a074a981eb2920c5f9a711c253565d4844858)) +* **deps:** Allow pyarrow < 11 ([#1393](https://github.com/googleapis/python-bigquery/issues/1393)) ([c898546](https://github.com/googleapis/python-bigquery/commit/c898546d3292f9ec1ba6120cd3f9e2805aa087bb)) +* **deps:** Require requests>=2.21.0 ([#1388](https://github.com/googleapis/python-bigquery/issues/1388)) ([e398336](https://github.com/googleapis/python-bigquery/commit/e39833673582e4a7a34103cfc45603932c9c33b3)) +* Refactor to adapt to changes to shapely dependency ([#1376](https://github.com/googleapis/python-bigquery/issues/1376)) ([2afd278](https://github.com/googleapis/python-bigquery/commit/2afd278febe1eb247adc6278ab59903962a5bb6c)) + + +### Documentation + +* Fix typos ([#1372](https://github.com/googleapis/python-bigquery/issues/1372)) ([21cc525](https://github.com/googleapis/python-bigquery/commit/21cc525a86a06acfe73e5c5a74ec5f0b61e410f2)) + + +### Miscellaneous Chores + +* release 3.3.6 ([4fce1d9](https://github.com/googleapis/python-bigquery/commit/4fce1d93b1763703b115a0480a2b97021786aff7)) + ## [3.3.4](https://github.com/googleapis/python-bigquery/compare/v3.3.3...v3.3.4) (2022-09-29) diff --git a/google/cloud/bigquery/_pandas_helpers.py b/google/cloud/bigquery/_pandas_helpers.py index cc0ee75ff..0d05f53a3 100644 --- a/google/cloud/bigquery/_pandas_helpers.py +++ b/google/cloud/bigquery/_pandas_helpers.py @@ -57,15 +57,9 @@ if pandas is not None: # pragma: NO COVER def _to_wkb(): - # Create a closure that: - # - Adds a not-null check. This allows the returned function to - # be used directly with apply, unlike `shapely.wkb.dumps`. - # - Avoid extra work done by `shapely.wkb.dumps` that we don't need. - # - Caches the WKBWriter (and write method lookup :) ) - # - Avoids adding WKBWriter, lgeos, and notnull to the module namespace. - from shapely.geos import WKBWriter, lgeos # type: ignore - - write = WKBWriter(lgeos).write + from shapely import wkb # type: ignore + + write = wkb.dumps notnull = pandas.notnull def _to_wkb(v): diff --git a/google/cloud/bigquery/_tqdm_helpers.py b/google/cloud/bigquery/_tqdm_helpers.py index ae289d8a6..456ca2530 100644 --- a/google/cloud/bigquery/_tqdm_helpers.py +++ b/google/cloud/bigquery/_tqdm_helpers.py @@ -15,6 +15,7 @@ """Shared helper functions for tqdm progress bar.""" import concurrent.futures +import sys import time import typing from typing import Optional @@ -22,6 +23,7 @@ try: import tqdm # type: ignore + import tqdm.notebook as notebook # type: ignore except ImportError: # pragma: NO COVER tqdm = None @@ -47,9 +49,22 @@ def get_progress_bar(progress_bar_type, description, total, unit): try: if progress_bar_type == "tqdm": - return tqdm.tqdm(desc=description, total=total, unit=unit) + return tqdm.tqdm( + bar_format="{l_bar}{bar}|", + colour="green", + desc=description, + file=sys.stdout, + total=total, + unit=unit, + ) elif progress_bar_type == "tqdm_notebook": - return tqdm.notebook.tqdm(desc=description, total=total, unit=unit) + return notebook.tqdm( + bar_format="{l_bar}{bar}|", + desc=description, + file=sys.stdout, + total=total, + unit=unit, + ) elif progress_bar_type == "tqdm_gui": return tqdm.tqdm_gui(desc=description, total=total, unit=unit) except (KeyError, TypeError): @@ -80,7 +95,7 @@ def wait_for_query( """ default_total = 1 current_stage = None - start_time = time.time() + start_time = time.perf_counter() progress_bar = get_progress_bar( progress_bar_type, "Query is running", default_total, "query" @@ -95,11 +110,7 @@ def wait_for_query( current_stage = query_job.query_plan[i] progress_bar.total = len(query_job.query_plan) progress_bar.set_description( - "Query executing stage {} and status {} : {:0.2f}s".format( - current_stage.name, - current_stage.status, - time.time() - start_time, - ), + f"Query executing stage {current_stage.name} and status {current_stage.status} : {time.perf_counter() - start_time:.2f}s" ) try: query_result = query_job.result( @@ -107,7 +118,7 @@ def wait_for_query( ) progress_bar.update(default_total) progress_bar.set_description( - "Query complete after {:0.2f}s".format(time.time() - start_time), + f"Job ID {query_job.job_id} successfully executed", ) break except concurrent.futures.TimeoutError: diff --git a/google/cloud/bigquery/job/query.py b/google/cloud/bigquery/job/query.py index c2d304e30..b0286deae 100644 --- a/google/cloud/bigquery/job/query.py +++ b/google/cloud/bigquery/job/query.py @@ -1556,9 +1556,9 @@ def to_arrow( No progress bar. ``'tqdm'`` Use the :func:`tqdm.tqdm` function to print a progress bar - to :data:`sys.stderr`. + to :data:`sys.stdout`. ``'tqdm_notebook'`` - Use the :func:`tqdm.tqdm_notebook` function to display a + Use the :func:`tqdm.notebook.tqdm` function to display a progress bar as a Jupyter notebook widget. ``'tqdm_gui'`` Use the :func:`tqdm.tqdm_gui` function to display a diff --git a/google/cloud/bigquery/magics/magics.py b/google/cloud/bigquery/magics/magics.py index 14819aa59..613cc1b58 100644 --- a/google/cloud/bigquery/magics/magics.py +++ b/google/cloud/bigquery/magics/magics.py @@ -125,7 +125,7 @@ def __init__(self): self._default_query_job_config = bigquery.QueryJobConfig() self._bigquery_client_options = client_options.ClientOptions() self._bqstorage_client_options = client_options.ClientOptions() - self._progress_bar_type = "tqdm" + self._progress_bar_type = "tqdm_notebook" @property def credentials(self): @@ -269,7 +269,7 @@ def progress_bar_type(self): Manually setting the progress_bar_type: >>> from google.cloud.bigquery import magics - >>> magics.context.progress_bar_type = "tqdm" + >>> magics.context.progress_bar_type = "tqdm_notebook" """ return self._progress_bar_type @@ -286,7 +286,7 @@ def _handle_error(error, destination_var=None): Args: error (Exception): - An exception that ocurred during the query exectution. + An exception that ocurred during the query execution. destination_var (Optional[str]): The name of the IPython session variable to store the query job. """ @@ -329,22 +329,25 @@ def _run_query(client, query, job_config=None): Query complete after 2.07s 'bf633912-af2c-4780-b568-5d868058632b' """ - start_time = time.time() + start_time = time.perf_counter() query_job = client.query(query, job_config=job_config) if job_config and job_config.dry_run: return query_job - print("Executing query with job ID: {}".format(query_job.job_id)) + print(f"Executing query with job ID: {query_job.job_id}") while True: - print("\rQuery executing: {:0.2f}s".format(time.time() - start_time), end="") + print( + f"\rQuery executing: {time.perf_counter() - start_time:.2f}s".format(), + end="", + ) try: query_job.result(timeout=0.5) break except futures.TimeoutError: continue - print("\nQuery complete after {:0.2f}s".format(time.time() - start_time)) + print(f"\nJob ID {query_job.job_id} successfully executed") return query_job @@ -365,7 +368,7 @@ def _create_dataset_if_necessary(client, dataset_id): pass dataset = bigquery.Dataset(dataset_reference) dataset.location = client.location - print("Creating dataset: {}".format(dataset_id)) + print(f"Creating dataset: {dataset_id}") dataset = client.create_dataset(dataset) @@ -500,7 +503,7 @@ def _create_dataset_if_necessary(client, dataset_id): default=None, help=( "Sets progress bar type to display a progress bar while executing the query." - "Defaults to use tqdm. Install the ``tqdm`` package to use this feature." + "Defaults to use tqdm_notebook. Install the ``tqdm`` package to use this feature." ), ) def _cell_magic(line, query): diff --git a/google/cloud/bigquery/query.py b/google/cloud/bigquery/query.py index 0469cb271..944ad884e 100644 --- a/google/cloud/bigquery/query.py +++ b/google/cloud/bigquery/query.py @@ -795,7 +795,7 @@ def _key(self): Used to compute this instance's hashcode and evaluate equality. Returns: - Tuple: The contents of this :class:`~google.cloud.biquery.ArrayQueryParameter`. + Tuple: The contents of this :class:`~google.cloud.bigquery.ArrayQueryParameter`. """ return (self.name, self.struct_types, self.struct_values) diff --git a/google/cloud/bigquery/schema.py b/google/cloud/bigquery/schema.py index 5580a2ae9..1df78424d 100644 --- a/google/cloud/bigquery/schema.py +++ b/google/cloud/bigquery/schema.py @@ -144,7 +144,7 @@ def from_api_repr(cls, api_repr: dict) -> "SchemaField": :meth:`to_api_repr`. Returns: - google.cloud.biquery.schema.SchemaField: The ``SchemaField`` object. + google.cloud.bigquery.schema.SchemaField: The ``SchemaField`` object. """ field_type = api_repr["type"].upper() diff --git a/google/cloud/bigquery/table.py b/google/cloud/bigquery/table.py index 72eb1baf6..4fd77dd21 100644 --- a/google/cloud/bigquery/table.py +++ b/google/cloud/bigquery/table.py @@ -39,11 +39,12 @@ _COORDINATE_REFERENCE_SYSTEM = "EPSG:4326" try: - import shapely.geos # type: ignore + import shapely # type: ignore + from shapely import wkt # type: ignore except ImportError: shapely = None else: - _read_wkt = shapely.geos.WKTReader(shapely.geos.lgeos).read + _read_wkt = wkt.loads import google.api_core.exceptions from google.api_core.page_iterator import HTTPIterator @@ -1728,9 +1729,9 @@ def to_arrow( No progress bar. ``'tqdm'`` Use the :func:`tqdm.tqdm` function to print a progress bar - to :data:`sys.stderr`. + to :data:`sys.stdout`. ``'tqdm_notebook'`` - Use the :func:`tqdm.tqdm_notebook` function to display a + Use the :func:`tqdm.notebook.tqdm` function to display a progress bar as a Jupyter notebook widget. ``'tqdm_gui'`` Use the :func:`tqdm.tqdm_gui` function to display a @@ -1921,9 +1922,9 @@ def to_dataframe( No progress bar. ``'tqdm'`` Use the :func:`tqdm.tqdm` function to print a progress bar - to :data:`sys.stderr`. + to :data:`sys.stdout`. ``'tqdm_notebook'`` - Use the :func:`tqdm.tqdm_notebook` function to display a + Use the :func:`tqdm.notebook.tqdm` function to display a progress bar as a Jupyter notebook widget. ``'tqdm_gui'`` Use the :func:`tqdm.tqdm_gui` function to display a @@ -2075,9 +2076,9 @@ def to_geodataframe( No progress bar. ``'tqdm'`` Use the :func:`tqdm.tqdm` function to print a progress bar - to :data:`sys.stderr`. + to :data:`sys.stdout`. ``'tqdm_notebook'`` - Use the :func:`tqdm.tqdm_notebook` function to display a + Use the :func:`tqdm.notebook.tqdm` function to display a progress bar as a Jupyter notebook widget. ``'tqdm_gui'`` Use the :func:`tqdm.tqdm_gui` function to display a diff --git a/google/cloud/bigquery/version.py b/google/cloud/bigquery/version.py index 3e1a9869c..43360a201 100644 --- a/google/cloud/bigquery/version.py +++ b/google/cloud/bigquery/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "3.3.4" +__version__ = "3.3.6" diff --git a/noxfile.py b/noxfile.py index 0b0800d35..a91e60a5f 100644 --- a/noxfile.py +++ b/noxfile.py @@ -81,7 +81,7 @@ def default(session, install_extras=True): ) if install_extras and session.python == "3.10": - install_target = ".[bqstorage,pandas,tqdm,opentelemetry]" + install_target = ".[bqstorage,ipywidgets,pandas,tqdm,opentelemetry]" elif install_extras: install_target = ".[all]" else: @@ -186,7 +186,7 @@ def system(session): session.install("google-cloud-datacatalog", "-c", constraints_path) if session.python == "3.10": - extras = "[bqstorage,pandas,tqdm,opentelemetry]" + extras = "[bqstorage,ipywidgets,pandas,tqdm,opentelemetry]" else: extras = "[all]" session.install("-e", f".{extras}", "-c", constraints_path) @@ -235,7 +235,7 @@ def snippets(session): session.install("grpcio", "-c", constraints_path) if session.python == "3.10": - extras = "[bqstorage,pandas,tqdm,opentelemetry]" + extras = "[bqstorage,ipywidgets,pandas,tqdm,opentelemetry]" else: extras = "[all]" session.install("-e", f".{extras}", "-c", constraints_path) @@ -387,7 +387,7 @@ def blacken(session): def docs(session): """Build the docs.""" - session.install("recommonmark", "sphinx==4.0.1", "sphinx_rtd_theme") + session.install("recommonmark", "sphinx==4.0.2", "sphinx_rtd_theme") session.install("google-cloud-storage") session.install("-e", ".[all]") @@ -412,7 +412,7 @@ def docfx(session): session.install("-e", ".") session.install( - "sphinx==4.0.1", "alabaster", "recommonmark", "gcp-sphinx-docfx-yaml" + "sphinx==4.0.2", "alabaster", "recommonmark", "gcp-sphinx-docfx-yaml" ) shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index 6b14f90ab..798de6bb5 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -1,5 +1,5 @@ attrs==22.1.0 -certifi==2022.9.14 +certifi==2022.9.24 cffi==1.15.1 charset-normalizer==2.1.1 click==8.1.3 @@ -7,26 +7,26 @@ click-plugins==1.1.1 cligj==0.7.2 dataclasses==0.8; python_version < '3.7' db-dtypes==1.0.4 -Fiona==1.8.21 +Fiona==1.8.22 geojson==2.5.0 geopandas===0.10.2; python_version == '3.7' -geopandas==0.11.1; python_version >= '3.8' -google-api-core==2.10.1 -google-auth==2.11.1 -google-cloud-bigquery==3.3.2 -google-cloud-bigquery-storage==2.16.0 +geopandas==0.12.1; python_version >= '3.8' +google-api-core==2.10.2 +google-auth==2.13.0 +google-cloud-bigquery==3.3.5 +google-cloud-bigquery-storage==2.16.2 google-cloud-core==2.3.2 google-crc32c==1.5.0 -google-resumable-media==2.3.3 +google-resumable-media==2.4.0 googleapis-common-protos==1.56.4 -grpcio==1.49.0 +grpcio==1.50.0 idna==3.4 libcst==0.4.7 munch==2.5.0 mypy-extensions==0.4.3 packaging==21.3 pandas===1.3.5; python_version == '3.7' -pandas==1.5.0; python_version >= '3.8' +pandas==1.5.1; python_version >= '3.8' proto-plus==1.22.1 pyarrow==9.0.0 pyasn1==0.4.8 @@ -34,12 +34,12 @@ pyasn1-modules==0.2.8 pycparser==2.21 pyparsing==3.0.9 python-dateutil==2.8.2 -pytz==2022.2.1 +pytz==2022.5 PyYAML==6.0 requests==2.28.1 rsa==4.9 -Shapely==1.8.4 +Shapely==1.8.5.post1 six==1.16.0 -typing-extensions==4.3.0 +typing-extensions==4.4.0 typing-inspect==0.8.0 urllib3==1.26.12 diff --git a/samples/magics/requirements.txt b/samples/magics/requirements.txt index dd61784f1..bdd026ce5 100644 --- a/samples/magics/requirements.txt +++ b/samples/magics/requirements.txt @@ -1,14 +1,15 @@ db-dtypes==1.0.4 -google-cloud-bigquery-storage==2.16.0 +google-cloud-bigquery-storage==2.16.2 google-auth-oauthlib==0.5.3 -grpcio==1.49.0 +grpcio==1.50.0 +ipywidgets==8.0.2 ipython===7.31.1; python_version == '3.7' ipython===8.0.1; python_version == '3.8' ipython==8.5.0; python_version >= '3.9' matplotlib===3.5.3; python_version == '3.7' -matplotlib==3.6.0; python_version >= '3.8' +matplotlib==3.6.1; python_version >= '3.8' pandas===1.3.5; python_version == '3.7' -pandas==1.5.0; python_version >= '3.8' +pandas==1.5.1; python_version >= '3.8' pyarrow==9.0.0 -pytz==2022.2.1 -typing-extensions==4.3.0 +pytz==2022.5 +typing-extensions==4.4.0 diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 1e91a5ec7..ebf892279 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,15 +1,16 @@ db-dtypes==1.0.4 -google-cloud-bigquery==3.3.2 -google-cloud-bigquery-storage==2.16.0 +google-cloud-bigquery==3.3.5 +google-cloud-bigquery-storage==2.16.2 google-auth-oauthlib==0.5.3 -grpcio==1.49.0 +grpcio==1.50.0 +ipywidgets==8.0.2 ipython===7.31.1; python_version == '3.7' ipython===8.0.1; python_version == '3.8' ipython==8.5.0; python_version >= '3.9' matplotlib===3.5.3; python_version == '3.7' -matplotlib==3.6.0; python_version >= '3.8' +matplotlib==3.6.1; python_version >= '3.8' pandas===1.3.5; python_version == '3.7' -pandas==1.5.0; python_version >= '3.8' +pandas==1.5.1; python_version >= '3.8' pyarrow==9.0.0 -pytz==2022.2.1 -typing-extensions==4.3.0 +pytz==2022.5 +typing-extensions==4.4.0 diff --git a/setup.py b/setup.py index be02dc409..c8bf640c2 100644 --- a/setup.py +++ b/setup.py @@ -42,16 +42,17 @@ "google-cloud-core >= 1.4.1, <3.0.0dev", "google-resumable-media >= 0.6.0, < 3.0dev", "packaging >= 14.3, <22.0.0dev", - "protobuf >= 3.20.2, <5.0.0dev", # For the legacy proto-based types. + "protobuf>=3.19.5,<5.0.0dev,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5", # For the legacy proto-based types. "python-dateutil >= 2.7.2, <3.0dev", - "pyarrow >= 3.0.0, < 10.0dev", - "requests >= 2.18.0, < 3.0.0dev", + "pyarrow >= 3.0.0, < 11.0dev", + "requests >= 2.21.0, < 3.0.0dev", ] extras = { # Keep the no-op bqstorage extra for backward compatibility. # See: https://github.com/googleapis/python-bigquery/issues/757 "bqstorage": [], "pandas": ["pandas>=1.0.0", "db-dtypes>=0.3.0,<2.0.0dev"], + "ipywidgets": ["ipywidgets==7.7.1"], "geopandas": ["geopandas>=0.9.0, <1.0dev", "Shapely>=1.6.0, <2.0dev"], "ipython": ["ipython>=7.0.1,!=8.1.0"], "tqdm": ["tqdm >= 4.7.4, <5.0.0dev"], diff --git a/testing/constraints-3.7.txt b/testing/constraints-3.7.txt index 3b07dc9fa..57928714f 100644 --- a/testing/constraints-3.7.txt +++ b/testing/constraints-3.7.txt @@ -12,16 +12,17 @@ google-cloud-bigquery-storage==2.0.0 google-cloud-core==1.4.1 google-resumable-media==0.6.0 grpcio==1.47.0 +ipywidgets==7.7.1 ipython==7.0.1 opentelemetry-api==1.1.0 opentelemetry-instrumentation==0.20b0 opentelemetry-sdk==1.1.0 pandas==1.1.0 proto-plus==1.22.0 -protobuf==3.20.2 +protobuf==3.19.5 pyarrow==3.0.0 python-dateutil==2.7.3 -requests==2.18.0 +requests==2.21.0 Shapely==1.6.4.post2 six==1.13.0 tqdm==4.7.4 diff --git a/tests/system/test_magics.py b/tests/system/test_magics.py index 78c15cb50..3d761cd35 100644 --- a/tests/system/test_magics.py +++ b/tests/system/test_magics.py @@ -71,8 +71,7 @@ def test_bigquery_magic(ipython_interactive): # Removes blanks & terminal code (result of display clearing) updates = list(filter(lambda x: bool(x) and x != "\x1b[2K", lines)) assert re.match("Executing query with job ID: .*", updates[0]) - assert all(re.match("Query executing: .*s", line) for line in updates[1:-1]) - assert re.match("Query complete after .*s", updates[-1]) + assert (re.match("Query executing: .*s", line) for line in updates[1:-1]) assert isinstance(result, pandas.DataFrame) assert len(result) == 10 # verify row count assert list(result) == ["url", "view_count"] # verify column names diff --git a/tests/unit/job/test_query_pandas.py b/tests/unit/job/test_query_pandas.py index 84aab3aca..a45401664 100644 --- a/tests/unit/job/test_query_pandas.py +++ b/tests/unit/job/test_query_pandas.py @@ -37,7 +37,7 @@ except (ImportError, AttributeError): # pragma: NO COVER geopandas = None try: - from tqdm import tqdm + import tqdm except (ImportError, AttributeError): # pragma: NO COVER tqdm = None @@ -301,7 +301,8 @@ def test_to_arrow_max_results_no_progress_bar(): @pytest.mark.skipif(tqdm is None, reason="Requires `tqdm`") -def test_to_arrow_w_tqdm_w_query_plan(): +@mock.patch("google.cloud.bigquery._tqdm_helpers.tqdm") +def test_to_arrow_w_tqdm_w_query_plan(tqdm_mock): from google.cloud.bigquery import table from google.cloud.bigquery.job import QueryJob as target_class from google.cloud.bigquery.schema import SchemaField @@ -344,20 +345,20 @@ def test_to_arrow_w_tqdm_w_query_plan(): row_iterator, ], ) - - with result_patch as result_patch_tqdm, reload_patch: + with result_patch as tqdm_mock, reload_patch: tbl = job.to_arrow(progress_bar_type="tqdm", create_bqstorage_client=False) - assert result_patch_tqdm.call_count == 3 + assert tqdm_mock.call_count == 3 assert isinstance(tbl, pyarrow.Table) assert tbl.num_rows == 2 - result_patch_tqdm.assert_called_with( + tqdm_mock.assert_called_with( timeout=_PROGRESS_BAR_UPDATE_INTERVAL, max_results=None ) @pytest.mark.skipif(tqdm is None, reason="Requires `tqdm`") -def test_to_arrow_w_tqdm_w_pending_status(): +@mock.patch("google.cloud.bigquery._tqdm_helpers.tqdm") +def test_to_arrow_w_tqdm_w_pending_status(tqdm_mock): from google.cloud.bigquery import table from google.cloud.bigquery.job import QueryJob as target_class from google.cloud.bigquery.schema import SchemaField @@ -396,20 +397,20 @@ def test_to_arrow_w_tqdm_w_pending_status(): "google.cloud.bigquery.job.QueryJob.result", side_effect=[concurrent.futures.TimeoutError, row_iterator], ) - - with result_patch as result_patch_tqdm, reload_patch: + with result_patch as tqdm_mock, reload_patch: tbl = job.to_arrow(progress_bar_type="tqdm", create_bqstorage_client=False) - assert result_patch_tqdm.call_count == 2 + assert tqdm_mock.call_count == 2 assert isinstance(tbl, pyarrow.Table) assert tbl.num_rows == 2 - result_patch_tqdm.assert_called_with( + tqdm_mock.assert_called_with( timeout=_PROGRESS_BAR_UPDATE_INTERVAL, max_results=None ) @pytest.mark.skipif(tqdm is None, reason="Requires `tqdm`") -def test_to_arrow_w_tqdm_wo_query_plan(): +@mock.patch("google.cloud.bigquery._tqdm_helpers.tqdm") +def test_to_arrow_w_tqdm_wo_query_plan(tqdm_mock): from google.cloud.bigquery import table from google.cloud.bigquery.job import QueryJob as target_class from google.cloud.bigquery.schema import SchemaField @@ -439,14 +440,13 @@ def test_to_arrow_w_tqdm_wo_query_plan(): "google.cloud.bigquery.job.QueryJob.result", side_effect=[concurrent.futures.TimeoutError, row_iterator], ) - - with result_patch as result_patch_tqdm, reload_patch: + with result_patch as tqdm_mock, reload_patch: tbl = job.to_arrow(progress_bar_type="tqdm", create_bqstorage_client=False) - assert result_patch_tqdm.call_count == 2 + assert tqdm_mock.call_count == 2 assert isinstance(tbl, pyarrow.Table) assert tbl.num_rows == 2 - result_patch_tqdm.assert_called() + tqdm_mock.assert_called() def _make_job(schema=(), rows=()): @@ -720,7 +720,7 @@ def test_to_dataframe_column_date_dtypes(): @pytest.mark.skipif(tqdm is None, reason="Requires `tqdm`") -@mock.patch("tqdm.tqdm") +@mock.patch("google.cloud.bigquery._tqdm_helpers.tqdm") def test_to_dataframe_with_progress_bar(tqdm_mock): from google.cloud.bigquery.job import QueryJob as target_class @@ -744,14 +744,15 @@ def test_to_dataframe_with_progress_bar(tqdm_mock): job = target_class.from_api_repr(begun_resource, client) job.to_dataframe(progress_bar_type=None, create_bqstorage_client=False) - tqdm_mock.assert_not_called() + tqdm_mock.tqdm.assert_not_called() job.to_dataframe(progress_bar_type="tqdm", create_bqstorage_client=False) - tqdm_mock.assert_called() + tqdm_mock.tqdm.assert_called() @pytest.mark.skipif(tqdm is None, reason="Requires `tqdm`") -def test_to_dataframe_w_tqdm_pending(): +@mock.patch("google.cloud.bigquery._tqdm_helpers.tqdm") +def test_to_dataframe_w_tqdm_pending(tqdm_mock): from google.cloud.bigquery import table from google.cloud.bigquery.job import QueryJob as target_class from google.cloud.bigquery.schema import SchemaField @@ -780,7 +781,7 @@ def test_to_dataframe_w_tqdm_pending(): job._properties["statistics"] = { "query": { "queryPlan": [ - {"name": "S00: Input", "id": "0", "status": "PRNDING"}, + {"name": "S00: Input", "id": "0", "status": "PENDING"}, {"name": "S01: Output", "id": "1", "status": "COMPLETE"}, ] }, @@ -792,21 +793,21 @@ def test_to_dataframe_w_tqdm_pending(): "google.cloud.bigquery.job.QueryJob.result", side_effect=[concurrent.futures.TimeoutError, row_iterator], ) - - with result_patch as result_patch_tqdm, reload_patch: + with result_patch as tqdm_mock, reload_patch: df = job.to_dataframe(progress_bar_type="tqdm", create_bqstorage_client=False) - assert result_patch_tqdm.call_count == 2 + assert tqdm_mock.call_count == 2 assert isinstance(df, pandas.DataFrame) assert len(df) == 4 # verify the number of rows assert list(df) == ["name", "age"] # verify the column names - result_patch_tqdm.assert_called_with( + tqdm_mock.assert_called_with( timeout=_PROGRESS_BAR_UPDATE_INTERVAL, max_results=None ) @pytest.mark.skipif(tqdm is None, reason="Requires `tqdm`") -def test_to_dataframe_w_tqdm(): +@mock.patch("google.cloud.bigquery._tqdm_helpers.tqdm") +def test_to_dataframe_w_tqdm(tqdm_mock): from google.cloud.bigquery import table from google.cloud.bigquery.job import QueryJob as target_class from google.cloud.bigquery.schema import SchemaField @@ -852,20 +853,21 @@ def test_to_dataframe_w_tqdm(): ], ) - with result_patch as result_patch_tqdm, reload_patch: + with result_patch as tqdm_mock, reload_patch: df = job.to_dataframe(progress_bar_type="tqdm", create_bqstorage_client=False) - assert result_patch_tqdm.call_count == 3 + assert tqdm_mock.call_count == 3 assert isinstance(df, pandas.DataFrame) assert len(df) == 4 # verify the number of rows assert list(df), ["name", "age"] # verify the column names - result_patch_tqdm.assert_called_with( + tqdm_mock.assert_called_with( timeout=_PROGRESS_BAR_UPDATE_INTERVAL, max_results=None ) @pytest.mark.skipif(tqdm is None, reason="Requires `tqdm`") -def test_to_dataframe_w_tqdm_max_results(): +@mock.patch("google.cloud.bigquery._tqdm_helpers.tqdm") +def test_to_dataframe_w_tqdm_max_results(tqdm_mock): from google.cloud.bigquery import table from google.cloud.bigquery.job import QueryJob as target_class from google.cloud.bigquery.schema import SchemaField @@ -901,16 +903,13 @@ def test_to_dataframe_w_tqdm_max_results(): "google.cloud.bigquery.job.QueryJob.result", side_effect=[concurrent.futures.TimeoutError, row_iterator], ) - - with result_patch as result_patch_tqdm, reload_patch: + with result_patch as tqdm_mock, reload_patch: job.to_dataframe( progress_bar_type="tqdm", create_bqstorage_client=False, max_results=3 ) - assert result_patch_tqdm.call_count == 2 - result_patch_tqdm.assert_called_with( - timeout=_PROGRESS_BAR_UPDATE_INTERVAL, max_results=3 - ) + assert tqdm_mock.call_count == 2 + tqdm_mock.assert_called_with(timeout=_PROGRESS_BAR_UPDATE_INTERVAL, max_results=3) @pytest.mark.skipif(pandas is None, reason="Requires `pandas`") diff --git a/tests/unit/test_magics.py b/tests/unit/test_magics.py index ea8fe568f..fdfb16d16 100644 --- a/tests/unit/test_magics.py +++ b/tests/unit/test_magics.py @@ -278,7 +278,6 @@ def test__run_query(): assert len(execution_updates) == 3 # one update per API response for line in execution_updates: assert re.match("Query executing: .*s", line) - assert re.match("Query complete after .*s", updates[-1]) def test__run_query_dry_run_without_errors_is_silent(): @@ -597,7 +596,7 @@ def warning_match(warning): query_job_mock.to_dataframe.assert_called_once_with( bqstorage_client=bqstorage_instance_mock, create_bqstorage_client=mock.ANY, - progress_bar_type="tqdm", + progress_bar_type="tqdm_notebook", ) assert isinstance(return_value, pandas.DataFrame) @@ -641,7 +640,7 @@ def test_bigquery_magic_with_rest_client_requested(monkeypatch): query_job_mock.to_dataframe.assert_called_once_with( bqstorage_client=None, create_bqstorage_client=False, - progress_bar_type="tqdm", + progress_bar_type="tqdm_notebook", ) assert isinstance(return_value, pandas.DataFrame) diff --git a/tests/unit/test_table.py b/tests/unit/test_table.py index fca43f1ee..f542c7523 100644 --- a/tests/unit/test_table.py +++ b/tests/unit/test_table.py @@ -15,6 +15,7 @@ import datetime import logging import re +from sys import version_info import time import types import unittest @@ -1969,7 +1970,10 @@ def test_to_geodataframe(self): df = row_iterator.to_geodataframe(create_bqstorage_client=False) self.assertIsInstance(df, geopandas.GeoDataFrame) self.assertEqual(len(df), 0) # verify the number of rows - self.assertIsNone(df.crs) + if version_info.major == 3 and version_info.minor > 7: + assert not hasattr(df, "crs") # used with Python > 3.7 + else: + self.assertIsNone(df.crs) # used with Python == 3.7 class TestRowIterator(unittest.TestCase):