diff --git a/.coveragerc b/.coveragerc
index 098720f67..b178b094a 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -1,3 +1,4 @@
+# Generated by synthtool. DO NOT EDIT!
[run]
branch = True
@@ -14,3 +15,5 @@ exclude_lines =
omit =
*/gapic/*.py
*/proto/*.py
+ */core/*.py
+ */site-packages/*.py
\ No newline at end of file
diff --git a/.flake8 b/.flake8
index 61766fa84..0268ecc9c 100644
--- a/.flake8
+++ b/.flake8
@@ -1,3 +1,4 @@
+# Generated by synthtool. DO NOT EDIT!
[flake8]
ignore = E203, E266, E501, W503
exclude =
diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md
new file mode 100644
index 000000000..939e5341e
--- /dev/null
+++ b/.github/CONTRIBUTING.md
@@ -0,0 +1,28 @@
+# How to Contribute
+
+We'd love to accept your patches and contributions to this project. There are
+just a few small guidelines you need to follow.
+
+## Contributor License Agreement
+
+Contributions to this project must be accompanied by a Contributor License
+Agreement. You (or your employer) retain the copyright to your contribution;
+this simply gives us permission to use and redistribute your contributions as
+part of the project. Head over to to see
+your current agreements on file or to sign a new one.
+
+You generally only need to submit a CLA once, so if you've already submitted one
+(even if it was for a different project), you probably don't need to do it
+again.
+
+## Code reviews
+
+All submissions, including submissions by project members, require review. We
+use GitHub pull requests for this purpose. Consult
+[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
+information on using pull requests.
+
+## Community Guidelines
+
+This project follows [Google's Open Source Community
+Guidelines](https://opensource.google.com/conduct/).
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
new file mode 100644
index 000000000..222dc82a4
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -0,0 +1,44 @@
+---
+name: Bug report
+about: Create a report to help us improve
+
+---
+
+Thanks for stopping by to let us know something could be better!
+
+**PLEASE READ**: If you have a support contract with Google, please create an issue in the [support console](https://cloud.google.com/support/) instead of filing on GitHub. This will ensure a timely response.
+
+Please run down the following list and make sure you've tried the usual "quick fixes":
+
+ - Search the issues already opened: https://github.com/googleapis/python-bigquery/issues
+ - Search the issues on our "catch-all" repository: https://github.com/googleapis/google-cloud-python
+ - Search StackOverflow: http://stackoverflow.com/questions/tagged/google-cloud-platform+python
+
+If you are still having issues, please be sure to include as much information as possible:
+
+#### Environment details
+
+ - OS type and version:
+ - Python version: `python --version`
+ - pip version: `pip --version`
+ - `google-cloud-bigquery` version: `pip show google-cloud-bigquery`
+
+#### Steps to reproduce
+
+ 1. ?
+ 2. ?
+
+#### Code example
+
+```python
+# example
+```
+
+#### Stack trace
+```
+# example
+```
+
+Making sure to follow these steps will guarantee the quickest resolution possible.
+
+Thanks!
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
new file mode 100644
index 000000000..6365857f3
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -0,0 +1,18 @@
+---
+name: Feature request
+about: Suggest an idea for this library
+
+---
+
+Thanks for stopping by to let us know something could be better!
+
+**PLEASE READ**: If you have a support contract with Google, please create an issue in the [support console](https://cloud.google.com/support/) instead of filing on GitHub. This will ensure a timely response.
+
+ **Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+ **Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+ **Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+ **Additional context**
+Add any other context or screenshots about the feature request here.
diff --git a/.github/ISSUE_TEMPLATE/support_request.md b/.github/ISSUE_TEMPLATE/support_request.md
new file mode 100644
index 000000000..995869032
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/support_request.md
@@ -0,0 +1,7 @@
+---
+name: Support request
+about: If you have a support contract with Google, please create an issue in the Google Cloud Support console.
+
+---
+
+**PLEASE READ**: If you have a support contract with Google, please create an issue in the [support console](https://cloud.google.com/support/) instead of filing on GitHub. This will ensure a timely response.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
new file mode 100644
index 000000000..65ceeeb5e
--- /dev/null
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -0,0 +1,7 @@
+Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly:
+- [ ] Make sure to open an issue as a [bug/issue](https://github.com/googleapis/python-bigquery/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea
+- [ ] Ensure the tests and linter pass
+- [ ] Code coverage does not decrease (if any source code was changed)
+- [ ] Appropriate docs were updated (if necessary)
+
+Fixes # 🦕
diff --git a/.github/release-please.yml b/.github/release-please.yml
new file mode 100644
index 000000000..4507ad059
--- /dev/null
+++ b/.github/release-please.yml
@@ -0,0 +1 @@
+releaseType: python
diff --git a/.gitignore b/.gitignore
index 9e3a5f257..3fb06e09c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,58 @@
-docs/_build
\ No newline at end of file
+*.py[cod]
+*.sw[op]
+
+# C extensions
+*.so
+
+# Packages
+*.egg
+*.egg-info
+dist
+build
+eggs
+parts
+bin
+var
+sdist
+develop-eggs
+.installed.cfg
+lib
+lib64
+__pycache__
+
+# Installer logs
+pip-log.txt
+
+# Unit test / coverage reports
+.coverage
+.nox
+.cache
+.pytest_cache
+
+
+# Mac
+.DS_Store
+
+# JetBrains
+.idea
+
+# VS Code
+.vscode
+
+# emacs
+*~
+
+# Built documentation
+docs/_build
+bigquery/docs/generated
+
+# Virtual environment
+env/
+coverage.xml
+
+# System test environment variables.
+system_tests/local_test_setup
+
+# Make sure a generated file isn't accidentally committed.
+pylintrc
+pylintrc.test
\ No newline at end of file
diff --git a/.kokoro/build.sh b/.kokoro/build.sh
new file mode 100755
index 000000000..d3749e290
--- /dev/null
+++ b/.kokoro/build.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eo pipefail
+
+cd github/python-bigquery
+
+# Disable buffering, so that the logs stream through.
+export PYTHONUNBUFFERED=1
+
+# Debug: show build environment
+env | grep KOKORO
+
+# Setup service account credentials.
+export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json
+
+# Setup project id.
+export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json")
+
+# Remove old nox
+python3.6 -m pip uninstall --yes --quiet nox-automation
+
+# Install nox
+python3.6 -m pip install --upgrade --quiet nox
+python3.6 -m nox --version
+
+python3.6 -m nox
diff --git a/.kokoro/continuous/common.cfg b/.kokoro/continuous/common.cfg
new file mode 100644
index 000000000..1f46f6270
--- /dev/null
+++ b/.kokoro/continuous/common.cfg
@@ -0,0 +1,27 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Build logs will be here
+action {
+ define_artifacts {
+ regex: "**/*sponge_log.xml"
+ }
+}
+
+# Download trampoline resources.
+gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
+
+# Download resources for system tests (service account key, etc.)
+gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-python"
+
+# Use the trampoline script to run in docker.
+build_file: "python-bigquery/.kokoro/trampoline.sh"
+
+# Configure the docker image for kokoro-trampoline.
+env_vars: {
+ key: "TRAMPOLINE_IMAGE"
+ value: "gcr.io/cloud-devrel-kokoro-resources/python-multi"
+}
+env_vars: {
+ key: "TRAMPOLINE_BUILD_FILE"
+ value: "github/python-bigquery/.kokoro/build.sh"
+}
diff --git a/.kokoro/continuous/continuous.cfg b/.kokoro/continuous/continuous.cfg
new file mode 100644
index 000000000..8f43917d9
--- /dev/null
+++ b/.kokoro/continuous/continuous.cfg
@@ -0,0 +1 @@
+# Format: //devtools/kokoro/config/proto/build.proto
\ No newline at end of file
diff --git a/.kokoro/docs/common.cfg b/.kokoro/docs/common.cfg
new file mode 100644
index 000000000..229abf075
--- /dev/null
+++ b/.kokoro/docs/common.cfg
@@ -0,0 +1,48 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Build logs will be here
+action {
+ define_artifacts {
+ regex: "**/*sponge_log.xml"
+ }
+}
+
+# Download trampoline resources.
+gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
+
+# Use the trampoline script to run in docker.
+build_file: "python-bigquery/.kokoro/trampoline.sh"
+
+# Configure the docker image for kokoro-trampoline.
+env_vars: {
+ key: "TRAMPOLINE_IMAGE"
+ value: "gcr.io/cloud-devrel-kokoro-resources/python-multi"
+}
+env_vars: {
+ key: "TRAMPOLINE_BUILD_FILE"
+ value: "github/python-bigquery/.kokoro/publish-docs.sh"
+}
+
+env_vars: {
+ key: "STAGING_BUCKET"
+ value: "docs-staging"
+}
+
+# Fetch the token needed for reporting release status to GitHub
+before_action {
+ fetch_keystore {
+ keystore_resource {
+ keystore_config_id: 73713
+ keyname: "yoshi-automation-github-key"
+ }
+ }
+}
+
+before_action {
+ fetch_keystore {
+ keystore_resource {
+ keystore_config_id: 73713
+ keyname: "docuploader_service_account"
+ }
+ }
+}
\ No newline at end of file
diff --git a/.kokoro/docs/docs.cfg b/.kokoro/docs/docs.cfg
new file mode 100644
index 000000000..8f43917d9
--- /dev/null
+++ b/.kokoro/docs/docs.cfg
@@ -0,0 +1 @@
+# Format: //devtools/kokoro/config/proto/build.proto
\ No newline at end of file
diff --git a/.kokoro/presubmit/common.cfg b/.kokoro/presubmit/common.cfg
new file mode 100644
index 000000000..1f46f6270
--- /dev/null
+++ b/.kokoro/presubmit/common.cfg
@@ -0,0 +1,27 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Build logs will be here
+action {
+ define_artifacts {
+ regex: "**/*sponge_log.xml"
+ }
+}
+
+# Download trampoline resources.
+gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
+
+# Download resources for system tests (service account key, etc.)
+gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-python"
+
+# Use the trampoline script to run in docker.
+build_file: "python-bigquery/.kokoro/trampoline.sh"
+
+# Configure the docker image for kokoro-trampoline.
+env_vars: {
+ key: "TRAMPOLINE_IMAGE"
+ value: "gcr.io/cloud-devrel-kokoro-resources/python-multi"
+}
+env_vars: {
+ key: "TRAMPOLINE_BUILD_FILE"
+ value: "github/python-bigquery/.kokoro/build.sh"
+}
diff --git a/.kokoro/presubmit/presubmit.cfg b/.kokoro/presubmit/presubmit.cfg
new file mode 100644
index 000000000..8f43917d9
--- /dev/null
+++ b/.kokoro/presubmit/presubmit.cfg
@@ -0,0 +1 @@
+# Format: //devtools/kokoro/config/proto/build.proto
\ No newline at end of file
diff --git a/.kokoro/publish-docs.sh b/.kokoro/publish-docs.sh
new file mode 100755
index 000000000..de3549ef8
--- /dev/null
+++ b/.kokoro/publish-docs.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+
+set -eo pipefail
+
+# Disable buffering, so that the logs stream through.
+export PYTHONUNBUFFERED=1
+
+cd github/python-bigquery
+
+# Remove old nox
+python3.6 -m pip uninstall --yes --quiet nox-automation
+
+# Install nox
+python3.6 -m pip install --upgrade --quiet nox
+python3.6 -m nox --version
+
+# build docs
+nox -s docs
+
+python3 -m pip install gcp-docuploader
+
+# install a json parser
+sudo apt-get update
+sudo apt-get -y install software-properties-common
+sudo add-apt-repository universe
+sudo apt-get update
+sudo apt-get -y install jq
+
+# create metadata
+python3 -m docuploader create-metadata \
+ --name=$(jq --raw-output '.name // empty' .repo-metadata.json) \
+ --version=$(python3 setup.py --version) \
+ --language=$(jq --raw-output '.language // empty' .repo-metadata.json) \
+ --distribution-name=$(python3 setup.py --name) \
+ --product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \
+ --github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \
+ --issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json)
+
+cat docs.metadata
+
+# upload docs
+python3 -m docuploader upload docs/_build/html --metadata-file docs.metadata --staging-bucket docs-staging
diff --git a/.kokoro/release.sh b/.kokoro/release.sh
new file mode 100755
index 000000000..55233bd89
--- /dev/null
+++ b/.kokoro/release.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+
+set -eo pipefail
+
+# Start the releasetool reporter
+python3 -m pip install gcp-releasetool
+python3 -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script
+
+# Ensure that we have the latest versions of Twine, Wheel, and Setuptools.
+python3 -m pip install --upgrade twine wheel setuptools
+
+# Disable buffering, so that the logs stream through.
+export PYTHONUNBUFFERED=1
+
+# Move into the package, build the distribution and upload.
+TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google_cloud_pypi_password")
+cd github/python-bigquery
+python3 setup.py sdist bdist_wheel
+twine upload --username gcloudpypi --password "${TWINE_PASSWORD}" dist/*
diff --git a/.kokoro/release/common.cfg b/.kokoro/release/common.cfg
new file mode 100644
index 000000000..661a04481
--- /dev/null
+++ b/.kokoro/release/common.cfg
@@ -0,0 +1,64 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Build logs will be here
+action {
+ define_artifacts {
+ regex: "**/*sponge_log.xml"
+ }
+}
+
+# Download trampoline resources.
+gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
+
+# Use the trampoline script to run in docker.
+build_file: "python-bigquery/.kokoro/trampoline.sh"
+
+# Configure the docker image for kokoro-trampoline.
+env_vars: {
+ key: "TRAMPOLINE_IMAGE"
+ value: "gcr.io/cloud-devrel-kokoro-resources/python-multi"
+}
+env_vars: {
+ key: "TRAMPOLINE_BUILD_FILE"
+ value: "github/python-bigquery/.kokoro/release.sh"
+}
+
+# Fetch the token needed for reporting release status to GitHub
+before_action {
+ fetch_keystore {
+ keystore_resource {
+ keystore_config_id: 73713
+ keyname: "yoshi-automation-github-key"
+ }
+ }
+}
+
+# Fetch PyPI password
+before_action {
+ fetch_keystore {
+ keystore_resource {
+ keystore_config_id: 73713
+ keyname: "google_cloud_pypi_password"
+ }
+ }
+}
+
+# Fetch magictoken to use with Magic Github Proxy
+before_action {
+ fetch_keystore {
+ keystore_resource {
+ keystore_config_id: 73713
+ keyname: "releasetool-magictoken"
+ }
+ }
+}
+
+# Fetch api key to use with Magic Github Proxy
+before_action {
+ fetch_keystore {
+ keystore_resource {
+ keystore_config_id: 73713
+ keyname: "magic-github-proxy-api-key"
+ }
+ }
+}
diff --git a/.kokoro/release/release.cfg b/.kokoro/release/release.cfg
new file mode 100644
index 000000000..8f43917d9
--- /dev/null
+++ b/.kokoro/release/release.cfg
@@ -0,0 +1 @@
+# Format: //devtools/kokoro/config/proto/build.proto
\ No newline at end of file
diff --git a/.kokoro/trampoline.sh b/.kokoro/trampoline.sh
new file mode 100755
index 000000000..e8c4251f3
--- /dev/null
+++ b/.kokoro/trampoline.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eo pipefail
+
+python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py" || ret_code=$?
+
+chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh
+${KOKORO_GFILE_DIR}/trampoline_cleanup.sh || true
+
+exit ${ret_code}
diff --git a/.repo-metadata.json b/.repo-metadata.json
index 5b4734b8e..f50dbbeb2 100644
--- a/.repo-metadata.json
+++ b/.repo-metadata.json
@@ -6,7 +6,7 @@
"issue_tracker": "https://issuetracker.google.com/savedsearches/559654",
"release_level": "ga",
"language": "python",
- "repo": "googleapis/google-cloud-python",
+ "repo": "googleapis/python-bigquery",
"distribution_name": "google-cloud-bigquery",
"api_id": "bigquery.googleapis.com",
"requires_billing": false
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0da745204..7506ed438 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,29 @@
[1]: https://pypi.org/project/google-cloud-bigquery/#history
+## [1.25.0](https://www.github.com/googleapis/python-bigquery/compare/v1.24.0...v1.25.0) (2020-06-06)
+
+
+### Features
+
+* add BigQuery storage client support to DB API ([#36](https://www.github.com/googleapis/python-bigquery/issues/36)) ([ba9b2f8](https://www.github.com/googleapis/python-bigquery/commit/ba9b2f87e36320d80f6f6460b77e6daddb0fa214))
+* **bigquery:** add create job method ([#32](https://www.github.com/googleapis/python-bigquery/issues/32)) ([2abdef8](https://www.github.com/googleapis/python-bigquery/commit/2abdef82bed31601d1ca1aa92a10fea1e09f5297))
+* **bigquery:** add support of model for extract job ([#71](https://www.github.com/googleapis/python-bigquery/issues/71)) ([4a7a514](https://www.github.com/googleapis/python-bigquery/commit/4a7a514659a9f6f9bbd8af46bab3f8782d6b4b98))
+* add HOUR support for time partitioning interval ([#91](https://www.github.com/googleapis/python-bigquery/issues/91)) ([0dd90b9](https://www.github.com/googleapis/python-bigquery/commit/0dd90b90e3714c1d18f8a404917a9454870e338a))
+* add support for policy tags ([#77](https://www.github.com/googleapis/python-bigquery/issues/77)) ([38a5c01](https://www.github.com/googleapis/python-bigquery/commit/38a5c01ca830daf165592357c45f2fb4016aad23))
+* make AccessEntry objects hashable ([#93](https://www.github.com/googleapis/python-bigquery/issues/93)) ([23a173b](https://www.github.com/googleapis/python-bigquery/commit/23a173bc5a25c0c8200adc5af62eb05624c9099e))
+* **bigquery:** expose start index parameter for query result ([#121](https://www.github.com/googleapis/python-bigquery/issues/121)) ([be86de3](https://www.github.com/googleapis/python-bigquery/commit/be86de330a3c3801653a0ccef90e3d9bdb3eee7a))
+* **bigquery:** unit and system test for dataframe with int column with Nan values ([#39](https://www.github.com/googleapis/python-bigquery/issues/39)) ([5fd840e](https://www.github.com/googleapis/python-bigquery/commit/5fd840e9d4c592c4f736f2fd4792c9670ba6795e))
+
+
+### Bug Fixes
+
+* allow partial streaming_buffer statistics ([#37](https://www.github.com/googleapis/python-bigquery/issues/37)) ([645f0fd](https://www.github.com/googleapis/python-bigquery/commit/645f0fdb35ee0e81ee70f7459e796a42a1f03210))
+* distinguish server timeouts from transport timeouts ([#43](https://www.github.com/googleapis/python-bigquery/issues/43)) ([a17be5f](https://www.github.com/googleapis/python-bigquery/commit/a17be5f01043f32d9fbfb2ddf456031ea9205c8f))
+* improve cell magic error message on missing query ([#58](https://www.github.com/googleapis/python-bigquery/issues/58)) ([6182cf4](https://www.github.com/googleapis/python-bigquery/commit/6182cf48aef8f463bb96891cfc44a96768121dbc))
+* **bigquery:** fix repr of model reference ([#66](https://www.github.com/googleapis/python-bigquery/issues/66)) ([26c6204](https://www.github.com/googleapis/python-bigquery/commit/26c62046f4ec8880cf6561cc90a8b821dcc84ec5))
+* **bigquery:** fix start index with page size for list rows ([#27](https://www.github.com/googleapis/python-bigquery/issues/27)) ([400673b](https://www.github.com/googleapis/python-bigquery/commit/400673b5d0f2a6a3d828fdaad9d222ca967ffeff))
+
## 1.24.0
02-03-2020 01:38 PST
diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
new file mode 100644
index 000000000..b3d1f6029
--- /dev/null
+++ b/CODE_OF_CONDUCT.md
@@ -0,0 +1,44 @@
+
+# Contributor Code of Conduct
+
+As contributors and maintainers of this project,
+and in the interest of fostering an open and welcoming community,
+we pledge to respect all people who contribute through reporting issues,
+posting feature requests, updating documentation,
+submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project
+a harassment-free experience for everyone,
+regardless of level of experience, gender, gender identity and expression,
+sexual orientation, disability, personal appearance,
+body size, race, ethnicity, age, religion, or nationality.
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery
+* Personal attacks
+* Trolling or insulting/derogatory comments
+* Public or private harassment
+* Publishing other's private information,
+such as physical or electronic
+addresses, without explicit permission
+* Other unethical or unprofessional conduct.
+
+Project maintainers have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct.
+By adopting this Code of Conduct,
+project maintainers commit themselves to fairly and consistently
+applying these principles to every aspect of managing this project.
+Project maintainers who do not follow or enforce the Code of Conduct
+may be permanently removed from the project team.
+
+This code of conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior
+may be reported by opening an issue
+or contacting one or more of the project maintainers.
+
+This Code of Conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org), version 1.2.0,
+available at [http://contributor-covenant.org/version/1/2/0/](http://contributor-covenant.org/version/1/2/0/)
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
new file mode 100644
index 000000000..c812edbd1
--- /dev/null
+++ b/CONTRIBUTING.rst
@@ -0,0 +1,279 @@
+.. Generated by synthtool. DO NOT EDIT!
+############
+Contributing
+############
+
+#. **Please sign one of the contributor license agreements below.**
+#. Fork the repo, develop and test your code changes, add docs.
+#. Make sure that your commit messages clearly describe the changes.
+#. Send a pull request. (Please Read: `Faster Pull Request Reviews`_)
+
+.. _Faster Pull Request Reviews: https://github.com/kubernetes/community/blob/master/contributors/guide/pull-requests.md#best-practices-for-faster-reviews
+
+.. contents:: Here are some guidelines for hacking on the Google Cloud Client libraries.
+
+***************
+Adding Features
+***************
+
+In order to add a feature:
+
+- The feature must be documented in both the API and narrative
+ documentation.
+
+- The feature must work fully on the following CPython versions: 2.7,
+ 3.5, 3.6, and 3.7 on both UNIX and Windows.
+
+- The feature must not add unnecessary dependencies (where
+ "unnecessary" is of course subjective, but new dependencies should
+ be discussed).
+
+****************************
+Using a Development Checkout
+****************************
+
+You'll have to create a development environment using a Git checkout:
+
+- While logged into your GitHub account, navigate to the
+ ``python-bigquery`` `repo`_ on GitHub.
+
+- Fork and clone the ``python-bigquery`` repository to your GitHub account by
+ clicking the "Fork" button.
+
+- Clone your fork of ``python-bigquery`` from your GitHub account to your local
+ computer, substituting your account username and specifying the destination
+ as ``hack-on-python-bigquery``. E.g.::
+
+ $ cd ${HOME}
+ $ git clone git@github.com:USERNAME/python-bigquery.git hack-on-python-bigquery
+ $ cd hack-on-python-bigquery
+ # Configure remotes such that you can pull changes from the googleapis/python-bigquery
+ # repository into your local repository.
+ $ git remote add upstream git@github.com:googleapis/python-bigquery.git
+ # fetch and merge changes from upstream into master
+ $ git fetch upstream
+ $ git merge upstream/master
+
+Now your local repo is set up such that you will push changes to your GitHub
+repo, from which you can submit a pull request.
+
+To work on the codebase and run the tests, we recommend using ``nox``,
+but you can also use a ``virtualenv`` of your own creation.
+
+.. _repo: https://github.com/googleapis/python-bigquery
+
+Using ``nox``
+=============
+
+We use `nox `__ to instrument our tests.
+
+- To test your changes, run unit tests with ``nox``::
+
+ $ nox -s unit-2.7
+ $ nox -s unit-3.7
+ $ ...
+
+ .. note::
+
+ The unit tests and system tests are described in the
+ ``noxfile.py`` files in each directory.
+
+.. nox: https://pypi.org/project/nox/
+
+Note on Editable Installs / Develop Mode
+========================================
+
+- As mentioned previously, using ``setuptools`` in `develop mode`_
+ or a ``pip`` `editable install`_ is not possible with this
+ library. This is because this library uses `namespace packages`_.
+ For context see `Issue #2316`_ and the relevant `PyPA issue`_.
+
+ Since ``editable`` / ``develop`` mode can't be used, packages
+ need to be installed directly. Hence your changes to the source
+ tree don't get incorporated into the **already installed**
+ package.
+
+.. _namespace packages: https://www.python.org/dev/peps/pep-0420/
+.. _Issue #2316: https://github.com/GoogleCloudPlatform/google-cloud-python/issues/2316
+.. _PyPA issue: https://github.com/pypa/packaging-problems/issues/12
+.. _develop mode: https://setuptools.readthedocs.io/en/latest/setuptools.html#development-mode
+.. _editable install: https://pip.pypa.io/en/stable/reference/pip_install/#editable-installs
+
+*****************************************
+I'm getting weird errors... Can you help?
+*****************************************
+
+If the error mentions ``Python.h`` not being found,
+install ``python-dev`` and try again.
+On Debian/Ubuntu::
+
+ $ sudo apt-get install python-dev
+
+************
+Coding Style
+************
+
+- PEP8 compliance, with exceptions defined in the linter configuration.
+ If you have ``nox`` installed, you can test that you have not introduced
+ any non-compliant code via::
+
+ $ nox -s lint
+
+- In order to make ``nox -s lint`` run faster, you can set some environment
+ variables::
+
+ export GOOGLE_CLOUD_TESTING_REMOTE="upstream"
+ export GOOGLE_CLOUD_TESTING_BRANCH="master"
+
+ By doing this, you are specifying the location of the most up-to-date
+ version of ``python-bigquery``. The the suggested remote name ``upstream``
+ should point to the official ``googleapis`` checkout and the
+ the branch should be the main branch on that remote (``master``).
+
+Exceptions to PEP8:
+
+- Many unit tests use a helper method, ``_call_fut`` ("FUT" is short for
+ "Function-Under-Test"), which is PEP8-incompliant, but more readable.
+ Some also use a local variable, ``MUT`` (short for "Module-Under-Test").
+
+********************
+Running System Tests
+********************
+
+- To run system tests, you can execute::
+
+ $ nox -s system-3.7
+ $ nox -s system-2.7
+
+ .. note::
+
+ System tests are only configured to run under Python 2.7 and
+ Python 3.7. For expediency, we do not run them in older versions
+ of Python 3.
+
+ This alone will not run the tests. You'll need to change some local
+ auth settings and change some configuration in your project to
+ run all the tests.
+
+- System tests will be run against an actual project and
+ so you'll need to provide some environment variables to facilitate
+ authentication to your project:
+
+ - ``GOOGLE_APPLICATION_CREDENTIALS``: The path to a JSON key file;
+ Such a file can be downloaded directly from the developer's console by clicking
+ "Generate new JSON key". See private key
+ `docs `__
+ for more details.
+
+- Once you have downloaded your json keys, set the environment variable
+ ``GOOGLE_APPLICATION_CREDENTIALS`` to the absolute path of the json file::
+
+ $ export GOOGLE_APPLICATION_CREDENTIALS="/Users//path/to/app_credentials.json"
+
+
+*************
+Test Coverage
+*************
+
+- The codebase *must* have 100% test statement coverage after each commit.
+ You can test coverage via ``nox -s cover``.
+
+******************************************************
+Documentation Coverage and Building HTML Documentation
+******************************************************
+
+If you fix a bug, and the bug requires an API or behavior modification, all
+documentation in this package which references that API or behavior must be
+changed to reflect the bug fix, ideally in the same commit that fixes the bug
+or adds the feature.
+
+Build the docs via:
+
+ $ nox -s docs
+
+********************************************
+Note About ``README`` as it pertains to PyPI
+********************************************
+
+The `description on PyPI`_ for the project comes directly from the
+``README``. Due to the reStructuredText (``rst``) parser used by
+PyPI, relative links which will work on GitHub (e.g. ``CONTRIBUTING.rst``
+instead of
+``https://github.com/googleapis/python-bigquery/blob/master/CONTRIBUTING.rst``)
+may cause problems creating links or rendering the description.
+
+.. _description on PyPI: https://pypi.org/project/google-cloud-bigquery
+
+
+*************************
+Supported Python Versions
+*************************
+
+We support:
+
+- `Python 3.5`_
+- `Python 3.6`_
+- `Python 3.7`_
+
+.. _Python 3.5: https://docs.python.org/3.5/
+.. _Python 3.6: https://docs.python.org/3.6/
+.. _Python 3.7: https://docs.python.org/3.7/
+
+
+Supported versions can be found in our ``noxfile.py`` `config`_.
+
+.. _config: https://github.com/googleapis/python-bigquery/blob/master/noxfile.py
+
+We explicitly decided not to support `Python 2.5`_ due to `decreased usage`_
+and lack of continuous integration `support`_.
+
+.. _Python 2.5: https://docs.python.org/2.5/
+.. _decreased usage: https://caremad.io/2013/10/a-look-at-pypi-downloads/
+.. _support: https://blog.travis-ci.com/2013-11-18-upcoming-build-environment-updates/
+
+We have `dropped 2.6`_ as a supported version as well since Python 2.6 is no
+longer supported by the core development team.
+
+Python 2.7 support is deprecated. All code changes should maintain Python 2.7 compatibility until January 1, 2020.
+
+We also explicitly decided to support Python 3 beginning with version
+3.5. Reasons for this include:
+
+- Encouraging use of newest versions of Python 3
+- Taking the lead of `prominent`_ open-source `projects`_
+- `Unicode literal support`_ which allows for a cleaner codebase that
+ works in both Python 2 and Python 3
+
+.. _prominent: https://docs.djangoproject.com/en/1.9/faq/install/#what-python-version-can-i-use-with-django
+.. _projects: http://flask.pocoo.org/docs/0.10/python3/
+.. _Unicode literal support: https://www.python.org/dev/peps/pep-0414/
+.. _dropped 2.6: https://github.com/googleapis/google-cloud-python/issues/995
+
+**********
+Versioning
+**********
+
+This library follows `Semantic Versioning`_.
+
+.. _Semantic Versioning: http://semver.org/
+
+Some packages are currently in major version zero (``0.y.z``), which means that
+anything may change at any time and the public API should not be considered
+stable.
+
+******************************
+Contributor License Agreements
+******************************
+
+Before we can accept your pull requests you'll need to sign a Contributor
+License Agreement (CLA):
+
+- **If you are an individual writing original source code** and **you own the
+ intellectual property**, then you'll need to sign an
+ `individual CLA `__.
+- **If you work for a company that wants to allow you to contribute your work**,
+ then you'll need to sign a
+ `corporate CLA `__.
+
+You can sign these electronically (just scroll to the bottom). After that,
+we'll be able to accept your pull requests.
diff --git a/MANIFEST.in b/MANIFEST.in
index 9cbf175af..cd011be27 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,3 +1,4 @@
+# Generated by synthtool. DO NOT EDIT!
include README.rst LICENSE
recursive-include google *.json *.proto
recursive-include tests *
diff --git a/docs/conf.py b/docs/conf.py
index 1b83501d1..87501ce66 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -1,17 +1,4 @@
# -*- coding: utf-8 -*-
-# Copyright 2019 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
#
# google-cloud-bigquery documentation build configuration file
#
@@ -26,23 +13,19 @@
import sys
import os
-import shutil
-
-from sphinx.util import logging
-
-logger = logging.getLogger(__name__)
+import shlex
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath(".."))
-__version__ = "0.1.0"
+__version__ = ""
# -- General configuration ------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
-# needs_sphinx = '1.0'
+needs_sphinx = "1.6.3"
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
@@ -53,22 +36,23 @@
"sphinx.ext.intersphinx",
"sphinx.ext.coverage",
"sphinx.ext.napoleon",
+ "sphinx.ext.todo",
"sphinx.ext.viewcode",
+ "recommonmark",
]
# autodoc/autosummary flags
autoclass_content = "both"
-autodoc_default_flags = ["members"]
+autodoc_default_flags = ["members", "inherited-members"]
autosummary_generate = True
-# Add any paths that contain templates here, relative to this directory.
-templates_path = ["_templates", os.path.join("..", "..", "docs", "_templates")]
-# Allow markdown includes (so releases.md can include CHANGLEOG.md)
-# http://www.sphinx-doc.org/en/master/markdown.html
-source_parsers = {".md": "recommonmark.parser.CommonMarkParser"}
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+# source_suffix = ['.rst', '.md']
source_suffix = [".rst", ".md"]
# The encoding of source files.
@@ -79,7 +63,7 @@
# General information about the project.
project = u"google-cloud-bigquery"
-copyright = u"2015, Google"
+copyright = u"2019, Google"
author = u"Google APIs"
# The version info for the project you're documenting, acts as replacement for
@@ -135,6 +119,7 @@
# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True
+
# -- Options for HTML output ----------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
@@ -144,7 +129,15 @@
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
-# html_theme_options = {}
+html_theme_options = {
+ "description": "Google Cloud Client Libraries for google-cloud-bigquery",
+ "github_user": "googleapis",
+ "github_repo": "python-bigquery",
+ "github_banner": True,
+ "font_family": "'Roboto', Georgia, sans",
+ "head_font_family": "'Roboto', Georgia, serif",
+ "code_font_family": "'Roboto Mono', 'Consolas', monospace",
+}
# Add any paths that contain custom themes here, relative to this directory.
# html_theme_path = []
@@ -233,6 +226,18 @@
# Output file base name for HTML help builder.
htmlhelp_basename = "google-cloud-bigquery-doc"
+# -- Options for warnings ------------------------------------------------------
+
+
+suppress_warnings = [
+ # Temporarily suppress this to avoid "more than one target found for
+ # cross-reference" warning, which are intractable for us to avoid while in
+ # a mono-repo.
+ # See https://github.com/sphinx-doc/sphinx/blob
+ # /2a65ffeef5c107c19084fabdd706cdff3f52d93c/sphinx/domains/python.py#L843
+ "ref.python"
+]
+
# -- Options for LaTeX output ---------------------------------------------
latex_elements = {
@@ -279,6 +284,7 @@
# If false, no module index is generated.
# latex_domain_indices = True
+
# -- Options for manual page output ---------------------------------------
# One entry per manual page. List of tuples
@@ -296,6 +302,7 @@
# If true, show URL addresses after external links.
# man_show_urls = False
+
# -- Options for Texinfo output -------------------------------------------
# Grouping the document tree into Texinfo files. List of tuples
@@ -308,6 +315,7 @@
u"google-cloud-bigquery Documentation",
author,
"google-cloud-bigquery",
+ "google-cloud-bigquery Library",
"APIs",
)
]
@@ -324,13 +332,16 @@
# If true, do not generate a @detailmenu in the "Top" node's menu.
# texinfo_no_detailmenu = False
+
# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {
- "gax": ("https://gax-python.readthedocs.org/en/latest/", None),
- "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None),
"python": ("http://python.readthedocs.org/en/latest/", None),
+ "google-auth": ("https://google-auth.readthedocs.io/en/stable", None),
+ "google.api_core": ("https://googleapis.dev/python/google-api-core/latest/", None),
+ "grpc": ("https://grpc.io/grpc/python/", None),
}
+
# Napoleon settings
napoleon_google_docstring = True
napoleon_numpy_docstring = True
@@ -342,22 +353,3 @@
napoleon_use_ivar = False
napoleon_use_param = True
napoleon_use_rtype = True
-
-# Static HTML pages, e.g. to support redirects
-# See: https://tech.signavio.com/2017/managing-sphinx-redirects
-# HTML pages to be copied from source to target
-static_html_pages = ["usage.html", "generated/google.cloud.bigquery.magics.html"]
-
-
-def copy_static_html_pages(app, exception):
- if exception is None and app.builder.name == "html":
- for static_html_page in static_html_pages:
- target_path = app.outdir + "/" + static_html_page
- src_path = app.srcdir + "/" + static_html_page
- if os.path.isfile(src_path):
- logger.info("Copying static html: %s -> %s", src_path, target_path)
- shutil.copyfile(src_path, target_path)
-
-
-def setup(app):
- app.connect("build-finished", copy_static_html_pages)
diff --git a/docs/gapic/v2/types.rst b/docs/gapic/v2/types.rst
index 97938768a..99b954eca 100644
--- a/docs/gapic/v2/types.rst
+++ b/docs/gapic/v2/types.rst
@@ -2,4 +2,5 @@ Types for BigQuery API Client
=============================
.. automodule:: google.cloud.bigquery_v2.types
- :members:
\ No newline at end of file
+ :members:
+ :noindex:
\ No newline at end of file
diff --git a/docs/snippets.py b/docs/snippets.py
index 4981a1e18..bc6b58020 100644
--- a/docs/snippets.py
+++ b/docs/snippets.py
@@ -25,7 +25,6 @@
import time
import pytest
-import six
try:
import fastparquet
@@ -124,7 +123,8 @@ def test_create_client_default_credentials():
def test_create_table_nested_repeated_schema(client, to_delete):
dataset_id = "create_table_nested_repeated_{}".format(_millis())
- dataset_ref = client.dataset(dataset_id)
+ project = client.project
+ dataset_ref = bigquery.DatasetReference(project, dataset_id)
dataset = bigquery.Dataset(dataset_ref)
client.create_dataset(dataset)
to_delete.append(dataset)
@@ -132,7 +132,8 @@ def test_create_table_nested_repeated_schema(client, to_delete):
# [START bigquery_nested_repeated_schema]
# from google.cloud import bigquery
# client = bigquery.Client()
- # dataset_ref = client.dataset('my_dataset')
+ # project = client.project
+ # dataset_ref = bigquery.DatasetReference(project, 'my_dataset')
schema = [
bigquery.SchemaField("id", "STRING", mode="NULLABLE"),
@@ -163,7 +164,9 @@ def test_create_table_nested_repeated_schema(client, to_delete):
def test_create_table_cmek(client, to_delete):
dataset_id = "create_table_cmek_{}".format(_millis())
- dataset = bigquery.Dataset(client.dataset(dataset_id))
+ project = client.project
+ dataset_ref = bigquery.DatasetReference(project, dataset_id)
+ dataset = bigquery.Dataset(dataset_ref)
client.create_dataset(dataset)
to_delete.append(dataset)
@@ -172,7 +175,7 @@ def test_create_table_cmek(client, to_delete):
# client = bigquery.Client()
# dataset_id = 'my_dataset'
- table_ref = client.dataset(dataset_id).table("my_table")
+ table_ref = dataset.table("my_table")
table = bigquery.Table(table_ref)
# Set the encryption key to use for the table.
@@ -192,14 +195,16 @@ def test_create_table_cmek(client, to_delete):
def test_create_partitioned_table(client, to_delete):
dataset_id = "create_table_partitioned_{}".format(_millis())
- dataset_ref = bigquery.Dataset(client.dataset(dataset_id))
+ project = client.project
+ dataset_ref = bigquery.DatasetReference(project, dataset_id)
dataset = client.create_dataset(dataset_ref)
to_delete.append(dataset)
# [START bigquery_create_table_partitioned]
# from google.cloud import bigquery
# client = bigquery.Client()
- # dataset_ref = client.dataset('my_dataset')
+ # project = client.project
+ # dataset_ref = bigquery.DatasetReference(project, 'my_dataset')
table_ref = dataset_ref.table("my_partitioned_table")
schema = [
@@ -237,7 +242,9 @@ def test_create_partitioned_table(client, to_delete):
def test_manage_table_labels(client, to_delete):
dataset_id = "label_table_dataset_{}".format(_millis())
table_id = "label_table_{}".format(_millis())
- dataset = bigquery.Dataset(client.dataset(dataset_id))
+ project = client.project
+ dataset_ref = bigquery.DatasetReference(project, dataset_id)
+ dataset = bigquery.Dataset(dataset_ref)
client.create_dataset(dataset)
to_delete.append(dataset)
@@ -247,7 +254,9 @@ def test_manage_table_labels(client, to_delete):
# [START bigquery_label_table]
# from google.cloud import bigquery
# client = bigquery.Client()
- # table_ref = client.dataset('my_dataset').table('my_table')
+ # project = client.project
+ # dataset_ref = bigquery.DatasetReference(project, dataset_id)
+ # table_ref = dataset_ref.table('my_table')
# table = client.get_table(table_ref) # API request
assert table.labels == {}
@@ -265,7 +274,8 @@ def test_manage_table_labels(client, to_delete):
# dataset_id = 'my_dataset'
# table_id = 'my_table'
- dataset_ref = client.dataset(dataset_id)
+ project = client.project
+ dataset_ref = bigquery.DatasetReference(project, dataset_id)
table_ref = dataset_ref.table(table_id)
table = client.get_table(table_ref) # API Request
@@ -283,7 +293,9 @@ def test_manage_table_labels(client, to_delete):
# [START bigquery_delete_label_table]
# from google.cloud import bigquery
# client = bigquery.Client()
- # table_ref = client.dataset('my_dataset').table('my_table')
+ # project = client.project
+ # dataset_ref = bigquery.DatasetReference(project, dataset_id)
+ # table_ref = dataset_ref.table('my_table')
# table = client.get_table(table_ref) # API request
# This example table starts with one label
@@ -307,7 +319,9 @@ def test_update_table_description(client, to_delete):
"""Update a table's description."""
dataset_id = "update_table_description_dataset_{}".format(_millis())
table_id = "update_table_description_table_{}".format(_millis())
- dataset = bigquery.Dataset(client.dataset(dataset_id))
+ project = client.project
+ dataset_ref = bigquery.DatasetReference(project, dataset_id)
+ dataset = bigquery.Dataset(dataset_ref)
client.create_dataset(dataset)
to_delete.append(dataset)
@@ -318,7 +332,9 @@ def test_update_table_description(client, to_delete):
# [START bigquery_update_table_description]
# from google.cloud import bigquery
# client = bigquery.Client()
- # table_ref = client.dataset('my_dataset').table('my_table')
+ # project = client.project
+ # dataset_ref = bigquery.DatasetReference(project, dataset_id)
+ # table_ref = dataset_ref.table('my_table')
# table = client.get_table(table_ref) # API request
assert table.description == "Original description."
@@ -340,7 +356,9 @@ def test_update_table_expiration(client, to_delete):
"""Update a table's expiration time."""
dataset_id = "update_table_expiration_dataset_{}".format(_millis())
table_id = "update_table_expiration_table_{}".format(_millis())
- dataset = bigquery.Dataset(client.dataset(dataset_id))
+ project = client.project
+ dataset_ref = bigquery.DatasetReference(project, dataset_id)
+ dataset = bigquery.Dataset(dataset_ref)
client.create_dataset(dataset)
to_delete.append(dataset)
@@ -353,7 +371,9 @@ def test_update_table_expiration(client, to_delete):
# from google.cloud import bigquery
# client = bigquery.Client()
- # table_ref = client.dataset('my_dataset').table('my_table')
+ # project = client.project
+ # dataset_ref = bigquery.DatasetReference(project, dataset_id)
+ # table_ref = dataset_ref.table('my_table')
# table = client.get_table(table_ref) # API request
assert table.expires is None
@@ -379,7 +399,9 @@ def test_relax_column(client, to_delete):
"""Updates a schema field from required to nullable."""
dataset_id = "relax_column_dataset_{}".format(_millis())
table_id = "relax_column_table_{}".format(_millis())
- dataset = bigquery.Dataset(client.dataset(dataset_id))
+ project = client.project
+ dataset_ref = bigquery.DatasetReference(project, dataset_id)
+ dataset = bigquery.Dataset(dataset_ref)
dataset = client.create_dataset(dataset)
to_delete.append(dataset)
@@ -393,7 +415,9 @@ def test_relax_column(client, to_delete):
bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"),
bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"),
]
- table_ref = client.dataset(dataset_id).table(table_id)
+
+ dataset_ref = bigquery.DatasetReference(project, dataset_id)
+ table_ref = dataset_ref.table(table_id)
table = bigquery.Table(table_ref, schema=original_schema)
table = client.create_table(table)
assert all(field.mode == "REQUIRED" for field in table.schema)
@@ -421,7 +445,9 @@ def test_update_table_cmek(client, to_delete):
"""Patch a table's metadata."""
dataset_id = "update_table_cmek_{}".format(_millis())
table_id = "update_table_cmek_{}".format(_millis())
- dataset = bigquery.Dataset(client.dataset(dataset_id))
+ project = client.project
+ dataset_ref = bigquery.DatasetReference(project, dataset_id)
+ dataset = bigquery.Dataset(dataset_ref)
client.create_dataset(dataset)
to_delete.append(dataset)
@@ -465,7 +491,7 @@ def test_update_table_cmek(client, to_delete):
def test_manage_views(client, to_delete):
project = client.project
source_dataset_id = "source_dataset_{}".format(_millis())
- source_dataset_ref = client.dataset(source_dataset_id)
+ source_dataset_ref = bigquery.DatasetReference(project, source_dataset_id)
source_dataset = bigquery.Dataset(source_dataset_ref)
source_dataset = client.create_dataset(source_dataset)
to_delete.append(source_dataset)
@@ -484,7 +510,7 @@ def test_manage_views(client, to_delete):
load_job.result()
shared_dataset_id = "shared_dataset_{}".format(_millis())
- shared_dataset_ref = client.dataset(shared_dataset_id)
+ shared_dataset_ref = bigquery.DatasetReference(project, shared_dataset_id)
shared_dataset = bigquery.Dataset(shared_dataset_ref)
shared_dataset = client.create_dataset(shared_dataset)
to_delete.append(shared_dataset)
@@ -495,7 +521,7 @@ def test_manage_views(client, to_delete):
# project = 'my-project'
# source_dataset_id = 'my_source_dataset'
# source_table_id = 'us_states'
- # shared_dataset_ref = client.dataset('my_shared_dataset')
+ # shared_dataset_ref = bigquery.DatasetReference(project, 'my_shared_dataset')
# This example shows how to create a shared view of a source table of
# US States. The source table contains all 50 states, while the view will
@@ -515,7 +541,7 @@ def test_manage_views(client, to_delete):
# project = 'my-project'
# source_dataset_id = 'my_source_dataset'
# source_table_id = 'us_states'
- # shared_dataset_ref = client.dataset('my_shared_dataset')
+ # shared_dataset_ref = bigquery.DatasetReference(project, 'my_shared_dataset')
# This example shows how to update a shared view of a source table of
# US States. The view's query will be updated to contain only states with
@@ -531,8 +557,9 @@ def test_manage_views(client, to_delete):
# from google.cloud import bigquery
# client = bigquery.Client()
# shared_dataset_id = 'my_shared_dataset'
-
- view_ref = client.dataset(shared_dataset_id).table("my_shared_view")
+ project = client.project
+ shared_dataset_ref = bigquery.DatasetReference(project, shared_dataset_id)
+ view_ref = shared_dataset_ref.table("my_shared_view")
view = client.get_table(view_ref) # API Request
# Display view properties
@@ -549,9 +576,9 @@ def test_manage_views(client, to_delete):
# Assign access controls to the dataset containing the view
# shared_dataset_id = 'my_shared_dataset'
# analyst_group_email = 'data_analysts@example.com'
- shared_dataset = client.get_dataset(
- client.dataset(shared_dataset_id)
- ) # API request
+ project = client.project
+ shared_dataset_ref = bigquery.DatasetReference(project, shared_dataset_id)
+ shared_dataset = client.get_dataset(shared_dataset_ref) # API request
access_entries = shared_dataset.access_entries
access_entries.append(
bigquery.AccessEntry("READER", "groupByEmail", analyst_group_email)
@@ -564,9 +591,9 @@ def test_manage_views(client, to_delete):
# Authorize the view to access the source dataset
# project = 'my-project'
# source_dataset_id = 'my_source_dataset'
- source_dataset = client.get_dataset(
- client.dataset(source_dataset_id)
- ) # API request
+ project = client.project
+ source_dataset_ref = bigquery.DatasetReference(project, source_dataset_id)
+ source_dataset = client.get_dataset(source_dataset_ref) # API request
view_reference = {
"projectId": project,
"datasetId": shared_dataset_id,
@@ -581,181 +608,17 @@ def test_manage_views(client, to_delete):
# [END bigquery_grant_view_access]
-def test_load_table_from_uri_autodetect(client, to_delete, capsys):
- """Load table from a GCS URI using various formats and auto-detected schema
- Each file format has its own tested load from URI sample. Because most of
- the code is common for autodetect, append, and truncate, this sample
- includes snippets for all supported formats but only calls a single load
- job.
- This code snippet is made up of shared code, then format-specific code,
- followed by more shared code. Note that only the last format in the
- format-specific code section will be tested in this test.
- """
- dataset_id = "load_table_from_uri_auto_{}".format(_millis())
- dataset = bigquery.Dataset(client.dataset(dataset_id))
- client.create_dataset(dataset)
- to_delete.append(dataset)
-
- # Shared code
- # [START bigquery_load_table_gcs_csv_autodetect]
- # [START bigquery_load_table_gcs_json_autodetect]
- # from google.cloud import bigquery
- # client = bigquery.Client()
- # dataset_id = 'my_dataset'
-
- dataset_ref = client.dataset(dataset_id)
- job_config = bigquery.LoadJobConfig()
- job_config.autodetect = True
- # [END bigquery_load_table_gcs_csv_autodetect]
- # [END bigquery_load_table_gcs_json_autodetect]
-
- # Format-specific code
- # [START bigquery_load_table_gcs_csv_autodetect]
- job_config.skip_leading_rows = 1
- # The source format defaults to CSV, so the line below is optional.
- job_config.source_format = bigquery.SourceFormat.CSV
- uri = "gs://cloud-samples-data/bigquery/us-states/us-states.csv"
- # [END bigquery_load_table_gcs_csv_autodetect]
- # unset csv-specific attribute
- del job_config._properties["load"]["skipLeadingRows"]
-
- # [START bigquery_load_table_gcs_json_autodetect]
- job_config.source_format = bigquery.SourceFormat.NEWLINE_DELIMITED_JSON
- uri = "gs://cloud-samples-data/bigquery/us-states/us-states.json"
- # [END bigquery_load_table_gcs_json_autodetect]
-
- # Shared code
- # [START bigquery_load_table_gcs_csv_autodetect]
- # [START bigquery_load_table_gcs_json_autodetect]
- load_job = client.load_table_from_uri(
- uri, dataset_ref.table("us_states"), job_config=job_config
- ) # API request
- print("Starting job {}".format(load_job.job_id))
-
- load_job.result() # Waits for table load to complete.
- print("Job finished.")
-
- destination_table = client.get_table(dataset_ref.table("us_states"))
- print("Loaded {} rows.".format(destination_table.num_rows))
- # [END bigquery_load_table_gcs_csv_autodetect]
- # [END bigquery_load_table_gcs_json_autodetect]
-
- out, _ = capsys.readouterr()
- assert "Loaded 50 rows." in out
-
-
-def test_load_table_from_uri_truncate(client, to_delete, capsys):
- """Replaces table data with data from a GCS URI using various formats
- Each file format has its own tested load from URI sample. Because most of
- the code is common for autodetect, append, and truncate, this sample
- includes snippets for all supported formats but only calls a single load
- job.
- This code snippet is made up of shared code, then format-specific code,
- followed by more shared code. Note that only the last format in the
- format-specific code section will be tested in this test.
- """
- dataset_id = "load_table_from_uri_trunc_{}".format(_millis())
- dataset = bigquery.Dataset(client.dataset(dataset_id))
- client.create_dataset(dataset)
- to_delete.append(dataset)
-
- job_config = bigquery.LoadJobConfig()
- job_config.schema = [
- bigquery.SchemaField("name", "STRING"),
- bigquery.SchemaField("post_abbr", "STRING"),
- ]
- table_ref = dataset.table("us_states")
- body = six.BytesIO(b"Washington,WA")
- client.load_table_from_file(body, table_ref, job_config=job_config).result()
- previous_rows = client.get_table(table_ref).num_rows
- assert previous_rows > 0
-
- # Shared code
- # [START bigquery_load_table_gcs_avro_truncate]
- # [START bigquery_load_table_gcs_csv_truncate]
- # [START bigquery_load_table_gcs_json_truncate]
- # [START bigquery_load_table_gcs_parquet_truncate]
- # [START bigquery_load_table_gcs_orc_truncate]
- # from google.cloud import bigquery
- # client = bigquery.Client()
- # table_ref = client.dataset('my_dataset').table('existing_table')
-
- job_config = bigquery.LoadJobConfig()
- job_config.write_disposition = bigquery.WriteDisposition.WRITE_TRUNCATE
- # [END bigquery_load_table_gcs_avro_truncate]
- # [END bigquery_load_table_gcs_csv_truncate]
- # [END bigquery_load_table_gcs_json_truncate]
- # [END bigquery_load_table_gcs_parquet_truncate]
- # [END bigquery_load_table_gcs_orc_truncate]
-
- # Format-specific code
- # [START bigquery_load_table_gcs_avro_truncate]
- job_config.source_format = bigquery.SourceFormat.AVRO
- uri = "gs://cloud-samples-data/bigquery/us-states/us-states.avro"
- # [END bigquery_load_table_gcs_avro_truncate]
-
- # [START bigquery_load_table_gcs_csv_truncate]
- job_config.skip_leading_rows = 1
- # The source format defaults to CSV, so the line below is optional.
- job_config.source_format = bigquery.SourceFormat.CSV
- uri = "gs://cloud-samples-data/bigquery/us-states/us-states.csv"
- # [END bigquery_load_table_gcs_csv_truncate]
- # unset csv-specific attribute
- del job_config._properties["load"]["skipLeadingRows"]
-
- # [START bigquery_load_table_gcs_json_truncate]
- job_config.source_format = bigquery.SourceFormat.NEWLINE_DELIMITED_JSON
- uri = "gs://cloud-samples-data/bigquery/us-states/us-states.json"
- # [END bigquery_load_table_gcs_json_truncate]
-
- # [START bigquery_load_table_gcs_parquet_truncate]
- job_config.source_format = bigquery.SourceFormat.PARQUET
- uri = "gs://cloud-samples-data/bigquery/us-states/us-states.parquet"
- # [END bigquery_load_table_gcs_parquet_truncate]
-
- # [START bigquery_load_table_gcs_orc_truncate]
- job_config.source_format = bigquery.SourceFormat.ORC
- uri = "gs://cloud-samples-data/bigquery/us-states/us-states.orc"
- # [END bigquery_load_table_gcs_orc_truncate]
-
- # Shared code
- # [START bigquery_load_table_gcs_avro_truncate]
- # [START bigquery_load_table_gcs_csv_truncate]
- # [START bigquery_load_table_gcs_json_truncate]
- # [START bigquery_load_table_gcs_parquet_truncate]
- # [START bigquery_load_table_gcs_orc_truncate]
- load_job = client.load_table_from_uri(
- uri, table_ref, job_config=job_config
- ) # API request
- print("Starting job {}".format(load_job.job_id))
-
- load_job.result() # Waits for table load to complete.
- print("Job finished.")
-
- destination_table = client.get_table(table_ref)
- print("Loaded {} rows.".format(destination_table.num_rows))
- # [END bigquery_load_table_gcs_avro_truncate]
- # [END bigquery_load_table_gcs_csv_truncate]
- # [END bigquery_load_table_gcs_json_truncate]
- # [END bigquery_load_table_gcs_parquet_truncate]
- # [END bigquery_load_table_gcs_orc_truncate]
-
- out, _ = capsys.readouterr()
- assert "Loaded 50 rows." in out
-
-
def test_load_table_add_column(client, to_delete):
dataset_id = "load_table_add_column_{}".format(_millis())
- dataset_ref = client.dataset(dataset_id)
+ project = client.project
+ dataset_ref = bigquery.DatasetReference(project, dataset_id)
dataset = bigquery.Dataset(dataset_ref)
dataset.location = "US"
dataset = client.create_dataset(dataset)
to_delete.append(dataset)
snippets_dir = os.path.abspath(os.path.dirname(__file__))
- filepath = os.path.join(
- snippets_dir, "..", "..", "bigquery", "tests", "data", "people.csv"
- )
+ filepath = os.path.join(snippets_dir, "..", "tests", "data", "people.csv")
table_ref = dataset_ref.table("my_table")
old_schema = [bigquery.SchemaField("full_name", "STRING", mode="REQUIRED")]
table = client.create_table(bigquery.Table(table_ref, schema=old_schema))
@@ -763,7 +626,8 @@ def test_load_table_add_column(client, to_delete):
# [START bigquery_add_column_load_append]
# from google.cloud import bigquery
# client = bigquery.Client()
- # dataset_ref = client.dataset('my_dataset')
+ # project = client.project
+ # dataset_ref = bigquery.DatasetReference(project, 'my_dataset')
# filepath = 'path/to/your_file.csv'
# Retrieves the destination table and checks the length of the schema
@@ -814,16 +678,15 @@ def test_load_table_add_column(client, to_delete):
def test_load_table_relax_column(client, to_delete):
dataset_id = "load_table_relax_column_{}".format(_millis())
- dataset_ref = client.dataset(dataset_id)
+ project = client.project
+ dataset_ref = bigquery.DatasetReference(project, dataset_id)
dataset = bigquery.Dataset(dataset_ref)
dataset.location = "US"
dataset = client.create_dataset(dataset)
to_delete.append(dataset)
snippets_dir = os.path.abspath(os.path.dirname(__file__))
- filepath = os.path.join(
- snippets_dir, "..", "..", "bigquery", "tests", "data", "people.csv"
- )
+ filepath = os.path.join(snippets_dir, "..", "tests", "data", "people.csv")
table_ref = dataset_ref.table("my_table")
old_schema = [
bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"),
@@ -835,7 +698,8 @@ def test_load_table_relax_column(client, to_delete):
# [START bigquery_relax_column_load_append]
# from google.cloud import bigquery
# client = bigquery.Client()
- # dataset_ref = client.dataset('my_dataset')
+ # project = client.project
+ # dataset_ref = bigquery.DatasetReference(project, 'my_dataset')
# filepath = 'path/to/your_file.csv'
# Retrieves the destination table and checks the number of required fields
@@ -904,7 +768,7 @@ def test_extract_table(client, to_delete):
table_id = "shakespeare"
destination_uri = "gs://{}/{}".format(bucket_name, "shakespeare.csv")
- dataset_ref = client.dataset(dataset_id, project=project)
+ dataset_ref = bigquery.DatasetReference(project, dataset_id)
table_ref = dataset_ref.table(table_id)
extract_job = client.extract_table(
@@ -931,6 +795,8 @@ def test_extract_table_json(client, to_delete):
storage_client = storage.Client()
bucket = retry_storage_errors(storage_client.create_bucket)(bucket_name)
to_delete.append(bucket)
+ project = "bigquery-public-data"
+ dataset_id = "samples"
# [START bigquery_extract_table_json]
# from google.cloud import bigquery
@@ -938,7 +804,7 @@ def test_extract_table_json(client, to_delete):
# bucket_name = 'my-bucket'
destination_uri = "gs://{}/{}".format(bucket_name, "shakespeare.json")
- dataset_ref = client.dataset("samples", project="bigquery-public-data")
+ dataset_ref = bigquery.DatasetReference(project, dataset_id)
table_ref = dataset_ref.table("shakespeare")
job_config = bigquery.job.ExtractJobConfig()
job_config.destination_format = bigquery.DestinationFormat.NEWLINE_DELIMITED_JSON
@@ -964,6 +830,8 @@ def test_extract_table_compressed(client, to_delete):
storage_client = storage.Client()
bucket = retry_storage_errors(storage_client.create_bucket)(bucket_name)
to_delete.append(bucket)
+ project = "bigquery-public-data"
+ dataset_id = "samples"
# [START bigquery_extract_table_compressed]
# from google.cloud import bigquery
@@ -971,7 +839,7 @@ def test_extract_table_compressed(client, to_delete):
# bucket_name = 'my-bucket'
destination_uri = "gs://{}/{}".format(bucket_name, "shakespeare.csv.gz")
- dataset_ref = client.dataset("samples", project="bigquery-public-data")
+ dataset_ref = bigquery.DatasetReference(project, dataset_id)
table_ref = dataset_ref.table("shakespeare")
job_config = bigquery.job.ExtractJobConfig()
job_config.compression = bigquery.Compression.GZIP
@@ -1058,7 +926,9 @@ def test_manage_job(client):
def test_query_external_gcs_permanent_table(client, to_delete):
dataset_id = "query_external_gcs_{}".format(_millis())
- dataset = bigquery.Dataset(client.dataset(dataset_id))
+ project = client.project
+ dataset_ref = bigquery.DatasetReference(project, dataset_id)
+ dataset = bigquery.Dataset(dataset_ref)
client.create_dataset(dataset)
to_delete.append(dataset)
@@ -1068,7 +938,7 @@ def test_query_external_gcs_permanent_table(client, to_delete):
# dataset_id = 'my_dataset'
# Configure the external data source
- dataset_ref = client.dataset(dataset_id)
+ dataset_ref = bigquery.DatasetReference(project, dataset_id)
table_id = "us_states"
schema = [
bigquery.SchemaField("name", "STRING"),
@@ -1101,7 +971,8 @@ def test_ddl_create_view(client, to_delete, capsys):
project = client.project
dataset_id = "ddl_view_{}".format(_millis())
table_id = "new_view"
- dataset = bigquery.Dataset(client.dataset(dataset_id))
+ dataset_ref = bigquery.DatasetReference(project, dataset_id)
+ dataset = bigquery.Dataset(dataset_ref)
client.create_dataset(dataset)
to_delete.append(dataset)
@@ -1179,8 +1050,10 @@ def test_list_rows_as_dataframe(client):
# [START bigquery_list_rows_dataframe]
# from google.cloud import bigquery
# client = bigquery.Client()
+ project = "bigquery-public-data"
+ dataset_id = "samples"
- dataset_ref = client.dataset("samples", project="bigquery-public-data")
+ dataset_ref = bigquery.DatasetReference(project, dataset_id)
table_ref = dataset_ref.table("shakespeare")
table = client.get_table(table_ref)
diff --git a/docs/usage/tables.rst b/docs/usage/tables.rst
index 45145cd19..27af7c7df 100644
--- a/docs/usage/tables.rst
+++ b/docs/usage/tables.rst
@@ -132,6 +132,22 @@ Load an ORC file from Cloud Storage:
See also: `Loading ORC data from Cloud Storage
`_.
+Load a CSV file from Cloud Storage and auto-detect schema:
+
+.. literalinclude:: ../samples/load_table_uri_autodetect_csv.py
+ :language: python
+ :dedent: 4
+ :start-after: [START bigquery_load_table_gcs_csv_autodetect]
+ :end-before: [END bigquery_load_table_gcs_csv_autodetect]
+
+Load a JSON file from Cloud Storage and auto-detect schema:
+
+.. literalinclude:: ../samples/load_table_uri_autodetect_json.py
+ :language: python
+ :dedent: 4
+ :start-after: [START bigquery_load_table_gcs_json_autodetect]
+ :end-before: [END bigquery_load_table_gcs_json_autodetect]
+
Updating a Table
^^^^^^^^^^^^^^^^
@@ -220,3 +236,46 @@ Restore a deleted table from a snapshot by using the
:dedent: 4
:start-after: [START bigquery_undelete_table]
:end-before: [END bigquery_undelete_table]
+
+Overwrite a Table
+^^^^^^^^^^^^^^^^^
+
+Replace the table data with an Avro file from Cloud Storage:
+
+.. literalinclude:: ../samples/load_table_uri_truncate_avro.py
+ :language: python
+ :dedent: 4
+ :start-after: [START bigquery_load_table_gcs_avro_truncate]
+ :end-before: [END bigquery_load_table_gcs_avro_truncate]
+
+Replace the table data with a CSV file from Cloud Storage:
+
+.. literalinclude:: ../samples/load_table_uri_truncate_csv.py
+ :language: python
+ :dedent: 4
+ :start-after: [START bigquery_load_table_gcs_csv_truncate]
+ :end-before: [END bigquery_load_table_gcs_csv_truncate]
+
+Replace the table data with a JSON file from Cloud Storage:
+
+.. literalinclude:: ../samples/load_table_uri_truncate_json.py
+ :language: python
+ :dedent: 4
+ :start-after: [START bigquery_load_table_gcs_json_truncate]
+ :end-before: [END bigquery_load_table_gcs_json_truncate]
+
+Replace the table data with an ORC file from Cloud Storage:
+
+.. literalinclude:: ../samples/load_table_uri_truncate_orc.py
+ :language: python
+ :dedent: 4
+ :start-after: [START bigquery_load_table_gcs_orc_truncate]
+ :end-before: [END bigquery_load_table_gcs_orc_truncate]
+
+Replace the table data with a Parquet file from Cloud Storage:
+
+.. literalinclude:: ../samples/load_table_uri_truncate_parquet.py
+ :language: python
+ :dedent: 4
+ :start-after: [START bigquery_load_table_gcs_parquet_truncate]
+ :end-before: [END bigquery_load_table_gcs_parquet_truncate]
\ No newline at end of file
diff --git a/google/cloud/bigquery/__init__.py b/google/cloud/bigquery/__init__.py
index 3982c1175..63d71694c 100644
--- a/google/cloud/bigquery/__init__.py
+++ b/google/cloud/bigquery/__init__.py
@@ -38,6 +38,7 @@
from google.cloud.bigquery.dataset import DatasetReference
from google.cloud.bigquery import enums
from google.cloud.bigquery.enums import StandardSqlDataTypes
+from google.cloud.bigquery.exceptions import PyarrowMissingWarning
from google.cloud.bigquery.external_config import ExternalConfig
from google.cloud.bigquery.external_config import BigtableOptions
from google.cloud.bigquery.external_config import BigtableColumnFamily
@@ -142,6 +143,8 @@
"WriteDisposition",
# EncryptionConfiguration
"EncryptionConfiguration",
+ # Errors and warnings
+ "PyarrowMissingWarning",
]
diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py
index 5da12990b..da5b30a35 100644
--- a/google/cloud/bigquery/client.py
+++ b/google/cloud/bigquery/client.py
@@ -22,7 +22,6 @@
except ImportError: # Python 2.7
import collections as collections_abc
-import concurrent.futures
import copy
import functools
import gzip
@@ -48,22 +47,25 @@
import google.api_core.client_options
import google.api_core.exceptions
from google.api_core import page_iterator
-from google.auth.transport.requests import TimeoutGuard
import google.cloud._helpers
from google.cloud import exceptions
from google.cloud.client import ClientWithProject
+from google.cloud.bigquery._helpers import _get_sub_prop
from google.cloud.bigquery._helpers import _record_field_to_json
from google.cloud.bigquery._helpers import _str_or_none
from google.cloud.bigquery._helpers import _verify_job_config_type
+from google.cloud.bigquery._helpers import _del_sub_prop
from google.cloud.bigquery._http import Connection
from google.cloud.bigquery import _pandas_helpers
from google.cloud.bigquery.dataset import Dataset
from google.cloud.bigquery.dataset import DatasetListItem
from google.cloud.bigquery.dataset import DatasetReference
+from google.cloud.bigquery.exceptions import PyarrowMissingWarning
from google.cloud.bigquery import job
from google.cloud.bigquery.model import Model
from google.cloud.bigquery.model import ModelReference
+from google.cloud.bigquery.model import _model_arg_to_model_ref
from google.cloud.bigquery.query import _QueryResults
from google.cloud.bigquery.retry import DEFAULT_RETRY
from google.cloud.bigquery.routine import Routine
@@ -1314,6 +1316,78 @@ def job_from_resource(self, resource):
return job.QueryJob.from_api_repr(resource, self)
return job.UnknownJob.from_api_repr(resource, self)
+ def create_job(self, job_config, retry=DEFAULT_RETRY):
+ """Create a new job.
+ Arguments:
+ job_config (dict): configuration job representation returned from the API.
+
+ Keyword Arguments:
+ retry (google.api_core.retry.Retry):
+ (Optional) How to retry the RPC.
+
+ Returns:
+ Union[ \
+ google.cloud.bigquery.job.LoadJob, \
+ google.cloud.bigquery.job.CopyJob, \
+ google.cloud.bigquery.job.ExtractJob, \
+ google.cloud.bigquery.job.QueryJob \
+ ]:
+ A new job instance.
+ """
+
+ if "load" in job_config:
+ load_job_config = google.cloud.bigquery.job.LoadJobConfig.from_api_repr(
+ job_config
+ )
+ destination = _get_sub_prop(job_config, ["load", "destinationTable"])
+ source_uris = _get_sub_prop(job_config, ["load", "sourceUris"])
+ return self.load_table_from_uri(
+ source_uris, destination, job_config=load_job_config, retry=retry
+ )
+ elif "copy" in job_config:
+ copy_job_config = google.cloud.bigquery.job.CopyJobConfig.from_api_repr(
+ job_config
+ )
+ destination = _get_sub_prop(job_config, ["copy", "destinationTable"])
+ sources = []
+ source_configs = _get_sub_prop(job_config, ["copy", "sourceTables"])
+
+ if source_configs is None:
+ source_configs = [_get_sub_prop(job_config, ["copy", "sourceTable"])]
+ for source_config in source_configs:
+ table_ref = TableReference.from_api_repr(source_config)
+ sources.append(table_ref)
+ return self.copy_table(
+ sources, destination, job_config=copy_job_config, retry=retry
+ )
+ elif "extract" in job_config:
+ extract_job_config = google.cloud.bigquery.job.ExtractJobConfig.from_api_repr(
+ job_config
+ )
+ source = _get_sub_prop(job_config, ["extract", "sourceTable"])
+ source_type = "Table"
+ if not source:
+ source = _get_sub_prop(job_config, ["extract", "sourceModel"])
+ source_type = "Model"
+ destination_uris = _get_sub_prop(job_config, ["extract", "destinationUris"])
+ return self.extract_table(
+ source,
+ destination_uris,
+ job_config=extract_job_config,
+ retry=retry,
+ source_type=source_type,
+ )
+ elif "query" in job_config:
+ copy_config = copy.deepcopy(job_config)
+ _del_sub_prop(copy_config, ["query", "destinationTable"])
+ query_job_config = google.cloud.bigquery.job.QueryJobConfig.from_api_repr(
+ copy_config
+ )
+ query = _get_sub_prop(copy_config, ["query", "query"])
+ return self.query(query, job_config=query_job_config, retry=retry)
+ else:
+ raise TypeError("Invalid job configuration received.")
+
def get_job(
self, job_id, project=None, location=None, retry=DEFAULT_RETRY, timeout=None
):
@@ -1712,6 +1786,14 @@ def load_table_from_dataframe(
Similar to :meth:`load_table_from_uri`, this method creates, starts and
returns a :class:`~google.cloud.bigquery.job.LoadJob`.
+ .. note::
+
+ Due to the way REPEATED fields are encoded in the ``parquet`` file
+ format, a mismatch with the existing table schema can occur, and
+ 100% compatibility cannot be guaranteed for REPEATED fields.
+
+ https://github.com/googleapis/python-bigquery/issues/17
+
Arguments:
dataframe (pandas.DataFrame):
A :class:`~pandas.DataFrame` containing the data to load.
@@ -1842,6 +1924,15 @@ def load_table_from_dataframe(
parquet_compression=parquet_compression,
)
else:
+ if not pyarrow:
+ warnings.warn(
+ "Loading dataframe data without pyarrow installed is "
+ "deprecated and will become unsupported in the future. "
+ "Please install the pyarrow package.",
+ PyarrowMissingWarning,
+ stacklevel=2,
+ )
+
if job_config.schema:
warnings.warn(
"job_config.schema is set, but not used to assist in "
@@ -2200,6 +2291,7 @@ def extract_table(
job_config=None,
retry=DEFAULT_RETRY,
timeout=None,
+ source_type="Table",
):
"""Start a job to extract a table into Cloud Storage files.
@@ -2210,9 +2302,11 @@ def extract_table(
source (Union[ \
google.cloud.bigquery.table.Table, \
google.cloud.bigquery.table.TableReference, \
+ google.cloud.bigquery.model.Model, \
+ google.cloud.bigquery.model.ModelReference, \
src, \
]):
- Table to be extracted.
+ Table or Model to be extracted.
destination_uris (Union[str, Sequence[str]]):
URIs of Cloud Storage file(s) into which table data is to be
extracted; in format
@@ -2237,9 +2331,9 @@ def extract_table(
timeout (Optional[float]):
The number of seconds to wait for the underlying HTTP transport
before using ``retry``.
- Args:
- source (google.cloud.bigquery.table.TableReference): table to be extracted.
-
+ source_type (str):
+ (Optional) Type of source to be extracted.``Table`` or ``Model``.
+ Defaults to ``Table``.
Returns:
google.cloud.bigquery.job.ExtractJob: A new extract job instance.
@@ -2247,7 +2341,9 @@ def extract_table(
TypeError:
If ``job_config`` is not an instance of :class:`~google.cloud.bigquery.job.ExtractJobConfig`
class.
- """
+ ValueError:
+ If ``source_type`` is not among ``Table``,``Model``.
+ """
job_id = _make_job_id(job_id, job_id_prefix)
if project is None:
@@ -2257,7 +2353,17 @@ def extract_table(
location = self.location
job_ref = job._JobReference(job_id, project=project, location=location)
- source = _table_arg_to_table_ref(source, default_project=self.project)
+ src = source_type.lower()
+ if src == "table":
+ source = _table_arg_to_table_ref(source, default_project=self.project)
+ elif src == "model":
+ source = _model_arg_to_model_ref(source, default_project=self.project)
+ else:
+ raise ValueError(
+ "Cannot pass `{}` as a ``source_type``, pass Table or Model".format(
+ source_type
+ )
+ )
if isinstance(destination_uris, six.string_types):
destination_uris = [destination_uris]
@@ -2590,27 +2696,22 @@ def list_partitions(self, table, retry=DEFAULT_RETRY, timeout=None):
timeout (Optional[float]):
The number of seconds to wait for the underlying HTTP transport
before using ``retry``.
- If multiple requests are made under the hood, ``timeout`` is
- interpreted as the approximate total time of **all** requests.
+ If multiple requests are made under the hood, ``timeout``
+ applies to each individual request.
Returns:
List[str]:
A list of the partition ids present in the partitioned table
"""
table = _table_arg_to_table_ref(table, default_project=self.project)
-
- with TimeoutGuard(
- timeout, timeout_error_type=concurrent.futures.TimeoutError
- ) as guard:
- meta_table = self.get_table(
- TableReference(
- DatasetReference(table.project, table.dataset_id),
- "%s$__PARTITIONS_SUMMARY__" % table.table_id,
- ),
- retry=retry,
- timeout=timeout,
- )
- timeout = guard.remaining_timeout
+ meta_table = self.get_table(
+ TableReference(
+ DatasetReference(table.project, table.dataset_id),
+ "%s$__PARTITIONS_SUMMARY__" % table.table_id,
+ ),
+ retry=retry,
+ timeout=timeout,
+ )
subset = [col for col in meta_table.schema if col.name == "partition_id"]
return [
@@ -2677,8 +2778,8 @@ def list_rows(
timeout (Optional[float]):
The number of seconds to wait for the underlying HTTP transport
before using ``retry``.
- If multiple requests are made under the hood, ``timeout`` is
- interpreted as the approximate total time of **all** requests.
+ If multiple requests are made under the hood, ``timeout``
+ applies to each individual request.
Returns:
google.cloud.bigquery.table.RowIterator:
@@ -2703,11 +2804,7 @@ def list_rows(
# No schema, but no selected_fields. Assume the developer wants all
# columns, so get the table resource for them rather than failing.
elif len(schema) == 0:
- with TimeoutGuard(
- timeout, timeout_error_type=concurrent.futures.TimeoutError
- ) as guard:
- table = self.get_table(table.reference, retry=retry, timeout=timeout)
- timeout = guard.remaining_timeout
+ table = self.get_table(table.reference, retry=retry, timeout=timeout)
schema = table.schema
params = {}
diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py
index 99c47026f..40489a38b 100644
--- a/google/cloud/bigquery/dataset.py
+++ b/google/cloud/bigquery/dataset.py
@@ -20,6 +20,7 @@
import copy
import google.cloud._helpers
+
from google.cloud.bigquery import _helpers
from google.cloud.bigquery.model import ModelReference
from google.cloud.bigquery.routine import RoutineReference
@@ -145,38 +146,60 @@ def __init__(self, role, entity_type, entity_id):
"Role must be set for entity " "type %r" % (entity_type,)
)
- self.role = role
- self.entity_type = entity_type
- self.entity_id = entity_id
+ self._role = role
+ self._entity_type = entity_type
+ self._entity_id = entity_id
+
+ @property
+ def role(self):
+ """str: The role of the entry."""
+ return self._role
+
+ @property
+ def entity_type(self):
+ """str: The entity_type of the entry."""
+ return self._entity_type
+
+ @property
+ def entity_id(self):
+ """str: The entity_id of the entry."""
+ return self._entity_id
def __eq__(self, other):
if not isinstance(other, AccessEntry):
return NotImplemented
- return (
- self.role == other.role
- and self.entity_type == other.entity_type
- and self.entity_id == other.entity_id
- )
+ return self._key() == other._key()
def __ne__(self, other):
return not self == other
def __repr__(self):
return "" % (
- self.role,
- self.entity_type,
- self.entity_id,
+ self._role,
+ self._entity_type,
+ self._entity_id,
)
+ def _key(self):
+ """ A tuple key that uniquely describes this field.
+ Used to compute this instance's hashcode and evaluate equality.
+ Returns:
+ Tuple: The contents of this :class:`~google.cloud.bigquery.dataset.AccessEntry`.
+ """
+ return (self._role, self._entity_type, self._entity_id)
+
+ def __hash__(self):
+ return hash(self._key())
+
def to_api_repr(self):
"""Construct the API resource representation of this access entry
Returns:
Dict[str, object]: Access entry represented as an API resource
"""
- resource = {self.entity_type: self.entity_id}
- if self.role is not None:
- resource["role"] = self.role
+ resource = {self._entity_type: self._entity_id}
+ if self._role is not None:
+ resource["role"] = self._role
return resource
@classmethod
diff --git a/google/cloud/bigquery/dbapi/_helpers.py b/google/cloud/bigquery/dbapi/_helpers.py
index 651880fea..6558177d7 100644
--- a/google/cloud/bigquery/dbapi/_helpers.py
+++ b/google/cloud/bigquery/dbapi/_helpers.py
@@ -24,6 +24,7 @@
import six
from google.cloud import bigquery
+from google.cloud.bigquery import table
from google.cloud.bigquery.dbapi import exceptions
@@ -218,3 +219,22 @@ def array_like(value):
return isinstance(value, collections_abc.Sequence) and not isinstance(
value, (six.text_type, six.binary_type, bytearray)
)
+
+
+def to_bq_table_rows(rows_iterable):
+ """Convert table rows to BigQuery table Row instances.
+
+ Args:
+ rows_iterable (Iterable[Mapping]):
+ An iterable of row data items to convert to ``Row`` instances.
+
+ Returns:
+ Iterable[google.cloud.bigquery.table.Row]
+ """
+
+ def to_table_row(row):
+ values = tuple(row.values())
+ keys_to_index = {key: i for i, key in enumerate(row.keys())}
+ return table.Row(values, keys_to_index)
+
+ return (to_table_row(row_data) for row_data in rows_iterable)
diff --git a/google/cloud/bigquery/dbapi/connection.py b/google/cloud/bigquery/dbapi/connection.py
index ee7d0dc3c..b8eaf2f9b 100644
--- a/google/cloud/bigquery/dbapi/connection.py
+++ b/google/cloud/bigquery/dbapi/connection.py
@@ -23,10 +23,24 @@ class Connection(object):
Args:
client (google.cloud.bigquery.Client): A client used to connect to BigQuery.
+ bqstorage_client(\
+ Optional[google.cloud.bigquery_storage_v1beta1.BigQueryStorageClient] \
+ ):
+ [Beta] An alternative client that uses the faster BigQuery Storage
+ API to fetch rows from BigQuery. If both clients are given,
+ ``bqstorage_client`` is used first to fetch query results,
+ with a fallback on ``client``, if necessary.
+
+ .. note::
+ There is a known issue with the BigQuery Storage API with small
+ anonymous result sets, which results in such fallback.
+
+ https://github.com/googleapis/python-bigquery-storage/issues/2
"""
- def __init__(self, client):
+ def __init__(self, client, bqstorage_client=None):
self._client = client
+ self._bqstorage_client = bqstorage_client
def close(self):
"""No-op."""
@@ -43,17 +57,30 @@ def cursor(self):
return cursor.Cursor(self)
-def connect(client=None):
+def connect(client=None, bqstorage_client=None):
"""Construct a DB-API connection to Google BigQuery.
Args:
- client (google.cloud.bigquery.Client):
- (Optional) A client used to connect to BigQuery. If not passed, a
- client is created using default options inferred from the environment.
+ client (Optional[google.cloud.bigquery.Client]):
+ A client used to connect to BigQuery. If not passed, a client is
+ created using default options inferred from the environment.
+ bqstorage_client(\
+ Optional[google.cloud.bigquery_storage_v1beta1.BigQueryStorageClient] \
+ ):
+ [Beta] An alternative client that uses the faster BigQuery Storage
+ API to fetch rows from BigQuery. If both clients are given,
+ ``bqstorage_client`` is used first to fetch query results,
+ with a fallback on ``client``, if necessary.
+
+ .. note::
+ There is a known issue with the BigQuery Storage API with small
+ anonymous result sets, which results in such fallback.
+
+ https://github.com/googleapis/python-bigquery-storage/issues/2
Returns:
google.cloud.bigquery.dbapi.Connection: A new DB-API connection to BigQuery.
"""
if client is None:
client = bigquery.Client()
- return Connection(client)
+ return Connection(client, bqstorage_client)
diff --git a/google/cloud/bigquery/dbapi/cursor.py b/google/cloud/bigquery/dbapi/cursor.py
index a3e6ea5be..eb73b3d56 100644
--- a/google/cloud/bigquery/dbapi/cursor.py
+++ b/google/cloud/bigquery/dbapi/cursor.py
@@ -21,6 +21,8 @@
except ImportError: # Python 2.7
import collections as collections_abc
+import logging
+
import six
from google.cloud.bigquery import job
@@ -28,6 +30,9 @@
from google.cloud.bigquery.dbapi import exceptions
import google.cloud.exceptions
+
+_LOGGER = logging.getLogger(__name__)
+
# Per PEP 249: A 7-item sequence containing information describing one result
# column. The first two items (name and type_code) are mandatory, the other
# five are optional and are set to None if no meaningful values can be
@@ -212,6 +217,30 @@ def _try_fetch(self, size=None):
if self._query_data is None:
client = self.connection._client
+ bqstorage_client = self.connection._bqstorage_client
+
+ if bqstorage_client is not None:
+ try:
+ rows_iterable = self._bqstorage_fetch(bqstorage_client)
+ self._query_data = _helpers.to_bq_table_rows(rows_iterable)
+ return
+ except google.api_core.exceptions.GoogleAPICallError as exc:
+ # NOTE: Forbidden is a subclass of GoogleAPICallError
+ if isinstance(exc, google.api_core.exceptions.Forbidden):
+ # Don't hide errors such as insufficient permissions to create
+ # a read session, or the API is not enabled. Both of those are
+ # clearly problems if the developer has explicitly asked for
+ # BigQuery Storage API support.
+ raise
+
+ # There is an issue with reading from small anonymous
+ # query results tables. If such an error occurs, we silence
+ # it in order to try again with the tabledata.list API.
+ _LOGGER.debug(
+ "Error fetching data with BigQuery Storage API, "
+ "falling back to tabledata.list API."
+ )
+
rows_iter = client.list_rows(
self._query_job.destination,
selected_fields=self._query_job._query_results.schema,
@@ -219,6 +248,45 @@ def _try_fetch(self, size=None):
)
self._query_data = iter(rows_iter)
+ def _bqstorage_fetch(self, bqstorage_client):
+ """Start fetching data with the BigQuery Storage API.
+
+ The method assumes that the data about the relevant query job already
+ exists internally.
+
+ Args:
+ bqstorage_client(\
+ google.cloud.bigquery_storage_v1beta1.BigQueryStorageClient \
+ ):
+ A client tha know how to talk to the BigQuery Storage API.
+
+ Returns:
+ Iterable[Mapping]:
+ A sequence of rows, represented as dictionaries.
+ """
+ # NOTE: Given that BQ storage client instance is passed in, it means
+ # that bigquery_storage_v1beta1 library is available (no ImportError).
+ from google.cloud import bigquery_storage_v1beta1
+
+ table_reference = self._query_job.destination
+
+ read_session = bqstorage_client.create_read_session(
+ table_reference.to_bqstorage(),
+ "projects/{}".format(table_reference.project),
+ # a single stream only, as DB API is not well-suited for multithreading
+ requested_streams=1,
+ )
+
+ if not read_session.streams:
+ return iter([]) # empty table, nothing to read
+
+ read_position = bigquery_storage_v1beta1.types.StreamPosition(
+ stream=read_session.streams[0],
+ )
+ read_rows_stream = bqstorage_client.read_rows(read_position)
+ rows_iterable = read_rows_stream.rows(read_session)
+ return rows_iterable
+
def fetchone(self):
"""Fetch a single row from the results of the last ``execute*()`` call.
diff --git a/google/cloud/bigquery/exceptions.py b/google/cloud/bigquery/exceptions.py
new file mode 100644
index 000000000..93490ef97
--- /dev/null
+++ b/google/cloud/bigquery/exceptions.py
@@ -0,0 +1,17 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class PyarrowMissingWarning(DeprecationWarning):
+ pass
diff --git a/google/cloud/bigquery/job.py b/google/cloud/bigquery/job.py
index 5861febe8..7a1178a8c 100644
--- a/google/cloud/bigquery/job.py
+++ b/google/cloud/bigquery/job.py
@@ -26,7 +26,6 @@
from six.moves import http_client
import google.api_core.future.polling
-from google.auth.transport.requests import TimeoutGuard
from google.cloud import exceptions
from google.cloud.exceptions import NotFound
from google.cloud.bigquery.dataset import Dataset
@@ -55,7 +54,6 @@
_DONE_STATE = "DONE"
_STOPPED_REASON = "stopped"
_TIMEOUT_BUFFER_SECS = 0.1
-_SERVER_TIMEOUT_MARGIN_SECS = 1.0
_CONTAINS_ORDER_BY = re.compile(r"ORDER\s+BY", re.IGNORECASE)
_ERROR_REASON_TO_EXCEPTION = {
@@ -796,8 +794,8 @@ def result(self, retry=DEFAULT_RETRY, timeout=None):
timeout (Optional[float]):
The number of seconds to wait for the underlying HTTP transport
before using ``retry``.
- If multiple requests are made under the hood, ``timeout`` is
- interpreted as the approximate total time of **all** requests.
+ If multiple requests are made under the hood, ``timeout``
+ applies to each individual request.
Returns:
_AsyncJob: This instance.
@@ -809,11 +807,7 @@ def result(self, retry=DEFAULT_RETRY, timeout=None):
if the job did not complete in the given timeout.
"""
if self.state is None:
- with TimeoutGuard(
- timeout, timeout_error_type=concurrent.futures.TimeoutError
- ) as guard:
- self._begin(retry=retry, timeout=timeout)
- timeout = guard.remaining_timeout
+ self._begin(retry=retry, timeout=timeout)
# TODO: modify PollingFuture so it can pass a retry argument to done().
return super(_AsyncJob, self).result(timeout=timeout)
@@ -1996,8 +1990,11 @@ class ExtractJob(_AsyncJob):
Args:
job_id (str): the job's ID.
- source (google.cloud.bigquery.table.TableReference):
- Table into which data is to be loaded.
+ source (Union[ \
+ google.cloud.bigquery.table.TableReference, \
+ google.cloud.bigquery.model.ModelReference \
+ ]):
+ Table or Model from which data is to be loaded or extracted.
destination_uris (List[str]):
URIs describing where the extracted data will be written in Cloud
@@ -2073,14 +2070,20 @@ def destination_uri_file_counts(self):
def to_api_repr(self):
"""Generate a resource for :meth:`_begin`."""
+ configuration = self._configuration.to_api_repr()
source_ref = {
"projectId": self.source.project,
"datasetId": self.source.dataset_id,
- "tableId": self.source.table_id,
}
- configuration = self._configuration.to_api_repr()
- _helpers._set_sub_prop(configuration, ["extract", "sourceTable"], source_ref)
+ source = "sourceTable"
+ if isinstance(self.source, TableReference):
+ source_ref["tableId"] = self.source.table_id
+ else:
+ source_ref["modelId"] = self.source.model_id
+ source = "sourceModel"
+
+ _helpers._set_sub_prop(configuration, ["extract", source], source_ref)
_helpers._set_sub_prop(
configuration, ["extract", "destinationUris"], self.destination_uris
)
@@ -2118,10 +2121,20 @@ def from_api_repr(cls, resource, client):
source_config = _helpers._get_sub_prop(
config_resource, ["extract", "sourceTable"]
)
- dataset = DatasetReference(
- source_config["projectId"], source_config["datasetId"]
- )
- source = dataset.table(source_config["tableId"])
+ if source_config:
+ dataset = DatasetReference(
+ source_config["projectId"], source_config["datasetId"]
+ )
+ source = dataset.table(source_config["tableId"])
+ else:
+ source_config = _helpers._get_sub_prop(
+ config_resource, ["extract", "sourceModel"]
+ )
+ dataset = DatasetReference(
+ source_config["projectId"], source_config["datasetId"]
+ )
+ source = dataset.model(source_config["modelId"])
+
destination_uris = _helpers._get_sub_prop(
config_resource, ["extract", "destinationUris"]
)
@@ -2602,6 +2615,7 @@ def __init__(self, job_id, query, client, job_config=None):
self._configuration = job_config
self._query_results = None
self._done_timeout = None
+ self._transport_timeout = None
@property
def allow_large_results(self):
@@ -3059,19 +3073,9 @@ def done(self, retry=DEFAULT_RETRY, timeout=None):
self._done_timeout = max(0, self._done_timeout)
timeout_ms = int(api_timeout * 1000)
- # If the server-side processing timeout (timeout_ms) is specified and
- # would be picked as the total request timeout, we want to add a small
- # margin to it - we don't want to timeout the connection just as the
- # server-side processing might have completed, but instead slightly
- # after the server-side deadline.
- # However, if `timeout` is specified, and is shorter than the adjusted
- # server timeout, the former prevails.
- if timeout_ms is not None and timeout_ms > 0:
- server_timeout_with_margin = timeout_ms / 1000 + _SERVER_TIMEOUT_MARGIN_SECS
- if timeout is not None:
- timeout = min(server_timeout_with_margin, timeout)
- else:
- timeout = server_timeout_with_margin
+ # If an explicit timeout is not given, fall back to the transport timeout
+ # stored in _blocking_poll() in the process of polling for job completion.
+ transport_timeout = timeout if timeout is not None else self._transport_timeout
# Do not refresh if the state is already done, as the job will not
# change once complete.
@@ -3082,19 +3086,20 @@ def done(self, retry=DEFAULT_RETRY, timeout=None):
project=self.project,
timeout_ms=timeout_ms,
location=self.location,
- timeout=timeout,
+ timeout=transport_timeout,
)
# Only reload the job once we know the query is complete.
# This will ensure that fields such as the destination table are
# correctly populated.
if self._query_results.complete:
- self.reload(retry=retry, timeout=timeout)
+ self.reload(retry=retry, timeout=transport_timeout)
return self.state == _DONE_STATE
def _blocking_poll(self, timeout=None):
self._done_timeout = timeout
+ self._transport_timeout = timeout
super(QueryJob, self)._blocking_poll(timeout=timeout)
@staticmethod
@@ -3111,7 +3116,7 @@ def _format_for_exception(query, job_id):
template = "\n\n(job ID: {job_id})\n\n{header}\n\n{ruler}\n{body}\n{ruler}"
lines = query.splitlines()
- max_line_len = max(len(l) for l in lines)
+ max_line_len = max(len(line) for line in lines)
header = "-----Query Job SQL Follows-----"
header = "{:^{total_width}}".format(header, total_width=max_line_len + 5)
@@ -3155,7 +3160,12 @@ def _begin(self, client=None, retry=DEFAULT_RETRY, timeout=None):
raise
def result(
- self, page_size=None, max_results=None, retry=DEFAULT_RETRY, timeout=None
+ self,
+ page_size=None,
+ max_results=None,
+ retry=DEFAULT_RETRY,
+ timeout=None,
+ start_index=None,
):
"""Start the job and wait for it to complete and get the result.
@@ -3170,8 +3180,10 @@ def result(
timeout (Optional[float]):
The number of seconds to wait for the underlying HTTP transport
before using ``retry``.
- If multiple requests are made under the hood, ``timeout`` is
- interpreted as the approximate total time of **all** requests.
+ If multiple requests are made under the hood, ``timeout``
+ applies to each individual request.
+ start_index (Optional[int]):
+ The zero-based index of the starting row to read.
Returns:
google.cloud.bigquery.table.RowIterator:
@@ -3182,6 +3194,9 @@ def result(
set** (this is distinct from the total number of rows in the
current page: ``iterator.page.num_items``).
+ If the query is a special query that produces no results, e.g.
+ a DDL query, an ``_EmptyRowIterator`` instance is returned.
+
Raises:
google.cloud.exceptions.GoogleCloudError:
If the job failed.
@@ -3189,27 +3204,17 @@ def result(
If the job did not complete in the given timeout.
"""
try:
- guard = TimeoutGuard(
- timeout, timeout_error_type=concurrent.futures.TimeoutError
- )
- with guard:
- super(QueryJob, self).result(retry=retry, timeout=timeout)
- timeout = guard.remaining_timeout
+ super(QueryJob, self).result(retry=retry, timeout=timeout)
# Return an iterator instead of returning the job.
if not self._query_results:
- guard = TimeoutGuard(
- timeout, timeout_error_type=concurrent.futures.TimeoutError
+ self._query_results = self._client._get_query_results(
+ self.job_id,
+ retry,
+ project=self.project,
+ location=self.location,
+ timeout=timeout,
)
- with guard:
- self._query_results = self._client._get_query_results(
- self.job_id,
- retry,
- project=self.project,
- location=self.location,
- timeout=timeout,
- )
- timeout = guard.remaining_timeout
except exceptions.GoogleCloudError as exc:
exc.message += self._format_for_exception(self.query, self.job_id)
exc.query_job = self
@@ -3232,6 +3237,7 @@ def result(
dest_table,
page_size=page_size,
max_results=max_results,
+ start_index=start_index,
retry=retry,
timeout=timeout,
)
diff --git a/google/cloud/bigquery/magics.py b/google/cloud/bigquery/magics.py
index 39608b19f..4f2a16cca 100644
--- a/google/cloud/bigquery/magics.py
+++ b/google/cloud/bigquery/magics.py
@@ -65,8 +65,18 @@
serializable. The variable reference is indicated by a ``$`` before
the variable name (ex. ``$my_dict_var``). See ``In[6]`` and ``In[7]``
in the Examples section below.
+
+ .. note::
+
+ Due to the way IPython argument parser works, negative numbers in
+ dictionaries are incorrectly "recognized" as additional arguments,
+ resulting in an error ("unrecognized arguments"). To get around this,
+ pass such dictionary as a JSON string variable.
+
* ```` (required, cell argument):
- SQL query to run.
+ SQL query to run. If the query does not contain any whitespace (aside
+ from leading and trailing whitespace), it is assumed to represent a
+ fully-qualified table ID, and the latter's data will be fetched.
Returns:
A :class:`pandas.DataFrame` with the query results.
@@ -506,6 +516,11 @@ def _cell_magic(line, query):
query = query.strip()
+ if not query:
+ error = ValueError("Query is missing.")
+ _handle_error(error, args.destination_var)
+ return
+
# Any query that does not contain whitespace (aside from leading and trailing whitespace)
# is assumed to be a table id
if not re.search(r"\s", query):
diff --git a/google/cloud/bigquery/model.py b/google/cloud/bigquery/model.py
index d39ec5f2f..eb459f57a 100644
--- a/google/cloud/bigquery/model.py
+++ b/google/cloud/bigquery/model.py
@@ -430,6 +430,18 @@ def __hash__(self):
return hash(self._key())
def __repr__(self):
- return "ModelReference(project='{}', dataset_id='{}', project_id='{}')".format(
+ return "ModelReference(project_id='{}', dataset_id='{}', model_id='{}')".format(
self.project, self.dataset_id, self.model_id
)
+
+
+def _model_arg_to_model_ref(value, default_project=None):
+ """Helper to convert a string or Model to ModelReference.
+
+ This function keeps ModelReference and other kinds of objects unchanged.
+ """
+ if isinstance(value, six.string_types):
+ return ModelReference.from_string(value, default_project=default_project)
+ if isinstance(value, Model):
+ return value.reference
+ return value
diff --git a/google/cloud/bigquery/schema.py b/google/cloud/bigquery/schema.py
index 3878a80a9..0eaf1201b 100644
--- a/google/cloud/bigquery/schema.py
+++ b/google/cloud/bigquery/schema.py
@@ -62,14 +62,26 @@ class SchemaField(object):
fields (Tuple[google.cloud.bigquery.schema.SchemaField]):
subfields (requires ``field_type`` of 'RECORD').
+
+ policy_tags (Optional[PolicyTagList]): The policy tag list for the field.
+
"""
- def __init__(self, name, field_type, mode="NULLABLE", description=None, fields=()):
+ def __init__(
+ self,
+ name,
+ field_type,
+ mode="NULLABLE",
+ description=None,
+ fields=(),
+ policy_tags=None,
+ ):
self._name = name
self._field_type = field_type
self._mode = mode
self._description = description
self._fields = tuple(fields)
+ self._policy_tags = policy_tags
@classmethod
def from_api_repr(cls, api_repr):
@@ -87,12 +99,14 @@ def from_api_repr(cls, api_repr):
mode = api_repr.get("mode", "NULLABLE")
description = api_repr.get("description")
fields = api_repr.get("fields", ())
+
return cls(
field_type=api_repr["type"].upper(),
fields=[cls.from_api_repr(f) for f in fields],
mode=mode.upper(),
description=description,
name=api_repr["name"],
+ policy_tags=PolicyTagList.from_api_repr(api_repr.get("policyTags")),
)
@property
@@ -136,6 +150,13 @@ def fields(self):
"""
return self._fields
+ @property
+ def policy_tags(self):
+ """Optional[google.cloud.bigquery.schema.PolicyTagList]: Policy tag list
+ definition for this field.
+ """
+ return self._policy_tags
+
def to_api_repr(self):
"""Return a dictionary representing this schema field.
@@ -155,6 +176,10 @@ def to_api_repr(self):
if self.field_type.upper() in _STRUCT_TYPES:
answer["fields"] = [f.to_api_repr() for f in self.fields]
+ # If this contains a policy tag definition, include that as well:
+ if self.policy_tags is not None:
+ answer["policyTags"] = self.policy_tags.to_api_repr()
+
# Done; return the serialized dictionary.
return answer
@@ -172,6 +197,7 @@ def _key(self):
self._mode.upper(),
self._description,
self._fields,
+ self._policy_tags,
)
def to_standard_sql(self):
@@ -244,7 +270,10 @@ def _parse_schema_resource(info):
mode = r_field.get("mode", "NULLABLE")
description = r_field.get("description")
sub_fields = _parse_schema_resource(r_field)
- schema.append(SchemaField(name, field_type, mode, description, sub_fields))
+ policy_tags = PolicyTagList.from_api_repr(r_field.get("policyTags"))
+ schema.append(
+ SchemaField(name, field_type, mode, description, sub_fields, policy_tags)
+ )
return schema
@@ -291,3 +320,88 @@ def _to_schema_fields(schema):
field if isinstance(field, SchemaField) else SchemaField.from_api_repr(field)
for field in schema
]
+
+
+class PolicyTagList(object):
+ """Define Policy Tags for a column.
+
+ Args:
+ names (
+ Optional[Tuple[str]]): list of policy tags to associate with
+ the column. Policy tag identifiers are of the form
+ `projects/*/locations/*/taxonomies/*/policyTags/*`.
+ """
+
+ def __init__(self, names=()):
+ self._properties = {}
+ self._properties["names"] = tuple(names)
+
+ @property
+ def names(self):
+ """Tuple[str]: Policy tags associated with this definition.
+ """
+ return self._properties.get("names", ())
+
+ def _key(self):
+ """A tuple key that uniquely describes this PolicyTagList.
+
+ Used to compute this instance's hashcode and evaluate equality.
+
+ Returns:
+ Tuple: The contents of this :class:`~google.cloud.bigquery.schema.PolicyTagList`.
+ """
+ return tuple(sorted(self._properties.items()))
+
+ def __eq__(self, other):
+ if not isinstance(other, PolicyTagList):
+ return NotImplemented
+ return self._key() == other._key()
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __hash__(self):
+ return hash(self._key())
+
+ def __repr__(self):
+ return "PolicyTagList{}".format(self._key())
+
+ @classmethod
+ def from_api_repr(cls, api_repr):
+ """Return a :class:`PolicyTagList` object deserialized from a dict.
+
+ This method creates a new ``PolicyTagList`` instance that points to
+ the ``api_repr`` parameter as its internal properties dict. This means
+ that when a ``PolicyTagList`` instance is stored as a property of
+ another object, any changes made at the higher level will also appear
+ here.
+
+ Args:
+ api_repr (Mapping[str, str]):
+ The serialized representation of the PolicyTagList, such as
+ what is output by :meth:`to_api_repr`.
+
+ Returns:
+ Optional[google.cloud.bigquery.schema.PolicyTagList]:
+ The ``PolicyTagList`` object or None.
+ """
+ if api_repr is None:
+ return None
+ names = api_repr.get("names", ())
+ return cls(names=names)
+
+ def to_api_repr(self):
+ """Return a dictionary representing this object.
+
+ This method returns the properties dict of the ``PolicyTagList``
+ instance rather than making a copy. This means that when a
+ ``PolicyTagList`` instance is stored as a property of another
+ object, any changes made at the higher level will also appear here.
+
+ Returns:
+ dict:
+ A dictionary representing the PolicyTagList object in
+ serialized form.
+ """
+ answer = {"names": [name for name in self.names]}
+ return answer
diff --git a/google/cloud/bigquery/table.py b/google/cloud/bigquery/table.py
index 555f529f3..e66d24e74 100644
--- a/google/cloud/bigquery/table.py
+++ b/google/cloud/bigquery/table.py
@@ -54,6 +54,7 @@
from google.cloud.bigquery.schema import _build_schema_resource
from google.cloud.bigquery.schema import _parse_schema_resource
from google.cloud.bigquery.schema import _to_schema_fields
+from google.cloud.bigquery.exceptions import PyarrowMissingWarning
from google.cloud.bigquery.external_config import ExternalConfig
from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration
@@ -582,8 +583,6 @@ def partitioning_type(self):
"""Union[str, None]: Time partitioning of the table if it is
partitioned (Defaults to :data:`None`).
- The only partitioning type that is currently supported is
- :attr:`~google.cloud.bigquery.table.TimePartitioningType.DAY`.
"""
warnings.warn(
"This method will be deprecated in future versions. Please use "
@@ -1139,12 +1138,17 @@ class StreamingBuffer(object):
"""
def __init__(self, resource):
- self.estimated_bytes = int(resource["estimatedBytes"])
- self.estimated_rows = int(resource["estimatedRows"])
- # time is in milliseconds since the epoch.
- self.oldest_entry_time = google.cloud._helpers._datetime_from_microseconds(
- 1000.0 * int(resource["oldestEntryTime"])
- )
+ self.estimated_bytes = None
+ if "estimatedBytes" in resource:
+ self.estimated_bytes = int(resource["estimatedBytes"])
+ self.estimated_rows = None
+ if "estimatedRows" in resource:
+ self.estimated_rows = int(resource["estimatedRows"])
+ self.oldest_entry_time = None
+ if "oldestEntryTime" in resource:
+ self.oldest_entry_time = google.cloud._helpers._datetime_from_microseconds(
+ 1000.0 * int(resource["oldestEntryTime"])
+ )
class Row(object):
@@ -1362,6 +1366,8 @@ def _get_next_page_response(self):
"""
params = self._get_query_params()
if self._page_size is not None:
+ if self.page_number and "startIndex" in params:
+ del params["startIndex"]
params["maxResults"] = self._page_size
return self.api_request(
method=self._HTTP_METHOD, path=self.path, query_params=params
@@ -1732,6 +1738,14 @@ def to_dataframe(
for column in dtypes:
df[column] = pandas.Series(df[column], dtype=dtypes[column])
return df
+ else:
+ warnings.warn(
+ "Converting to a dataframe without pyarrow installed is "
+ "often slower and will become unsupported in the future. "
+ "Please install the pyarrow package.",
+ PyarrowMissingWarning,
+ stacklevel=2,
+ )
# The bqstorage_client is only used if pyarrow is available, so the
# rest of this method only needs to account for tabledata.list.
@@ -1964,6 +1978,9 @@ class TimePartitioningType(object):
DAY = "DAY"
"""str: Generates one partition per day."""
+ HOUR = "HOUR"
+ """str: Generates one partition per hour."""
+
class TimePartitioning(object):
"""Configures time-based partitioning for a table.
diff --git a/noxfile.py b/noxfile.py
index 17a2dee41..b2d26568c 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -20,8 +20,6 @@
import nox
-LOCAL_DEPS = (os.path.join("..", "api_core[grpc]"), os.path.join("..", "core"))
-
BLACK_PATHS = ("docs", "google", "samples", "tests", "noxfile.py", "setup.py")
@@ -34,28 +32,15 @@ def default(session):
run the tests.
"""
# Install all test dependencies, then install local packages in-place.
- session.install("mock", "pytest", "pytest-cov", "freezegun")
- for local_dep in LOCAL_DEPS:
- session.install("-e", local_dep)
-
- session.install("-e", os.path.join("..", "test_utils"))
-
- coverage_fail_under = "--cov-fail-under=97"
+ session.install(
+ "mock", "pytest", "google-cloud-testutils", "pytest-cov", "freezegun"
+ )
+ session.install("grpcio")
# fastparquet is not included in .[all] because, in general, it's redundant
# with pyarrow. We still want to run some unit tests with fastparquet
# serialization, though.
- dev_install = ".[all,fastparquet]"
-
- # There is no pyarrow or fastparquet wheel for Python 3.8.
- if session.python == "3.8":
- # Since many tests are skipped due to missing dependencies, test
- # coverage is much lower in Python 3.8. Remove once we can test with
- # pyarrow.
- coverage_fail_under = "--cov-fail-under=92"
- dev_install = ".[pandas,tqdm]"
-
- session.install("-e", dev_install)
+ session.install("-e", ".[all,fastparquet]")
# IPython does not support Python 2 after version 5.x
if session.python == "2.7":
@@ -72,9 +57,9 @@ def default(session):
"--cov-append",
"--cov-config=.coveragerc",
"--cov-report=",
- coverage_fail_under,
+ "--cov-fail-under=0",
os.path.join("tests", "unit"),
- *session.posargs
+ *session.posargs,
)
@@ -84,7 +69,7 @@ def unit(session):
default(session)
-@nox.session(python=["2.7", "3.7"])
+@nox.session(python=["2.7", "3.8"])
def system(session):
"""Run the system test suite."""
@@ -96,11 +81,9 @@ def system(session):
session.install("--pre", "grpcio")
# Install all test dependencies, then install local packages in place.
- session.install("mock", "pytest", "psutil")
- for local_dep in LOCAL_DEPS:
- session.install("-e", local_dep)
- session.install("-e", os.path.join("..", "storage"))
- session.install("-e", os.path.join("..", "test_utils"))
+ session.install("mock", "pytest", "psutil", "google-cloud-testutils")
+ session.install("google-cloud-storage")
+ session.install("fastavro")
session.install("-e", ".[all]")
# IPython does not support Python 2 after version 5.x
@@ -115,7 +98,7 @@ def system(session):
)
-@nox.session(python=["2.7", "3.7"])
+@nox.session(python=["2.7", "3.8"])
def snippets(session):
"""Run the snippets test suite."""
@@ -124,11 +107,9 @@ def snippets(session):
session.skip("Credentials must be set via environment variable.")
# Install all test dependencies, then install local packages in place.
- session.install("mock", "pytest")
- for local_dep in LOCAL_DEPS:
- session.install("-e", local_dep)
- session.install("-e", os.path.join("..", "storage"))
- session.install("-e", os.path.join("..", "test_utils"))
+ session.install("mock", "pytest", "google-cloud-testutils")
+ session.install("google-cloud-storage")
+ session.install("grpcio")
session.install("-e", ".[all]")
# Run py.test against the snippets tests.
@@ -136,7 +117,7 @@ def snippets(session):
session.run("py.test", "samples", *session.posargs)
-@nox.session(python="3.7")
+@nox.session(python="3.8")
def cover(session):
"""Run the final coverage report.
@@ -148,7 +129,7 @@ def cover(session):
session.run("coverage", "erase")
-@nox.session(python="3.7")
+@nox.session(python="3.8")
def lint(session):
"""Run linters.
@@ -157,8 +138,6 @@ def lint(session):
"""
session.install("black", "flake8")
- for local_dep in LOCAL_DEPS:
- session.install("-e", local_dep)
session.install("-e", ".")
session.run("flake8", os.path.join("google", "cloud", "bigquery"))
session.run("flake8", "tests")
@@ -167,7 +146,7 @@ def lint(session):
session.run("black", "--check", *BLACK_PATHS)
-@nox.session(python="3.7")
+@nox.session(python="3.8")
def lint_setup_py(session):
"""Verify that setup.py is valid (including RST check)."""
@@ -188,14 +167,12 @@ def blacken(session):
session.run("black", *BLACK_PATHS)
-@nox.session(python="3.7")
+@nox.session(python="3.8")
def docs(session):
"""Build the docs."""
session.install("ipython", "recommonmark", "sphinx", "sphinx_rtd_theme")
- for local_dep in LOCAL_DEPS:
- session.install("-e", local_dep)
- session.install("-e", os.path.join("..", "storage"))
+ session.install("google-cloud-storage")
session.install("-e", ".[all]")
shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True)
diff --git a/renovate.json b/renovate.json
new file mode 100644
index 000000000..4fa949311
--- /dev/null
+++ b/renovate.json
@@ -0,0 +1,5 @@
+{
+ "extends": [
+ "config:base", ":preserveSemverRanges"
+ ]
+}
diff --git a/samples/create_dataset.py b/samples/create_dataset.py
index e47d68a96..6af3c67eb 100644
--- a/samples/create_dataset.py
+++ b/samples/create_dataset.py
@@ -30,9 +30,9 @@ def create_dataset(dataset_id):
# TODO(developer): Specify the geographic location where the dataset should reside.
dataset.location = "US"
- # Send the dataset to the API for creation.
+ # Send the dataset to the API for creation, with an explicit timeout.
# Raises google.api_core.exceptions.Conflict if the Dataset already
# exists within the project.
- dataset = client.create_dataset(dataset) # Make an API request.
+ dataset = client.create_dataset(dataset, timeout=30) # Make an API request.
print("Created dataset {}.{}".format(client.project, dataset.dataset_id))
# [END bigquery_create_dataset]
diff --git a/samples/load_table_uri_autodetect_csv.py b/samples/load_table_uri_autodetect_csv.py
new file mode 100644
index 000000000..09a5d708d
--- /dev/null
+++ b/samples/load_table_uri_autodetect_csv.py
@@ -0,0 +1,45 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def load_table_uri_autodetect_csv(table_id):
+
+ # [START bigquery_load_table_gcs_csv_autodetect]
+ from google.cloud import bigquery
+
+ # Construct a BigQuery client object.
+ client = bigquery.Client()
+
+ # TODO(developer): Set table_id to the ID of the table to create.
+ # table_id = "your-project.your_dataset.your_table_name
+
+ # Set the encryption key to use for the destination.
+ # TODO: Replace this key with a key you have created in KMS.
+ # kms_key_name = "projects/{}/locations/{}/keyRings/{}/cryptoKeys/{}".format(
+ # "cloud-samples-tests", "us", "test", "test"
+ # )
+ job_config = bigquery.LoadJobConfig(
+ autodetect=True,
+ skip_leading_rows=1,
+ # The source format defaults to CSV, so the line below is optional.
+ source_format=bigquery.SourceFormat.CSV,
+ )
+ uri = "gs://cloud-samples-data/bigquery/us-states/us-states.csv"
+ load_job = client.load_table_from_uri(
+ uri, table_id, job_config=job_config
+ ) # Make an API request.
+ load_job.result() # Waits for the job to complete.
+ destination_table = client.get_table(table_id)
+ print("Loaded {} rows.".format(destination_table.num_rows))
+ # [END bigquery_load_table_gcs_csv_autodetect]
diff --git a/samples/load_table_uri_autodetect_json.py b/samples/load_table_uri_autodetect_json.py
new file mode 100644
index 000000000..61b7aab12
--- /dev/null
+++ b/samples/load_table_uri_autodetect_json.py
@@ -0,0 +1,42 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def load_table_uri_autodetect_json(table_id):
+
+ # [START bigquery_load_table_gcs_json_autodetect]
+ from google.cloud import bigquery
+
+ # Construct a BigQuery client object.
+ client = bigquery.Client()
+
+ # TODO(developer): Set table_id to the ID of the table to create.
+ # table_id = "your-project.your_dataset.your_table_name
+
+ # Set the encryption key to use for the destination.
+ # TODO: Replace this key with a key you have created in KMS.
+ # kms_key_name = "projects/{}/locations/{}/keyRings/{}/cryptoKeys/{}".format(
+ # "cloud-samples-tests", "us", "test", "test"
+ # )
+ job_config = bigquery.LoadJobConfig(
+ autodetect=True, source_format=bigquery.SourceFormat.NEWLINE_DELIMITED_JSON
+ )
+ uri = "gs://cloud-samples-data/bigquery/us-states/us-states.json"
+ load_job = client.load_table_from_uri(
+ uri, table_id, job_config=job_config
+ ) # Make an API request.
+ load_job.result() # Waits for the job to complete.
+ destination_table = client.get_table(table_id)
+ print("Loaded {} rows.".format(destination_table.num_rows))
+ # [END bigquery_load_table_gcs_json_autodetect]
diff --git a/samples/load_table_uri_truncate_avro.py b/samples/load_table_uri_truncate_avro.py
new file mode 100644
index 000000000..98a791477
--- /dev/null
+++ b/samples/load_table_uri_truncate_avro.py
@@ -0,0 +1,55 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def load_table_uri_truncate_avro(table_id):
+
+ # [START bigquery_load_table_gcs_avro_truncate]
+ import six
+
+ from google.cloud import bigquery
+
+ # Construct a BigQuery client object.
+ client = bigquery.Client()
+
+ # TODO(developer): Set table_id to the ID of the table to create.
+ # table_id = "your-project.your_dataset.your_table_name
+
+ job_config = bigquery.LoadJobConfig(
+ schema=[
+ bigquery.SchemaField("name", "STRING"),
+ bigquery.SchemaField("post_abbr", "STRING"),
+ ],
+ )
+
+ body = six.BytesIO(b"Washington,WA")
+ client.load_table_from_file(body, table_id, job_config=job_config).result()
+ previous_rows = client.get_table(table_id).num_rows
+ assert previous_rows > 0
+
+ job_config = bigquery.LoadJobConfig(
+ write_disposition=bigquery.WriteDisposition.WRITE_TRUNCATE,
+ source_format=bigquery.SourceFormat.AVRO,
+ )
+
+ uri = "gs://cloud-samples-data/bigquery/us-states/us-states.avro"
+ load_job = client.load_table_from_uri(
+ uri, table_id, job_config=job_config
+ ) # Make an API request.
+
+ load_job.result() # Waits for the job to complete.
+
+ destination_table = client.get_table(table_id)
+ print("Loaded {} rows.".format(destination_table.num_rows))
+ # [END bigquery_load_table_gcs_avro_truncate]
diff --git a/samples/load_table_uri_truncate_csv.py b/samples/load_table_uri_truncate_csv.py
new file mode 100644
index 000000000..73de7a8c1
--- /dev/null
+++ b/samples/load_table_uri_truncate_csv.py
@@ -0,0 +1,56 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def load_table_uri_truncate_csv(table_id):
+
+ # [START bigquery_load_table_gcs_csv_truncate]
+ import six
+
+ from google.cloud import bigquery
+
+ # Construct a BigQuery client object.
+ client = bigquery.Client()
+
+ # TODO(developer): Set table_id to the ID of the table to create.
+ # table_id = "your-project.your_dataset.your_table_name
+
+ job_config = bigquery.LoadJobConfig(
+ schema=[
+ bigquery.SchemaField("name", "STRING"),
+ bigquery.SchemaField("post_abbr", "STRING"),
+ ],
+ )
+
+ body = six.BytesIO(b"Washington,WA")
+ client.load_table_from_file(body, table_id, job_config=job_config).result()
+ previous_rows = client.get_table(table_id).num_rows
+ assert previous_rows > 0
+
+ job_config = bigquery.LoadJobConfig(
+ write_disposition=bigquery.WriteDisposition.WRITE_TRUNCATE,
+ source_format=bigquery.SourceFormat.CSV,
+ skip_leading_rows=1,
+ )
+
+ uri = "gs://cloud-samples-data/bigquery/us-states/us-states.csv"
+ load_job = client.load_table_from_uri(
+ uri, table_id, job_config=job_config
+ ) # Make an API request.
+
+ load_job.result() # Waits for the job to complete.
+
+ destination_table = client.get_table(table_id)
+ print("Loaded {} rows.".format(destination_table.num_rows))
+ # [END bigquery_load_table_gcs_csv_truncate]
diff --git a/samples/load_table_uri_truncate_json.py b/samples/load_table_uri_truncate_json.py
new file mode 100644
index 000000000..a30fae736
--- /dev/null
+++ b/samples/load_table_uri_truncate_json.py
@@ -0,0 +1,55 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def load_table_uri_truncate_json(table_id):
+
+ # [START bigquery_load_table_gcs_json_truncate]
+ import six
+
+ from google.cloud import bigquery
+
+ # Construct a BigQuery client object.
+ client = bigquery.Client()
+
+ # TODO(developer): Set table_id to the ID of the table to create.
+ # table_id = "your-project.your_dataset.your_table_name
+
+ job_config = bigquery.LoadJobConfig(
+ schema=[
+ bigquery.SchemaField("name", "STRING"),
+ bigquery.SchemaField("post_abbr", "STRING"),
+ ],
+ )
+
+ body = six.BytesIO(b"Washington,WA")
+ client.load_table_from_file(body, table_id, job_config=job_config).result()
+ previous_rows = client.get_table(table_id).num_rows
+ assert previous_rows > 0
+
+ job_config = bigquery.LoadJobConfig(
+ write_disposition=bigquery.WriteDisposition.WRITE_TRUNCATE,
+ source_format=bigquery.SourceFormat.NEWLINE_DELIMITED_JSON,
+ )
+
+ uri = "gs://cloud-samples-data/bigquery/us-states/us-states.json"
+ load_job = client.load_table_from_uri(
+ uri, table_id, job_config=job_config
+ ) # Make an API request.
+
+ load_job.result() # Waits for the job to complete.
+
+ destination_table = client.get_table(table_id)
+ print("Loaded {} rows.".format(destination_table.num_rows))
+ # [END bigquery_load_table_gcs_json_truncate]
diff --git a/samples/load_table_uri_truncate_orc.py b/samples/load_table_uri_truncate_orc.py
new file mode 100644
index 000000000..18f963be2
--- /dev/null
+++ b/samples/load_table_uri_truncate_orc.py
@@ -0,0 +1,55 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def load_table_uri_truncate_orc(table_id):
+
+ # [START bigquery_load_table_gcs_orc_truncate]
+ import six
+
+ from google.cloud import bigquery
+
+ # Construct a BigQuery client object.
+ client = bigquery.Client()
+
+ # TODO(developer): Set table_id to the ID of the table to create.
+ # table_id = "your-project.your_dataset.your_table_name
+
+ job_config = bigquery.LoadJobConfig(
+ schema=[
+ bigquery.SchemaField("name", "STRING"),
+ bigquery.SchemaField("post_abbr", "STRING"),
+ ],
+ )
+
+ body = six.BytesIO(b"Washington,WA")
+ client.load_table_from_file(body, table_id, job_config=job_config).result()
+ previous_rows = client.get_table(table_id).num_rows
+ assert previous_rows > 0
+
+ job_config = bigquery.LoadJobConfig(
+ write_disposition=bigquery.WriteDisposition.WRITE_TRUNCATE,
+ source_format=bigquery.SourceFormat.ORC,
+ )
+
+ uri = "gs://cloud-samples-data/bigquery/us-states/us-states.orc"
+ load_job = client.load_table_from_uri(
+ uri, table_id, job_config=job_config
+ ) # Make an API request.
+
+ load_job.result() # Waits for the job to complete.
+
+ destination_table = client.get_table(table_id)
+ print("Loaded {} rows.".format(destination_table.num_rows))
+ # [END bigquery_load_table_gcs_orc_truncate]
diff --git a/samples/load_table_uri_truncate_parquet.py b/samples/load_table_uri_truncate_parquet.py
new file mode 100644
index 000000000..28692d840
--- /dev/null
+++ b/samples/load_table_uri_truncate_parquet.py
@@ -0,0 +1,55 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def load_table_uri_truncate_parquet(table_id):
+
+ # [START bigquery_load_table_gcs_parquet_truncate]
+ import six
+
+ from google.cloud import bigquery
+
+ # Construct a BigQuery client object.
+ client = bigquery.Client()
+
+ # TODO(developer): Set table_id to the ID of the table to create.
+ # table_id = "your-project.your_dataset.your_table_name
+
+ job_config = bigquery.LoadJobConfig(
+ schema=[
+ bigquery.SchemaField("name", "STRING"),
+ bigquery.SchemaField("post_abbr", "STRING"),
+ ],
+ )
+
+ body = six.BytesIO(b"Washington,WA")
+ client.load_table_from_file(body, table_id, job_config=job_config).result()
+ previous_rows = client.get_table(table_id).num_rows
+ assert previous_rows > 0
+
+ job_config = bigquery.LoadJobConfig(
+ write_disposition=bigquery.WriteDisposition.WRITE_TRUNCATE,
+ source_format=bigquery.SourceFormat.PARQUET,
+ )
+
+ uri = "gs://cloud-samples-data/bigquery/us-states/us-states.parquet"
+ load_job = client.load_table_from_uri(
+ uri, table_id, job_config=job_config
+ ) # Make an API request.
+
+ load_job.result() # Waits for the job to complete.
+
+ destination_table = client.get_table(table_id)
+ print("Loaded {} rows.".format(destination_table.num_rows))
+ # [END bigquery_load_table_gcs_parquet_truncate]
diff --git a/samples/tests/test_load_table_uri_autodetect_csv.py b/samples/tests/test_load_table_uri_autodetect_csv.py
new file mode 100644
index 000000000..a40719783
--- /dev/null
+++ b/samples/tests/test_load_table_uri_autodetect_csv.py
@@ -0,0 +1,22 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .. import load_table_uri_autodetect_csv
+
+
+def test_load_table_uri_autodetect_csv(capsys, random_table_id):
+
+ load_table_uri_autodetect_csv.load_table_uri_autodetect_csv(random_table_id)
+ out, err = capsys.readouterr()
+ assert "Loaded 50 rows." in out
diff --git a/samples/tests/test_load_table_uri_autodetect_json.py b/samples/tests/test_load_table_uri_autodetect_json.py
new file mode 100644
index 000000000..df14d26ed
--- /dev/null
+++ b/samples/tests/test_load_table_uri_autodetect_json.py
@@ -0,0 +1,22 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .. import load_table_uri_autodetect_json
+
+
+def test_load_table_uri_autodetect_csv(capsys, random_table_id):
+
+ load_table_uri_autodetect_json.load_table_uri_autodetect_json(random_table_id)
+ out, err = capsys.readouterr()
+ assert "Loaded 50 rows." in out
diff --git a/samples/tests/test_load_table_uri_truncate_avro.py b/samples/tests/test_load_table_uri_truncate_avro.py
new file mode 100644
index 000000000..ba680cabd
--- /dev/null
+++ b/samples/tests/test_load_table_uri_truncate_avro.py
@@ -0,0 +1,21 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .. import load_table_uri_truncate_avro
+
+
+def test_load_table_uri_truncate_avro(capsys, random_table_id):
+ load_table_uri_truncate_avro.load_table_uri_truncate_avro(random_table_id)
+ out, _ = capsys.readouterr()
+ assert "Loaded 50 rows." in out
diff --git a/samples/tests/test_load_table_uri_truncate_csv.py b/samples/tests/test_load_table_uri_truncate_csv.py
new file mode 100644
index 000000000..5c1da7dce
--- /dev/null
+++ b/samples/tests/test_load_table_uri_truncate_csv.py
@@ -0,0 +1,21 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .. import load_table_uri_truncate_csv
+
+
+def test_load_table_uri_truncate_csv(capsys, random_table_id):
+ load_table_uri_truncate_csv.load_table_uri_truncate_csv(random_table_id)
+ out, _ = capsys.readouterr()
+ assert "Loaded 50 rows." in out
diff --git a/samples/tests/test_load_table_uri_truncate_json.py b/samples/tests/test_load_table_uri_truncate_json.py
new file mode 100644
index 000000000..180ca7f40
--- /dev/null
+++ b/samples/tests/test_load_table_uri_truncate_json.py
@@ -0,0 +1,21 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .. import load_table_uri_truncate_json
+
+
+def test_load_table_uri_truncate_json(capsys, random_table_id):
+ load_table_uri_truncate_json.load_table_uri_truncate_json(random_table_id)
+ out, _ = capsys.readouterr()
+ assert "Loaded 50 rows." in out
diff --git a/samples/tests/test_load_table_uri_truncate_orc.py b/samples/tests/test_load_table_uri_truncate_orc.py
new file mode 100644
index 000000000..322bf3127
--- /dev/null
+++ b/samples/tests/test_load_table_uri_truncate_orc.py
@@ -0,0 +1,21 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .. import load_table_uri_truncate_orc
+
+
+def test_load_table_uri_truncate_orc(capsys, random_table_id):
+ load_table_uri_truncate_orc.load_table_uri_truncate_orc(random_table_id)
+ out, _ = capsys.readouterr()
+ assert "Loaded 50 rows." in out
diff --git a/samples/tests/test_load_table_uri_truncate_parquet.py b/samples/tests/test_load_table_uri_truncate_parquet.py
new file mode 100644
index 000000000..ca901defa
--- /dev/null
+++ b/samples/tests/test_load_table_uri_truncate_parquet.py
@@ -0,0 +1,21 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .. import load_table_uri_truncate_parquet
+
+
+def test_load_table_uri_truncate_parquet(capsys, random_table_id):
+ load_table_uri_truncate_parquet.load_table_uri_truncate_parquet(random_table_id)
+ out, _ = capsys.readouterr()
+ assert "Loaded 50 rows." in out
diff --git a/setup.cfg b/setup.cfg
index 2a9acf13d..3bd555500 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,2 +1,3 @@
+# Generated by synthtool. DO NOT EDIT!
[bdist_wheel]
universal = 1
diff --git a/setup.py b/setup.py
index 378c4fc1b..3ec2ba0bd 100644
--- a/setup.py
+++ b/setup.py
@@ -22,7 +22,7 @@
name = "google-cloud-bigquery"
description = "Google BigQuery API client library"
-version = "1.24.0"
+version = "1.25.0"
# Should be one of:
# 'Development Status :: 3 - Alpha'
# 'Development Status :: 4 - Beta'
@@ -40,9 +40,12 @@
extras = {
"bqstorage": [
"google-cloud-bigquery-storage >= 0.6.0, <2.0.0dev",
- # Bad Linux release for 0.14.0.
- # https://issues.apache.org/jira/browse/ARROW-5868
- "pyarrow>=0.13.0, != 0.14.0",
+ # Due to an issue in pip's dependency resolver, the `grpc` extra is not
+ # installed, even though `google-cloud-bigquery-storage` specifies it
+ # as `google-api-core[grpc]`. We thus need to explicitly specify it here.
+ # See: https://github.com/googleapis/python-bigquery/issues/83
+ "grpcio >= 1.8.2, < 2.0dev",
+ "pyarrow>=0.16.0, < 2.0dev",
],
"pandas": ["pandas>=0.17.1"],
# Exclude PyArrow dependency from Windows Python 2.7.
@@ -52,7 +55,14 @@
"pyarrow>=0.4.1, != 0.14.0"
],
"tqdm": ["tqdm >= 4.0.0, <5.0.0dev"],
- "fastparquet": ["fastparquet", "python-snappy"],
+ "fastparquet": [
+ "fastparquet",
+ "python-snappy",
+ # llvmlite >= 0.32.0 cannot be installed on Python 3.5 and below
+ # (building the wheel fails), thus needs to be restricted.
+ # See: https://github.com/googleapis/python-bigquery/issues/78
+ "llvmlite <= 0.31.0",
+ ],
}
all_extras = []
@@ -95,7 +105,7 @@
author="Google LLC",
author_email="googleapis-packages@google.com",
license="Apache 2.0",
- url="https://github.com/GoogleCloudPlatform/google-cloud-python",
+ url="https://github.com/googleapis/python-bigquery",
classifiers=[
release_status,
"Intended Audience :: Developers",
@@ -107,6 +117,7 @@
"Programming Language :: Python :: 3.5",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
+ "Programming Language :: Python :: 3.8",
"Operating System :: OS Independent",
"Topic :: Internet",
],
diff --git a/synth.metadata b/synth.metadata
index ef9fc79c5..86ecc1ffa 100644
--- a/synth.metadata
+++ b/synth.metadata
@@ -1,5 +1,5 @@
{
- "updateTime": "2020-01-29T13:17:11.693204Z",
+ "updateTime": "2020-02-04T11:46:46.343511Z",
"sources": [
{
"generator": {
@@ -12,9 +12,15 @@
"git": {
"name": "googleapis",
"remote": "https://github.com/googleapis/googleapis.git",
- "sha": "cf3b61102ed5f36b827bc82ec39be09525f018c8",
- "internalRef": "292034635",
- "log": "cf3b61102ed5f36b827bc82ec39be09525f018c8\n Fix to protos for v1p1beta1 release of Cloud Security Command Center\n\nPiperOrigin-RevId: 292034635\n\n4e1cfaa7c0fede9e65d64213ca3da1b1255816c0\nUpdate the public proto to support UTF-8 encoded id for CatalogService API, increase the ListCatalogItems deadline to 300s and some minor documentation change\n\nPiperOrigin-RevId: 292030970\n\n9c483584f8fd5a1b862ae07973f4cc7bb3e46648\nasset: add annotations to v1p1beta1\n\nPiperOrigin-RevId: 292009868\n\ne19209fac29731d0baf6d9ac23da1164f7bdca24\nAdd the google.rpc.context.AttributeContext message to the open source\ndirectories.\n\nPiperOrigin-RevId: 291999930\n\nae5662960573f279502bf98a108a35ba1175e782\noslogin API: move file level option on top of the file to avoid protobuf.js bug.\n\nPiperOrigin-RevId: 291990506\n\neba3897fff7c49ed85d3c47fc96fe96e47f6f684\nAdd cc_proto_library and cc_grpc_library targets for Spanner and IAM protos.\n\nPiperOrigin-RevId: 291988651\n\n8e981acfd9b97ea2f312f11bbaa7b6c16e412dea\nBeta launch for PersonDetection and FaceDetection features.\n\nPiperOrigin-RevId: 291821782\n\n994e067fae3b21e195f7da932b08fff806d70b5d\nasset: add annotations to v1p2beta1\n\nPiperOrigin-RevId: 291815259\n\n244e1d2c89346ca2e0701b39e65552330d68545a\nAdd Playable Locations service\n\nPiperOrigin-RevId: 291806349\n\n909f8f67963daf45dd88d020877fb9029b76788d\nasset: add annotations to v1beta2\n\nPiperOrigin-RevId: 291805301\n\n3c39a1d6e23c1ef63c7fba4019c25e76c40dfe19\nKMS: add file-level message for CryptoKeyPath, it is defined in gapic yaml but not\nin proto files.\n\nPiperOrigin-RevId: 291420695\n\nc6f3f350b8387f8d1b85ed4506f30187ebaaddc3\ncontaineranalysis: update v1beta1 and bazel build with annotations\n\nPiperOrigin-RevId: 291401900\n\n92887d74b44e4e636252b7b8477d0d2570cd82db\nfix: fix the location of grpc config file.\n\nPiperOrigin-RevId: 291396015\n\ne26cab8afd19d396b929039dac5d874cf0b5336c\nexpr: add default_host and method_signature annotations to CelService\n\nPiperOrigin-RevId: 291240093\n\n06093ae3952441c34ec176d1f7431b8765cec0be\nirm: fix v1alpha2 bazel build by adding missing proto imports\n\nPiperOrigin-RevId: 291227940\n\na8a2514af326e4673063f9a3c9d0ef1091c87e6c\nAdd proto annotation for cloud/irm API\n\nPiperOrigin-RevId: 291217859\n\n8d16f76de065f530d395a4c7eabbf766d6a120fd\nGenerate Memcache v1beta2 API protos and gRPC ServiceConfig files\n\nPiperOrigin-RevId: 291008516\n\n3af1dabd93df9a9f17bf3624d3b875c11235360b\ngrafeas: Add containeranalysis default_host to Grafeas service\n\nPiperOrigin-RevId: 290965849\n\nbe2663fa95e31cba67d0cd62611a6674db9f74b7\nfix(google/maps/roads): add missing opening bracket\n\nPiperOrigin-RevId: 290964086\n\nfacc26550a0af0696e0534bc9cae9df14275aa7c\nUpdating v2 protos with the latest inline documentation (in comments) and adding a per-service .yaml file.\n\nPiperOrigin-RevId: 290952261\n\ncda99c1f7dc5e4ca9b1caeae1dc330838cbc1461\nChange api_name to 'asset' for v1p1beta1\n\nPiperOrigin-RevId: 290800639\n\n94e9e90c303a820ce40643d9129e7f0d2054e8a1\nAdds Google Maps Road service\n\nPiperOrigin-RevId: 290795667\n\na3b23dcb2eaecce98c600c7d009451bdec52dbda\nrpc: new message ErrorInfo, other comment updates\n\nPiperOrigin-RevId: 290781668\n\n26420ef4e46c37f193c0fbe53d6ebac481de460e\nAdd proto definition for Org Policy v1.\n\nPiperOrigin-RevId: 290771923\n\n7f0dab8177cf371ae019a082e2512de7ac102888\nPublish Routes Preferred API v1 service definitions.\n\nPiperOrigin-RevId: 290326986\n\nad6e508d0728e1d1bca6e3f328cd562718cb772d\nFix: Qualify resource type references with \"jobs.googleapis.com/\"\n\nPiperOrigin-RevId: 290285762\n\n58e770d568a2b78168ddc19a874178fee8265a9d\ncts client library\n\nPiperOrigin-RevId: 290146169\n\naf9daa4c3b4c4a8b7133b81588dd9ffd37270af2\nAdd more programming language options to public proto\n\nPiperOrigin-RevId: 290144091\n\nd9f2bbf2df301ef84641d4cec7c828736a0bd907\ntalent: add missing resource.proto dep to Bazel build target\n\nPiperOrigin-RevId: 290143164\n\n3b3968237451d027b42471cd28884a5a1faed6c7\nAnnotate Talent API.\nAdd gRPC service config for retry.\nUpdate bazel file with google.api.resource dependency.\n\nPiperOrigin-RevId: 290125172\n\n0735b4b096872960568d1f366bfa75b7b0e1f1a3\nWeekly library update.\n\nPiperOrigin-RevId: 289939042\n\n8760d3d9a4543d7f9c0d1c7870aca08b116e4095\nWeekly library update.\n\nPiperOrigin-RevId: 289939020\n\n8607df842f782a901805187e02fff598145b0b0e\nChange Talent API timeout to 30s.\n\nPiperOrigin-RevId: 289912621\n\n908155991fe32570653bcb72ecfdcfc896642f41\nAdd Recommendations AI V1Beta1\n\nPiperOrigin-RevId: 289901914\n\n5c9a8c2bebd8b71aa66d1cc473edfaac837a2c78\nAdding no-arg method signatures for ListBillingAccounts and ListServices\n\nPiperOrigin-RevId: 289891136\n\n50b0e8286ac988b0593bd890eb31fef6ea2f5767\nlongrunning: add grpc service config and default_host annotation to operations.proto\n\nPiperOrigin-RevId: 289876944\n\n6cac27dabe51c54807b0401698c32d34998948a9\n Updating default deadline for Cloud Security Command Center's v1 APIs.\n\nPiperOrigin-RevId: 289875412\n\nd99df0d67057a233c711187e0689baa4f8e6333d\nFix: Correct spelling in C# namespace option\n\nPiperOrigin-RevId: 289709813\n\n2fa8d48165cc48e35b0c62e6f7bdade12229326c\nfeat: Publish Recommender v1 to GitHub.\n\nCommitter: @lukesneeringer\nPiperOrigin-RevId: 289619243\n\n9118db63d1ab493a2e44a3b4973fde810a835c49\nfirestore: don't retry reads that fail with Aborted\n\nFor transaction reads that fail with ABORTED, we need to rollback and start a new transaction. Our current configuration makes it so that GAPIC retries ABORTED reads multiple times without making any progress. Instead, we should retry at the transaction level.\n\nPiperOrigin-RevId: 289532382\n\n1dbfd3fe4330790b1e99c0bb20beb692f1e20b8a\nFix bazel build\nAdd other langauges (Java was already there) for bigquery/storage/v1alpha2 api.\n\nPiperOrigin-RevId: 289519766\n\nc06599cdd7d11f8d3fd25f8d3249e5bb1a3d5d73\nInitial commit of google.cloud.policytroubleshooter API, The API helps in troubleshooting GCP policies. Refer https://cloud.google.com/iam/docs/troubleshooting-access for more information\n\nPiperOrigin-RevId: 289491444\n\nfce7d80fa16ea241e87f7bc33d68595422e94ecd\nDo not pass samples option for Artman config of recommender v1 API.\n\nPiperOrigin-RevId: 289477403\n\nef179e8c61436297e6bb124352e47e45c8c80cb1\nfix: Address missing Bazel dependency.\n\nBazel builds stopped working in 06ec6d5 because\nthe google/longrunning/operations.proto file took\nan import from google/api/client.proto, but that\nimport was not added to BUILD.bazel.\n\nCommitter: @lukesneeringer\nPiperOrigin-RevId: 289446074\n\n8841655b242c84fd691d77d7bcf21b61044f01ff\nMigrate Data Labeling v1beta1 to GAPIC v2.\n\nCommitter: @lukesneeringer\nPiperOrigin-RevId: 289446026\n\n06ec6d5d053fff299eaa6eaa38afdd36c5e2fc68\nAdd annotations to google.longrunning.v1\n\nCommitter: @lukesneeringer\nPiperOrigin-RevId: 289413169\n\n0480cf40be1d3cc231f4268a2fdb36a8dd60e641\nMigrate IAM Admin v1 to GAPIC v2.\n\nCommitter: @lukesneeringer\nPiperOrigin-RevId: 289411084\n\n1017173e9adeb858587639af61889ad970c728b1\nSpecify a C# namespace for BigQuery Connection v1beta1\n\nPiperOrigin-RevId: 289396763\n\nb08714b378e8e5b0c4ecdde73f92c36d6303b4b6\nfix: Integrate latest proto-docs-plugin fix.\nFixes dialogflow v2\n\nPiperOrigin-RevId: 289189004\n\n51217a67e79255ee1f2e70a6a3919df082513327\nCreate BUILD file for recommender v1\n\nPiperOrigin-RevId: 289183234\n\nacacd87263c0a60e458561b8b8ce9f67c760552a\nGenerate recommender v1 API protos and gRPC ServiceConfig files\n\nPiperOrigin-RevId: 289177510\n\n9d2f7133b97720b1fa3601f6dcd30760ba6d8a1e\nFix kokoro build script\n\nPiperOrigin-RevId: 289166315\n\nc43a67530d2a47a0220cad20ca8de39b3fbaf2c5\ncloudtasks: replace missing RPC timeout config for v2beta2 and v2beta3\n\nPiperOrigin-RevId: 289162391\n\n4cefc229a9197236fc0adf02d69b71c0c5cf59de\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 289158456\n\n56f263fe959c50786dab42e3c61402d32d1417bd\nCatalog API: Adding config necessary to build client libraries\n\nPiperOrigin-RevId: 289149879\n\n4543762b23a57fc3c53d409efc3a9affd47b6ab3\nFix Bazel build\nbilling/v1 and dialogflow/v2 remain broken (not bazel-related issues).\nBilling has wrong configuration, dialogflow failure is caused by a bug in documentation plugin.\n\nPiperOrigin-RevId: 289140194\n\nc9dce519127b97e866ca133a01157f4ce27dcceb\nUpdate Bigtable docs\n\nPiperOrigin-RevId: 289114419\n\n802c5c5f2bf94c3facb011267d04e71942e0d09f\nMigrate DLP to proto annotations (but not GAPIC v2).\n\nCommitter: @lukesneeringer\nPiperOrigin-RevId: 289102579\n\n6357f30f2ec3cff1d8239d18b707ff9d438ea5da\nRemove gRPC configuration file that was in the wrong place.\n\nPiperOrigin-RevId: 289096111\n\n360a8792ed62f944109d7e22d613a04a010665b4\n Protos for v1p1beta1 release of Cloud Security Command Center\n\nPiperOrigin-RevId: 289011995\n\na79211c20c4f2807eec524d00123bf7c06ad3d6e\nRoll back containeranalysis v1 to GAPIC v1.\n\nCommitter: @lukesneeringer\nPiperOrigin-RevId: 288999068\n\n9e60345ba603e03484a8aaa33ce5ffa19c1c652b\nPublish Routes Preferred API v1 proto definitions.\n\nPiperOrigin-RevId: 288941399\n\nd52885b642ad2aa1f42b132ee62dbf49a73e1e24\nMigrate the service management API to GAPIC v2.\n\nCommitter: @lukesneeringer\nPiperOrigin-RevId: 288909426\n\n6ace586805c08896fef43e28a261337fcf3f022b\ncloudtasks: replace missing RPC timeout config\n\nPiperOrigin-RevId: 288783603\n\n51d906cabee4876b12497054b15b05d4a50ad027\nImport of Grafeas from Github.\n\nUpdate BUILD.bazel accordingly.\n\nPiperOrigin-RevId: 288783426\n\n5ef42bcd363ba0440f0ee65b3c80b499e9067ede\nMigrate Recommender v1beta1 to GAPIC v2.\n\nCommitter: @lukesneeringer\nPiperOrigin-RevId: 288713066\n\n94f986afd365b7d7e132315ddcd43d7af0e652fb\nMigrate Container Analysis v1 to GAPIC v2.\n\nCommitter: @lukesneeringer\nPiperOrigin-RevId: 288708382\n\n7a751a279184970d3b6ba90e4dd4d22a382a0747\nRemove Container Analysis v1alpha1 (nobody publishes it).\n\nCommitter: @lukesneeringer\nPiperOrigin-RevId: 288707473\n\n3c0d9c71242e70474b2b640e15bb0a435fd06ff0\nRemove specious annotation from BigQuery Data Transfer before\nanyone accidentally does anything that uses it.\n\nCommitter: @lukesneeringer\nPiperOrigin-RevId: 288701604\n\n1af307a4764bd415ef942ac5187fa1def043006f\nMigrate BigQuery Connection to GAPIC v2.\n\nCommitter: @lukesneeringer\nPiperOrigin-RevId: 288698681\n\n08b488e0660c59842a7dee0e3e2b65d9e3a514a9\nExposing cloud_catalog.proto (This API is already available through REST)\n\nPiperOrigin-RevId: 288625007\n\na613482977e11ac09fa47687a5d1b5a01efcf794\nUpdate the OS Login v1beta API description to render better in the UI.\n\nPiperOrigin-RevId: 288547940\n\n5e182b8d9943f1b17008d69d4c7e865dc83641a7\nUpdate the OS Login API description to render better in the UI.\n\nPiperOrigin-RevId: 288546443\n\ncb79155f596e0396dd900da93872be7066f6340d\nFix: Add a resource annotation for Agent\nFix: Correct the service name in annotations for Intent and SessionEntityType\n\nPiperOrigin-RevId: 288441307\n\nf7f6e9daec3315fd47cb638789bd8415bf4a27cc\nAdded cloud asset api v1p1beta1\n\nPiperOrigin-RevId: 288427239\n\nf2880f5b342c6345f3dcaad24fcb3c6ca9483654\nBilling account API: Adding config necessary to build client libraries\n\nPiperOrigin-RevId: 288351810\n\ndc250ffe071729f8f8bef9d6fd0fbbeb0254c666\nFix: Remove incorrect resource annotations in requests\n\nPiperOrigin-RevId: 288321208\n\n91ef2d9dd69807b0b79555f22566fb2d81e49ff9\nAdd GAPIC annotations to Cloud KMS (but do not migrate the GAPIC config yet).\n\nCommitter: @lukesneeringer\nPiperOrigin-RevId: 287999179\n\n4d45a6399e9444fbddaeb1c86aabfde210723714\nRefreshing Cloud Billing API protos.\n\nThis exposes the following API methods:\n- UpdateBillingAccount\n- CreateBillingAccount\n- GetIamPolicy\n- SetIamPolicy\n- TestIamPermissions\n\nThere are also some new fields to support the management of sub-accounts.\n\nPiperOrigin-RevId: 287908369\n\nec285d3d230810147ebbf8d5b691ee90320c6d2d\nHide not yet implemented update_transforms message\n\nPiperOrigin-RevId: 287608953\n\na202fb3b91cd0e4231be878b0348afd17067cbe2\nBigQuery Storage Write API v1alpha2 clients. The service is enabled by whitelist only.\n\nPiperOrigin-RevId: 287379998\n\n650d7f1f8adb0cfaf37b3ce2241c3168f24efd4d\nUpdate Readme.md to match latest Bazel updates\n090d98aea20270e3be4b64240775588f7ce50ff8\ndocs(bigtable): Fix library release level listed in generated documentation\n\nPiperOrigin-RevId: 287308849\n\n2c28f646ca77b1d57550368be22aa388adde2e66\nfirestore: retry reads that fail with contention\n\nPiperOrigin-RevId: 287250665\n\nfd3091fbe9b2083cabc53dc50c78035658bfc4eb\nSync timeout in grpc config back to 10s for tasks API with github googelapis gapic config.\n\nPiperOrigin-RevId: 287207067\n\n49dd7d856a6f77c0cf7e5cb3334423e5089a9e8a\nbazel: Integrate bazel-2.0.0 compatibility fixes\n\nPiperOrigin-RevId: 287205644\n\n46e52fd64973e815cae61e78b14608fe7aa7b1df\nbazel: Integrate bazel build file generator\n\nTo generate/update BUILD.bazel files for any particular client or a batch of clients:\n```\nbazel run //:build_gen -- --src=google/example/library\n```\n\nPiperOrigin-RevId: 286958627\n\n1a380ea21dea9b6ac6ad28c60ad96d9d73574e19\nBigQuery Storage Read API v1beta2 clients.\n\nPiperOrigin-RevId: 286616241\n\n5f3f1d0f1c06b6475a17d995e4f7a436ca67ec9e\nAdd Artman config for secretmanager.\n\nPiperOrigin-RevId: 286598440\n\n50af0530730348f1e3697bf3c70261f7daaf2981\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 286491002\n\n91818800384f4ed26961aea268910b1a2ec58cc8\nFor Data Catalog API,\n1. Add support for marking a tag template field as required when creating a new tag template.\n2. Add support for updating a tag template field from required to optional.\n\nPiperOrigin-RevId: 286490262\n\nff4a2047b3d66f38c9b22197c370ed0d02fc0238\nWeekly library update.\n\nPiperOrigin-RevId: 286484215\n\n192c14029861752a911ed434fd6ee5b850517cd9\nWeekly library update.\n\nPiperOrigin-RevId: 286484165\n\nd9e328eaf790d4e4346fbbf32858160f497a03e0\nFix bazel build (versions 1.x)\n\nBump gapic-generator and resource names plugins to the latest version.\n\nPiperOrigin-RevId: 286469287\n\n0ca305403dcc50e31ad9477c9b6241ddfd2056af\nsecretmanager client package name option updates for java and go\n\nPiperOrigin-RevId: 286439553\n\nade4803e8a1a9e3efd249c8c86895d2f12eb2aaa\niam credentials: publish v1 protos containing annotations\n\nPiperOrigin-RevId: 286418383\n\n03e5708e5f8d1909dcb74b25520309e59ebf24be\nsecuritycenter: add missing proto deps for Bazel build\n\nPiperOrigin-RevId: 286417075\n\n8b991eb3eb82483b0ca1f1361a9c8e5b375c4747\nAdd secretmanager client package name options.\n\nPiperOrigin-RevId: 286415883\n\nd400cb8d45df5b2ae796b909f098a215b2275c1d\ndialogflow: add operation_info annotations to BatchUpdateEntities and BatchDeleteEntities.\n\nPiperOrigin-RevId: 286312673\n\nf2b25232db397ebd4f67eb901a2a4bc99f7cc4c6\nIncreased the default timeout time for all the Cloud Security Command Center client libraries.\n\nPiperOrigin-RevId: 286263771\n\ncb2f1eefd684c7efd56fd375cde8d4084a20439e\nExposing new Resource fields in the SecurityCenterProperties proto, added more comments to the filter logic for these Resource fields, and updated the response proto for the ListFindings API with the new Resource fields.\n\nPiperOrigin-RevId: 286263092\n\n73cebb20432b387c3d8879bb161b517d60cf2552\nUpdate v1beta2 clusters and jobs to include resource ids in GRPC header.\n\nPiperOrigin-RevId: 286261392\n\n1b4e453d51c0bd77e7b73896cdd8357d62768d83\nsecuritycenter: publish v1beta1 protos with annotations\n\nPiperOrigin-RevId: 286228860\n\na985eeda90ae98e8519d2320bee4dec148eb8ccb\nAdd default retry configurations for speech_v1p1beta1.\n\nSettings are copied from speech_gapic.legacy.yaml. The Python client library is being generated with timeouts that are too low. See https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2578\n\nPiperOrigin-RevId: 286191318\n\n3352100a15ede383f5ab3c34599f7a10a3d066fe\nMake importing rule with the same name (but different aliases) from different repositories possible.\n\nThis is needed to allow monolitic gapic-generator and microgenerators coexist during transition period.\n\nTo plug a microgenerator:\n\n1) Add corresponding rules bidnings under `switched_rules_by_language` in repository_rules.bzl:\n rules[\"go_gapic_library2\"] = _switch(\n go and grpc and gapic,\n \"@gapic_generator_go//rules_go_gapic/go_gapic.bzl\",\n \"go_gapic_library\",\n )\n\n2) Import microgenerator in WORKSPACE (the above example assumes that the generator was imported under name \"gapic_generator_go\").\n\n3) To migrate an API from monolith to micro generator (this is done per API and per language) modify the corresponding load statement in the API's BUILD.bazel file. For example, for the example above, to migrate to go microgenerator modify the go-specific load statement in BUILD.bazel file of a specific API (which you want to migrate) to the following:\n\nload(\n \"@com_google_googleapis_imports//:imports.bzl\",\n \"go_gapic_assembly_pkg\",\n go_gapic_library = \"go_gapic_library2\",\n \"go_proto_library\",\n \"go_test\",\n)\n\nPiperOrigin-RevId: 286065440\n\n6ad2bb13bc4b0f3f785517f0563118f6ca52ddfd\nUpdated v1beta1 protos for the client:\n- added support for GenericSignedAttestation which has a generic Signature\n- added support for CVSSv3 and WindowsDetail in Vulnerability\n- documentation updates\n\nPiperOrigin-RevId: 286008145\n\nfe1962e49999a832eed8162c45f23096336a9ced\nAdMob API v1 20191210\n\nBasic account info, mediation and network report available. See https://developers.google.com/admob/api/release-notes for more details.\n\nPiperOrigin-RevId: 285894502\n\n41fc1403738b61427f3a798ca9750ef47eb9c0f2\nAnnotate the required fields for the Monitoring Dashboards API\n\nPiperOrigin-RevId: 285824386\n\n27d0e0f202cbe91bf155fcf36824a87a5764ef1e\nRemove inappropriate resource_reference annotations for UpdateWorkflowTemplateRequest.template.\n\nPiperOrigin-RevId: 285802643\n\ne5c4d3a2b5b5bef0a30df39ebb27711dc98dee64\nAdd Artman BUILD.bazel file for the Monitoring Dashboards API\n\nPiperOrigin-RevId: 285445602\n\n2085a0d3c76180ee843cf2ecef2b94ca5266be31\nFix path in the artman config for Monitoring Dashboard API.\n\nPiperOrigin-RevId: 285233245\n\n2da72dfe71e4cca80902f9e3e125c40f02c2925b\nAdd Artman and GAPIC configs for the Monitoring Dashboards API.\n\nPiperOrigin-RevId: 285211544\n\n9f6eeebf1f30f51ffa02acea5a71680fe592348e\nAdd annotations to Dataproc v1. (Also forwarding comment changes from internal source control.)\n\nCommitter: @lukesneeringer\nPiperOrigin-RevId: 285197557\n\n19c4589a3cb44b3679f7b3fba88365b3d055d5f8\noslogin: fix v1beta retry configuration\n\nPiperOrigin-RevId: 285013366\n\nee3f02926d0f8a0bc13f8d716581aad20f575751\nAdd Monitoring Dashboards API protocol buffers to Google Cloud Monitoring API.\n\nPiperOrigin-RevId: 284982647\n\ne47fdd266542386e5e7346697f90476e96dc7ee8\nbigquery datatransfer: Remove non-publicly available DataSourceService.\n\nPiperOrigin-RevId: 284822593\n\n6156f433fd1d9d5e4a448d6c6da7f637921d92ea\nAdds OSConfig v1beta protos and initial client library config\n\nPiperOrigin-RevId: 284799663\n\n6cc9499e225a4f6a5e34fe07e390f67055d7991c\nAdd datetime.proto to google/type/BUILD.bazel\n\nPiperOrigin-RevId: 284643689\n\nfe7dd5277e39ffe0075729c61e8d118d7527946d\nCosmetic changes to proto comment as part of testing internal release instructions.\n\nPiperOrigin-RevId: 284608712\n\n68d109adad726b89f74276d2f4b2ba6aac6ec04a\nAdd annotations to securitycenter v1, but leave GAPIC v1 in place.\n\nCommitter: @lukesneeringer\nPiperOrigin-RevId: 284580511\n\ndf8a1707a910fc17c71407a75547992fd1864c51\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 284568564\n\na69a974976221ce3bb944901b739418b85d6408c\nclient library update\n\nPiperOrigin-RevId: 284463979\n\na4adac3a12aca6e3a792c9c35ee850435fe7cf7e\nAdded DateTime, TimeZone, and Month proto files to google/type\n\nPiperOrigin-RevId: 284277770\n\ned5dec392906078db4f7745fe4f11d34dd401ae9\nchange common resources from message-level annotations to file-level annotations.\n\nPiperOrigin-RevId: 284236794\n\na00e2c575ef1b637667b4ebe96b8c228b2ddb273\nbigquerydatatransfer: change resource type TransferRun to Run to be consistent with gapic configs\nbigquerydatatransfer: add missing patterns for DataSource, TransferConfig and Run (to allow the location segment)\nbigquerydatatransfer: add file-level Parent resource type (to allow the location segement)\nbigquerydatatransfer: update grpc service config with correct retry delays\n\nPiperOrigin-RevId: 284234378\n\nb10e4547017ca529ac8d183e839f3c272e1c13de\ncloud asset: replace required fields for batchgetassethistory. Correct the time out duration.\n\nPiperOrigin-RevId: 284059574\n\n6690161e3dcc3367639a2ec10db67bf1cf392550\nAdd default retry configurations for speech_v1.\n\nSettings are copied from speech_gapic.legacy.yaml. The Python client library is being generated with timeouts that are too low. See https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2578\n\nPiperOrigin-RevId: 284035915\n\n9b2635ef91e114f0357bdb87652c26a8f59316d5\ncloudtasks: fix gapic v2 config\n\nPiperOrigin-RevId: 284020555\n\ne5676ba8b863951a8ed0bfd6046e1db38062743c\nReinstate resource name handling in GAPIC config for Asset v1.\n\nPiperOrigin-RevId: 283993903\n\nf337f7fb702c85833b7b6ca56afaf9a1bf32c096\nOSConfig AgentEndpoint: add LookupEffectiveGuestPolicy rpc\n\nPiperOrigin-RevId: 283989762\n\nc0ac9b55f2e2efd0ee525b3a6591a1b09330e55a\nInclude real time feed api into v1 version\n\nPiperOrigin-RevId: 283845474\n\n2427a3a0f6f4222315362d973d91a082a3a884a7\nfirestore admin: update v1 protos with annotations & retry config\n\nPiperOrigin-RevId: 283826605\n\n555e844dbe04af50a8f55fe1217fa9d39a0a80b2\nchore: publish retry configs for iam admin, cloud asset, and remoteworkers\n\nPiperOrigin-RevId: 283801979\n\n6311dc536668849142d1fe5cd9fc46da66d1f77f\nfirestore: update v1beta1 protos with annotations and retry config\n\nPiperOrigin-RevId: 283794315\n\nda0edeeef953b05eb1524d514d2e9842ac2df0fd\nfeat: publish several retry config files for client generation\n\nPiperOrigin-RevId: 283614497\n\n59a78053537e06190f02d0a7ffb792c34e185c5a\nRemoving TODO comment\n\nPiperOrigin-RevId: 283592535\n\n8463992271d162e2aff1d5da5b78db11f2fb5632\nFix bazel build\n\nPiperOrigin-RevId: 283589351\n\n3bfcb3d8df10dfdba58f864d3bdb8ccd69364669\nPublic client library for bebop_jobs_api_20191118_1_RC3 release.\n\nPiperOrigin-RevId: 283568877\n\n27ab0db61021d267c452b34d149161a7bf0d9f57\nfirestore: publish annotated protos and new retry config\n\nPiperOrigin-RevId: 283565148\n\n38dc36a2a43cbab4a2a9183a43dd0441670098a9\nfeat: add http annotations for operations calls\n\nPiperOrigin-RevId: 283384331\n\n366caab94906975af0e17822e372f1d34e319d51\ndatastore: add a legacy artman config for PHP generation\n\nPiperOrigin-RevId: 283378578\n\n82944da21578a53b74e547774cf62ed31a05b841\nMigrate container v1beta1 to GAPIC v2.\n\nPiperOrigin-RevId: 283342796\n\n584dcde5826dd11ebe222016b7b208a4e1196f4b\nRemove resource name annotation for UpdateKeyRequest.key, because it's the resource, not a name.\n\nPiperOrigin-RevId: 283167368\n\n6ab0171e3688bfdcf3dbc4056e2df6345e843565\nAdded resource annotation for Key message.\n\nPiperOrigin-RevId: 283066965\n\n86c1a2db1707a25cec7d92f8850cc915163ec3c3\nExpose Admin API methods for Key manipulation.\n\nPiperOrigin-RevId: 282988776\n\n3ddad085965896ffb205d44cb0c0616fe3def10b\nC++ targets: correct deps so they build, rename them from trace* to cloudtrace*\nto match the proto names.\n\nPiperOrigin-RevId: 282857635\n\ne9389365a971ad6457ceb9646c595e79dfdbdea5\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 282810797\n\ne42eaaa9abed3c4d63d64f790bd3191448dbbca6\nPut back C++ targets for cloud trace v2 api.\n\nPiperOrigin-RevId: 282803841\n\nd8896a3d8a191702a9e39f29cf4c2e16fa05f76d\nAdd initial BUILD.bazel for secretmanager.googleapis.com\n\nPiperOrigin-RevId: 282674885\n\n2cc56cb83ea3e59a6364e0392c29c9e23ad12c3a\nCreate sample for list recommendations\n\nPiperOrigin-RevId: 282665402\n\nf88e2ca65790e3b44bb3455e4779b41de1bf7136\nbump Go to ga\n\nPiperOrigin-RevId: 282651105\n\naac86d932b3cefd7d746f19def6935d16d6235e0\nDocumentation update. Add location_id in preparation for regionalization.\n\nPiperOrigin-RevId: 282586371\n\n5b501cd384f6b842486bd41acce77854876158e7\nMigrate Datastore Admin to GAPIC v2.\n\nCommitter: @lukesneeringer\nPiperOrigin-RevId: 282570874\n\n6a16d474d5be201b20a27646e2009c4dfde30452\nMigrate Datastore to GAPIC v2.\n\nCommitter: @lukesneeringer\nPiperOrigin-RevId: 282564329\n\n74bd9b95ac8c70b883814e4765a725cffe43d77c\nmark Go lib ga\n\nPiperOrigin-RevId: 282562558\n\nf7b3d434f44f6a77cf6c37cae5474048a0639298\nAdd secretmanager.googleapis.com protos\n\nPiperOrigin-RevId: 282546399\n\nc34a911aaa0660a45f5a556578f764f135e6e060\niot: bump Go GAPIC to GA release level\n\nPiperOrigin-RevId: 282494787\n\n79b7f1c5ba86859dbf70aa6cd546057c1002cdc0\nPut back C++ targets.\nPrevious change overrode custom C++ targets made by external teams. This PR puts those targets back.\n\nPiperOrigin-RevId: 282458292\n\n06a840781d2dc1b0a28e03e30fb4b1bfb0b29d1e\nPopulate BAZEL.build files for around 100 APIs (all APIs we publish) in all 7 langauges.\n\nPiperOrigin-RevId: 282449910\n\n777b580a046c4fa84a35e1d00658b71964120bb0\nCreate BUILD file for recommender v1beta1\n\nPiperOrigin-RevId: 282068850\n\n48b385b6ef71dfe2596490ea34c9a9a434e74243\nGenerate recommender v1beta1 gRPC ServiceConfig file\n\nPiperOrigin-RevId: 282067795\n\n8395b0f1435a4d7ce8737b3b55392627758bd20c\nfix: Set timeout to 25s, because Tasks fails for any deadline above 30s.\n\nCommitter: @lukesneeringer\nPiperOrigin-RevId: 282017295\n\n3ba7ddc4b2acf532bdfb0004ca26311053c11c30\nfix: Shift Ruby and PHP to legacy GAPIC YAMLs for back-compat.\n\nCommitter: @lukesneeringer\nPiperOrigin-RevId: 281852671\n\nad6f0c002194c3ec6c13d592d911d122d2293931\nRemove unneeded yaml files\n\nPiperOrigin-RevId: 281835839\n\n1f42588e4373750588152cdf6f747de1cadbcbef\nrefactor: Migrate Tasks beta 2 to GAPIC v2.\n\nCommitter: @lukesneeringer\nPiperOrigin-RevId: 281769558\n\n902b51f2073e9958a2aba441f7f7ac54ea00966d\nrefactor: Migrate Tasks to GAPIC v2 (for real this time).\n\nCommitter: @lukesneeringer\nPiperOrigin-RevId: 281769522\n\n17561f59970eede87f61ef6e9c322fa1198a2f4d\nMigrate Tasks Beta 3 to GAPIC v2.\n\nCommitter: @lukesneeringer\nPiperOrigin-RevId: 281769519\n\nf95883b15a1ddd58eb7e3583fdefe7b00505faa3\nRegenerate recommender v1beta1 protos and sanitized yaml\n\nPiperOrigin-RevId: 281765245\n\n9a52df54c626b36699a058013d1735a166933167\nadd gRPC ServiceConfig for grafeas v1\n\nPiperOrigin-RevId: 281762754\n\n7a79d682ef40c5ca39c3fca1c0901a8e90021f8a\nfix: Roll back Tasks GAPIC v2 while we investigate C# issue.\n\nCommitter: @lukesneeringer\nPiperOrigin-RevId: 281758548\n\n3fc31491640a90f029f284289e7e97f78f442233\nMigrate Tasks to GAPIC v2.\n\nCommitter: @lukesneeringer\nPiperOrigin-RevId: 281751187\n\n5bc0fecee454f857cec042fb99fe2d22e1bff5bc\nfix: adds operation HTTP rules back to v1p1beta1 config\n\nPiperOrigin-RevId: 281635572\n\n5364a19284a1333b3ffe84e4e78a1919363d9f9c\nbazel: Fix build\n\n1) Update to latest gapic-generator (has iam resource names fix for java).\n2) Fix non-trivial issues with oslogin (resources defined in sibling package to the one they are used from) and monitoring.\n3) Fix trivial missing dependencies in proto_library targets for other apis.\n\nThis is to prepare the repository to being populated with BUILD.bazel files for all supported apis (101 API) in all 7 languages.\n\nPiperOrigin-RevId: 281618750\n\n0aa77cbe45538d5e5739eb637db3f2940b912789\nUpdating common proto files in google/type/ with their latest versions.\n\nPiperOrigin-RevId: 281603926\n\nd47e1b4485b3effbb2298eb10dd13a544c0f66dc\nfix: replace Speech Recognize RPC retry_codes_name for non-standard assignment\n\nPiperOrigin-RevId: 281594037\n\n16543773103e2619d2b5f52456264de5bb9be104\nRegenerating public protos for datacatalog, also adding gRPC service config.\n\nPiperOrigin-RevId: 281423227\n\n328ebe76adb06128d12547ed70107fb841aebf4e\nChange custom data type from String to google.protobuf.Struct to be consistent with other docs such as\nhttps://developers.google.com/actions/smarthome/develop/process-intents#response_format\n\nPiperOrigin-RevId: 281402467\n\n5af83f47b9656261cafcf88b0b3334521ab266b3\n(internal change without visible public changes)\n\nPiperOrigin-RevId: 281334391\n\nc53ed56649583a149382bd88d3c427be475b91b6\nFix typo in protobuf docs.\n\nPiperOrigin-RevId: 281293109\n\nd8dd7fe8d5304f7bd1c52207703d7f27d5328c5a\nFix build by adding missing deps.\n\nPiperOrigin-RevId: 281088257\n\n3ef5ffd7351809d75c1332d2eaad1f24d9c318e4\nMigrate Error Reporting v1beta1 to proto annotations / GAPIC v2.\n\nCommitter: @lukesneeringer\nPiperOrigin-RevId: 281075722\n\n418ee8e24a56b5959e1c1defa4b6c97f883be379\nTrace v2: Add remaining proto annotations, migrate to GAPIC v2.\n\nCommitter: @lukesneeringer\nPiperOrigin-RevId: 281068859\n\nc89394342a9ef70acaf73a6959e04b943fbc817b\nThis change updates an outdated comment for the feature importance proto field since they are no longer in [0, 1] for online predictions.\n\nPiperOrigin-RevId: 280761373\n\n1ec8b8e2c3c8f41d7d2b22c594c025276d6a4ae6\nCode refactoring\n\nPiperOrigin-RevId: 280760149\n\n427a22b04039f93b769d89accd6f487413f667c1\nImport automl operation protos.\n\nPiperOrigin-RevId: 280703572\n\n45749a04dac104e986f6cc47da3baf7c8bb6f9b0\nfix: bigqueryconnection_gapic.yaml to reflect proto annotations\n\n* remove connection_credential resource\n* make CreateCredentialRequest.connection_id optional\n* shuffle field ordering in CreateCredential flattening\n\nPiperOrigin-RevId: 280685438\n\n8385366aa1e5d7796793db02a9c5e167d1fd8f17\nRevert the Trace v2 GAPIC for now.\nCommitter: @lukesneeringer\n\nPiperOrigin-RevId: 280669295\n\n5c8ab2c072d557c2f4c4e54b544394e2d62202d5\nMigrate Trace v1 and Trace v2 to GAPIC v2.\n\nCommitter: @lukesneeringer\nPiperOrigin-RevId: 280667429\n\nf6808ff4e8b966cd571e99279d4a2780ed97dff2\nRename the `endpoint_urls` field to `endpoint_uris` to be consistent with\nGoogle API nomenclature.\n\nPiperOrigin-RevId: 280581337\n\n1935fb8889686f5c9d107f11b3c6870fc3aa7cdc\nComment updates\n\nPiperOrigin-RevId: 280451656\n\n0797fd5b9029d630e68a0899734715d62ad38e33\nComment updates\n\nPiperOrigin-RevId: 280451600\n\n9bc8d07b8b749e791d16c8d559526928ceaf1994\nRollback of \"Migrate Cloud Error Reporting to proto annotations & GAPIC v2.\"\n\nPiperOrigin-RevId: 280445975\n\nf8720321aecf4aab42e03602ac2c67f9777d9170\nfix: bigtable retry config in GAPIC v2\n\nPiperOrigin-RevId: 280434856\n\nb11664ba64f92d96d748e0dd9724d006dcafd120\nMigrate Cloud Error Reporting to proto annotations & GAPIC v2.\n\nPiperOrigin-RevId: 280432937\n\n4f747bda9b099b4426f495985680d16d0227fa5f\n1. Change DataCatalog package name in java from com.google.cloud.datacatalog to com.google.cloud.datacatalog.v1beta1 (API version is included in the package). *This is a breaking change.*\n\n2. Add API for Taxonomies (PolicyTagManager and PolicyTagManagerSerialization services).\n\n3. Minor changes to documentation.\n\nPiperOrigin-RevId: 280394936\n\nbc76ffd87360ce1cd34e3a6eac28afd5e1efda76\nUse rules_proto bzl files to load proto_library\n\nThis makes googleapis forward compatible with Bazel incompatible change https://github.com/bazelbuild/bazel/issues/8922.\n\nThis CL was created by adding @rules_proto to the WORKSPACE file and then running:\n\nfind . -name BUILD.bazel | \\\n while read build; do \\\n buildifier --lint=fix --warnings=load $build; \\\n done\n\nSince buildifier cannot be told not to reformat the BUILD file, some files are reformatted.\n\nPiperOrigin-RevId: 280356106\n\n218164b3deba1075979c9dca5f71461379e42dd1\nMake the `permissions` argument in TestIamPermissions required.\n\nPiperOrigin-RevId: 280279014\n\ndec8fd8ea5dc464496606189ba4b8949188639c8\nUpdating Cloud Billing Budget API documentation for clarity.\n\nPiperOrigin-RevId: 280225437\n\na667ffab90deb5e2669eb40ec7b61ec96a3d0454\nIntroduced detailed status message for CreateTimeSeries: CreateTimeSeriesSummary replaces CreateTimeSeriesError, which is now deprecated and unused.\n\nPiperOrigin-RevId: 280221707\n\nbe0a25eceec8916633447a37af0ecea801b85186\nMigrate Bigtable API to GAPIC v2 config.\n\nPiperOrigin-RevId: 280199643\n\n88bbf96b90089994ed16208a0f38cdd07f743742\nFix location of monitoring.yaml in Artman config for monitoring v3.\n\nPiperOrigin-RevId: 280134477\n\ndbaa01a20303758eed0c5a95ad2239ea306ad9a5\nUpdate namespace for PHP.\n\nPiperOrigin-RevId: 280085199\n\nf73b3796a635b2026a590d5133af7fa1f0eb807b\nStandardize pub/sub client default settings across clients:\n- Add retry codes for streaming pull\n- Decrease publish's max_rpc_timeout (mini-timeout) from 10 mins to 1 min\n- Decrease publish's total timeout from 10 mins to 1 min\n- Increase publish batching threshold from 10 to 100 elements\n- Increase publish batching size threshold from 1 KiB to 1 MiB\n\nPiperOrigin-RevId: 280044012\n\n822172613e1d93bede3beaf78b123c42a5876e2b\nReplace local_repository with http_archive in WORKSPACE\n\nPiperOrigin-RevId: 280039052\n\n6a8c7914d1b79bd832b5157a09a9332e8cbd16d4\nAdded notification_supported_by_agent to indicate whether the agent is sending notifications to Google or not.\n\nPiperOrigin-RevId: 279991530\n\n675de3dc9ab98cc1cf54216ad58c933ede54e915\nAdd an endpoint_urls field to the instance admin proto and adds a field_mask field to the GetInstanceRequest.\n\nPiperOrigin-RevId: 279982263\n\nf69562be0608904932bdcfbc5ad8b9a22d9dceb8\nAdds some clarification to IAM Policy public proto comments about the policy versioning compliance check for etag-less SetIamPolicy requests.\n\nPiperOrigin-RevId: 279774957\n\n4e86b2538758e3155e867d1cb4155ee91de7c6e9\nDocumentation update. Add the new action for sending metrics to Stackdriver.\n\nPiperOrigin-RevId: 279768476\n\neafaf30b7a3af0bc72f323fe6a6827327d3cad75\nfix: Restore deleted field to avoid a breaking change.\n\nPiperOrigin-RevId: 279760458\n\ned13a73f3054a29b764f104feaa503820b75140a\nAdd GAPIC annotations to the GKE API.\n\nPiperOrigin-RevId: 279734275\n\n6b125955bf0d6377b96f205e5d187e9d524b7ea2\nUpdate timeouts to 1 hour for default and streaming RPCs.\n\nPiperOrigin-RevId: 279657866\n\n989b304c8a6cfe72bdd7cb264e0d71b784db9421\nAdd Service Monitoring (Service and ServiceLevelObjective) protocol buffers to Google Cloud Monitoring API.\n\nPiperOrigin-RevId: 279649144\n\n1ef3bed9594674bb571ce20418af307505e3f609\nUpdating configs for AgentEndpoint to fix the client library generation.\n\nPiperOrigin-RevId: 279518887\n\n34e661f58d58fa57da8ed113a3d8bb3de26b307d\nUpdate v1beta2 clusters and jobs to include resource ids in GRPC header.\n\nPiperOrigin-RevId: 279417429\n\n248abde06efb7e5a3d81b84de02c8272122b0c3b\nIntegrate GAPIC Python Bazel Extensions\n\nAlso configure python build for the following clients as an example:\n\ndiaglogflow/v2\nlanguage/v1\ntexttospeech/v1\nfirestore/v1beta1\npubsub/v1\n\nPiperOrigin-RevId: 279406526\n\n7ffbf721e29b8806e0c8947c5dd0cdddc02de72a\nOSConfig Agentendpoint: Rename ReportTaskStart to StartNextTask\n\nPiperOrigin-RevId: 279389774\n\n2642d8688bab8981c8a5153b7578f9ff8460a37c\nAgentendpoint API: minor doc updates, addition of exclusive_packages|patches to PatchConfigs.\n\nPiperOrigin-RevId: 279326626\n\nd323b287c782802242005072d15f1474d7d10819\nDocumentation changes.\n\nPiperOrigin-RevId: 279234903\n\n29927f71d92d59551a42272ab7c6e97e8413af78\nPublishing Billing Budgets v1alpha1 API.\n\nPiperOrigin-RevId: 279176561\n\nff413d36f8358818d76fa92006f2d8f608843093\nAdding gRPC service config for Billing Budgets API.\n\nPiperOrigin-RevId: 279175129\n\n3eb91187709cc96bb890c110f518505f65ffd95d\nagentendpoint: removes all gapic languages except Go from artman config\n\nPiperOrigin-RevId: 279173968\n\na34950f968c7944a1036551b545557edcc18c767\nFix bazel build.\n\nUpdate gapic-generator and protoc-java-resource-name plugin dependencies to the latest versions.\n\nThe following clients remain broken because of bugs in gapic-generator and/or corresponding configs\n\ngoogle/cloud/iot/v1\ngoogle/cloud/oslogin/v1\ngoogle/spanner/admin/instance/v1\ngoogle/cloud/oslogin/v1\n\nPiperOrigin-RevId: 279171061\n\n0ed34e9fdf601dfc37eb24c40e17495b86771ff4\nAdds agentendpoint protos and initial client library config\n\nPiperOrigin-RevId: 279147036\n\ncad1d3b365a90c2a9f014b84a2a1acb55c15480f\nUpdates to MediaCard\n\nPiperOrigin-RevId: 279100776\n\n05556c26b633c153f2eca62aeafbcd62705f41b7\nUpdates to MediaCard\n\nPiperOrigin-RevId: 279100278\n\n2275670a746ab2bc03ebba0d914b45320ea15af4\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 278922329\n\n5691fcb7c1a926b52577aa1834f31d9c50efda54\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 278731899\n\ncb542d6f5f1c9431ec4181d9cfd7f8d8c953e60b\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 278688708\n\n311e73f017a474c9a41f2a41b00d5d704ff191c5\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 278658917\n\n521ce65c04266df83dde9e2cfd8b2caf057cab45\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 278656745\n\nf06bab1c11b7a6dcd15c50525da44c4b2ff3ef3d\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 278627678\n\n8c6569ced063c08a48272de2e887860d0c40d388\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 278552094\n\n21262f41c4445d24bf441e2a5c250a4207348008\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 278486499\n\ndf366ed5ee26ebb73511127b4c329a98ecdd1f7b\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 278469200\n\n58bc0f51b1270975b532f5847d9e9e0ff5cdc592\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 278368388\n\ne0935db8bfe6fd901ee5d2104b0e1865682899f7\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 278368327\n\naf4a739e9d810eb033903f1aa44c615ab729760d\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 278132545\n\naac770126e2def40dcc387f50e8007b21c869e58\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 278016738\n\n271fed175d16501fb988e02b891166e9718ff141\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 277992079\n\n597951d86beb120bc18428f70ffe0d5b97c70620\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 277991975\n\nbba93d7148ff203d400a4929cd0fbc7dafd8dae2\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 277920288\n\n5b86376273637f5ce3844f29bf8cb1c4aceaea2d\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 277850256\n\n8bc65fb6973a281e8fb9e5c12080644a550322c9\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 277813826\n\n30a6ca0f1a98f1777c94fc22094c892c2a43e0ef\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 277811318\n\n6bef7bd6184390a4e7aa8f09382d7d97afeccfc4\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 277789040\n\naa33c92d79760f2a03ba9b42f855f7a821ed9147\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 277759754\n\na4933867265e2b1cbc70f876a4312a92116c36ad\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 277759298\n\nb21f96290006525e039b9bd1acddeeae407ae1ff\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 277750396\n\n93661a24048eb64755fbbeedd7f6a207d1b4d8dc\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 277748718\n\nc0e494ca955a4fdd9ad460a5890a354ec3a3a0ff\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 277673798\n\n4e952e7e2bb0dd2ef389d552d48f44c8dc4b5f8f\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 277595731\n\n78883c8de959f7a9870c332ab0e3d788b13dd763\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 277528057\n\n7c4cf35d5fe3b8ad664bd219edd6d9f28a788b64\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 277334937\n\nf28342c58c1df57c92e967961e1eaa641d447dde\nSynchronize new proto/yaml changes.\n\nPiperOrigin-RevId: 277311984\n\n"
+ "sha": "69d9945330a5721cd679f17331a78850e2618226",
+ "internalRef": "293080182"
+ }
+ },
+ {
+ "template": {
+ "name": "python_split_library",
+ "origin": "synthtool.gcp",
+ "version": "2019.10.17"
}
}
],
@@ -29,742 +35,5 @@
"config": "google/cloud/bigquery/artman_bigquery_v2.yaml"
}
}
- ],
- "newFiles": [
- {
- "path": ".coveragerc"
- },
- {
- "path": ".flake8"
- },
- {
- "path": ".gitignore"
- },
- {
- "path": ".repo-metadata.json"
- },
- {
- "path": "CHANGELOG.md"
- },
- {
- "path": "LICENSE"
- },
- {
- "path": "MANIFEST.in"
- },
- {
- "path": "README.rst"
- },
- {
- "path": "benchmark/README.md"
- },
- {
- "path": "benchmark/benchmark.py"
- },
- {
- "path": "benchmark/queries.json"
- },
- {
- "path": "docs/.gitignore"
- },
- {
- "path": "docs/README.rst"
- },
- {
- "path": "docs/_static/custom.css"
- },
- {
- "path": "docs/_templates/layout.html"
- },
- {
- "path": "docs/changelog.md"
- },
- {
- "path": "docs/conf.py"
- },
- {
- "path": "docs/dbapi.rst"
- },
- {
- "path": "docs/gapic/v2/enums.rst"
- },
- {
- "path": "docs/gapic/v2/types.rst"
- },
- {
- "path": "docs/generated/google.cloud.bigquery.magics.html"
- },
- {
- "path": "docs/index.rst"
- },
- {
- "path": "docs/magics.rst"
- },
- {
- "path": "docs/reference.rst"
- },
- {
- "path": "docs/snippets.py"
- },
- {
- "path": "docs/usage.html"
- },
- {
- "path": "docs/usage/client.rst"
- },
- {
- "path": "docs/usage/datasets.rst"
- },
- {
- "path": "docs/usage/encryption.rst"
- },
- {
- "path": "docs/usage/index.rst"
- },
- {
- "path": "docs/usage/jobs.rst"
- },
- {
- "path": "docs/usage/pandas.rst"
- },
- {
- "path": "docs/usage/queries.rst"
- },
- {
- "path": "docs/usage/tables.rst"
- },
- {
- "path": "google/__init__.py"
- },
- {
- "path": "google/cloud/__init__.py"
- },
- {
- "path": "google/cloud/bigquery/__init__.py"
- },
- {
- "path": "google/cloud/bigquery/_helpers.py"
- },
- {
- "path": "google/cloud/bigquery/_http.py"
- },
- {
- "path": "google/cloud/bigquery/_pandas_helpers.py"
- },
- {
- "path": "google/cloud/bigquery/client.py"
- },
- {
- "path": "google/cloud/bigquery/dataset.py"
- },
- {
- "path": "google/cloud/bigquery/dbapi/__init__.py"
- },
- {
- "path": "google/cloud/bigquery/dbapi/_helpers.py"
- },
- {
- "path": "google/cloud/bigquery/dbapi/connection.py"
- },
- {
- "path": "google/cloud/bigquery/dbapi/cursor.py"
- },
- {
- "path": "google/cloud/bigquery/dbapi/exceptions.py"
- },
- {
- "path": "google/cloud/bigquery/dbapi/types.py"
- },
- {
- "path": "google/cloud/bigquery/encryption_configuration.py"
- },
- {
- "path": "google/cloud/bigquery/enums.py"
- },
- {
- "path": "google/cloud/bigquery/external_config.py"
- },
- {
- "path": "google/cloud/bigquery/job.py"
- },
- {
- "path": "google/cloud/bigquery/magics.py"
- },
- {
- "path": "google/cloud/bigquery/model.py"
- },
- {
- "path": "google/cloud/bigquery/query.py"
- },
- {
- "path": "google/cloud/bigquery/retry.py"
- },
- {
- "path": "google/cloud/bigquery/routine.py"
- },
- {
- "path": "google/cloud/bigquery/schema.py"
- },
- {
- "path": "google/cloud/bigquery/table.py"
- },
- {
- "path": "google/cloud/bigquery_v2/__init__.py"
- },
- {
- "path": "google/cloud/bigquery_v2/gapic/__init__.py"
- },
- {
- "path": "google/cloud/bigquery_v2/gapic/enums.py"
- },
- {
- "path": "google/cloud/bigquery_v2/proto/__init__.py"
- },
- {
- "path": "google/cloud/bigquery_v2/proto/encryption_config.proto"
- },
- {
- "path": "google/cloud/bigquery_v2/proto/encryption_config_pb2.py"
- },
- {
- "path": "google/cloud/bigquery_v2/proto/encryption_config_pb2_grpc.py"
- },
- {
- "path": "google/cloud/bigquery_v2/proto/location_metadata.proto"
- },
- {
- "path": "google/cloud/bigquery_v2/proto/location_metadata_pb2.py"
- },
- {
- "path": "google/cloud/bigquery_v2/proto/location_metadata_pb2_grpc.py"
- },
- {
- "path": "google/cloud/bigquery_v2/proto/model.proto"
- },
- {
- "path": "google/cloud/bigquery_v2/proto/model_pb2.py"
- },
- {
- "path": "google/cloud/bigquery_v2/proto/model_pb2_grpc.py"
- },
- {
- "path": "google/cloud/bigquery_v2/proto/model_reference.proto"
- },
- {
- "path": "google/cloud/bigquery_v2/proto/model_reference_pb2.py"
- },
- {
- "path": "google/cloud/bigquery_v2/proto/model_reference_pb2_grpc.py"
- },
- {
- "path": "google/cloud/bigquery_v2/proto/standard_sql.proto"
- },
- {
- "path": "google/cloud/bigquery_v2/proto/standard_sql_pb2.py"
- },
- {
- "path": "google/cloud/bigquery_v2/proto/standard_sql_pb2_grpc.py"
- },
- {
- "path": "google/cloud/bigquery_v2/types.py"
- },
- {
- "path": "noxfile.py"
- },
- {
- "path": "pylint.config.py"
- },
- {
- "path": "samples/__init__.py"
- },
- {
- "path": "samples/add_empty_column.py"
- },
- {
- "path": "samples/browse_table_data.py"
- },
- {
- "path": "samples/client_list_jobs.py"
- },
- {
- "path": "samples/client_load_partitioned_table.py"
- },
- {
- "path": "samples/client_query.py"
- },
- {
- "path": "samples/client_query_add_column.py"
- },
- {
- "path": "samples/client_query_batch.py"
- },
- {
- "path": "samples/client_query_destination_table.py"
- },
- {
- "path": "samples/client_query_destination_table_cmek.py"
- },
- {
- "path": "samples/client_query_destination_table_legacy.py"
- },
- {
- "path": "samples/client_query_dry_run.py"
- },
- {
- "path": "samples/client_query_legacy_sql.py"
- },
- {
- "path": "samples/client_query_relax_column.py"
- },
- {
- "path": "samples/client_query_w_array_params.py"
- },
- {
- "path": "samples/client_query_w_named_params.py"
- },
- {
- "path": "samples/client_query_w_positional_params.py"
- },
- {
- "path": "samples/client_query_w_struct_params.py"
- },
- {
- "path": "samples/client_query_w_timestamp_params.py"
- },
- {
- "path": "samples/copy_table.py"
- },
- {
- "path": "samples/copy_table_cmek.py"
- },
- {
- "path": "samples/copy_table_multiple_source.py"
- },
- {
- "path": "samples/create_dataset.py"
- },
- {
- "path": "samples/create_job.py"
- },
- {
- "path": "samples/create_routine.py"
- },
- {
- "path": "samples/create_routine_ddl.py"
- },
- {
- "path": "samples/create_table.py"
- },
- {
- "path": "samples/create_table_range_partitioned.py"
- },
- {
- "path": "samples/dataset_exists.py"
- },
- {
- "path": "samples/delete_dataset.py"
- },
- {
- "path": "samples/delete_dataset_labels.py"
- },
- {
- "path": "samples/delete_model.py"
- },
- {
- "path": "samples/delete_routine.py"
- },
- {
- "path": "samples/delete_table.py"
- },
- {
- "path": "samples/download_public_data.py"
- },
- {
- "path": "samples/download_public_data_sandbox.py"
- },
- {
- "path": "samples/get_dataset.py"
- },
- {
- "path": "samples/get_dataset_labels.py"
- },
- {
- "path": "samples/get_model.py"
- },
- {
- "path": "samples/get_routine.py"
- },
- {
- "path": "samples/get_table.py"
- },
- {
- "path": "samples/label_dataset.py"
- },
- {
- "path": "samples/list_datasets.py"
- },
- {
- "path": "samples/list_datasets_by_label.py"
- },
- {
- "path": "samples/list_models.py"
- },
- {
- "path": "samples/list_routines.py"
- },
- {
- "path": "samples/list_tables.py"
- },
- {
- "path": "samples/load_table_dataframe.py"
- },
- {
- "path": "samples/query_external_gcs_temporary_table.py"
- },
- {
- "path": "samples/query_external_sheets_permanent_table.py"
- },
- {
- "path": "samples/query_external_sheets_temporary_table.py"
- },
- {
- "path": "samples/query_no_cache.py"
- },
- {
- "path": "samples/query_pagination.py"
- },
- {
- "path": "samples/query_script.py"
- },
- {
- "path": "samples/query_to_arrow.py"
- },
- {
- "path": "samples/table_exists.py"
- },
- {
- "path": "samples/table_insert_rows.py"
- },
- {
- "path": "samples/table_insert_rows_explicit_none_insert_ids.py"
- },
- {
- "path": "samples/tests/__init__.py"
- },
- {
- "path": "samples/tests/conftest.py"
- },
- {
- "path": "samples/tests/test_add_empty_column.py"
- },
- {
- "path": "samples/tests/test_browse_table_data.py"
- },
- {
- "path": "samples/tests/test_client_list_jobs.py"
- },
- {
- "path": "samples/tests/test_client_load_partitioned_table.py"
- },
- {
- "path": "samples/tests/test_client_query.py"
- },
- {
- "path": "samples/tests/test_client_query_add_column.py"
- },
- {
- "path": "samples/tests/test_client_query_batch.py"
- },
- {
- "path": "samples/tests/test_client_query_destination_table.py"
- },
- {
- "path": "samples/tests/test_client_query_destination_table_cmek.py"
- },
- {
- "path": "samples/tests/test_client_query_destination_table_legacy.py"
- },
- {
- "path": "samples/tests/test_client_query_dry_run.py"
- },
- {
- "path": "samples/tests/test_client_query_legacy_sql.py"
- },
- {
- "path": "samples/tests/test_client_query_relax_column.py"
- },
- {
- "path": "samples/tests/test_client_query_w_array_params.py"
- },
- {
- "path": "samples/tests/test_client_query_w_named_params.py"
- },
- {
- "path": "samples/tests/test_client_query_w_positional_params.py"
- },
- {
- "path": "samples/tests/test_client_query_w_struct_params.py"
- },
- {
- "path": "samples/tests/test_client_query_w_timestamp_params.py"
- },
- {
- "path": "samples/tests/test_copy_table.py"
- },
- {
- "path": "samples/tests/test_copy_table_cmek.py"
- },
- {
- "path": "samples/tests/test_copy_table_multiple_source.py"
- },
- {
- "path": "samples/tests/test_create_dataset.py"
- },
- {
- "path": "samples/tests/test_create_job.py"
- },
- {
- "path": "samples/tests/test_create_table.py"
- },
- {
- "path": "samples/tests/test_create_table_range_partitioned.py"
- },
- {
- "path": "samples/tests/test_dataset_exists.py"
- },
- {
- "path": "samples/tests/test_dataset_label_samples.py"
- },
- {
- "path": "samples/tests/test_delete_dataset.py"
- },
- {
- "path": "samples/tests/test_delete_table.py"
- },
- {
- "path": "samples/tests/test_download_public_data.py"
- },
- {
- "path": "samples/tests/test_download_public_data_sandbox.py"
- },
- {
- "path": "samples/tests/test_get_dataset.py"
- },
- {
- "path": "samples/tests/test_get_table.py"
- },
- {
- "path": "samples/tests/test_list_datasets.py"
- },
- {
- "path": "samples/tests/test_list_datasets_by_label.py"
- },
- {
- "path": "samples/tests/test_list_tables.py"
- },
- {
- "path": "samples/tests/test_load_table_dataframe.py"
- },
- {
- "path": "samples/tests/test_model_samples.py"
- },
- {
- "path": "samples/tests/test_query_external_gcs_temporary_table.py"
- },
- {
- "path": "samples/tests/test_query_external_sheets_permanent_table.py"
- },
- {
- "path": "samples/tests/test_query_external_sheets_temporary_table.py"
- },
- {
- "path": "samples/tests/test_query_no_cache.py"
- },
- {
- "path": "samples/tests/test_query_pagination.py"
- },
- {
- "path": "samples/tests/test_query_script.py"
- },
- {
- "path": "samples/tests/test_query_to_arrow.py"
- },
- {
- "path": "samples/tests/test_routine_samples.py"
- },
- {
- "path": "samples/tests/test_table_exists.py"
- },
- {
- "path": "samples/tests/test_table_insert_rows.py"
- },
- {
- "path": "samples/tests/test_table_insert_rows_explicit_none_insert_ids.py"
- },
- {
- "path": "samples/tests/test_undelete_table.py"
- },
- {
- "path": "samples/tests/test_update_dataset_access.py"
- },
- {
- "path": "samples/tests/test_update_dataset_default_partition_expiration.py"
- },
- {
- "path": "samples/tests/test_update_dataset_default_table_expiration.py"
- },
- {
- "path": "samples/tests/test_update_dataset_description.py"
- },
- {
- "path": "samples/tests/test_update_table_require_partition_filter.py"
- },
- {
- "path": "samples/undelete_table.py"
- },
- {
- "path": "samples/update_dataset_access.py"
- },
- {
- "path": "samples/update_dataset_default_partition_expiration.py"
- },
- {
- "path": "samples/update_dataset_default_table_expiration.py"
- },
- {
- "path": "samples/update_dataset_description.py"
- },
- {
- "path": "samples/update_model.py"
- },
- {
- "path": "samples/update_routine.py"
- },
- {
- "path": "samples/update_table_require_partition_filter.py"
- },
- {
- "path": "setup.cfg"
- },
- {
- "path": "setup.py"
- },
- {
- "path": "synth.metadata"
- },
- {
- "path": "synth.py"
- },
- {
- "path": "tests/__init__.py"
- },
- {
- "path": "tests/data/characters.json"
- },
- {
- "path": "tests/data/characters.jsonl"
- },
- {
- "path": "tests/data/colors.avro"
- },
- {
- "path": "tests/data/people.csv"
- },
- {
- "path": "tests/data/schema.json"
- },
- {
- "path": "tests/scrub_datasets.py"
- },
- {
- "path": "tests/system.py"
- },
- {
- "path": "tests/unit/__init__.py"
- },
- {
- "path": "tests/unit/enums/__init__.py"
- },
- {
- "path": "tests/unit/enums/test_standard_sql_data_types.py"
- },
- {
- "path": "tests/unit/helpers.py"
- },
- {
- "path": "tests/unit/model/__init__.py"
- },
- {
- "path": "tests/unit/model/test_model.py"
- },
- {
- "path": "tests/unit/model/test_model_reference.py"
- },
- {
- "path": "tests/unit/routine/__init__.py"
- },
- {
- "path": "tests/unit/routine/test_routine.py"
- },
- {
- "path": "tests/unit/routine/test_routine_argument.py"
- },
- {
- "path": "tests/unit/routine/test_routine_reference.py"
- },
- {
- "path": "tests/unit/test__helpers.py"
- },
- {
- "path": "tests/unit/test__http.py"
- },
- {
- "path": "tests/unit/test__pandas_helpers.py"
- },
- {
- "path": "tests/unit/test_client.py"
- },
- {
- "path": "tests/unit/test_dataset.py"
- },
- {
- "path": "tests/unit/test_dbapi__helpers.py"
- },
- {
- "path": "tests/unit/test_dbapi_connection.py"
- },
- {
- "path": "tests/unit/test_dbapi_cursor.py"
- },
- {
- "path": "tests/unit/test_dbapi_types.py"
- },
- {
- "path": "tests/unit/test_encryption_configuration.py"
- },
- {
- "path": "tests/unit/test_external_config.py"
- },
- {
- "path": "tests/unit/test_job.py"
- },
- {
- "path": "tests/unit/test_magics.py"
- },
- {
- "path": "tests/unit/test_query.py"
- },
- {
- "path": "tests/unit/test_retry.py"
- },
- {
- "path": "tests/unit/test_schema.py"
- },
- {
- "path": "tests/unit/test_signature_compatibility.py"
- },
- {
- "path": "tests/unit/test_table.py"
- }
]
}
\ No newline at end of file
diff --git a/synth.py b/synth.py
index a20426d39..d26c61489 100644
--- a/synth.py
+++ b/synth.py
@@ -17,16 +17,14 @@
import synthtool as s
from synthtool import gcp
-gapic = gcp.GAPICGenerator()
-
+gapic = gcp.GAPICBazel()
+common = gcp.CommonTemplates()
version = 'v2'
library = gapic.py_library(
- 'bigquery',
- version,
- config_path='/google/cloud/bigquery/'
- 'artman_bigquery_v2.yaml',
- artman_output_name='bigquery-v2',
+ service='bigquery',
+ version=version,
+ bazel_target=f"//google/cloud/bigquery/{version}:bigquery-{version}-py",
include_protos=True,
)
@@ -57,4 +55,10 @@
# Format quoted strings as plain text.
s.replace("google/cloud/bigquery_v2/proto/*.py", "[“”]", '``')
+# ----------------------------------------------------------------------------
+# Add templated files
+# ----------------------------------------------------------------------------
+templated_files = common.py_library(cov_level=100)
+s.move(templated_files, excludes=["noxfile.py"])
+
s.shell.run(["nox", "-s", "blacken"], hide_output=False)
diff --git a/tests/system.py b/tests/system.py
index 4a1c03271..66d7ee259 100644
--- a/tests/system.py
+++ b/tests/system.py
@@ -31,11 +31,18 @@
import psutil
import pytest
import pytz
+import pkg_resources
try:
from google.cloud import bigquery_storage_v1beta1
except ImportError: # pragma: NO COVER
bigquery_storage_v1beta1 = None
+
+try:
+ import fastavro # to parse BQ storage client results
+except ImportError: # pragma: NO COVER
+ fastavro = None
+
try:
import pandas
except ImportError: # pragma: NO COVER
@@ -119,6 +126,9 @@
(TooManyRequests, InternalServerError, ServiceUnavailable)
)
+PANDAS_MINIMUM_VERSION = pkg_resources.parse_version("1.0.0")
+PANDAS_INSTALLED_VERSION = pkg_resources.get_distribution("pandas").parsed_version
+
def _has_rows(result):
return len(result) > 0
@@ -237,12 +247,14 @@ def test_create_dataset(self):
def test_get_dataset(self):
dataset_id = _make_dataset_id("get_dataset")
client = Config.CLIENT
- dataset_arg = Dataset(client.dataset(dataset_id))
+ project = client.project
+ dataset_ref = bigquery.DatasetReference(project, dataset_id)
+ dataset_arg = Dataset(dataset_ref)
dataset_arg.friendly_name = "Friendly"
dataset_arg.description = "Description"
dataset = retry_403(client.create_dataset)(dataset_arg)
self.to_delete.append(dataset)
- dataset_ref = client.dataset(dataset_id)
+ dataset_ref = bigquery.DatasetReference(project, dataset_id)
# Get with a reference.
got = client.get_dataset(dataset_ref)
@@ -329,6 +341,57 @@ def test_create_table(self):
self.assertTrue(_table_exists(table))
self.assertEqual(table.table_id, table_id)
+ def test_create_table_with_policy(self):
+ from google.cloud.bigquery.schema import PolicyTagList
+
+ dataset = self.temp_dataset(_make_dataset_id("create_table_with_policy"))
+ table_id = "test_table"
+ policy_1 = PolicyTagList(
+ names=[
+ "projects/{}/locations/us/taxonomies/1/policyTags/2".format(
+ Config.CLIENT.project
+ ),
+ ]
+ )
+ policy_2 = PolicyTagList(
+ names=[
+ "projects/{}/locations/us/taxonomies/3/policyTags/4".format(
+ Config.CLIENT.project
+ ),
+ ]
+ )
+
+ schema = [
+ bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"),
+ bigquery.SchemaField(
+ "secret_int", "INTEGER", mode="REQUIRED", policy_tags=policy_1
+ ),
+ ]
+ table_arg = Table(dataset.table(table_id), schema=schema)
+ self.assertFalse(_table_exists(table_arg))
+
+ table = retry_403(Config.CLIENT.create_table)(table_arg)
+ self.to_delete.insert(0, table)
+
+ self.assertTrue(_table_exists(table))
+ self.assertEqual(policy_1, table.schema[1].policy_tags)
+
+ # Amend the schema to replace the policy tags
+ new_schema = table.schema[:]
+ old_field = table.schema[1]
+ new_schema[1] = bigquery.SchemaField(
+ name=old_field.name,
+ field_type=old_field.field_type,
+ mode=old_field.mode,
+ description=old_field.description,
+ fields=old_field.fields,
+ policy_tags=policy_2,
+ )
+
+ table.schema = new_schema
+ table2 = Config.CLIENT.update_table(table, ["schema"])
+ self.assertEqual(policy_2, table2.schema[1].policy_tags)
+
def test_create_table_w_time_partitioning_w_clustering_fields(self):
from google.cloud.bigquery.table import TimePartitioning
from google.cloud.bigquery.table import TimePartitioningType
@@ -355,7 +418,8 @@ def test_create_table_w_time_partitioning_w_clustering_fields(self):
def test_delete_dataset_with_string(self):
dataset_id = _make_dataset_id("delete_table_true")
- dataset_ref = Config.CLIENT.dataset(dataset_id)
+ project = Config.CLIENT.project
+ dataset_ref = bigquery.DatasetReference(project, dataset_id)
retry_403(Config.CLIENT.create_dataset)(Dataset(dataset_ref))
self.assertTrue(_dataset_exists(dataset_ref))
Config.CLIENT.delete_dataset(dataset_id)
@@ -363,9 +427,9 @@ def test_delete_dataset_with_string(self):
def test_delete_dataset_delete_contents_true(self):
dataset_id = _make_dataset_id("delete_table_true")
- dataset = retry_403(Config.CLIENT.create_dataset)(
- Dataset(Config.CLIENT.dataset(dataset_id))
- )
+ project = Config.CLIENT.project
+ dataset_ref = bigquery.DatasetReference(project, dataset_id)
+ dataset = retry_403(Config.CLIENT.create_dataset)(Dataset(dataset_ref))
table_id = "test_table"
table_arg = Table(dataset.table(table_id), schema=SCHEMA)
@@ -736,6 +800,66 @@ def test_load_table_from_dataframe_w_automatic_schema(self):
)
self.assertEqual(table.num_rows, 3)
+ @unittest.skipIf(
+ pandas is None or PANDAS_INSTALLED_VERSION < PANDAS_MINIMUM_VERSION,
+ "Only `pandas version >=1.0.0` is supported",
+ )
+ @unittest.skipIf(pyarrow is None, "Requires `pyarrow`")
+ def test_load_table_from_dataframe_w_nullable_int64_datatype(self):
+ """Test that a DataFrame containing column with None-type values and int64 datatype
+ can be uploaded if a BigQuery schema is specified.
+
+ https://github.com/googleapis/python-bigquery/issues/22
+ """
+
+ dataset_id = _make_dataset_id("bq_load_test")
+ self.temp_dataset(dataset_id)
+ table_id = "{}.{}.load_table_from_dataframe_w_nullable_int64_datatype".format(
+ Config.CLIENT.project, dataset_id
+ )
+ table_schema = (bigquery.SchemaField("x", "INTEGER", mode="NULLABLE"),)
+ table = retry_403(Config.CLIENT.create_table)(
+ Table(table_id, schema=table_schema)
+ )
+ self.to_delete.insert(0, table)
+
+ df_data = collections.OrderedDict(
+ [("x", pandas.Series([1, 2, None, 4], dtype="Int64"))]
+ )
+ dataframe = pandas.DataFrame(df_data, columns=df_data.keys())
+ load_job = Config.CLIENT.load_table_from_dataframe(dataframe, table_id)
+ load_job.result()
+ table = Config.CLIENT.get_table(table_id)
+ self.assertEqual(tuple(table.schema), (bigquery.SchemaField("x", "INTEGER"),))
+ self.assertEqual(table.num_rows, 4)
+
+ @unittest.skipIf(
+ pandas is None or PANDAS_INSTALLED_VERSION < PANDAS_MINIMUM_VERSION,
+ "Only `pandas version >=1.0.0` is supported",
+ )
+ @unittest.skipIf(pyarrow is None, "Requires `pyarrow`")
+ def test_load_table_from_dataframe_w_nullable_int64_datatype_automatic_schema(self):
+ """Test that a DataFrame containing column with None-type values and int64 datatype
+ can be uploaded without specifying a schema.
+
+ https://github.com/googleapis/python-bigquery/issues/22
+ """
+
+ dataset_id = _make_dataset_id("bq_load_test")
+ self.temp_dataset(dataset_id)
+ table_id = "{}.{}.load_table_from_dataframe_w_nullable_int64_datatype".format(
+ Config.CLIENT.project, dataset_id
+ )
+ df_data = collections.OrderedDict(
+ [("x", pandas.Series([1, 2, None, 4], dtype="Int64"))]
+ )
+ dataframe = pandas.DataFrame(df_data, columns=df_data.keys())
+ load_job = Config.CLIENT.load_table_from_dataframe(dataframe, table_id)
+ load_job.result()
+ table = Config.CLIENT.get_table(table_id)
+ self.assertEqual(tuple(table.schema), (bigquery.SchemaField("x", "INTEGER"),))
+ self.assertEqual(table.num_rows, 4)
+
@unittest.skipIf(pandas is None, "Requires `pandas`")
@unittest.skipIf(pyarrow is None, "Requires `pyarrow`")
def test_load_table_from_dataframe_w_nulls(self):
@@ -852,11 +976,16 @@ def test_load_table_from_dataframe_w_required(self):
@unittest.skipIf(pyarrow is None, "Requires `pyarrow`")
def test_load_table_from_dataframe_w_explicit_schema(self):
# Schema with all scalar types.
+ # TODO: Uploading DATETIME columns currently fails, thus that field type
+ # is temporarily removed from the test.
+ # See:
+ # https://github.com/googleapis/python-bigquery/issues/61
+ # https://issuetracker.google.com/issues/151765076
scalars_schema = (
bigquery.SchemaField("bool_col", "BOOLEAN"),
bigquery.SchemaField("bytes_col", "BYTES"),
bigquery.SchemaField("date_col", "DATE"),
- bigquery.SchemaField("dt_col", "DATETIME"),
+ # bigquery.SchemaField("dt_col", "DATETIME"),
bigquery.SchemaField("float_col", "FLOAT"),
bigquery.SchemaField("geo_col", "GEOGRAPHY"),
bigquery.SchemaField("int_col", "INTEGER"),
@@ -882,14 +1011,14 @@ def test_load_table_from_dataframe_w_explicit_schema(self):
"date_col",
[datetime.date(1, 1, 1), None, datetime.date(9999, 12, 31)],
),
- (
- "dt_col",
- [
- datetime.datetime(1, 1, 1, 0, 0, 0),
- None,
- datetime.datetime(9999, 12, 31, 23, 59, 59, 999999),
- ],
- ),
+ # (
+ # "dt_col",
+ # [
+ # datetime.datetime(1, 1, 1, 0, 0, 0),
+ # None,
+ # datetime.datetime(9999, 12, 31, 23, 59, 59, 999999),
+ # ],
+ # ),
("float_col", [float("-inf"), float("nan"), float("inf")]),
(
"geo_col",
@@ -1237,7 +1366,9 @@ def test_extract_table(self):
source_blob_name = "person_ages.csv"
dataset_id = _make_dataset_id("load_gcs_then_extract")
table_id = "test_table"
- table_ref = Config.CLIENT.dataset(dataset_id).table(table_id)
+ project = Config.CLIENT.project
+ dataset_ref = bigquery.DatasetReference(project, dataset_id)
+ table_ref = dataset_ref.table(table_id)
table = Table(table_ref)
self.to_delete.insert(0, table)
bucket = self._create_bucket(bucket_name)
@@ -1420,8 +1551,10 @@ def test_query_w_wrong_config(self):
rows = list(Config.CLIENT.query("SELECT 1;").result())
assert rows[0][0] == 1
+ project = Config.CLIENT.project
+ dataset_ref = bigquery.DatasetReference(project, "dset")
bad_config = LoadJobConfig()
- bad_config.destination = Config.CLIENT.dataset("dset").table("tbl")
+ bad_config.destination = dataset_ref.table("tbl")
with self.assertRaises(Exception):
Config.CLIENT.query(good_query, job_config=bad_config).result()
@@ -1444,6 +1577,18 @@ def test_query_w_page_size(self):
iterator = query_job.result(page_size=page_size)
self.assertEqual(next(iterator.pages).num_items, page_size)
+ def test_query_w_start_index(self):
+ start_index = 164652
+ query_job = Config.CLIENT.query(
+ "SELECT word FROM `bigquery-public-data.samples.shakespeare`;",
+ job_id_prefix="test_query_w_start_index_",
+ )
+ result1 = query_job.result(start_index=start_index)
+ total_rows = result1.total_rows
+
+ self.assertEqual(result1.extra_params["startIndex"], start_index)
+ self.assertEqual(len(list(result1)), total_rows - start_index)
+
def test_query_statistics(self):
"""
A system test to exercise some of the extended query statistics.
@@ -1543,6 +1688,100 @@ def test_dbapi_fetchall(self):
row_tuples = [r.values() for r in rows]
self.assertEqual(row_tuples, [(1, 2), (3, 4), (5, 6)])
+ @unittest.skipIf(
+ bigquery_storage_v1beta1 is None, "Requires `google-cloud-bigquery-storage`"
+ )
+ def test_dbapi_fetch_w_bqstorage_client_small_result_set(self):
+ bqstorage_client = bigquery_storage_v1beta1.BigQueryStorageClient(
+ credentials=Config.CLIENT._credentials
+ )
+ cursor = dbapi.connect(Config.CLIENT, bqstorage_client).cursor()
+
+ # Reading small result sets causes an issue with BQ storage client,
+ # and the DB API should transparently fall back to the default client.
+ cursor.execute(
+ """
+ SELECT id, `by`, time_ts
+ FROM `bigquery-public-data.hacker_news.comments`
+ ORDER BY `id` ASC
+ LIMIT 10
+ """
+ )
+
+ result_rows = [cursor.fetchone(), cursor.fetchone(), cursor.fetchone()]
+
+ field_name = operator.itemgetter(0)
+ fetched_data = [sorted(row.items(), key=field_name) for row in result_rows]
+
+ expected_data = [
+ [
+ ("by", "sama"),
+ ("id", 15),
+ ("time_ts", datetime.datetime(2006, 10, 9, 19, 51, 1, tzinfo=UTC)),
+ ],
+ [
+ ("by", "pg"),
+ ("id", 17),
+ ("time_ts", datetime.datetime(2006, 10, 9, 19, 52, 45, tzinfo=UTC)),
+ ],
+ [
+ ("by", "pg"),
+ ("id", 22),
+ ("time_ts", datetime.datetime(2006, 10, 10, 2, 18, 22, tzinfo=UTC)),
+ ],
+ ]
+ self.assertEqual(fetched_data, expected_data)
+
+ @unittest.skipIf(
+ bigquery_storage_v1beta1 is None, "Requires `google-cloud-bigquery-storage`"
+ )
+ @unittest.skipIf(fastavro is None, "Requires `fastavro`")
+ def test_dbapi_fetch_w_bqstorage_client_large_result_set(self):
+ bqstorage_client = bigquery_storage_v1beta1.BigQueryStorageClient(
+ credentials=Config.CLIENT._credentials
+ )
+ cursor = dbapi.connect(Config.CLIENT, bqstorage_client).cursor()
+
+ # Pick a large enouhg LIMIT value to assure that the fallback to the
+ # default client is not needed due to the result set being too small
+ # (a known issue that causes problems when reding such result sets with
+ # BQ storage client).
+ cursor.execute(
+ """
+ SELECT id, `by`, time_ts
+ FROM `bigquery-public-data.hacker_news.comments`
+ ORDER BY `id` ASC
+ LIMIT 100000
+ """
+ )
+
+ result_rows = [cursor.fetchone(), cursor.fetchone(), cursor.fetchone()]
+
+ field_name = operator.itemgetter(0)
+ fetched_data = [sorted(row.items(), key=field_name) for row in result_rows]
+
+ # Since DB API is not thread safe, only a single result stream should be
+ # requested by the BQ storage client, meaning that results should arrive
+ # in the sorted order.
+ expected_data = [
+ [
+ ("by", "sama"),
+ ("id", 15),
+ ("time_ts", datetime.datetime(2006, 10, 9, 19, 51, 1, tzinfo=UTC)),
+ ],
+ [
+ ("by", "pg"),
+ ("id", 17),
+ ("time_ts", datetime.datetime(2006, 10, 9, 19, 52, 45, tzinfo=UTC)),
+ ],
+ [
+ ("by", "pg"),
+ ("id", 22),
+ ("time_ts", datetime.datetime(2006, 10, 10, 2, 18, 22, tzinfo=UTC)),
+ ],
+ ]
+ self.assertEqual(fetched_data, expected_data)
+
def _load_table_for_dml(self, rows, dataset_id, table_id):
from google.cloud._testing import _NamedTemporaryFile
from google.cloud.bigquery.job import CreateDisposition
@@ -2458,7 +2697,9 @@ def test_list_rows_max_results_w_bqstorage(self):
self.assertEqual(len(dataframe.index), 100)
def temp_dataset(self, dataset_id, location=None):
- dataset = Dataset(Config.CLIENT.dataset(dataset_id))
+ project = Config.CLIENT.project
+ dataset_ref = bigquery.DatasetReference(project, dataset_id)
+ dataset = Dataset(dataset_ref)
if location:
dataset.location = location
dataset = retry_403(Config.CLIENT.create_dataset)(dataset)
diff --git a/tests/unit/model/test_model.py b/tests/unit/model/test_model.py
index bbb93ef9e..90fc09e66 100644
--- a/tests/unit/model/test_model.py
+++ b/tests/unit/model/test_model.py
@@ -316,5 +316,5 @@ def test_repr(target_class):
got = repr(model)
assert got == (
"Model(reference=ModelReference("
- "project='my-proj', dataset_id='my_dset', project_id='my_model'))"
+ "project_id='my-proj', dataset_id='my_dset', model_id='my_model'))"
)
diff --git a/tests/unit/model/test_model_reference.py b/tests/unit/model/test_model_reference.py
index ff1d1df7d..39dabb55d 100644
--- a/tests/unit/model/test_model_reference.py
+++ b/tests/unit/model/test_model_reference.py
@@ -136,5 +136,5 @@ def test_repr(target_class):
got = repr(model)
assert (
got
- == "ModelReference(project='my-proj', dataset_id='my_dset', project_id='my_model')"
+ == "ModelReference(project_id='my-proj', dataset_id='my_dset', model_id='my_model')"
)
diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py
index 2227183a9..f1dc4e816 100644
--- a/tests/unit/test_client.py
+++ b/tests/unit/test_client.py
@@ -24,13 +24,13 @@
import unittest
import warnings
-import freezegun
import mock
import requests
import six
from six.moves import http_client
import pytest
import pytz
+import pkg_resources
try:
import fastparquet
@@ -57,6 +57,9 @@
bigquery_storage_v1beta1 = None
from tests.unit.helpers import make_connection
+PANDAS_MINIUM_VERSION = pkg_resources.parse_version("1.0.0")
+PANDAS_INSTALLED_VERSION = pkg_resources.get_distribution("pandas").parsed_version
+
def _make_credentials():
import google.auth.credentials
@@ -2796,6 +2799,191 @@ def test_delete_table_w_not_found_ok_true(self):
conn.api_request.assert_called_with(method="DELETE", path=path, timeout=None)
+ def _create_job_helper(self, job_config, client_method):
+ creds = _make_credentials()
+ http = object()
+ client = self._make_one(project=self.PROJECT, credentials=creds, _http=http)
+
+ client._connection = make_connection()
+ rf1 = mock.Mock()
+ get_config_patch = mock.patch(
+ "google.cloud.bigquery.job._JobConfig.from_api_repr", return_value=rf1,
+ )
+ load_patch = mock.patch(client_method, autospec=True)
+
+ with load_patch as client_method, get_config_patch:
+ client.create_job(job_config=job_config)
+ client_method.assert_called_once()
+
+ def test_create_job_load_config(self):
+ configuration = {
+ "load": {
+ "destinationTable": {
+ "projectId": self.PROJECT,
+ "datasetId": self.DS_ID,
+ "tableId": "source_table",
+ },
+ "sourceUris": ["gs://test_bucket/src_object*"],
+ }
+ }
+
+ self._create_job_helper(
+ configuration, "google.cloud.bigquery.client.Client.load_table_from_uri"
+ )
+
+ def test_create_job_copy_config(self):
+ configuration = {
+ "copy": {
+ "sourceTables": [
+ {
+ "projectId": self.PROJECT,
+ "datasetId": self.DS_ID,
+ "tableId": "source_table",
+ }
+ ],
+ "destinationTable": {
+ "projectId": self.PROJECT,
+ "datasetId": self.DS_ID,
+ "tableId": "destination_table",
+ },
+ }
+ }
+
+ self._create_job_helper(
+ configuration, "google.cloud.bigquery.client.Client.copy_table",
+ )
+
+ def test_create_job_copy_config_w_single_source(self):
+ configuration = {
+ "copy": {
+ "sourceTable": {
+ "projectId": self.PROJECT,
+ "datasetId": self.DS_ID,
+ "tableId": "source_table",
+ },
+ "destinationTable": {
+ "projectId": self.PROJECT,
+ "datasetId": self.DS_ID,
+ "tableId": "destination_table",
+ },
+ }
+ }
+
+ self._create_job_helper(
+ configuration, "google.cloud.bigquery.client.Client.copy_table",
+ )
+
+ def test_create_job_extract_config(self):
+ configuration = {
+ "extract": {
+ "sourceTable": {
+ "projectId": self.PROJECT,
+ "datasetId": self.DS_ID,
+ "tableId": "source_table",
+ },
+ "destinationUris": ["gs://test_bucket/dst_object*"],
+ }
+ }
+ self._create_job_helper(
+ configuration, "google.cloud.bigquery.client.Client.extract_table",
+ )
+
+ def test_create_job_extract_config_for_model(self):
+ configuration = {
+ "extract": {
+ "sourceModel": {
+ "projectId": self.PROJECT,
+ "datasetId": self.DS_ID,
+ "modelId": "source_model",
+ },
+ "destinationUris": ["gs://test_bucket/dst_object*"],
+ }
+ }
+ self._create_job_helper(
+ configuration, "google.cloud.bigquery.client.Client.extract_table",
+ )
+
+ def test_create_job_query_config(self):
+ configuration = {
+ "query": {"query": "query", "destinationTable": {"tableId": "table_id"}}
+ }
+ self._create_job_helper(
+ configuration, "google.cloud.bigquery.client.Client.query",
+ )
+
+ def test_create_job_query_config_w_rateLimitExceeded_error(self):
+ from google.cloud.exceptions import Forbidden
+ from google.cloud.bigquery.retry import DEFAULT_RETRY
+
+ query = "select count(*) from persons"
+ configuration = {
+ "query": {
+ "query": query,
+ "useLegacySql": False,
+ "destinationTable": {"tableId": "table_id"},
+ }
+ }
+ resource = {
+ "jobReference": {"projectId": self.PROJECT, "jobId": mock.ANY},
+ "configuration": {
+ "query": {
+ "query": query,
+ "useLegacySql": False,
+ "destinationTable": {
+ "projectId": self.PROJECT,
+ "datasetId": self.DS_ID,
+ "tableId": "query_destination_table",
+ },
+ }
+ },
+ }
+ data_without_destination = {
+ "jobReference": {"projectId": self.PROJECT, "jobId": mock.ANY},
+ "configuration": {"query": {"query": query, "useLegacySql": False}},
+ }
+
+ creds = _make_credentials()
+ http = object()
+ retry = DEFAULT_RETRY.with_deadline(1).with_predicate(
+ lambda exc: isinstance(exc, Forbidden)
+ )
+ client = self._make_one(project=self.PROJECT, credentials=creds, _http=http)
+
+ api_request_patcher = mock.patch.object(
+ client._connection,
+ "api_request",
+ side_effect=[
+ Forbidden("", errors=[{"reason": "rateLimitExceeded"}]),
+ resource,
+ ],
+ )
+
+ with api_request_patcher as fake_api_request:
+ job = client.create_job(job_config=configuration, retry=retry)
+
+ self.assertEqual(job.destination.table_id, "query_destination_table")
+ self.assertEqual(len(fake_api_request.call_args_list), 2) # was retried once
+ self.assertEqual(
+ fake_api_request.call_args_list[1],
+ mock.call(
+ method="POST",
+ path="/projects/PROJECT/jobs",
+ data=data_without_destination,
+ timeout=None,
+ ),
+ )
+
+ def test_create_job_w_invalid_job_config(self):
+ configuration = {"unknown": {}}
+ creds = _make_credentials()
+ http = object()
+ client = self._make_one(project=self.PROJECT, credentials=creds, _http=http)
+
+ with self.assertRaises(TypeError) as exc:
+ client.create_job(job_config=configuration)
+
+ self.assertIn("Invalid job configuration", exc.exception.args[0])
+
def test_job_from_resource_unknown_type(self):
from google.cloud.bigquery.job import UnknownJob
@@ -4048,6 +4236,140 @@ def test_extract_table_w_destination_uris(self):
self.assertEqual(job.source, source)
self.assertEqual(list(job.destination_uris), [DESTINATION1, DESTINATION2])
+ def test_extract_table_for_source_type_model(self):
+ from google.cloud.bigquery.job import ExtractJob
+
+ JOB = "job_id"
+ SOURCE = "source_model"
+ DESTINATION = "gs://bucket_name/object_name"
+ RESOURCE = {
+ "jobReference": {"projectId": self.PROJECT, "jobId": JOB},
+ "configuration": {
+ "extract": {
+ "sourceModel": {
+ "projectId": self.PROJECT,
+ "datasetId": self.DS_ID,
+ "modelId": SOURCE,
+ },
+ "destinationUris": [DESTINATION],
+ }
+ },
+ }
+ creds = _make_credentials()
+ http = object()
+ client = self._make_one(project=self.PROJECT, credentials=creds, _http=http)
+ conn = client._connection = make_connection(RESOURCE)
+ dataset = DatasetReference(self.PROJECT, self.DS_ID)
+ source = dataset.model(SOURCE)
+
+ job = client.extract_table(
+ source, DESTINATION, job_id=JOB, timeout=7.5, source_type="Model"
+ )
+
+ # Check that extract_table actually starts the job.
+ conn.api_request.assert_called_once_with(
+ method="POST", path="/projects/PROJECT/jobs", data=RESOURCE, timeout=7.5,
+ )
+
+ # Check the job resource.
+ self.assertIsInstance(job, ExtractJob)
+ self.assertIs(job._client, client)
+ self.assertEqual(job.job_id, JOB)
+ self.assertEqual(job.source, source)
+ self.assertEqual(list(job.destination_uris), [DESTINATION])
+
+ def test_extract_table_for_source_type_model_w_string_model_id(self):
+ JOB = "job_id"
+ source_id = "source_model"
+ DESTINATION = "gs://bucket_name/object_name"
+ RESOURCE = {
+ "jobReference": {"projectId": self.PROJECT, "jobId": JOB},
+ "configuration": {
+ "extract": {
+ "sourceModel": {
+ "projectId": self.PROJECT,
+ "datasetId": self.DS_ID,
+ "modelId": source_id,
+ },
+ "destinationUris": [DESTINATION],
+ }
+ },
+ }
+ creds = _make_credentials()
+ http = object()
+ client = self._make_one(project=self.PROJECT, credentials=creds, _http=http)
+ conn = client._connection = make_connection(RESOURCE)
+
+ client.extract_table(
+ # Test with string for model ID.
+ "{}.{}".format(self.DS_ID, source_id),
+ DESTINATION,
+ job_id=JOB,
+ timeout=7.5,
+ source_type="Model",
+ )
+
+ # Check that extract_table actually starts the job.
+ conn.api_request.assert_called_once_with(
+ method="POST", path="/projects/PROJECT/jobs", data=RESOURCE, timeout=7.5,
+ )
+
+ def test_extract_table_for_source_type_model_w_model_object(self):
+ from google.cloud.bigquery.model import Model
+
+ JOB = "job_id"
+ DESTINATION = "gs://bucket_name/object_name"
+ model_id = "{}.{}.{}".format(self.PROJECT, self.DS_ID, self.MODEL_ID)
+ model = Model(model_id)
+ RESOURCE = {
+ "jobReference": {"projectId": self.PROJECT, "jobId": JOB},
+ "configuration": {
+ "extract": {
+ "sourceModel": {
+ "projectId": self.PROJECT,
+ "datasetId": self.DS_ID,
+ "modelId": self.MODEL_ID,
+ },
+ "destinationUris": [DESTINATION],
+ }
+ },
+ }
+ creds = _make_credentials()
+ http = object()
+ client = self._make_one(project=self.PROJECT, credentials=creds, _http=http)
+ conn = client._connection = make_connection(RESOURCE)
+
+ client.extract_table(
+ # Test with Model class object.
+ model,
+ DESTINATION,
+ job_id=JOB,
+ timeout=7.5,
+ source_type="Model",
+ )
+
+ # Check that extract_table actually starts the job.
+ conn.api_request.assert_called_once_with(
+ method="POST", path="/projects/PROJECT/jobs", data=RESOURCE, timeout=7.5,
+ )
+
+ def test_extract_table_for_invalid_source_type_model(self):
+ JOB = "job_id"
+ SOURCE = "source_model"
+ DESTINATION = "gs://bucket_name/object_name"
+ creds = _make_credentials()
+ http = object()
+ client = self._make_one(project=self.PROJECT, credentials=creds, _http=http)
+ dataset = DatasetReference(self.PROJECT, self.DS_ID)
+ source = dataset.model(SOURCE)
+
+ with self.assertRaises(ValueError) as exc:
+ client.extract_table(
+ source, DESTINATION, job_id=JOB, timeout=7.5, source_type="foo"
+ )
+
+ self.assertIn("Cannot pass", exc.exception.args[0])
+
def test_query_defaults(self):
from google.cloud.bigquery.job import QueryJob
@@ -5496,43 +5818,6 @@ def test_list_partitions_with_string_id(self):
self.assertEqual(len(partition_list), 0)
- def test_list_partitions_splitting_timout_between_requests(self):
- from google.cloud.bigquery.table import Table
-
- row_count = 2
- meta_info = _make_list_partitons_meta_info(
- self.PROJECT, self.DS_ID, self.TABLE_ID, row_count
- )
-
- data = {
- "totalRows": str(row_count),
- "rows": [{"f": [{"v": "20180101"}]}, {"f": [{"v": "20180102"}]}],
- }
- creds = _make_credentials()
- http = object()
- client = self._make_one(project=self.PROJECT, credentials=creds, _http=http)
- client._connection = make_connection(meta_info, data)
- table = Table(self.TABLE_REF)
-
- with freezegun.freeze_time("2019-01-01 00:00:00", tick=False) as frozen_time:
-
- def delayed_get_table(*args, **kwargs):
- frozen_time.tick(delta=1.4)
- return orig_get_table(*args, **kwargs)
-
- orig_get_table = client.get_table
- client.get_table = mock.Mock(side_effect=delayed_get_table)
-
- client.list_partitions(table, timeout=5.0)
-
- client.get_table.assert_called_once()
- _, kwargs = client.get_table.call_args
- self.assertEqual(kwargs.get("timeout"), 5.0)
-
- client._connection.api_request.assert_called()
- _, kwargs = client._connection.api_request.call_args
- self.assertAlmostEqual(kwargs.get("timeout"), 3.6, places=5)
-
def test_list_rows(self):
import datetime
from google.cloud._helpers import UTC
@@ -5612,6 +5897,71 @@ def _bigquery_timestamp_float_repr(ts_float):
method="GET", path="/%s" % PATH, query_params={}, timeout=7.5
)
+ def test_list_rows_w_start_index_w_page_size(self):
+ from google.cloud.bigquery.schema import SchemaField
+ from google.cloud.bigquery.table import Table
+ from google.cloud.bigquery.table import Row
+
+ PATH = "projects/%s/datasets/%s/tables/%s/data" % (
+ self.PROJECT,
+ self.DS_ID,
+ self.TABLE_ID,
+ )
+
+ page_1 = {
+ "totalRows": 4,
+ "pageToken": "some-page-token",
+ "rows": [
+ {"f": [{"v": "Phred Phlyntstone"}]},
+ {"f": [{"v": "Bharney Rhubble"}]},
+ ],
+ }
+ page_2 = {
+ "totalRows": 4,
+ "rows": [
+ {"f": [{"v": "Wylma Phlyntstone"}]},
+ {"f": [{"v": "Bhettye Rhubble"}]},
+ ],
+ }
+ creds = _make_credentials()
+ http = object()
+ client = self._make_one(project=self.PROJECT, credentials=creds, _http=http)
+ conn = client._connection = make_connection(page_1, page_2)
+ full_name = SchemaField("full_name", "STRING", mode="REQUIRED")
+ table = Table(self.TABLE_REF, schema=[full_name])
+ iterator = client.list_rows(table, max_results=4, page_size=2, start_index=1)
+ pages = iterator.pages
+ rows = list(six.next(pages))
+ extra_params = iterator.extra_params
+ f2i = {"full_name": 0}
+ self.assertEqual(len(rows), 2)
+ self.assertEqual(rows[0], Row(("Phred Phlyntstone",), f2i))
+ self.assertEqual(rows[1], Row(("Bharney Rhubble",), f2i))
+
+ rows = list(six.next(pages))
+
+ self.assertEqual(len(rows), 2)
+ self.assertEqual(rows[0], Row(("Wylma Phlyntstone",), f2i))
+ self.assertEqual(rows[1], Row(("Bhettye Rhubble",), f2i))
+ self.assertEqual(extra_params, {"startIndex": 1})
+
+ conn.api_request.assert_has_calls(
+ [
+ mock.call(
+ method="GET",
+ path="/%s" % PATH,
+ query_params={"startIndex": 1, "maxResults": 2},
+ timeout=None,
+ ),
+ mock.call(
+ method="GET",
+ path="/%s" % PATH,
+ query_params={"pageToken": "some-page-token", "maxResults": 2},
+ timeout=None,
+ ),
+ ]
+ )
+
def test_list_rows_empty_table(self):
response = {"totalRows": "0", "rows": []}
creds = _make_credentials()
@@ -5853,46 +6203,6 @@ def test_list_rows_with_missing_schema(self):
self.assertEqual(rows[1].age, 31, msg=repr(table))
self.assertIsNone(rows[2].age, msg=repr(table))
- def test_list_rows_splitting_timout_between_requests(self):
- from google.cloud.bigquery.schema import SchemaField
- from google.cloud.bigquery.table import Table
-
- response = {"totalRows": "0", "rows": []}
- creds = _make_credentials()
- http = object()
- client = self._make_one(project=self.PROJECT, credentials=creds, _http=http)
- client._connection = make_connection(response, response)
-
- table = Table(
- self.TABLE_REF, schema=[SchemaField("field_x", "INTEGER", mode="NULLABLE")]
- )
-
- with freezegun.freeze_time("1970-01-01 00:00:00", tick=False) as frozen_time:
-
- def delayed_get_table(*args, **kwargs):
- frozen_time.tick(delta=1.4)
- return table
-
- client.get_table = mock.Mock(side_effect=delayed_get_table)
-
- rows_iter = client.list_rows(
- "{}.{}.{}".format(
- self.TABLE_REF.project,
- self.TABLE_REF.dataset_id,
- self.TABLE_REF.table_id,
- ),
- timeout=5.0,
- )
- six.next(rows_iter.pages)
-
- client.get_table.assert_called_once()
- _, kwargs = client.get_table.call_args
- self.assertEqual(kwargs.get("timeout"), 5.0)
-
- client._connection.api_request.assert_called_once()
- _, kwargs = client._connection.api_request.call_args
- self.assertAlmostEqual(kwargs.get("timeout"), 3.6)
-
def test_list_rows_error(self):
creds = _make_credentials()
http = object()
@@ -6593,6 +6903,42 @@ def test_load_table_from_dataframe_unknown_table(self):
job_config=mock.ANY,
)
+ @unittest.skipIf(pandas is None, "Requires `pandas`")
+ @unittest.skipIf(fastparquet is None, "Requires `fastparquet`")
+ def test_load_table_from_dataframe_no_pyarrow_warning(self):
+ from google.cloud.bigquery.client import PyarrowMissingWarning
+
+ client = self._make_client()
+
+ # Pick at least one column type that translates to Pandas dtype
+ # "object". A string column matches that.
+ records = [{"name": "Monty", "age": 100}, {"name": "Python", "age": 60}]
+ dataframe = pandas.DataFrame(records)
+
+ get_table_patch = mock.patch(
+ "google.cloud.bigquery.client.Client.get_table",
+ autospec=True,
+ side_effect=google.api_core.exceptions.NotFound("Table not found"),
+ )
+ load_patch = mock.patch(
+ "google.cloud.bigquery.client.Client.load_table_from_file", autospec=True
+ )
+ pyarrow_patch = mock.patch("google.cloud.bigquery.client.pyarrow", None)
+ pyarrow_patch_helpers = mock.patch(
+ "google.cloud.bigquery._pandas_helpers.pyarrow", None
+ )
+ catch_warnings = warnings.catch_warnings(record=True)
+
+ with get_table_patch, load_patch, pyarrow_patch, pyarrow_patch_helpers, catch_warnings as warned:
+ client.load_table_from_dataframe(
+ dataframe, self.TABLE_REF, location=self.LOCATION
+ )
+
+ matches = [
+ warning for warning in warned if warning.category is PyarrowMissingWarning
+ ]
+ assert matches, "A missing pyarrow deprecation warning was not raised."
+
@unittest.skipIf(pandas is None, "Requires `pandas`")
@unittest.skipIf(fastparquet is None, "Requires `fastparquet`")
def test_load_table_from_dataframe_no_schema_warning_wo_pyarrow(self):
@@ -6631,6 +6977,98 @@ def test_load_table_from_dataframe_no_schema_warning_wo_pyarrow(self):
]
assert matches, "A missing schema deprecation warning was not raised."
+ @unittest.skipIf(
+ pandas is None or PANDAS_INSTALLED_VERSION < PANDAS_MINIUM_VERSION,
+ "Only `pandas version >=1.0.0` supported",
+ )
+ @unittest.skipIf(pyarrow is None, "Requires `pyarrow`")
+ def test_load_table_from_dataframe_w_nullable_int64_datatype(self):
+ from google.cloud.bigquery.client import _DEFAULT_NUM_RETRIES
+ from google.cloud.bigquery import job
+ from google.cloud.bigquery.schema import SchemaField
+
+ client = self._make_client()
+ dataframe = pandas.DataFrame({"x": [1, 2, None, 4]}, dtype="Int64")
+ load_patch = mock.patch(
+ "google.cloud.bigquery.client.Client.load_table_from_file", autospec=True
+ )
+
+ get_table_patch = mock.patch(
+ "google.cloud.bigquery.client.Client.get_table",
+ autospec=True,
+ return_value=mock.Mock(schema=[SchemaField("x", "INT64", "NULLABLE")]),
+ )
+
+ with load_patch as load_table_from_file, get_table_patch:
+ client.load_table_from_dataframe(
+ dataframe, self.TABLE_REF, location=self.LOCATION
+ )
+
+ load_table_from_file.assert_called_once_with(
+ client,
+ mock.ANY,
+ self.TABLE_REF,
+ num_retries=_DEFAULT_NUM_RETRIES,
+ rewind=True,
+ job_id=mock.ANY,
+ job_id_prefix=None,
+ location=self.LOCATION,
+ project=None,
+ job_config=mock.ANY,
+ )
+
+ sent_config = load_table_from_file.mock_calls[0][2]["job_config"]
+ assert sent_config.source_format == job.SourceFormat.PARQUET
+ assert tuple(sent_config.schema) == (
+ SchemaField("x", "INT64", "NULLABLE", None),
+ )
+
+ @unittest.skipIf(
+ pandas is None or PANDAS_INSTALLED_VERSION < PANDAS_MINIUM_VERSION,
+ "Only `pandas version >=1.0.0` supported",
+ )
+ @unittest.skipIf(pyarrow is None, "Requires `pyarrow`")
+ def test_load_table_from_dataframe_w_nullable_int64_datatype_automatic_schema(self):
+ from google.cloud.bigquery.client import _DEFAULT_NUM_RETRIES
+ from google.cloud.bigquery import job
+ from google.cloud.bigquery.schema import SchemaField
+
+ client = self._make_client()
+ dataframe = pandas.DataFrame({"x": [1, 2, None, 4]}, dtype="Int64")
+ load_patch = mock.patch(
+ "google.cloud.bigquery.client.Client.load_table_from_file", autospec=True
+ )
+
+ get_table_patch = mock.patch(
+ "google.cloud.bigquery.client.Client.get_table",
+ autospec=True,
+ side_effect=google.api_core.exceptions.NotFound("Table not found"),
+ )
+
+ with load_patch as load_table_from_file, get_table_patch:
+ client.load_table_from_dataframe(
+ dataframe, self.TABLE_REF, location=self.LOCATION
+ )
+
+ load_table_from_file.assert_called_once_with(
+ client,
+ mock.ANY,
+ self.TABLE_REF,
+ num_retries=_DEFAULT_NUM_RETRIES,
+ rewind=True,
+ job_id=mock.ANY,
+ job_id_prefix=None,
+ location=self.LOCATION,
+ project=None,
+ job_config=mock.ANY,
+ )
+
+ sent_config = load_table_from_file.mock_calls[0][2]["job_config"]
+ assert sent_config.source_format == job.SourceFormat.PARQUET
+ assert tuple(sent_config.schema) == (
+ SchemaField("x", "INT64", "NULLABLE", None),
+ )
+
@unittest.skipIf(pandas is None, "Requires `pandas`")
@unittest.skipIf(pyarrow is None, "Requires `pyarrow`")
def test_load_table_from_dataframe_struct_fields_error(self):
@@ -6867,7 +7305,9 @@ def test_load_table_from_dataframe_w_schema_wo_pyarrow(self):
assert warned # there should be at least one warning
for warning in warned:
assert "pyarrow" in str(warning)
- assert warning.category in (DeprecationWarning, PendingDeprecationWarning)
+ assert issubclass(
+ warning.category, (DeprecationWarning, PendingDeprecationWarning)
+ )
load_table_from_file.assert_called_once_with(
client,
diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py
index ac13e0093..e4977a270 100644
--- a/tests/unit/test_dataset.py
+++ b/tests/unit/test_dataset.py
@@ -84,6 +84,20 @@ def test__eq___type_mismatch(self):
self.assertNotEqual(entry, object())
self.assertEqual(entry, mock.ANY)
+ def test___hash__set_equality(self):
+ entry1 = self._make_one("OWNER", "userByEmail", "silly@example.com")
+ entry2 = self._make_one("OWNER", "userByEmail", "phred@example.com")
+ set_one = {entry1, entry2}
+ set_two = {entry1, entry2}
+ self.assertEqual(set_one, set_two)
+
+ def test___hash__not_equals(self):
+ entry1 = self._make_one("OWNER", "userByEmail", "silly@example.com")
+ entry2 = self._make_one("OWNER", "userByEmail", "phred@example.com")
+ set_one = {entry1}
+ set_two = {entry2}
+ self.assertNotEqual(set_one, set_two)
+
def test_to_api_repr(self):
entry = self._make_one("OWNER", "userByEmail", "salmon@example.com")
resource = entry.to_api_repr()
diff --git a/tests/unit/test_dbapi__helpers.py b/tests/unit/test_dbapi__helpers.py
index 45c690ede..8f98d0c53 100644
--- a/tests/unit/test_dbapi__helpers.py
+++ b/tests/unit/test_dbapi__helpers.py
@@ -15,9 +15,11 @@
import datetime
import decimal
import math
+import operator as op
import unittest
import google.cloud._helpers
+from google.cloud.bigquery import table
from google.cloud.bigquery.dbapi import _helpers
from google.cloud.bigquery.dbapi import exceptions
@@ -185,3 +187,35 @@ def test_to_query_parameters_w_list_dict_param(self):
def test_to_query_parameters_none_argument(self):
query_parameters = _helpers.to_query_parameters(None)
self.assertEqual(query_parameters, [])
+
+
+class TestToBqTableRows(unittest.TestCase):
+ def test_empty_iterable(self):
+ rows_iterable = iter([])
+ result = _helpers.to_bq_table_rows(rows_iterable)
+ self.assertEqual(list(result), [])
+
+ def test_non_empty_iterable(self):
+ rows_iterable = [
+ dict(one=1.1, four=1.4, two=1.2, three=1.3),
+ dict(one=2.1, four=2.4, two=2.2, three=2.3),
+ ]
+
+ result = _helpers.to_bq_table_rows(rows_iterable)
+
+ rows = list(result)
+ self.assertEqual(len(rows), 2)
+
+ row_1, row_2 = rows
+ self.assertIsInstance(row_1, table.Row)
+ self.assertIsInstance(row_2, table.Row)
+
+ field_value = op.itemgetter(1)
+
+ items = sorted(row_1.items(), key=field_value)
+ expected_items = [("one", 1.1), ("two", 1.2), ("three", 1.3), ("four", 1.4)]
+ self.assertEqual(items, expected_items)
+
+ items = sorted(row_2.items(), key=field_value)
+ expected_items = [("one", 2.1), ("two", 2.2), ("three", 2.3), ("four", 2.4)]
+ self.assertEqual(items, expected_items)
diff --git a/tests/unit/test_dbapi_connection.py b/tests/unit/test_dbapi_connection.py
index 19acec05b..595afd0fe 100644
--- a/tests/unit/test_dbapi_connection.py
+++ b/tests/unit/test_dbapi_connection.py
@@ -16,6 +16,11 @@
import mock
+try:
+ from google.cloud import bigquery_storage_v1beta1
+except ImportError: # pragma: NO COVER
+ bigquery_storage_v1beta1 = None
+
class TestConnection(unittest.TestCase):
@staticmethod
@@ -27,19 +32,41 @@ def _get_target_class():
def _make_one(self, *args, **kw):
return self._get_target_class()(*args, **kw)
- def _mock_client(self, rows=None, schema=None):
+ def _mock_client(self):
from google.cloud.bigquery import client
mock_client = mock.create_autospec(client.Client)
return mock_client
- def test_ctor(self):
+ def _mock_bqstorage_client(self):
+ from google.cloud.bigquery_storage_v1beta1 import client
+
+ mock_client = mock.create_autospec(client.BigQueryStorageClient)
+ return mock_client
+
+ def test_ctor_wo_bqstorage_client(self):
from google.cloud.bigquery.dbapi import Connection
mock_client = self._mock_client()
connection = self._make_one(client=mock_client)
self.assertIsInstance(connection, Connection)
self.assertIs(connection._client, mock_client)
+ self.assertIsNone(connection._bqstorage_client)
+
+ @unittest.skipIf(
+ bigquery_storage_v1beta1 is None, "Requires `google-cloud-bigquery-storage`"
+ )
+ def test_ctor_w_bqstorage_client(self):
+ from google.cloud.bigquery.dbapi import Connection
+
+ mock_client = self._mock_client()
+ mock_bqstorage_client = self._mock_bqstorage_client()
+ connection = self._make_one(
+ client=mock_client, bqstorage_client=mock_bqstorage_client,
+ )
+ self.assertIsInstance(connection, Connection)
+ self.assertIs(connection._client, mock_client)
+ self.assertIs(connection._bqstorage_client, mock_bqstorage_client)
@mock.patch("google.cloud.bigquery.Client", autospec=True)
def test_connect_wo_client(self, mock_client):
@@ -49,6 +76,7 @@ def test_connect_wo_client(self, mock_client):
connection = connect()
self.assertIsInstance(connection, Connection)
self.assertIsNotNone(connection._client)
+ self.assertIsNone(connection._bqstorage_client)
def test_connect_w_client(self):
from google.cloud.bigquery.dbapi import connect
@@ -58,6 +86,23 @@ def test_connect_w_client(self):
connection = connect(client=mock_client)
self.assertIsInstance(connection, Connection)
self.assertIs(connection._client, mock_client)
+ self.assertIsNone(connection._bqstorage_client)
+
+ @unittest.skipIf(
+ bigquery_storage_v1beta1 is None, "Requires `google-cloud-bigquery-storage`"
+ )
+ def test_connect_w_both_clients(self):
+ from google.cloud.bigquery.dbapi import connect
+ from google.cloud.bigquery.dbapi import Connection
+
+ mock_client = self._mock_client()
+ mock_bqstorage_client = self._mock_bqstorage_client()
+ connection = connect(
+ client=mock_client, bqstorage_client=mock_bqstorage_client,
+ )
+ self.assertIsInstance(connection, Connection)
+ self.assertIs(connection._client, mock_client)
+ self.assertIs(connection._bqstorage_client, mock_bqstorage_client)
def test_close(self):
connection = self._make_one(client=self._mock_client())
diff --git a/tests/unit/test_dbapi_cursor.py b/tests/unit/test_dbapi_cursor.py
index 4ccd5e71a..e53cc158a 100644
--- a/tests/unit/test_dbapi_cursor.py
+++ b/tests/unit/test_dbapi_cursor.py
@@ -12,9 +12,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import operator as op
import unittest
import mock
+import six
+
+from google.api_core import exceptions
+
+try:
+ from google.cloud import bigquery_storage_v1beta1
+except ImportError: # pragma: NO COVER
+ bigquery_storage_v1beta1 = None
class TestCursor(unittest.TestCase):
@@ -44,6 +53,29 @@ def _mock_client(self, rows=None, schema=None, num_dml_affected_rows=None):
mock_client.list_rows.return_value = rows
return mock_client
+ def _mock_bqstorage_client(self, rows=None, stream_count=0):
+ from google.cloud.bigquery_storage_v1beta1 import client
+ from google.cloud.bigquery_storage_v1beta1 import types
+
+ if rows is None:
+ rows = []
+
+ mock_client = mock.create_autospec(client.BigQueryStorageClient)
+
+ mock_read_session = mock.MagicMock(
+ streams=[
+ types.Stream(name="streams/stream_{}".format(i))
+ for i in range(stream_count)
+ ]
+ )
+ mock_client.create_read_session.return_value = mock_read_session
+
+ mock_rows_stream = mock.MagicMock()
+ mock_rows_stream.rows.return_value = iter(rows)
+ mock_client.read_rows.return_value = mock_rows_stream
+
+ return mock_client
+
def _mock_job(self, total_rows=0, schema=None, num_dml_affected_rows=None):
from google.cloud.bigquery import job
@@ -180,6 +212,156 @@ def test_fetchall_w_row(self):
self.assertEqual(len(rows), 1)
self.assertEqual(rows[0], (1,))
+ @unittest.skipIf(
+ bigquery_storage_v1beta1 is None, "Requires `google-cloud-bigquery-storage`"
+ )
+ def test_fetchall_w_bqstorage_client_fetch_success(self):
+ from google.cloud.bigquery import dbapi
+ from google.cloud.bigquery import table
+
+ # use unordered data to also test any non-determenistic key order in dicts
+ row_data = [
+ table.Row([1.4, 1.1, 1.3, 1.2], {"bar": 3, "baz": 2, "foo": 1, "quux": 0}),
+ table.Row([2.4, 2.1, 2.3, 2.2], {"bar": 3, "baz": 2, "foo": 1, "quux": 0}),
+ ]
+ bqstorage_streamed_rows = [
+ {"bar": 1.2, "foo": 1.1, "quux": 1.4, "baz": 1.3},
+ {"bar": 2.2, "foo": 2.1, "quux": 2.4, "baz": 2.3},
+ ]
+
+ mock_client = self._mock_client(rows=row_data)
+ mock_bqstorage_client = self._mock_bqstorage_client(
+ stream_count=1, rows=bqstorage_streamed_rows,
+ )
+
+ connection = dbapi.connect(
+ client=mock_client, bqstorage_client=mock_bqstorage_client,
+ )
+ cursor = connection.cursor()
+ cursor.execute("SELECT foo, bar FROM some_table")
+
+ rows = cursor.fetchall()
+
+ # the default client was not used
+ mock_client.list_rows.assert_not_called()
+
+ # check the data returned
+ field_value = op.itemgetter(1)
+ sorted_row_data = [sorted(row.items(), key=field_value) for row in rows]
+ expected_row_data = [
+ [("foo", 1.1), ("bar", 1.2), ("baz", 1.3), ("quux", 1.4)],
+ [("foo", 2.1), ("bar", 2.2), ("baz", 2.3), ("quux", 2.4)],
+ ]
+
+ self.assertEqual(sorted_row_data, expected_row_data)
+
+ @unittest.skipIf(
+ bigquery_storage_v1beta1 is None, "Requires `google-cloud-bigquery-storage`"
+ )
+ def test_fetchall_w_bqstorage_client_fetch_no_rows(self):
+ from google.cloud.bigquery import dbapi
+
+ mock_client = self._mock_client(rows=[])
+ mock_bqstorage_client = self._mock_bqstorage_client(stream_count=0)
+
+ connection = dbapi.connect(
+ client=mock_client, bqstorage_client=mock_bqstorage_client,
+ )
+ cursor = connection.cursor()
+ cursor.execute("SELECT foo, bar FROM some_table")
+
+ rows = cursor.fetchall()
+
+ # # the default client was not used
+ mock_client.list_rows.assert_not_called()
+
+ # check the data returned
+ self.assertEqual(rows, [])
+
+ @unittest.skipIf(
+ bigquery_storage_v1beta1 is None, "Requires `google-cloud-bigquery-storage`"
+ )
+ def test_fetchall_w_bqstorage_client_fetch_error_no_fallback(self):
+ from google.cloud.bigquery import dbapi
+ from google.cloud.bigquery import table
+
+ row_data = [table.Row([1.1, 1.2], {"foo": 0, "bar": 1})]
+
+ mock_client = self._mock_client(rows=row_data)
+ mock_bqstorage_client = self._mock_bqstorage_client(
+ stream_count=1, rows=row_data,
+ )
+ no_access_error = exceptions.Forbidden("invalid credentials")
+ mock_bqstorage_client.create_read_session.side_effect = no_access_error
+
+ connection = dbapi.connect(
+ client=mock_client, bqstorage_client=mock_bqstorage_client,
+ )
+ cursor = connection.cursor()
+ cursor.execute("SELECT foo, bar FROM some_table")
+
+ with six.assertRaisesRegex(self, exceptions.Forbidden, "invalid credentials"):
+ cursor.fetchall()
+
+ # the default client was not used
+ mock_client.list_rows.assert_not_called()
+
+ @unittest.skipIf(
+ bigquery_storage_v1beta1 is None, "Requires `google-cloud-bigquery-storage`"
+ )
+ def test_fetchall_w_bqstorage_client_fetch_error_fallback_on_client(self):
+ from google.cloud.bigquery import dbapi
+ from google.cloud.bigquery import table
+
+ # use unordered data to also test any non-determenistic key order in dicts
+ row_data = [
+ table.Row([1.4, 1.1, 1.3, 1.2], {"bar": 3, "baz": 2, "foo": 1, "quux": 0}),
+ table.Row([2.4, 2.1, 2.3, 2.2], {"bar": 3, "baz": 2, "foo": 1, "quux": 0}),
+ ]
+ bqstorage_streamed_rows = [
+ {"bar": 1.2, "foo": 1.1, "quux": 1.4, "baz": 1.3},
+ {"bar": 2.2, "foo": 2.1, "quux": 2.4, "baz": 2.3},
+ ]
+
+ mock_client = self._mock_client(rows=row_data)
+ mock_bqstorage_client = self._mock_bqstorage_client(
+ stream_count=1, rows=bqstorage_streamed_rows,
+ )
+ request_error = exceptions.BadRequest("BQ storage what??")
+ mock_bqstorage_client.create_read_session.side_effect = request_error
+
+ connection = dbapi.connect(
+ client=mock_client, bqstorage_client=mock_bqstorage_client,
+ )
+ cursor = connection.cursor()
+ cursor.execute("SELECT foo, bar FROM some_table")
+
+ logger_patcher = mock.patch("google.cloud.bigquery.dbapi.cursor._LOGGER")
+ with logger_patcher as mock_logger:
+ rows = cursor.fetchall()
+
+ # both client were used
+ mock_bqstorage_client.create_read_session.assert_called()
+ mock_client.list_rows.assert_called()
+
+ # fallback to default API should have been logged
+ relevant_calls = [
+ call
+ for call in mock_logger.debug.call_args_list
+ if call.args and "tabledata.list API" in call.args[0]
+ ]
+ self.assertTrue(relevant_calls)
+
+ # check the data returned
+ field_value = op.itemgetter(1)
+ sorted_row_data = [sorted(row.items(), key=field_value) for row in rows]
+ expected_row_data = [
+ [("foo", 1.1), ("bar", 1.2), ("baz", 1.3), ("quux", 1.4)],
+ [("foo", 2.1), ("bar", 2.2), ("baz", 2.3), ("quux", 2.4)],
+ ]
+
+ self.assertEqual(sorted_row_data, expected_row_data)
+
def test_execute_custom_job_id(self):
from google.cloud.bigquery.dbapi import connect
diff --git a/tests/unit/test_job.py b/tests/unit/test_job.py
index 6b0d4b8fb..c89cad749 100644
--- a/tests/unit/test_job.py
+++ b/tests/unit/test_job.py
@@ -994,24 +994,6 @@ def test_result_explicit_w_state(self, result):
begin.assert_not_called()
result.assert_called_once_with(timeout=timeout)
- @mock.patch("google.api_core.future.polling.PollingFuture.result")
- def test_result_splitting_timout_between_requests(self, result):
- client = _make_client(project=self.PROJECT)
- job = self._make_one(self.JOB_ID, client)
- begin = job._begin = mock.Mock()
- retry = mock.Mock()
-
- with freezegun.freeze_time("1970-01-01 00:00:00", tick=False) as frozen_time:
-
- def delayed_begin(*args, **kwargs):
- frozen_time.tick(delta=0.3)
-
- begin.side_effect = delayed_begin
- job.result(retry=retry, timeout=1.0)
-
- begin.assert_called_once_with(retry=retry, timeout=1.0)
- result.assert_called_once_with(timeout=0.7)
-
def test_cancelled_wo_error_result(self):
client = _make_client(project=self.PROJECT)
job = self._make_one(self.JOB_ID, client)
@@ -3194,10 +3176,16 @@ def _verifyResourceProperties(self, job, resource):
self.assertEqual(job.destination_uris, config["destinationUris"])
- table_ref = config["sourceTable"]
- self.assertEqual(job.source.project, table_ref["projectId"])
- self.assertEqual(job.source.dataset_id, table_ref["datasetId"])
- self.assertEqual(job.source.table_id, table_ref["tableId"])
+ if "sourceTable" in config:
+ table_ref = config["sourceTable"]
+ self.assertEqual(job.source.project, table_ref["projectId"])
+ self.assertEqual(job.source.dataset_id, table_ref["datasetId"])
+ self.assertEqual(job.source.table_id, table_ref["tableId"])
+ else:
+ model_ref = config["sourceModel"]
+ self.assertEqual(job.source.project, model_ref["projectId"])
+ self.assertEqual(job.source.dataset_id, model_ref["datasetId"])
+ self.assertEqual(job.source.model_id, model_ref["modelId"])
if "compression" in config:
self.assertEqual(job.compression, config["compression"])
@@ -3299,6 +3287,28 @@ def test_from_api_repr_bare(self):
self.assertIs(job._client, client)
self._verifyResourceProperties(job, RESOURCE)
+ def test_from_api_repr_for_model(self):
+ self._setUpConstants()
+ client = _make_client(project=self.PROJECT)
+ RESOURCE = {
+ "id": self.JOB_ID,
+ "jobReference": {"projectId": self.PROJECT, "jobId": self.JOB_ID},
+ "configuration": {
+ "extract": {
+ "sourceModel": {
+ "projectId": self.PROJECT,
+ "datasetId": self.DS_ID,
+ "modelId": "model_id",
+ },
+ "destinationUris": [self.DESTINATION_URI],
+ }
+ },
+ }
+ klass = self._get_target_class()
+ job = klass.from_api_repr(RESOURCE, client=client)
+ self.assertIs(job._client, client)
+ self._verifyResourceProperties(job, RESOURCE)
+
def test_from_api_repr_w_properties(self):
from google.cloud.bigquery.job import Compression
@@ -4011,33 +4021,6 @@ def test_done_w_timeout(self):
call_args = fake_reload.call_args
self.assertEqual(call_args.kwargs.get("timeout"), 42)
- def test_done_w_timeout_and_shorter_internal_api_timeout(self):
- from google.cloud.bigquery.job import _TIMEOUT_BUFFER_SECS
- from google.cloud.bigquery.job import _SERVER_TIMEOUT_MARGIN_SECS
-
- client = _make_client(project=self.PROJECT)
- resource = self._make_resource(ended=False)
- job = self._get_target_class().from_api_repr(resource, client)
- job._done_timeout = 8.8
-
- with mock.patch.object(
- client, "_get_query_results"
- ) as fake_get_results, mock.patch.object(job, "reload") as fake_reload:
- job.done(timeout=42)
-
- # The expected timeout used is the job's own done_timeout minus a
- # fixed amount (bigquery.job._TIMEOUT_BUFFER_SECS) increased by the
- # safety margin on top of server-side processing timeout - that's
- # because that final number is smaller than the given timeout (42 seconds).
- expected_timeout = 8.8 - _TIMEOUT_BUFFER_SECS + _SERVER_TIMEOUT_MARGIN_SECS
-
- fake_get_results.assert_called_once()
- call_args = fake_get_results.call_args
- self.assertAlmostEqual(call_args.kwargs.get("timeout"), expected_timeout)
-
- call_args = fake_reload.call_args
- self.assertAlmostEqual(call_args.kwargs.get("timeout"), expected_timeout)
-
def test_done_w_timeout_and_longer_internal_api_timeout(self):
client = _make_client(project=self.PROJECT)
resource = self._make_resource(ended=False)
@@ -4623,49 +4606,6 @@ def test_result_w_timeout(self):
self.assertEqual(query_request[1]["query_params"]["timeoutMs"], 900)
self.assertEqual(reload_request[1]["method"], "GET")
- @mock.patch("google.api_core.future.polling.PollingFuture.result")
- def test_result_splitting_timout_between_requests(self, polling_result):
- begun_resource = self._make_resource()
- query_resource = {
- "jobComplete": True,
- "jobReference": {"projectId": self.PROJECT, "jobId": self.JOB_ID},
- "schema": {"fields": [{"name": "col1", "type": "STRING"}]},
- "totalRows": "5",
- }
- done_resource = copy.deepcopy(begun_resource)
- done_resource["status"] = {"state": "DONE"}
-
- connection = _make_connection(begun_resource, query_resource, done_resource)
- client = _make_client(project=self.PROJECT, connection=connection)
- job = self._make_one(self.JOB_ID, self.QUERY, client)
-
- client.list_rows = mock.Mock()
-
- with freezegun.freeze_time("1970-01-01 00:00:00", tick=False) as frozen_time:
-
- def delayed_result(*args, **kwargs):
- frozen_time.tick(delta=0.8)
-
- polling_result.side_effect = delayed_result
-
- def delayed_get_results(*args, **kwargs):
- frozen_time.tick(delta=0.5)
- return orig_get_results(*args, **kwargs)
-
- orig_get_results = client._get_query_results
- client._get_query_results = mock.Mock(side_effect=delayed_get_results)
- job.result(timeout=2.0)
-
- polling_result.assert_called_once_with(timeout=2.0)
-
- client._get_query_results.assert_called_once()
- _, kwargs = client._get_query_results.call_args
- self.assertAlmostEqual(kwargs.get("timeout"), 1.2)
-
- client.list_rows.assert_called_once()
- _, kwargs = client.list_rows.call_args
- self.assertAlmostEqual(kwargs.get("timeout"), 0.7)
-
def test_result_w_page_size(self):
# Arrange
query_results_resource = {
@@ -4726,6 +4666,46 @@ def test_result_w_page_size(self):
]
)
+ def test_result_with_start_index(self):
+ from google.cloud.bigquery.table import RowIterator
+
+ query_resource = {
+ "jobComplete": True,
+ "jobReference": {"projectId": self.PROJECT, "jobId": self.JOB_ID},
+ "schema": {"fields": [{"name": "col1", "type": "STRING"}]},
+ "totalRows": "5",
+ }
+ tabledata_resource = {
+ "totalRows": "5",
+ "pageToken": None,
+ "rows": [
+ {"f": [{"v": "abc"}]},
+ {"f": [{"v": "def"}]},
+ {"f": [{"v": "ghi"}]},
+ {"f": [{"v": "jkl"}]},
+ ],
+ }
+ connection = _make_connection(query_resource, tabledata_resource)
+ client = _make_client(self.PROJECT, connection=connection)
+ resource = self._make_resource(ended=True)
+ job = self._get_target_class().from_api_repr(resource, client)
+
+ start_index = 1
+
+ result = job.result(start_index=start_index)
+
+ self.assertIsInstance(result, RowIterator)
+ self.assertEqual(result.total_rows, 5)
+
+ rows = list(result)
+
+ self.assertEqual(len(rows), 4)
+ self.assertEqual(len(connection.api_request.call_args_list), 2)
+ tabledata_list_request = connection.api_request.call_args_list[1]
+ self.assertEqual(
+ tabledata_list_request[1]["query_params"]["startIndex"], start_index
+ )
+
def test_result_error(self):
from google.cloud import exceptions
diff --git a/tests/unit/test_magics.py b/tests/unit/test_magics.py
index 3f66b2c4b..fd9d1d700 100644
--- a/tests/unit/test_magics.py
+++ b/tests/unit/test_magics.py
@@ -800,6 +800,22 @@ def test_bigquery_magic_w_table_id_invalid():
assert "Traceback (most recent call last)" not in output
+def test_bigquery_magic_w_missing_query():
+ ip = IPython.get_ipython()
+ ip.extension_manager.load_extension("google.cloud.bigquery")
+ magics.context._project = None
+
+ cell_body = " \n \n \t\t \n "
+
+ with io.capture_output() as captured_io:
+ ip.run_cell_magic("bigquery", "df", cell_body)
+
+ output = captured_io.stderr
+ assert "Could not save output to variable" in output
+ assert "Query is missing" in output
+ assert "Traceback (most recent call last)" not in output
+
+
@pytest.mark.usefixtures("ipython_interactive")
@pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
def test_bigquery_magic_w_table_id_and_destination_var():
diff --git a/tests/unit/test_schema.py b/tests/unit/test_schema.py
index e1bdd7b2f..9f7ee7bb3 100644
--- a/tests/unit/test_schema.py
+++ b/tests/unit/test_schema.py
@@ -63,11 +63,38 @@ def test_constructor_subfields(self):
self.assertIs(field._fields[0], sub_field1)
self.assertIs(field._fields[1], sub_field2)
+ def test_constructor_with_policy_tags(self):
+ from google.cloud.bigquery.schema import PolicyTagList
+
+ policy = PolicyTagList(names=("foo", "bar"))
+ field = self._make_one(
+ "test", "STRING", mode="REQUIRED", description="Testing", policy_tags=policy
+ )
+ self.assertEqual(field._name, "test")
+ self.assertEqual(field._field_type, "STRING")
+ self.assertEqual(field._mode, "REQUIRED")
+ self.assertEqual(field._description, "Testing")
+ self.assertEqual(field._fields, ())
+ self.assertEqual(field._policy_tags, policy)
+
def test_to_api_repr(self):
- field = self._make_one("foo", "INTEGER", "NULLABLE")
+ from google.cloud.bigquery.schema import PolicyTagList
+
+ policy = PolicyTagList(names=("foo", "bar"))
+ self.assertEqual(
+ policy.to_api_repr(), {"names": ["foo", "bar"]},
+ )
+
+ field = self._make_one("foo", "INTEGER", "NULLABLE", policy_tags=policy)
self.assertEqual(
field.to_api_repr(),
- {"mode": "NULLABLE", "name": "foo", "type": "INTEGER", "description": None},
+ {
+ "mode": "NULLABLE",
+ "name": "foo",
+ "type": "INTEGER",
+ "description": None,
+ "policyTags": {"names": ["foo", "bar"]},
+ },
)
def test_to_api_repr_with_subfield(self):
@@ -111,6 +138,23 @@ def test_from_api_repr(self):
self.assertEqual(field.fields[0].field_type, "INTEGER")
self.assertEqual(field.fields[0].mode, "NULLABLE")
+ def test_from_api_repr_policy(self):
+ field = self._get_target_class().from_api_repr(
+ {
+ "fields": [{"mode": "nullable", "name": "bar", "type": "integer"}],
+ "name": "foo",
+ "type": "record",
+ "policyTags": {"names": ["one", "two"]},
+ }
+ )
+ self.assertEqual(field.name, "foo")
+ self.assertEqual(field.field_type, "RECORD")
+ self.assertEqual(field.policy_tags.names, ("one", "two"))
+ self.assertEqual(len(field.fields), 1)
+ self.assertEqual(field.fields[0].name, "bar")
+ self.assertEqual(field.fields[0].field_type, "INTEGER")
+ self.assertEqual(field.fields[0].mode, "NULLABLE")
+
def test_from_api_repr_defaults(self):
field = self._get_target_class().from_api_repr(
{"name": "foo", "type": "record"}
@@ -408,7 +452,7 @@ def test___hash__not_equals(self):
def test___repr__(self):
field1 = self._make_one("field1", "STRING")
- expected = "SchemaField('field1', 'STRING', 'NULLABLE', None, ())"
+ expected = "SchemaField('field1', 'STRING', 'NULLABLE', None, (), None)"
self.assertEqual(repr(field1), expected)
@@ -632,3 +676,67 @@ def test_valid_mapping_representation(self):
result = self._call_fut(schema)
self.assertEqual(result, expected_schema)
+
+
+class TestPolicyTags(unittest.TestCase):
+ @staticmethod
+ def _get_target_class():
+ from google.cloud.bigquery.schema import PolicyTagList
+
+ return PolicyTagList
+
+ def _make_one(self, *args, **kw):
+ return self._get_target_class()(*args, **kw)
+
+ def test_constructor(self):
+ empty_policy_tags = self._make_one()
+ self.assertIsNotNone(empty_policy_tags.names)
+ self.assertEqual(len(empty_policy_tags.names), 0)
+ policy_tags = self._make_one(["foo", "bar"])
+ self.assertEqual(policy_tags.names, ("foo", "bar"))
+
+ def test_from_api_repr(self):
+ klass = self._get_target_class()
+ api_repr = {"names": ["foo"]}
+ policy_tags = klass.from_api_repr(api_repr)
+ self.assertEqual(policy_tags.to_api_repr(), api_repr)
+
+ # Ensure the None case correctly returns None, rather
+ # than an empty instance.
+ policy_tags2 = klass.from_api_repr(None)
+ self.assertIsNone(policy_tags2)
+
+ def test_to_api_repr(self):
+ taglist = self._make_one(names=["foo", "bar"])
+ self.assertEqual(
+ taglist.to_api_repr(), {"names": ["foo", "bar"]},
+ )
+ taglist2 = self._make_one(names=("foo", "bar"))
+ self.assertEqual(
+ taglist2.to_api_repr(), {"names": ["foo", "bar"]},
+ )
+
+ def test___eq___wrong_type(self):
+ policy = self._make_one(names=["foo"])
+ other = object()
+ self.assertNotEqual(policy, other)
+ self.assertEqual(policy, mock.ANY)
+
+ def test___eq___names_mismatch(self):
+ policy = self._make_one(names=["foo", "bar"])
+ other = self._make_one(names=["bar", "baz"])
+ self.assertNotEqual(policy, other)
+
+ def test___hash__set_equality(self):
+ policy1 = self._make_one(["foo", "bar"])
+ policy2 = self._make_one(["bar", "baz"])
+ set_one = {policy1, policy2}
+ set_two = {policy1, policy2}
+ self.assertEqual(set_one, set_two)
+
+ def test___hash__not_equals(self):
+ policy1 = self._make_one(["foo", "bar"])
+ policy2 = self._make_one(["bar", "baz"])
+ set_one = {policy1}
+ set_two = {policy2}
+ self.assertNotEqual(set_one, set_two)
diff --git a/tests/unit/test_table.py b/tests/unit/test_table.py
index 079ec6e00..72275fc53 100644
--- a/tests/unit/test_table.py
+++ b/tests/unit/test_table.py
@@ -855,6 +855,29 @@ def test_from_api_repr_w_properties(self):
table = klass.from_api_repr(RESOURCE)
self._verifyResourceProperties(table, RESOURCE)
+ def test_from_api_repr_w_partial_streamingbuffer(self):
+ import datetime
+ from google.cloud._helpers import UTC
+ from google.cloud._helpers import _millis
+
+ RESOURCE = self._make_resource()
+ self.OLDEST_TIME = datetime.datetime(2015, 8, 1, 23, 59, 59, tzinfo=UTC)
+ RESOURCE["streamingBuffer"] = {"oldestEntryTime": _millis(self.OLDEST_TIME)}
+ klass = self._get_target_class()
+ table = klass.from_api_repr(RESOURCE)
+ self.assertIsNotNone(table.streaming_buffer)
+ self.assertIsNone(table.streaming_buffer.estimated_rows)
+ self.assertIsNone(table.streaming_buffer.estimated_bytes)
+ self.assertEqual(table.streaming_buffer.oldest_entry_time, self.OLDEST_TIME)
+ # Another partial construction
+ RESOURCE["streamingBuffer"] = {"estimatedRows": 1}
+ klass = self._get_target_class()
+ table = klass.from_api_repr(RESOURCE)
+ self.assertIsNotNone(table.streaming_buffer)
+ self.assertEqual(table.streaming_buffer.estimated_rows, 1)
+ self.assertIsNone(table.streaming_buffer.estimated_bytes)
+ self.assertIsNone(table.streaming_buffer.oldest_entry_time)
+
def test_from_api_with_encryption(self):
self._setUpConstants()
RESOURCE = {
@@ -1001,11 +1024,11 @@ def test_time_partitioning_setter(self):
dataset = DatasetReference(self.PROJECT, self.DS_ID)
table_ref = dataset.table(self.TABLE_NAME)
table = self._make_one(table_ref)
- time_partitioning = TimePartitioning(type_=TimePartitioningType.DAY)
+ time_partitioning = TimePartitioning(type_=TimePartitioningType.HOUR)
table.time_partitioning = time_partitioning
- self.assertEqual(table.time_partitioning.type_, TimePartitioningType.DAY)
+ self.assertEqual(table.time_partitioning.type_, TimePartitioningType.HOUR)
# Both objects point to the same properties dict
self.assertIs(
table._properties["timePartitioning"], time_partitioning._properties
@@ -2216,6 +2239,38 @@ def test_to_dataframe(self):
self.assertEqual(df.name.dtype.name, "object")
self.assertEqual(df.age.dtype.name, "int64")
+ @unittest.skipIf(pandas is None, "Requires `pandas`")
+ def test_to_dataframe_warning_wo_pyarrow(self):
+ from google.cloud.bigquery.client import PyarrowMissingWarning
+ from google.cloud.bigquery.schema import SchemaField
+
+ schema = [
+ SchemaField("name", "STRING", mode="REQUIRED"),
+ SchemaField("age", "INTEGER", mode="REQUIRED"),
+ ]
+ rows = [
+ {"f": [{"v": "Phred Phlyntstone"}, {"v": "32"}]},
+ {"f": [{"v": "Bharney Rhubble"}, {"v": "33"}]},
+ ]
+ path = "/foo"
+ api_request = mock.Mock(return_value={"rows": rows})
+ row_iterator = self._make_one(_mock_client(), api_request, path, schema)
+
+ no_pyarrow_patch = mock.patch("google.cloud.bigquery.table.pyarrow", new=None)
+ catch_warnings = warnings.catch_warnings(record=True)
+
+ with no_pyarrow_patch, catch_warnings as warned:
+ df = row_iterator.to_dataframe()
+
+ self.assertIsInstance(df, pandas.DataFrame)
+ self.assertEqual(len(df), 2)
+ matches = [
+ warning for warning in warned if warning.category is PyarrowMissingWarning
+ ]
+ self.assertTrue(
+ matches, msg="A missing pyarrow deprecation warning was not raised."
+ )
+
@unittest.skipIf(pandas is None, "Requires `pandas`")
@unittest.skipIf(tqdm is None, "Requires `tqdm`")
@mock.patch("tqdm.tqdm_gui")