From ba76e7a425ba011526d9e85e70bf97c7376a887a Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Fri, 9 Jun 2023 10:47:44 -0400 Subject: [PATCH 01/15] chore(main): release 2.22.5-SNAPSHOT (#2056) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- gapic-google-cloud-storage-v2/pom.xml | 4 ++-- google-cloud-storage-bom/pom.xml | 10 +++++----- google-cloud-storage/pom.xml | 4 ++-- grpc-google-cloud-storage-v2/pom.xml | 4 ++-- pom.xml | 10 +++++----- proto-google-cloud-storage-v2/pom.xml | 4 ++-- samples/snapshot/pom.xml | 2 +- versions.txt | 8 ++++---- 8 files changed, 23 insertions(+), 23 deletions(-) diff --git a/gapic-google-cloud-storage-v2/pom.xml b/gapic-google-cloud-storage-v2/pom.xml index 481ed13b27..e8aa83e0f1 100644 --- a/gapic-google-cloud-storage-v2/pom.xml +++ b/gapic-google-cloud-storage-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc gapic-google-cloud-storage-v2 - 2.22.4-alpha + 2.22.5-alpha-SNAPSHOT gapic-google-cloud-storage-v2 GRPC library for gapic-google-cloud-storage-v2 com.google.cloud google-cloud-storage-parent - 2.22.4 + 2.22.5-SNAPSHOT diff --git a/google-cloud-storage-bom/pom.xml b/google-cloud-storage-bom/pom.xml index 7c8cfa7a51..356eca8e26 100644 --- a/google-cloud-storage-bom/pom.xml +++ b/google-cloud-storage-bom/pom.xml @@ -19,7 +19,7 @@ 4.0.0 com.google.cloud google-cloud-storage-bom - 2.22.4 + 2.22.5-SNAPSHOT pom com.google.cloud @@ -69,22 +69,22 @@ com.google.cloud google-cloud-storage - 2.22.4 + 2.22.5-SNAPSHOT com.google.api.grpc gapic-google-cloud-storage-v2 - 2.22.4-alpha + 2.22.5-alpha-SNAPSHOT com.google.api.grpc grpc-google-cloud-storage-v2 - 2.22.4-alpha + 2.22.5-alpha-SNAPSHOT com.google.api.grpc proto-google-cloud-storage-v2 - 2.22.4-alpha + 2.22.5-alpha-SNAPSHOT diff --git a/google-cloud-storage/pom.xml b/google-cloud-storage/pom.xml index f33742167c..825810ef27 100644 --- a/google-cloud-storage/pom.xml +++ b/google-cloud-storage/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-storage - 2.22.4 + 2.22.5-SNAPSHOT jar Google Cloud Storage https://github.com/googleapis/java-storage @@ -12,7 +12,7 @@ com.google.cloud google-cloud-storage-parent - 2.22.4 + 2.22.5-SNAPSHOT google-cloud-storage diff --git a/grpc-google-cloud-storage-v2/pom.xml b/grpc-google-cloud-storage-v2/pom.xml index 6f7566757e..1a6e14178f 100644 --- a/grpc-google-cloud-storage-v2/pom.xml +++ b/grpc-google-cloud-storage-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-storage-v2 - 2.22.4-alpha + 2.22.5-alpha-SNAPSHOT grpc-google-cloud-storage-v2 GRPC library for grpc-google-cloud-storage-v2 com.google.cloud google-cloud-storage-parent - 2.22.4 + 2.22.5-SNAPSHOT diff --git a/pom.xml b/pom.xml index f219d8441a..c5c6b54852 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-storage-parent pom - 2.22.4 + 2.22.5-SNAPSHOT Storage Parent https://github.com/googleapis/java-storage @@ -70,7 +70,7 @@ com.google.cloud google-cloud-storage - 2.22.4 + 2.22.5-SNAPSHOT com.google.apis @@ -111,17 +111,17 @@ com.google.api.grpc proto-google-cloud-storage-v2 - 2.22.4-alpha + 2.22.5-alpha-SNAPSHOT com.google.api.grpc grpc-google-cloud-storage-v2 - 2.22.4-alpha + 2.22.5-alpha-SNAPSHOT com.google.api.grpc gapic-google-cloud-storage-v2 - 2.22.4-alpha + 2.22.5-alpha-SNAPSHOT com.google.cloud diff --git a/proto-google-cloud-storage-v2/pom.xml b/proto-google-cloud-storage-v2/pom.xml index 5ea0d78d04..dc5d1647b8 100644 --- a/proto-google-cloud-storage-v2/pom.xml +++ b/proto-google-cloud-storage-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-storage-v2 - 2.22.4-alpha + 2.22.5-alpha-SNAPSHOT proto-google-cloud-storage-v2 PROTO library for proto-google-cloud-storage-v2 com.google.cloud google-cloud-storage-parent - 2.22.4 + 2.22.5-SNAPSHOT diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 4972a97043..84146d2a0e 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -28,7 +28,7 @@ com.google.cloud google-cloud-storage - 2.22.4 + 2.22.5-SNAPSHOT diff --git a/versions.txt b/versions.txt index 1328807ea6..84a23716b7 100644 --- a/versions.txt +++ b/versions.txt @@ -1,7 +1,7 @@ # Format: # module:released-version:current-version -google-cloud-storage:2.22.4:2.22.4 -gapic-google-cloud-storage-v2:2.22.4-alpha:2.22.4-alpha -grpc-google-cloud-storage-v2:2.22.4-alpha:2.22.4-alpha -proto-google-cloud-storage-v2:2.22.4-alpha:2.22.4-alpha +google-cloud-storage:2.22.4:2.22.5-SNAPSHOT +gapic-google-cloud-storage-v2:2.22.4-alpha:2.22.5-alpha-SNAPSHOT +grpc-google-cloud-storage-v2:2.22.4-alpha:2.22.5-alpha-SNAPSHOT +proto-google-cloud-storage-v2:2.22.4-alpha:2.22.5-alpha-SNAPSHOT From 29bf80da94258d43d15e449f4d736b510d8c899c Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Mon, 12 Jun 2023 17:33:07 +0200 Subject: [PATCH 02/15] chore(deps): update dependency com.google.cloud:google-cloud-storage to v2.22.4 (#2057) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update dependency com.google.cloud:google-cloud-storage to v2.22.4 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot --- README.md | 8 ++++---- samples/install-without-bom/pom.xml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index af4fd1c4de..699cb55fbd 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ If you are using Maven without the BOM, add this to your dependencies: com.google.cloud google-cloud-storage - 2.22.3 + 2.22.4 ``` @@ -57,13 +57,13 @@ implementation 'com.google.cloud:google-cloud-storage' If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation 'com.google.cloud:google-cloud-storage:2.22.3' +implementation 'com.google.cloud:google-cloud-storage:2.22.4' ``` If you are using SBT, add this to your dependencies: ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-storage" % "2.22.3" +libraryDependencies += "com.google.cloud" % "google-cloud-storage" % "2.22.4" ``` @@ -426,7 +426,7 @@ Java is a registered trademark of Oracle and/or its affiliates. [kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-storage/java11.html [stability-image]: https://img.shields.io/badge/stability-stable-green [maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-storage.svg -[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-storage/2.22.3 +[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-storage/2.22.4 [authentication]: https://github.com/googleapis/google-cloud-java#authentication [auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes [predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index cc9e1e719a..0e4b6ed0ff 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -30,7 +30,7 @@ com.google.cloud google-cloud-storage - 2.22.3 + 2.22.4 From 01cc33ad5ddc3baf4fc03cb1f409b8df5c2a5a22 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Mon, 12 Jun 2023 18:04:28 +0200 Subject: [PATCH 03/15] test(deps): update cross product test dependencies (#2058) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(deps): update cross product test dependencies * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot --- google-cloud-storage/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google-cloud-storage/pom.xml b/google-cloud-storage/pom.xml index 825810ef27..0054cd41a2 100644 --- a/google-cloud-storage/pom.xml +++ b/google-cloud-storage/pom.xml @@ -173,13 +173,13 @@ com.google.api.grpc proto-google-cloud-kms-v1 - 0.112.0 + 0.113.0 test com.google.cloud google-cloud-kms - 2.21.0 + 2.22.0 test From ebe389c65150cab8d5e468dd6bbe4055df623b70 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Tue, 13 Jun 2023 17:08:13 +0200 Subject: [PATCH 04/15] test(deps): update cross product test dependencies (#2061) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [![Mend Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Change | Age | Adoption | Passing | Confidence | |---|---|---|---|---|---| | [com.google.cloud:google-cloud-pubsub](https://togithub.com/googleapis/java-pubsub) | `1.123.13` -> `1.123.14` | [![age](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-pubsub/1.123.14/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-pubsub/1.123.14/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-pubsub/1.123.14/compatibility-slim/1.123.13)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-pubsub/1.123.14/confidence-slim/1.123.13)](https://docs.renovatebot.com/merge-confidence/) | | [com.google.api.grpc:proto-google-cloud-pubsub-v1](https://togithub.com/googleapis/java-pubsub/proto-google-cloud-pubsub-v1) ([source](https://togithub.com/googleapis/java-pubsub)) | `1.105.13` -> `1.105.14` | [![age](https://badges.renovateapi.com/packages/maven/com.google.api.grpc:proto-google-cloud-pubsub-v1/1.105.14/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/maven/com.google.api.grpc:proto-google-cloud-pubsub-v1/1.105.14/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/maven/com.google.api.grpc:proto-google-cloud-pubsub-v1/1.105.14/compatibility-slim/1.105.13)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/maven/com.google.api.grpc:proto-google-cloud-pubsub-v1/1.105.14/confidence-slim/1.105.13)](https://docs.renovatebot.com/merge-confidence/) | --- ### Release Notes
googleapis/java-pubsub ### [`v1.123.14`](https://togithub.com/googleapis/java-pubsub/blob/HEAD/CHANGELOG.md#​112314-httpsgithubcomgoogleapisjava-pubsubcomparev112313v112314-2023-06-12) [Compare Source](https://togithub.com/googleapis/java-pubsub/compare/v1.123.13...v1.123.14) ##### Dependencies - Update dependency com.google.cloud:google-cloud-core to v2.19.0 ([#​1604](https://togithub.com/googleapis/java-pubsub/issues/1604)) ([7ac609e](https://togithub.com/googleapis/java-pubsub/commit/7ac609e44c20db73460fe39919439c3a2b597454)) - Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.11.0 ([#​1605](https://togithub.com/googleapis/java-pubsub/issues/1605)) ([077ac04](https://togithub.com/googleapis/java-pubsub/commit/077ac04214be23f6693734c157925a5607ada869))
--- ### Configuration 📅 **Schedule**: Branch creation - At any time (no schedule defined), Automerge - At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 👻 **Immortal**: This PR will be recreated if closed unmerged. Get [config help](https://togithub.com/renovatebot/renovate/discussions) if that's undesired. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [Mend Renovate](https://www.mend.io/free-developer-tools/renovate/). View repository job log [here](https://app.renovatebot.com/dashboard#github/googleapis/java-storage). --- google-cloud-storage/pom.xml | 2 +- pom.xml | 2 +- samples/install-without-bom/pom.xml | 2 +- samples/native-image-sample/pom.xml | 2 +- samples/snapshot/pom.xml | 2 +- samples/snippets/pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/google-cloud-storage/pom.xml b/google-cloud-storage/pom.xml index 0054cd41a2..aa16af16d1 100644 --- a/google-cloud-storage/pom.xml +++ b/google-cloud-storage/pom.xml @@ -16,7 +16,7 @@ google-cloud-storage - 1.105.13 + 1.105.14 5.9.3 diff --git a/pom.xml b/pom.xml index c5c6b54852..b744479106 100644 --- a/pom.xml +++ b/pom.xml @@ -80,7 +80,7 @@ com.google.cloud google-cloud-pubsub - 1.123.13 + 1.123.14 test diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index 0e4b6ed0ff..6ad8b2bbb0 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -61,7 +61,7 @@ com.google.cloud google-cloud-pubsub - 1.123.13 + 1.123.14 test diff --git a/samples/native-image-sample/pom.xml b/samples/native-image-sample/pom.xml index 30a696b6d7..db4429ec91 100644 --- a/samples/native-image-sample/pom.xml +++ b/samples/native-image-sample/pom.xml @@ -61,7 +61,7 @@ com.google.cloud google-cloud-pubsub - 1.123.13 + 1.123.14 test diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 84146d2a0e..df4f78504c 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -52,7 +52,7 @@ com.google.cloud google-cloud-pubsub - 1.123.13 + 1.123.14 test diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index 6f9ab79386..57af5c9f26 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -72,7 +72,7 @@ com.google.cloud google-cloud-pubsub - 1.123.13 + 1.123.14 test From c57464c6ce11b7d9283c2c9938c6149d0afdce99 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Wed, 14 Jun 2023 21:36:43 +0200 Subject: [PATCH 05/15] chore(deps): update dependency com.google.cloud:libraries-bom to v26.17.0 (#2065) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update dependency com.google.cloud:libraries-bom to v26.17.0 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot --- README.md | 4 ++-- samples/native-image-sample/pom.xml | 2 +- samples/snippets/pom.xml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 699cb55fbd..1b1d586b02 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: com.google.cloud libraries-bom - 26.16.0 + 26.17.0 pom import @@ -50,7 +50,7 @@ If you are using Maven without the BOM, add this to your dependencies: If you are using Gradle 5.x or later, add this to your dependencies: ```Groovy -implementation platform('com.google.cloud:libraries-bom:26.16.0') +implementation platform('com.google.cloud:libraries-bom:26.17.0') implementation 'com.google.cloud:google-cloud-storage' ``` diff --git a/samples/native-image-sample/pom.xml b/samples/native-image-sample/pom.xml index db4429ec91..af1943d5c9 100644 --- a/samples/native-image-sample/pom.xml +++ b/samples/native-image-sample/pom.xml @@ -29,7 +29,7 @@ com.google.cloud libraries-bom - 26.16.0 + 26.17.0 pom import diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index 57af5c9f26..95f581a758 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -31,7 +31,7 @@ com.google.cloud libraries-bom - 26.16.0 + 26.17.0 pom import From e48862ab5d9a044d9b5d4c9e8465ff7d6d35efe4 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 16 Jun 2023 11:07:56 -0400 Subject: [PATCH 06/15] chore: Update `dependabot.yml` template (#1813) (#2073) * chore: Update `dependabot.yml` template not to touch pip dependencies Source-Link: https://github.com/googleapis/synthtool/commit/f961eb0fe51109238128055897ccba1b70dbd804 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-java:latest@sha256:af2eda87a54601ae7b7b2be5055c17b43ac98a7805b586772db314de8a7d4a1d Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 3 ++- .github/dependabot.yml | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index aadf54f643..73568a1e99 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,4 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-java:latest - digest: sha256:ad9cabee4c022f1aab04a71332369e0c23841062124818a4490f73337f790337 + digest: sha256:af2eda87a54601ae7b7b2be5055c17b43ac98a7805b586772db314de8a7d4a1d +# created: 2023-06-16T02:10:09.149325782Z diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c8f413b0da..fde1ced49f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,10 +5,13 @@ updates: schedule: interval: "daily" # Disable version updates for Maven dependencies - open-pull-requests-limit: 0 + # we use renovate-bot as well as shared-dependencies BOM to update maven dependencies. + ignore: "*" - package-ecosystem: "pip" directory: "/" schedule: interval: "daily" # Disable version updates for pip dependencies - open-pull-requests-limit: 0 \ No newline at end of file + # If a security vulnerability comes in, we will be notified about + # it via template in the synthtool repository. + ignore: "*" From 2ad196c063e67c7efdac344792b67de3479d789d Mon Sep 17 00:00:00 2001 From: BenWhitehead Date: Fri, 16 Jun 2023 11:13:01 -0400 Subject: [PATCH 07/15] feat: add new dedup utility method to Option classes (#2063) When providing options duplicates are not allowed, however there are times when you might end up providing two options which would collide. These new utility methods encapsulate the logic necessary to deduplicate any overlap wil returing an array which can be passed directly to the respective operation method. `dedupe` utility methods added to each of the following Option classes: * Storage.BlobGetOption.dedupe(); * Storage.BlobListOption.dedupe(); * Storage.BlobSourceOption.dedupe(); * Storage.BlobTargetOption.dedupe(); * Storage.BlobWriteOption.dedupe(); * Storage.BucketGetOption.dedupe(); * Storage.BucketListOption.dedupe(); * Storage.BucketSourceOption.dedupe(); * Storage.BucketTargetOption.dedupe(); * Storage.CreateHmacKeyOption.dedupe(); * Storage.DeleteHmacKeyOption.dedupe(); * Storage.GetHmacKeyOption.dedupe(); * Storage.ListHmacKeysOption.dedupe(); * Storage.UpdateHmacKeyOption.dedupe(); * Bucket.BlobTargetOption.dedupe(); * Bucket.BlobWriteOption.dedupe(); * Bucket.BucketSourceOption.dedupe(); * Blob.BlobSourceOption.dedupe(); There are overloads which accept a collection or array in addition to varargs. --- .../java/com/google/cloud/storage/Blob.java | 35 ++ .../java/com/google/cloud/storage/Bucket.java | 107 +++- .../java/com/google/cloud/storage/Option.java | 33 ++ .../com/google/cloud/storage/Storage.java | 484 +++++++++++++++++- .../cloud/storage/it/DedupeOptionTest.java | 163 ++++++ 5 files changed, 818 insertions(+), 4 deletions(-) create mode 100644 google-cloud-storage/src/test/java/com/google/cloud/storage/it/DedupeOptionTest.java diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java index 15d3c489fc..4bba8aef9c 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java @@ -40,6 +40,7 @@ import java.security.Key; import java.time.OffsetDateTime; import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Objects; @@ -163,6 +164,40 @@ public static BlobSourceOption shouldReturnRawInputStream(boolean shouldReturnRa return new BlobSourceOption(UnifiedOpts.returnRawInputStream(shouldReturnRawInputStream)); } + /** + * Deduplicate any options which are the same parameter. The value which comes last in {@code + * os} will be the value included in the return. + */ + @BetaApi + public static BlobSourceOption[] dedupe(BlobSourceOption... os) { + return Option.dedupe(BlobSourceOption[]::new, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static BlobSourceOption[] dedupe( + Collection collection, BlobSourceOption... os) { + return Option.dedupe(BlobSourceOption[]::new, collection, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static BlobSourceOption[] dedupe(BlobSourceOption[] array, BlobSourceOption... os) { + return Option.dedupe(BlobSourceOption[]::new, array, os); + } + static Storage.BlobSourceOption[] toSourceOptions( BlobInfo blobInfo, BlobSourceOption... options) { Storage.BlobSourceOption[] convertedOptions = new Storage.BlobSourceOption[options.length]; diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java index af5c2cc255..c8827c1fac 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java @@ -30,7 +30,6 @@ import com.google.cloud.storage.UnifiedOpts.BucketSourceOpt; import com.google.cloud.storage.UnifiedOpts.ObjectOptExtractor; import com.google.cloud.storage.UnifiedOpts.ObjectTargetOpt; -import com.google.cloud.storage.UnifiedOpts.OptionShim; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import java.io.IOException; @@ -41,6 +40,7 @@ import java.time.Duration; import java.time.OffsetDateTime; import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Objects; @@ -99,6 +99,41 @@ public static BucketSourceOption userProject(@NonNull String userProject) { return new BucketSourceOption(UnifiedOpts.userProject(userProject)); } + /** + * Deduplicate any options which are the same parameter. The value which comes last in {@code + * os} will be the value included in the return. + */ + @BetaApi + public static BucketSourceOption[] dedupe(BucketSourceOption... os) { + return Option.dedupe(BucketSourceOption[]::new, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static BucketSourceOption[] dedupe( + Collection collection, BucketSourceOption... os) { + return Option.dedupe(BucketSourceOption[]::new, collection, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static BucketSourceOption[] dedupe( + BucketSourceOption[] array, BucketSourceOption... os) { + return Option.dedupe(BucketSourceOption[]::new, array, os); + } + static Storage.BucketSourceOption[] toSourceOptions( BucketInfo bucketInfo, BucketSourceOption... options) { Storage.BucketSourceOption[] convertedOptions = @@ -237,6 +272,40 @@ public static BlobTargetOption userProject(@NonNull String userProject) { return new BlobTargetOption(UnifiedOpts.userProject(userProject)); } + /** + * Deduplicate any options which are the same parameter. The value which comes last in {@code + * os} will be the value included in the return. + */ + @BetaApi + public static BlobTargetOption[] dedupe(BlobTargetOption... os) { + return Option.dedupe(BlobTargetOption[]::new, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static BlobTargetOption[] dedupe( + Collection collection, BlobTargetOption... os) { + return Option.dedupe(BlobTargetOption[]::new, collection, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static BlobTargetOption[] dedupe(BlobTargetOption[] array, BlobTargetOption... os) { + return Option.dedupe(BlobTargetOption[]::new, array, os); + } + static Storage.BlobTargetOption[] toTargetOptions( BlobInfo blobInfo, BlobTargetOption... options) { Storage.BlobTargetOption[] targetOptions = new Storage.BlobTargetOption[options.length]; @@ -255,7 +324,7 @@ static Storage.BlobTargetOption[] toTargetOptions( } /** Class for specifying blob write options when {@code Bucket} methods are used. */ - public static class BlobWriteOption extends OptionShim implements Serializable { + public static class BlobWriteOption extends Option implements Serializable { private static final long serialVersionUID = 59762268190041584L; @@ -366,6 +435,40 @@ public static BlobWriteOption userProject(@NonNull String userProject) { return new BlobWriteOption(UnifiedOpts.userProject(userProject)); } + /** + * Deduplicate any options which are the same parameter. The value which comes last in {@code + * os} will be the value included in the return. + */ + @BetaApi + public static BlobWriteOption[] dedupe(BlobWriteOption... os) { + return Option.dedupe(BlobWriteOption[]::new, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static BlobWriteOption[] dedupe( + Collection collection, BlobWriteOption... os) { + return Option.dedupe(BlobWriteOption[]::new, collection, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static BlobWriteOption[] dedupe(BlobWriteOption[] array, BlobWriteOption... os) { + return Option.dedupe(BlobWriteOption[]::new, array, os); + } + static Storage.BlobWriteOption[] toWriteOptions(BlobInfo blobInfo, BlobWriteOption... options) { Storage.BlobWriteOption[] convertedOptions = new Storage.BlobWriteOption[options.length]; for (int i = 0; i < options.length; i++) { diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Option.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Option.java index 5fb7479c88..e526486ef3 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Option.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Option.java @@ -18,6 +18,11 @@ import com.google.cloud.storage.UnifiedOpts.Opt; import java.io.Serializable; +import java.util.Arrays; +import java.util.Collection; +import java.util.function.IntFunction; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** Base class for Storage operation option. */ @Deprecated @@ -29,4 +34,32 @@ public abstract class Option extends UnifiedOpts.OptionShim Option(O opt) { super(opt); } + + @SafeVarargs + static > O[] dedupe(IntFunction gen, O... os) { + return dedupe(gen, Arrays.stream(os)); + } + + @SafeVarargs + static > O[] dedupe(IntFunction gen, Collection collection, O... os) { + return dedupe(gen, Stream.of(collection.stream(), Arrays.stream(os)).flatMap(s -> s)); + } + + @SafeVarargs + static > O[] dedupe(IntFunction gen, O[] array, O... os) { + return dedupe(gen, Stream.of(Arrays.stream(array), Arrays.stream(os)).flatMap(s -> s)); + } + + /** + * All Options contain an {@link Opt}, {@code Opt}s are distinct classes allowing us to group + * based on those classes. Once grouped, we select the last element to provide last wins behavior. + * + *

Each of these helpers is an internal implementation detail, primarily due to the fact that + * generic arrays can not be instantiated in Java and requires a factory to be passed in. + */ + private static > O[] dedupe(IntFunction gen, Stream s) { + return s.collect(Collectors.groupingBy(o -> o.getOpt().getClass())).values().stream() + .map(l -> l.get(l.size() - 1)) + .toArray(gen); + } } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index d1838985a8..63a0793cfe 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -45,7 +45,6 @@ import com.google.cloud.storage.UnifiedOpts.ObjectListOpt; import com.google.cloud.storage.UnifiedOpts.ObjectSourceOpt; import com.google.cloud.storage.UnifiedOpts.ObjectTargetOpt; -import com.google.cloud.storage.UnifiedOpts.OptionShim; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; @@ -59,6 +58,7 @@ import java.nio.file.Path; import java.security.Key; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.LinkedHashSet; import java.util.LinkedList; @@ -358,6 +358,39 @@ public static BucketTargetOption userProject(@NonNull String userProject) { public static BucketTargetOption projection(@NonNull String projection) { return new BucketTargetOption(UnifiedOpts.projection(projection)); } + + /** + * Deduplicate any options which are the same parameter. The value which comes last in {@code + * os} will be the value included in the return. + */ + @BetaApi + public static BucketTargetOption[] dedupe(BucketTargetOption... os) { + return Option.dedupe(BucketTargetOption[]::new, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + public static BucketTargetOption[] dedupe( + Collection collection, BucketTargetOption... os) { + return Option.dedupe(BucketTargetOption[]::new, collection, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + public static BucketTargetOption[] dedupe( + BucketTargetOption[] array, BucketTargetOption... os) { + return Option.dedupe(BucketTargetOption[]::new, array, os); + } } /** Class for specifying bucket source options. */ @@ -400,6 +433,41 @@ public static BucketSourceOption userProject(@NonNull String userProject) { public static BucketSourceOption requestedPolicyVersion(long version) { return new BucketSourceOption(UnifiedOpts.requestedPolicyVersion(version)); } + + /** + * Deduplicate any options which are the same parameter. The value which comes last in {@code + * os} will be the value included in the return. + */ + @BetaApi + public static BucketSourceOption[] dedupe(BucketSourceOption... os) { + return Option.dedupe(BucketSourceOption[]::new, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static BucketSourceOption[] dedupe( + Collection collection, BucketSourceOption... os) { + return Option.dedupe(BucketSourceOption[]::new, collection, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static BucketSourceOption[] dedupe( + BucketSourceOption[] array, BucketSourceOption... os) { + return Option.dedupe(BucketSourceOption[]::new, array, os); + } } /** Class for specifying listHmacKeys options */ @@ -456,6 +524,41 @@ public static ListHmacKeysOption userProject(@NonNull String userProject) { public static ListHmacKeysOption projectId(@NonNull String projectId) { return new ListHmacKeysOption(UnifiedOpts.projectId(projectId)); } + + /** + * Deduplicate any options which are the same parameter. The value which comes last in {@code + * os} will be the value included in the return. + */ + @BetaApi + public static ListHmacKeysOption[] dedupe(ListHmacKeysOption... os) { + return Option.dedupe(ListHmacKeysOption[]::new, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static ListHmacKeysOption[] dedupe( + Collection collection, ListHmacKeysOption... os) { + return Option.dedupe(ListHmacKeysOption[]::new, collection, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static ListHmacKeysOption[] dedupe( + ListHmacKeysOption[] array, ListHmacKeysOption... os) { + return Option.dedupe(ListHmacKeysOption[]::new, array, os); + } } /** Class for specifying createHmacKey options */ @@ -482,6 +585,41 @@ public static CreateHmacKeyOption userProject(@NonNull String userProject) { public static CreateHmacKeyOption projectId(@NonNull String projectId) { return new CreateHmacKeyOption(UnifiedOpts.projectId(projectId)); } + + /** + * Deduplicate any options which are the same parameter. The value which comes last in {@code + * os} will be the value included in the return. + */ + @BetaApi + public static CreateHmacKeyOption[] dedupe(CreateHmacKeyOption... os) { + return Option.dedupe(CreateHmacKeyOption[]::new, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static CreateHmacKeyOption[] dedupe( + Collection collection, CreateHmacKeyOption... os) { + return Option.dedupe(CreateHmacKeyOption[]::new, collection, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static CreateHmacKeyOption[] dedupe( + CreateHmacKeyOption[] array, CreateHmacKeyOption... os) { + return Option.dedupe(CreateHmacKeyOption[]::new, array, os); + } } /** Class for specifying getHmacKey options */ @@ -507,6 +645,40 @@ public static GetHmacKeyOption userProject(@NonNull String userProject) { public static GetHmacKeyOption projectId(@NonNull String projectId) { return new GetHmacKeyOption(UnifiedOpts.projectId(projectId)); } + + /** + * Deduplicate any options which are the same parameter. The value which comes last in {@code + * os} will be the value included in the return. + */ + @BetaApi + public static GetHmacKeyOption[] dedupe(GetHmacKeyOption... os) { + return Option.dedupe(GetHmacKeyOption[]::new, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static GetHmacKeyOption[] dedupe( + Collection collection, GetHmacKeyOption... os) { + return Option.dedupe(GetHmacKeyOption[]::new, collection, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static GetHmacKeyOption[] dedupe(GetHmacKeyOption[] array, GetHmacKeyOption... os) { + return Option.dedupe(GetHmacKeyOption[]::new, array, os); + } } /** Class for specifying deleteHmacKey options */ @@ -523,6 +695,41 @@ private DeleteHmacKeyOption(HmacKeyTargetOpt opt) { public static DeleteHmacKeyOption userProject(@NonNull String userProject) { return new DeleteHmacKeyOption(UnifiedOpts.userProject(userProject)); } + + /** + * Deduplicate any options which are the same parameter. The value which comes last in {@code + * os} will be the value included in the return. + */ + @BetaApi + public static DeleteHmacKeyOption[] dedupe(DeleteHmacKeyOption... os) { + return Option.dedupe(DeleteHmacKeyOption[]::new, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static DeleteHmacKeyOption[] dedupe( + Collection collection, DeleteHmacKeyOption... os) { + return Option.dedupe(DeleteHmacKeyOption[]::new, collection, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static DeleteHmacKeyOption[] dedupe( + DeleteHmacKeyOption[] array, DeleteHmacKeyOption... os) { + return Option.dedupe(DeleteHmacKeyOption[]::new, array, os); + } } /** Class for specifying updateHmacKey options */ @@ -539,6 +746,41 @@ private UpdateHmacKeyOption(HmacKeyTargetOpt opt) { public static UpdateHmacKeyOption userProject(@NonNull String userProject) { return new UpdateHmacKeyOption(UnifiedOpts.userProject(userProject)); } + + /** + * Deduplicate any options which are the same parameter. The value which comes last in {@code + * os} will be the value included in the return. + */ + @BetaApi + public static UpdateHmacKeyOption[] dedupe(UpdateHmacKeyOption... os) { + return Option.dedupe(UpdateHmacKeyOption[]::new, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static UpdateHmacKeyOption[] dedupe( + Collection collection, UpdateHmacKeyOption... os) { + return Option.dedupe(UpdateHmacKeyOption[]::new, collection, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static UpdateHmacKeyOption[] dedupe( + UpdateHmacKeyOption[] array, UpdateHmacKeyOption... os) { + return Option.dedupe(UpdateHmacKeyOption[]::new, array, os); + } } /** Class for specifying bucket get options. */ @@ -593,6 +835,40 @@ public static BucketGetOption fields(BucketField... fields) { .build(); return new BucketGetOption(UnifiedOpts.fields(set)); } + + /** + * Deduplicate any options which are the same parameter. The value which comes last in {@code + * os} will be the value included in the return. + */ + @BetaApi + public static BucketGetOption[] dedupe(BucketGetOption... os) { + return Option.dedupe(BucketGetOption[]::new, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static BucketGetOption[] dedupe( + Collection collection, BucketGetOption... os) { + return Option.dedupe(BucketGetOption[]::new, collection, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static BucketGetOption[] dedupe(BucketGetOption[] array, BucketGetOption... os) { + return Option.dedupe(BucketGetOption[]::new, array, os); + } } /** Class for specifying blob target options. */ @@ -743,10 +1019,44 @@ public static BlobTargetOption encryptionKey(@NonNull String key) { public static BlobTargetOption kmsKeyName(@NonNull String kmsKeyName) { return new BlobTargetOption(UnifiedOpts.kmsKeyName(kmsKeyName)); } + + /** + * Deduplicate any options which are the same parameter. The value which comes last in {@code + * os} will be the value included in the return. + */ + @BetaApi + public static BlobTargetOption[] dedupe(BlobTargetOption... os) { + return Option.dedupe(BlobTargetOption[]::new, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static BlobTargetOption[] dedupe( + Collection collection, BlobTargetOption... os) { + return Option.dedupe(BlobTargetOption[]::new, collection, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static BlobTargetOption[] dedupe(BlobTargetOption[] array, BlobTargetOption... os) { + return Option.dedupe(BlobTargetOption[]::new, array, os); + } } /** Class for specifying blob write options. */ - class BlobWriteOption extends OptionShim implements Serializable { + class BlobWriteOption extends Option implements Serializable { private static final long serialVersionUID = 5536338021856320475L; @@ -921,6 +1231,40 @@ public static BlobWriteOption disableGzipContent() { public static BlobWriteOption detectContentType() { return new BlobWriteOption(UnifiedOpts.detectContentType()); } + + /** + * Deduplicate any options which are the same parameter. The value which comes last in {@code + * os} will be the value included in the return. + */ + @BetaApi + public static BlobWriteOption[] dedupe(BlobWriteOption... os) { + return Option.dedupe(BlobWriteOption[]::new, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static BlobWriteOption[] dedupe( + Collection collection, BlobWriteOption... os) { + return Option.dedupe(BlobWriteOption[]::new, collection, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static BlobWriteOption[] dedupe(BlobWriteOption[] array, BlobWriteOption... os) { + return Option.dedupe(BlobWriteOption[]::new, array, os); + } } /** Class for specifying blob source options. */ @@ -1034,6 +1378,40 @@ public static BlobSourceOption userProject(@NonNull String userProject) { public static BlobSourceOption shouldReturnRawInputStream(boolean shouldReturnRawInputStream) { return new BlobSourceOption(UnifiedOpts.returnRawInputStream(shouldReturnRawInputStream)); } + + /** + * Deduplicate any options which are the same parameter. The value which comes last in {@code + * os} will be the value included in the return. + */ + @BetaApi + public static BlobSourceOption[] dedupe(BlobSourceOption... os) { + return Option.dedupe(BlobSourceOption[]::new, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static BlobSourceOption[] dedupe( + Collection collection, BlobSourceOption... os) { + return Option.dedupe(BlobSourceOption[]::new, collection, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static BlobSourceOption[] dedupe(BlobSourceOption[] array, BlobSourceOption... os) { + return Option.dedupe(BlobSourceOption[]::new, array, os); + } } /** Class for specifying blob get options. */ @@ -1161,6 +1539,40 @@ public static BlobGetOption decryptionKey(@NonNull String key) { public static BlobGetOption shouldReturnRawInputStream(boolean shouldReturnRawInputStream) { return new BlobGetOption(UnifiedOpts.returnRawInputStream(shouldReturnRawInputStream)); } + + /** + * Deduplicate any options which are the same parameter. The value which comes last in {@code + * os} will be the value included in the return. + */ + @BetaApi + public static BlobGetOption[] dedupe(BlobGetOption... os) { + return Option.dedupe(BlobGetOption[]::new, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static BlobGetOption[] dedupe( + Collection collection, BlobGetOption... os) { + return Option.dedupe(BlobGetOption[]::new, collection, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static BlobGetOption[] dedupe(BlobGetOption[] array, BlobGetOption... os) { + return Option.dedupe(BlobGetOption[]::new, array, os); + } } /** Class for specifying bucket list options. */ @@ -1219,6 +1631,40 @@ public static BucketListOption fields(BucketField... fields) { .collect(ImmutableSet.toImmutableSet()); return new BucketListOption(UnifiedOpts.fields(set)); } + + /** + * Deduplicate any options which are the same parameter. The value which comes last in {@code + * os} will be the value included in the return. + */ + @BetaApi + public static BucketListOption[] dedupe(BucketListOption... os) { + return Option.dedupe(BucketListOption[]::new, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static BucketListOption[] dedupe( + Collection collection, BucketListOption... os) { + return Option.dedupe(BucketListOption[]::new, collection, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static BucketListOption[] dedupe(BucketListOption[] array, BucketListOption... os) { + return Option.dedupe(BucketListOption[]::new, array, os); + } } /** Class for specifying blob list options. */ @@ -1351,6 +1797,40 @@ public static BlobListOption fields(BlobField... fields) { .collect(ImmutableSet.toImmutableSet()); return new BlobListOption(UnifiedOpts.fields(set)); } + + /** + * Deduplicate any options which are the same parameter. The value which comes last in {@code + * os} will be the value included in the return. + */ + @BetaApi + public static BlobListOption[] dedupe(BlobListOption... os) { + return Option.dedupe(BlobListOption[]::new, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static BlobListOption[] dedupe( + Collection collection, BlobListOption... os) { + return Option.dedupe(BlobListOption[]::new, collection, os); + } + + /** + * Deduplicate any options which are the same parameter. + * + *

The value which comes last in {@code collection} and {@code os} will be the value included + * in the return. All options from {@code os} will override their counterparts in {@code + * collection}. + */ + @BetaApi + public static BlobListOption[] dedupe(BlobListOption[] array, BlobListOption... os) { + return Option.dedupe(BlobListOption[]::new, array, os); + } } /** Class for specifying Post Policy V4 options. * */ diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/DedupeOptionTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/DedupeOptionTest.java new file mode 100644 index 0000000000..b7b38df000 --- /dev/null +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/DedupeOptionTest.java @@ -0,0 +1,163 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.it; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.Bucket; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.Storage.BucketTargetOption; +import com.google.common.collect.ImmutableList; +import org.junit.Test; + +public final class DedupeOptionTest { + + @Test + public void dedupe_varargs() { + BucketTargetOption[] dedupe = + BucketTargetOption.dedupe( + BucketTargetOption.userProject("abc"), + BucketTargetOption.metagenerationMatch(), + BucketTargetOption.userProject("xyz")); + + assertThat(dedupe) + .asList() + .containsExactly( + BucketTargetOption.metagenerationMatch(), BucketTargetOption.userProject("xyz")); + } + + @Test + public void dedupe_collection_varargs() { + BucketTargetOption[] dedupe = + BucketTargetOption.dedupe( + ImmutableList.of( + BucketTargetOption.userProject("abc"), BucketTargetOption.metagenerationMatch()), + BucketTargetOption.userProject("xyz")); + + assertThat(dedupe) + .asList() + .containsExactly( + BucketTargetOption.metagenerationMatch(), BucketTargetOption.userProject("xyz")); + } + + @Test + public void dedupe_array_varargs() { + BucketTargetOption[] dedupe = + BucketTargetOption.dedupe( + new BucketTargetOption[] { + BucketTargetOption.userProject("abc"), BucketTargetOption.metagenerationMatch() + }, + BucketTargetOption.userProject("xyz")); + + assertThat(dedupe) + .asList() + .containsExactly( + BucketTargetOption.metagenerationMatch(), BucketTargetOption.userProject("xyz")); + } + + @Test + public void allClasses_varargs() { + Storage.BlobGetOption.dedupe(); + Storage.BlobListOption.dedupe(); + Storage.BlobSourceOption.dedupe(); + Storage.BlobTargetOption.dedupe(); + Storage.BlobWriteOption.dedupe(); + Storage.BucketGetOption.dedupe(); + Storage.BucketListOption.dedupe(); + Storage.BucketSourceOption.dedupe(); + Storage.BucketTargetOption.dedupe(); + Storage.CreateHmacKeyOption.dedupe(); + Storage.DeleteHmacKeyOption.dedupe(); + Storage.GetHmacKeyOption.dedupe(); + Storage.ListHmacKeysOption.dedupe(); + Storage.UpdateHmacKeyOption.dedupe(); + + Bucket.BlobTargetOption.dedupe(); + Bucket.BlobWriteOption.dedupe(); + Bucket.BucketSourceOption.dedupe(); + + Blob.BlobSourceOption.dedupe(); + } + + @Test + public void allClasses_collection_varargs() { + Storage.BlobGetOption.dedupe(ImmutableList.of()); + Storage.BlobListOption.dedupe(ImmutableList.of()); + Storage.BlobSourceOption.dedupe(ImmutableList.of()); + Storage.BlobTargetOption.dedupe(ImmutableList.of()); + Storage.BlobWriteOption.dedupe(ImmutableList.of()); + Storage.BucketGetOption.dedupe(ImmutableList.of()); + Storage.BucketListOption.dedupe(ImmutableList.of()); + Storage.BucketSourceOption.dedupe(ImmutableList.of()); + Storage.BucketTargetOption.dedupe(ImmutableList.of()); + Storage.CreateHmacKeyOption.dedupe(ImmutableList.of()); + Storage.DeleteHmacKeyOption.dedupe(ImmutableList.of()); + Storage.GetHmacKeyOption.dedupe(ImmutableList.of()); + Storage.ListHmacKeysOption.dedupe(ImmutableList.of()); + Storage.UpdateHmacKeyOption.dedupe(ImmutableList.of()); + + Bucket.BlobTargetOption.dedupe(ImmutableList.of()); + Bucket.BlobWriteOption.dedupe(ImmutableList.of()); + Bucket.BucketSourceOption.dedupe(ImmutableList.of()); + + Blob.BlobSourceOption.dedupe(ImmutableList.of()); + } + + @Test + public void allClasses_array_varargs() { + String p = "proj"; + Storage.BlobGetOption.dedupe( + new Storage.BlobGetOption[0], Storage.BlobGetOption.userProject(p)); + Storage.BlobListOption.dedupe( + new Storage.BlobListOption[0], Storage.BlobListOption.userProject(p)); + Storage.BlobSourceOption.dedupe( + new Storage.BlobSourceOption[0], Storage.BlobSourceOption.userProject(p)); + Storage.BlobTargetOption.dedupe( + new Storage.BlobTargetOption[0], Storage.BlobTargetOption.userProject(p)); + Storage.BlobWriteOption.dedupe( + new Storage.BlobWriteOption[0], Storage.BlobWriteOption.userProject(p)); + Storage.BucketGetOption.dedupe( + new Storage.BucketGetOption[0], Storage.BucketGetOption.userProject(p)); + Storage.BucketListOption.dedupe( + new Storage.BucketListOption[0], Storage.BucketListOption.userProject(p)); + Storage.BucketSourceOption.dedupe( + new Storage.BucketSourceOption[0], Storage.BucketSourceOption.userProject(p)); + Storage.BucketTargetOption.dedupe( + new Storage.BucketTargetOption[0], Storage.BucketTargetOption.userProject(p)); + Storage.CreateHmacKeyOption.dedupe( + new Storage.CreateHmacKeyOption[0], Storage.CreateHmacKeyOption.userProject(p)); + Storage.DeleteHmacKeyOption.dedupe( + new Storage.DeleteHmacKeyOption[0], Storage.DeleteHmacKeyOption.userProject(p)); + Storage.GetHmacKeyOption.dedupe( + new Storage.GetHmacKeyOption[0], Storage.GetHmacKeyOption.userProject(p)); + Storage.ListHmacKeysOption.dedupe( + new Storage.ListHmacKeysOption[0], Storage.ListHmacKeysOption.userProject(p)); + Storage.UpdateHmacKeyOption.dedupe( + new Storage.UpdateHmacKeyOption[0], Storage.UpdateHmacKeyOption.userProject(p)); + + Bucket.BlobTargetOption.dedupe( + new Bucket.BlobTargetOption[0], Bucket.BlobTargetOption.userProject(p)); + Bucket.BlobWriteOption.dedupe( + new Bucket.BlobWriteOption[0], Bucket.BlobWriteOption.userProject(p)); + Bucket.BucketSourceOption.dedupe( + new Bucket.BucketSourceOption[0], Bucket.BucketSourceOption.userProject(p)); + + Blob.BlobSourceOption.dedupe( + new Blob.BlobSourceOption[0], Blob.BlobSourceOption.userProject(p)); + } +} From 427f330793a20b0c3da4cbe5e85984a0df508c79 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Fri, 16 Jun 2023 18:34:49 +0200 Subject: [PATCH 08/15] deps: update dependency org.graalvm.buildtools:native-maven-plugin to v0.9.23 (#2074) --- samples/native-image-sample/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/native-image-sample/pom.xml b/samples/native-image-sample/pom.xml index af1943d5c9..14152b6c89 100644 --- a/samples/native-image-sample/pom.xml +++ b/samples/native-image-sample/pom.xml @@ -134,7 +134,7 @@ org.graalvm.buildtools native-maven-plugin - 0.9.22 + 0.9.23 true com.example.storage.NativeImageStorageSample From 95b23563da4678bfe5cab85fc59db73ad8f44ad9 Mon Sep 17 00:00:00 2001 From: BenWhitehead Date: Fri, 16 Jun 2023 16:19:51 -0400 Subject: [PATCH 09/15] chore: add new request idempotency header x-goog-gcs-idempotency-token (#2027) Page Bucket's token will be taken care of by the generator updates that will come in the near future. --- .../google/cloud/storage/GapicCopyWriter.java | 5 +- ...apicWritableByteChannelSessionBuilder.java | 6 +- .../google/cloud/storage/GrpcStorageImpl.java | 126 +++++++----- .../com/google/cloud/storage/Retrying.java | 13 ++ .../java/com/google/cloud/storage/Utils.java | 12 ++ .../cloud/storage/WriteFlushStrategy.java | 31 ++- .../cloud/storage/spi/v1/HttpStorageRpc.java | 1 + ...apicUnbufferedWritableByteChannelTest.java | 6 +- .../cloud/storage/WriteFlushStrategyTest.java | 16 +- .../cloud/storage/it/GrpcRequestAuditing.java | 80 ++++++++ .../it/ITGrpcIdempotencyTokenTest.java | 168 ++++++++++++++++ .../it/ITHttpIdempotencyTokenTest.java | 182 ++++++++++++++++++ .../cloud/storage/it/RequestAuditing.java | 15 ++ .../storage/it/runner/registry/Generator.java | 4 +- 14 files changed, 602 insertions(+), 63 deletions(-) create mode 100644 google-cloud-storage/src/test/java/com/google/cloud/storage/it/GrpcRequestAuditing.java create mode 100644 google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITGrpcIdempotencyTokenTest.java create mode 100644 google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITHttpIdempotencyTokenTest.java diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicCopyWriter.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicCopyWriter.java index d1e4e2f01e..cae70d6767 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicCopyWriter.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicCopyWriter.java @@ -16,6 +16,7 @@ package com.google.cloud.storage; +import com.google.api.gax.grpc.GrpcCallContext; import com.google.api.gax.retrying.ResultRetryAlgorithm; import com.google.api.gax.rpc.UnaryCallable; import com.google.cloud.RestorableState; @@ -78,7 +79,9 @@ public void copyChunk() { RewriteObjectRequest.newBuilder() .setRewriteToken(mostRecentResponse.getRewriteToken()) .build(); - mostRecentResponse = Retrying.run(options, alg, () -> callable.call(req), Decoder.identity()); + GrpcCallContext retryContext = Retrying.newCallContext(); + mostRecentResponse = + Retrying.run(options, alg, () -> callable.call(req, retryContext), Decoder.identity()); } } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicWritableByteChannelSessionBuilder.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicWritableByteChannelSessionBuilder.java index 329851dac2..a872560903 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicWritableByteChannelSessionBuilder.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicWritableByteChannelSessionBuilder.java @@ -281,7 +281,8 @@ UnbufferedWritableByteChannelSession build() { return new UnbufferedWriteSession<>( requireNonNull(start, "start must be non null"), bindFunction( - WriteFlushStrategy.fsyncEveryFlush(write, deps, alg), ResumableWrite::identity) + WriteFlushStrategy.fsyncEveryFlush(write, deps, alg, Retrying::newCallContext), + ResumableWrite::identity) .andThen(StorageByteChannels.writable()::createSynchronized)); } } @@ -309,7 +310,8 @@ BufferedWritableByteChannelSession build() { return new BufferedWriteSession<>( requireNonNull(start, "start must be non null"), bindFunction( - WriteFlushStrategy.fsyncEveryFlush(write, deps, alg), ResumableWrite::identity) + WriteFlushStrategy.fsyncEveryFlush(write, deps, alg, Retrying::newCallContext), + ResumableWrite::identity) .andThen(c -> new DefaultBufferedWritableByteChannel(bufferHandle, c)) .andThen(StorageByteChannels.writable()::createSynchronized)); } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java index e8faef4d2f..a4164eb216 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java @@ -205,10 +205,11 @@ public Bucket create(BucketInfo bucketInfo, BucketTargetOption... options) { .setBucketId(bucketInfo.getName()) .setParent("projects/_"); CreateBucketRequest req = opts.createBucketsRequest().apply(builder).build(); + GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext()); return Retrying.run( getOptions(), retryAlgorithmManager.getFor(req), - () -> storageClient.createBucketCallable().call(req, grpcCallContext), + () -> storageClient.createBucketCallable().call(req, merge), syntaxDecoders.bucket); } @@ -233,6 +234,7 @@ public Blob create( opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault()); WriteObjectRequest req = getWriteObjectRequest(blobInfo, opts); Hasher hasher = getHasherForRequest(req, Hasher.enabled()); + GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext()); return Retrying.run( getOptions(), retryAlgorithmManager.getFor(req), @@ -240,8 +242,7 @@ public Blob create( UnbufferedWritableByteChannelSession session = ResumableMedia.gapic() .write() - .byteChannel( - storageClient.writeObjectCallable().withDefaultCallContext(grpcCallContext)) + .byteChannel(storageClient.writeObjectCallable().withDefaultCallContext(merge)) .setByteStringStrategy(ByteStringStrategy.noCopy()) .setHasher(hasher) .direct() @@ -286,23 +287,21 @@ public Blob createFrom(BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOp WriteObjectRequest req = getWriteObjectRequest(blobInfo, opts); Hasher hasher = getHasherForRequest(req, Hasher.enabled()); - GapicWritableByteChannelSessionBuilder channelSessionBuilder = - ResumableMedia.gapic() - .write() - .byteChannel( - storageClient.writeObjectCallable().withDefaultCallContext(grpcCallContext)) - .setHasher(hasher) - .setByteStringStrategy(ByteStringStrategy.noCopy()); long size = Files.size(path); if (size < bufferSize) { // ignore the bufferSize argument if the file is smaller than it + GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext()); return Retrying.run( getOptions(), retryAlgorithmManager.getFor(req), () -> { BufferedWritableByteChannelSession session = - channelSessionBuilder + ResumableMedia.gapic() + .write() + .byteChannel(storageClient.writeObjectCallable().withDefaultCallContext(merge)) + .setHasher(hasher) + .setByteStringStrategy(ByteStringStrategy.noCopy()) .direct() .buffered(Buffers.allocate(size)) .setRequest(req) @@ -320,7 +319,12 @@ public Blob createFrom(BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOp } else { ApiFuture start = startResumableWrite(grpcCallContext, req); BufferedWritableByteChannelSession session = - channelSessionBuilder + ResumableMedia.gapic() + .write() + .byteChannel( + storageClient.writeObjectCallable().withDefaultCallContext(grpcCallContext)) + .setHasher(hasher) + .setByteStringStrategy(ByteStringStrategy.noCopy()) .resumable() .withRetryConfig(getOptions(), retryAlgorithmManager.idempotent()) .buffered(Buffers.allocateAligned(bufferSize, _256KiB)) @@ -396,10 +400,11 @@ public Bucket lockRetentionPolicy(BucketInfo bucket, BucketTargetOption... optio .setBucket(bucketNameCodec.encode(bucket.getName())); LockBucketRetentionPolicyRequest req = opts.lockBucketRetentionPolicyRequest().apply(builder).build(); + GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext()); return Retrying.run( getOptions(), retryAlgorithmManager.getFor(req), - () -> storageClient.lockBucketRetentionPolicyCallable().call(req, grpcCallContext), + () -> storageClient.lockBucketRetentionPolicyCallable().call(req, merge), syntaxDecoders.bucket); } @@ -431,10 +436,11 @@ public Page list(BucketListOption... options) { .apply(ListBucketsRequest.newBuilder()) .build(); try { + GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext()); return Retrying.run( getOptions(), retryAlgorithmManager.getFor(request), - () -> storageClient.listBucketsPagedCallable().call(request, grpcCallContext), + () -> storageClient.listBucketsPagedCallable().call(request, merge), resp -> new TransformingPageDecorator<>( resp.getPage(), @@ -455,10 +461,11 @@ public Page list(String bucket, BlobListOption... options) { ListObjectsRequest.newBuilder().setParent(bucketNameCodec.encode(bucket)); ListObjectsRequest req = opts.listObjectsRequest().apply(builder).build(); try { + GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext()); return Retrying.run( getOptions(), retryAlgorithmManager.getFor(req), - () -> storageClient.listObjectsCallable().call(req, grpcCallContext), + () -> storageClient.listObjectsCallable().call(req, merge), resp -> new ListObjectsWithSyntheticDirectoriesPage(grpcCallContext, req, resp)); } catch (Exception e) { throw StorageException.coalesce(e); @@ -484,10 +491,11 @@ public Bucket update(BucketInfo bucketInfo, BucketTargetOption... options) { .map(NamedField::getGrpcName) .collect(ImmutableList.toImmutableList())); UpdateBucketRequest req = builder.build(); + GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext()); return Retrying.run( getOptions(), retryAlgorithmManager.getFor(req), - () -> storageClient.updateBucketCallable().call(req, grpcCallContext), + () -> storageClient.updateBucketCallable().call(req, merge), syntaxDecoders.bucket); } @@ -510,10 +518,11 @@ public Blob update(BlobInfo blobInfo, BlobTargetOption... options) { .map(NamedField::getGrpcName) .collect(ImmutableList.toImmutableList())); UpdateObjectRequest req = builder.build(); + GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext()); return Retrying.run( getOptions(), retryAlgorithmManager.getFor(req), - () -> storageClient.updateObjectCallable().call(req, grpcCallContext), + () -> storageClient.updateObjectCallable().call(req, merge), syntaxDecoders.blob); } @@ -531,10 +540,11 @@ public boolean delete(String bucket, BucketSourceOption... options) { DeleteBucketRequest.newBuilder().setName(bucketNameCodec.encode(bucket)); DeleteBucketRequest req = opts.deleteBucketsRequest().apply(builder).build(); try { + GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext()); Retrying.run( getOptions(), retryAlgorithmManager.getFor(req), - () -> storageClient.deleteBucketCallable().call(req, grpcCallContext), + () -> storageClient.deleteBucketCallable().call(req, merge), Decoder.identity()); return true; } catch (StorageException e) { @@ -558,13 +568,14 @@ public boolean delete(BlobId blob, BlobSourceOption... options) { .setObject(blob.getName()); ifNonNull(blob.getGeneration(), builder::setGeneration); DeleteObjectRequest req = opts.deleteObjectsRequest().apply(builder).build(); + GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext()); return Boolean.TRUE.equals( Retrying.run( getOptions(), retryAlgorithmManager.getFor(req), () -> { try { - storageClient.deleteObjectCallable().call(req, grpcCallContext); + storageClient.deleteObjectCallable().call(req, merge); return true; } catch (NotFoundException e) { return false; @@ -593,10 +604,11 @@ public Blob compose(ComposeRequest composeRequest) { final Object target = codecs.blobInfo().encode(composeRequest.getTarget()); builder.setDestination(target); ComposeObjectRequest req = opts.composeObjectsRequest().apply(builder).build(); + GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext()); return Retrying.run( getOptions(), retryAlgorithmManager.getFor(req), - () -> storageClient.composeObjectCallable().call(req, grpcCallContext), + () -> storageClient.composeObjectCallable().call(req, merge), syntaxDecoders.blob); } @@ -646,10 +658,11 @@ public CopyWriter copy(CopyRequest copyRequest) { srcOpts.grpcMetadataMapper().apply(GrpcCallContext.createDefault()); UnaryCallable callable = storageClient.rewriteObjectCallable().withDefaultCallContext(grpcCallContext); + GrpcCallContext retryContext = Retrying.newCallContext(); return Retrying.run( getOptions(), retryAlgorithmManager.getFor(req), - () -> callable.call(req), + () -> callable.call(req, retryContext), (resp) -> new GapicCopyWriter(this, callable, retryAlgorithmManager.idempotent(), resp)); } @@ -687,7 +700,7 @@ public GrpcBlobReadChannel reader(BlobId blob, BlobSourceOption... options) { Opts opts = Opts.unwrap(options).resolveFrom(blob).prepend(defaultOpts); ReadObjectRequest request = getReadObjectRequest(blob, opts); Set codes = resultRetryAlgorithmToCodes(retryAlgorithmManager.getFor(request)); - GrpcCallContext grpcCallContext = GrpcCallContext.createDefault().withRetryableCodes(codes); + GrpcCallContext grpcCallContext = Retrying.newCallContext().withRetryableCodes(codes); return new GrpcBlobReadChannel( storageClient.readObjectCallable().withDefaultCallContext(grpcCallContext), request, @@ -1221,10 +1234,11 @@ public HmacKey createHmacKey(ServiceAccount serviceAccount, CreateHmacKeyOption. .apply(CreateHmacKeyRequest.newBuilder()) .setServiceAccountEmail(serviceAccount.getEmail()) .build(); + GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext()); return Retrying.run( getOptions(), retryAlgorithmManager.getFor(request), - () -> storageClient.createHmacKeyCallable().call(request, grpcCallContext), + () -> storageClient.createHmacKeyCallable().call(request, merge), resp -> { ByteString secretKeyBytes = resp.getSecretKeyBytes(); String b64SecretKey = BaseEncoding.base64().encode(secretKeyBytes.toByteArray()); @@ -1247,10 +1261,11 @@ public Page listHmacKeys(ListHmacKeysOption... options) { .apply(ListHmacKeysRequest.newBuilder()) .build(); try { + GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext()); return Retrying.run( getOptions(), retryAlgorithmManager.getFor(request), - () -> storageClient.listHmacKeysPagedCallable().call(request, grpcCallContext), + () -> storageClient.listHmacKeysPagedCallable().call(request, merge), resp -> new TransformingPageDecorator<>( resp.getPage(), @@ -1274,10 +1289,11 @@ public HmacKeyMetadata getHmacKey(String accessId, GetHmacKeyOption... options) .apply(GetHmacKeyRequest.newBuilder()) .setAccessId(accessId) .build(); + GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext()); return Retrying.run( getOptions(), retryAlgorithmManager.getFor(request), - () -> storageClient.getHmacKeyCallable().call(request, grpcCallContext), + () -> storageClient.getHmacKeyCallable().call(request, merge), codecs.hmacKeyMetadata()); } @@ -1291,11 +1307,12 @@ public void deleteHmacKey(HmacKeyMetadata hmacKeyMetadata, DeleteHmacKeyOption.. .setAccessId(hmacKeyMetadata.getAccessId()) .setProject(projectNameCodec.encode(hmacKeyMetadata.getProjectId())) .build(); + GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext()); Retrying.run( getOptions(), retryAlgorithmManager.getFor(req), () -> { - storageClient.deleteHmacKeyCallable().call(req, grpcCallContext); + storageClient.deleteHmacKeyCallable().call(req, merge); return null; }, Decoder.identity()); @@ -1314,10 +1331,11 @@ public HmacKeyMetadata updateHmacKeyState( opts.updateHmacKeysRequest().apply(UpdateHmacKeyRequest.newBuilder()).setHmacKey(encode); UpdateHmacKeyRequest request = builder.setUpdateMask(FieldMask.newBuilder().addPaths("state").build()).build(); + GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext()); return Retrying.run( getOptions(), retryAlgorithmManager.getFor(request), - () -> storageClient.updateHmacKeyCallable().call(request, grpcCallContext), + () -> storageClient.updateHmacKeyCallable().call(request, merge), codecs.hmacKeyMetadata()); } @@ -1329,10 +1347,11 @@ public Policy getIamPolicy(String bucket, BucketSourceOption... options) { GetIamPolicyRequest.Builder builder = GetIamPolicyRequest.newBuilder().setResource(bucketNameCodec.encode(bucket)); GetIamPolicyRequest req = opts.getIamPolicyRequest().apply(builder).build(); + GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext()); return Retrying.run( getOptions(), retryAlgorithmManager.getFor(req), - () -> storageClient.getIamPolicyCallable().call(req, grpcCallContext), + () -> storageClient.getIamPolicyCallable().call(req, merge), codecs.policyCodec()); } @@ -1346,10 +1365,11 @@ public Policy setIamPolicy(String bucket, Policy policy, BucketSourceOption... o .setResource(bucketNameCodec.encode(bucket)) .setPolicy(codecs.policyCodec().encode(policy)) .build(); + GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext()); return Retrying.run( getOptions(), retryAlgorithmManager.getFor(req), - () -> storageClient.setIamPolicyCallable().call(req, grpcCallContext), + () -> storageClient.setIamPolicyCallable().call(req, merge), codecs.policyCodec()); } @@ -1364,10 +1384,11 @@ public List testIamPermissions( .setResource(bucketNameCodec.encode(bucket)) .addAllPermissions(permissions) .build(); + GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext()); return Retrying.run( getOptions(), retryAlgorithmManager.getFor(req), - () -> storageClient.testIamPermissionsCallable().call(req, grpcCallContext), + () -> storageClient.testIamPermissionsCallable().call(req, merge), resp -> { Set heldPermissions = ImmutableSet.copyOf(resp.getPermissionsList()); return permissions.stream() @@ -1382,10 +1403,11 @@ public ServiceAccount getServiceAccount(String projectId) { GetServiceAccountRequest.newBuilder() .setProject(projectNameCodec.encode(projectId)) .build(); + GrpcCallContext retryContext = Retrying.newCallContext(); return Retrying.run( getOptions(), retryAlgorithmManager.getFor(req), - () -> storageClient.getServiceAccountCallable().call(req), + () -> storageClient.getServiceAccountCallable().call(req, retryContext), codecs.serviceAccount()); } @@ -1397,10 +1419,11 @@ public Notification createNotification(String bucket, NotificationInfo notificat .setParent(bucketNameCodec.encode(bucket)) .setNotificationConfig(encode) .build(); + GrpcCallContext retryContext = Retrying.newCallContext(); return Retrying.run( getOptions(), retryAlgorithmManager.getFor(req), - () -> storageClient.createNotificationConfigCallable().call(req), + () -> storageClient.createNotificationConfigCallable().call(req, retryContext), syntaxDecoders.notificationConfig); } @@ -1415,12 +1438,13 @@ public Notification getNotification(String bucket, String notificationId) { } GetNotificationConfigRequest req = GetNotificationConfigRequest.newBuilder().setName(name).build(); + GrpcCallContext retryContext = Retrying.newCallContext(); return Retrying.run( getOptions(), retryAlgorithmManager.getFor(req), () -> { try { - return storageClient.getNotificationConfigCallable().call(req); + return storageClient.getNotificationConfigCallable().call(req, retryContext); } catch (NotFoundException e) { return null; } @@ -1435,10 +1459,11 @@ public List listNotifications(String bucket) { .setParent(bucketNameCodec.encode(bucket)) .build(); ResultRetryAlgorithm algorithm = retryAlgorithmManager.getFor(req); + GrpcCallContext retryContext = Retrying.newCallContext(); return Retrying.run( getOptions(), algorithm, - () -> storageClient.listNotificationConfigsPagedCallable().call(req), + () -> storageClient.listNotificationConfigsPagedCallable().call(req, retryContext), resp -> { TransformingPageDecorator< ListNotificationConfigsRequest, @@ -1464,13 +1489,14 @@ public boolean deleteNotification(String bucket, String notificationId) { } DeleteNotificationConfigRequest req = DeleteNotificationConfigRequest.newBuilder().setName(name).build(); + GrpcCallContext retryContext = Retrying.newCallContext(); return Boolean.TRUE.equals( Retrying.run( getOptions(), retryAlgorithmManager.getFor(req), () -> { try { - storageClient.deleteNotificationConfigCallable().call(req); + storageClient.deleteNotificationConfigCallable().call(req, retryContext); return true; } catch (NotFoundException e) { return false; @@ -1551,11 +1577,12 @@ public Page getNextPage() { ListObjectsRequest nextPageReq = req.toBuilder().setPageToken(resp.getNextPageToken()).build(); try { + GrpcCallContext merge = Utils.merge(ctx, Retrying.newCallContext()); ListObjectsResponse nextPageResp = Retrying.run( GrpcStorageImpl.this.getOptions(), retryAlgorithmManager.getFor(nextPageReq), - () -> storageClient.listObjectsCallable().call(nextPageReq, ctx), + () -> storageClient.listObjectsCallable().call(nextPageReq, merge), Decoder.identity()); return new ListObjectsWithSyntheticDirectoriesPage(ctx, nextPageReq, nextPageResp); } catch (Exception e) { @@ -1649,6 +1676,7 @@ public Iterable iterateAll() { page, p -> p != null && p.hasNextPage(), prev -> { + // TODO: retry token header // explicitly define this callable rather than using the method reference to // prevent a javac 1.8 exception // https://bugs.java.com/bugdatabase/view_bug.do?bug_id=8056984 @@ -1773,7 +1801,7 @@ private UnbufferedReadableByteChannelSession unbufferedReadSession( ReadObjectRequest readObjectRequest = getReadObjectRequest(blob, opts); Set codes = resultRetryAlgorithmToCodes(retryAlgorithmManager.getFor(readObjectRequest)); - GrpcCallContext grpcCallContext = GrpcCallContext.createDefault().withRetryableCodes(codes); + GrpcCallContext grpcCallContext = Retrying.newCallContext().withRetryableCodes(codes); return ResumableMedia.gapic() .read() .byteChannel(storageClient.readObjectCallable().withDefaultCallContext(grpcCallContext)) @@ -1787,12 +1815,13 @@ private UnbufferedReadableByteChannelSession unbufferedReadSession( ApiFuture startResumableWrite( GrpcCallContext grpcCallContext, WriteObjectRequest req) { Set codes = resultRetryAlgorithmToCodes(retryAlgorithmManager.getFor(req)); + GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext()); return ResumableMedia.gapic() .write() .resumableWrite( storageClient .startResumableWriteCallable() - .withDefaultCallContext(grpcCallContext.withRetryableCodes(codes)), + .withDefaultCallContext(merge.withRetryableCodes(codes)), req); } @@ -1818,10 +1847,11 @@ private com.google.storage.v2.Bucket getBucketWithDefaultAcls(String bucketName) .setName(bucketNameCodec.encode(bucketName)) .build(); + GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext()); return Retrying.run( getOptions(), retryAlgorithmManager.getFor(req), - () -> storageClient.getBucketCallable().call(req, grpcCallContext), + () -> storageClient.getBucketCallable().call(req, merge), Decoder.identity()); } @@ -1837,19 +1867,21 @@ private com.google.storage.v2.Bucket getBucketWithAcls( .setName(bucketNameCodec.encode(bucketName)) .build(); + GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext()); return Retrying.run( getOptions(), retryAlgorithmManager.getFor(req), - () -> storageClient.getBucketCallable().call(req, grpcCallContext), + () -> storageClient.getBucketCallable().call(req, merge), Decoder.identity()); } private com.google.storage.v2.Bucket updateBucket(UpdateBucketRequest req) { GrpcCallContext grpcCallContext = GrpcCallContext.createDefault(); + GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext()); return Retrying.run( getOptions(), retryAlgorithmManager.getFor(req), - () -> storageClient.updateBucketCallable().call(req, grpcCallContext), + () -> storageClient.updateBucketCallable().call(req, merge), Decoder.identity()); } @@ -1899,10 +1931,11 @@ private Object getObjectWithAcls(Object obj) { .setObject(obj.getName()) .build(); + GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext()); return Retrying.run( getOptions(), retryAlgorithmManager.getFor(req), - () -> storageClient.getObjectCallable().call(req, grpcCallContext), + () -> storageClient.getObjectCallable().call(req, merge), Decoder.identity()); } @@ -1926,10 +1959,11 @@ private static UpdateObjectRequest createUpdateObjectAclRequest( private Object updateObject(UpdateObjectRequest req) { GrpcCallContext grpcCallContext = GrpcCallContext.createDefault(); + GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext()); return Retrying.run( getOptions(), retryAlgorithmManager.getFor(req), - () -> storageClient.updateObjectCallable().call(req, grpcCallContext), + () -> storageClient.updateObjectCallable().call(req, merge), Decoder.identity()); } @@ -1957,12 +1991,13 @@ private Blob internalBlobGet(BlobId blob, Opts unwrap) { .setObject(blob.getName()); ifNonNull(blob.getGeneration(), builder::setGeneration); GetObjectRequest req = opts.getObjectsRequest().apply(builder).build(); + GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext()); return Retrying.run( getOptions(), retryAlgorithmManager.getFor(req), () -> { try { - return storageClient.getObjectCallable().call(req, grpcCallContext); + return storageClient.getObjectCallable().call(req, merge); } catch (NotFoundException ignore) { return null; } @@ -1978,10 +2013,11 @@ private Bucket internalBucketGet(String bucket, Opts unwrap) { GetBucketRequest.Builder builder = GetBucketRequest.newBuilder().setName(bucketNameCodec.encode(bucket)); GetBucketRequest req = opts.getBucketsRequest().apply(builder).build(); + GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext()); return Retrying.run( getOptions(), retryAlgorithmManager.getFor(req), - () -> storageClient.getBucketCallable().call(req, grpcCallContext), + () -> storageClient.getBucketCallable().call(req, merge), syntaxDecoders.bucket.andThen(opts.clearBucketFields())); } } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Retrying.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Retrying.java index cfc961be80..f625b58b95 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Retrying.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Retrying.java @@ -20,14 +20,19 @@ import com.google.api.core.ApiClock; import com.google.api.core.NanoClock; +import com.google.api.gax.grpc.GrpcCallContext; import com.google.api.gax.retrying.BasicResultRetryAlgorithm; import com.google.api.gax.retrying.ResultRetryAlgorithm; import com.google.api.gax.retrying.RetrySettings; import com.google.cloud.RetryHelper.RetryHelperException; import com.google.cloud.storage.Conversions.Decoder; import com.google.cloud.storage.spi.v1.HttpRpcContext; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.util.UUID; import java.util.concurrent.Callable; import java.util.function.Function; +import org.checkerframework.checker.nullness.qual.NonNull; final class Retrying { @@ -115,6 +120,14 @@ static U run( } } + @NonNull + static GrpcCallContext newCallContext() { + return GrpcCallContext.createDefault() + .withExtraHeaders( + ImmutableMap.of( + "x-goog-gcs-idempotency-token", ImmutableList.of(UUID.randomUUID().toString()))); + } + static ResultRetryAlgorithm neverRetry() { return new BasicResultRetryAlgorithm() { @Override diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Utils.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Utils.java index 81d7fbb929..67bafab86f 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Utils.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Utils.java @@ -20,6 +20,8 @@ import com.google.api.client.util.DateTime; import com.google.api.core.InternalApi; +import com.google.api.gax.grpc.GrpcCallContext; +import com.google.api.gax.rpc.ApiCallContext; import com.google.cloud.storage.Conversions.Codec; import com.google.cloud.storage.UnifiedOpts.NamedField; import com.google.common.annotations.VisibleForTesting; @@ -289,4 +291,14 @@ private static int crc32cDecode(String from) { private static String crc32cEncode(int from) { return BaseEncoding.base64().encode(Ints.toByteArray(from)); } + + /** + * Type preserving method for {@link GrpcCallContext#merge(ApiCallContext)} + * + * @see GrpcCallContext#merge(ApiCallContext) + */ + @NonNull + static GrpcCallContext merge(@NonNull GrpcCallContext l, @NonNull GrpcCallContext r) { + return (GrpcCallContext) l.merge(r); + } } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/WriteFlushStrategy.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/WriteFlushStrategy.java index 7deb004342..d5a205edfa 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/WriteFlushStrategy.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/WriteFlushStrategy.java @@ -31,6 +31,7 @@ import java.util.concurrent.ExecutionException; import java.util.function.Consumer; import java.util.function.LongConsumer; +import java.util.function.Supplier; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; @@ -57,12 +58,19 @@ private WriteFlushStrategy() {} static FlusherFactory fsyncEveryFlush( ClientStreamingCallable write, RetryingDependencies deps, - ResultRetryAlgorithm alg) { + ResultRetryAlgorithm alg, + Supplier baseContextSupplier) { return (String bucketName, LongConsumer committedTotalBytesCallback, Consumer onSuccessCallback) -> new FsyncEveryFlusher( - write, deps, alg, bucketName, committedTotalBytesCallback, onSuccessCallback); + write, + deps, + alg, + bucketName, + committedTotalBytesCallback, + onSuccessCallback, + baseContextSupplier); } /** @@ -79,14 +87,14 @@ static FlusherFactory fsyncOnClose( new FsyncOnClose(write, bucketName, committedTotalBytesCallback, onSuccessCallback); } - private static GrpcCallContext contextWithBucketName(String bucketName) { - GrpcCallContext ret = GrpcCallContext.createDefault(); + private static GrpcCallContext contextWithBucketName( + String bucketName, GrpcCallContext baseContext) { if (bucketName != null && !bucketName.isEmpty()) { - return ret.withExtraHeaders( + return baseContext.withExtraHeaders( ImmutableMap.of( "x-goog-request-params", ImmutableList.of(String.format("bucket=%s", bucketName)))); } - return ret; + return baseContext; } /** @@ -138,6 +146,7 @@ private static final class FsyncEveryFlusher implements Flusher { private final String bucketName; private final LongConsumer sizeCallback; private final Consumer completeCallback; + private final Supplier baseContextSupplier; private FsyncEveryFlusher( ClientStreamingCallable write, @@ -145,13 +154,15 @@ private FsyncEveryFlusher( ResultRetryAlgorithm alg, String bucketName, LongConsumer sizeCallback, - Consumer completeCallback) { + Consumer completeCallback, + Supplier baseContextSupplier) { this.write = write; this.deps = deps; this.alg = alg; this.bucketName = bucketName; this.sizeCallback = sizeCallback; this.completeCallback = completeCallback; + this.baseContextSupplier = baseContextSupplier; } public void flush(@NonNull List segments) { @@ -160,7 +171,8 @@ public void flush(@NonNull List segments) { alg, () -> { Observer observer = new Observer(sizeCallback, completeCallback); - GrpcCallContext internalContext = contextWithBucketName(bucketName); + GrpcCallContext internalContext = + contextWithBucketName(bucketName, baseContextSupplier.get()); ApiStreamObserver write = this.write.withDefaultCallContext(internalContext).clientStreamingCall(observer); @@ -230,7 +242,8 @@ private void ensureOpen() { if (stream == null) { synchronized (this) { if (stream == null) { - GrpcCallContext internalContext = contextWithBucketName(bucketName); + GrpcCallContext internalContext = + contextWithBucketName(bucketName, GrpcCallContext.createDefault()); stream = this.write .withDefaultCallContext(internalContext) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java index 707ba4573e..18e95e193e 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java @@ -182,6 +182,7 @@ public void intercept(HttpRequest request) throws IOException { newValue = invocationEntry; } headers.set("x-goog-api-client", newValue); + headers.set("x-goog-gcs-idempotency-token", invocationId); } } } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGapicUnbufferedWritableByteChannelTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGapicUnbufferedWritableByteChannelTest.java index 6ee17f1172..08d228c27a 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGapicUnbufferedWritableByteChannelTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGapicUnbufferedWritableByteChannelTest.java @@ -191,7 +191,8 @@ public void resumableUpload() throws IOException, InterruptedException, Executio WriteFlushStrategy.fsyncEveryFlush( sc.writeObjectCallable(), RetryingDependencies.attemptOnce(), - Retrying.neverRetry())); + Retrying.neverRetry(), + Retrying::newCallContext)); ArrayList debugMessages = new ArrayList<>(); try { ImmutableList buffers = TestUtils.subDivide(bytes, 10); @@ -279,7 +280,8 @@ public void resumableUpload_chunkAutomaticRetry() public boolean shouldRetry(Throwable t, Object ignore) { return TestUtils.findThrowable(DataLossException.class, t) != null; } - }))) { + }, + Retrying::newCallContext))) { writeCtx = c.getWriteCtx(); ImmutableList buffers = TestUtils.subDivide(bytes, 10); c.write(buffers.get(0)); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/WriteFlushStrategyTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/WriteFlushStrategyTest.java index 4c96d43096..f4c037b18a 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/WriteFlushStrategyTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/WriteFlushStrategyTest.java @@ -18,6 +18,7 @@ import static com.google.common.truth.Truth.assertThat; +import com.google.api.gax.grpc.GrpcCallContext; import com.google.api.gax.rpc.ApiCallContext; import com.google.api.gax.rpc.ApiStreamObserver; import com.google.api.gax.rpc.ClientStreamingCallable; @@ -47,7 +48,10 @@ public void bucketNameAddedToXGoogRequestParams_nonNull_nonEmpty_fsyncEveryFlush doTest( write -> WriteFlushStrategy.fsyncEveryFlush( - write, RetryingDependencies.attemptOnce(), Retrying.neverRetry()), + write, + RetryingDependencies.attemptOnce(), + Retrying.neverRetry(), + GrpcCallContext::createDefault), "bucket-name", expectedHeaderNonNullNonEmpty); } @@ -62,7 +66,10 @@ public void bucketNameNotAddedToXGoogRequestParams_nonNull_empty_fsyncEveryFlush doTest( write -> WriteFlushStrategy.fsyncEveryFlush( - write, RetryingDependencies.attemptOnce(), Retrying.neverRetry()), + write, + RetryingDependencies.attemptOnce(), + Retrying.neverRetry(), + GrpcCallContext::createDefault), "", expectedHeaderNonNullEmpty); } @@ -77,7 +84,10 @@ public void bucketNameNotAddedToXGoogRequestParams_null_fsyncEveryFlush() { doTest( write -> WriteFlushStrategy.fsyncEveryFlush( - write, RetryingDependencies.attemptOnce(), Retrying.neverRetry()), + write, + RetryingDependencies.attemptOnce(), + Retrying.neverRetry(), + GrpcCallContext::createDefault), null, expectedHeaderNull); } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/GrpcRequestAuditing.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/GrpcRequestAuditing.java new file mode 100644 index 0000000000..a674a07924 --- /dev/null +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/GrpcRequestAuditing.java @@ -0,0 +1,80 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.it; + +import static com.google.common.truth.Truth.assertWithMessage; + +import com.google.common.collect.ImmutableList; +import com.google.common.truth.IterableSubject; +import io.grpc.Attributes; +import io.grpc.CallOptions; +import io.grpc.Channel; +import io.grpc.ClientCall; +import io.grpc.ClientInterceptor; +import io.grpc.ClientStreamTracer; +import io.grpc.ClientStreamTracer.StreamInfo; +import io.grpc.Metadata; +import io.grpc.MethodDescriptor; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +final class GrpcRequestAuditing implements ClientInterceptor { + + private final List requestHeaders; + + GrpcRequestAuditing() { + requestHeaders = Collections.synchronizedList(new ArrayList<>()); + } + + void clear() { + requestHeaders.clear(); + } + + @Override + public ClientCall interceptCall( + MethodDescriptor method, CallOptions callOptions, Channel next) { + CallOptions withStreamTracerFactory = callOptions.withStreamTracerFactory(new Factory()); + return next.newCall(method, withStreamTracerFactory); + } + + public IterableSubject assertRequestHeader(Metadata.Key key) { + ImmutableList actual = + requestHeaders.stream() + .map(m -> m.get(key)) + .filter(Objects::nonNull) + .distinct() + .collect(ImmutableList.toImmutableList()); + return assertWithMessage(String.format("Headers %s", key.name())).that(actual); + } + + private final class Factory extends ClientStreamTracer.Factory { + @Override + public ClientStreamTracer newClientStreamTracer(StreamInfo info, Metadata headers) { + return new Tracer(); + } + } + + private final class Tracer extends ClientStreamTracer { + + @Override + public void streamCreated(Attributes transportAttrs, Metadata headers) { + requestHeaders.add(headers); + } + } +} diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITGrpcIdempotencyTokenTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITGrpcIdempotencyTokenTest.java new file mode 100644 index 0000000000..ec5d54252b --- /dev/null +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITGrpcIdempotencyTokenTest.java @@ -0,0 +1,168 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.it; + +import static com.google.cloud.storage.TestUtils.assertAll; +import static com.google.cloud.storage.TestUtils.xxd; +import static com.google.common.truth.Truth.assertThat; + +import com.google.api.gax.paging.Page; +import com.google.cloud.WriteChannel; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Bucket; +import com.google.cloud.storage.BucketInfo; +import com.google.cloud.storage.DataGenerator; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.Storage.BlobListOption; +import com.google.cloud.storage.Storage.BlobSourceOption; +import com.google.cloud.storage.Storage.BlobTargetOption; +import com.google.cloud.storage.Storage.BlobWriteOption; +import com.google.cloud.storage.StorageOptions; +import com.google.cloud.storage.it.runner.StorageITRunner; +import com.google.cloud.storage.it.runner.annotations.Backend; +import com.google.cloud.storage.it.runner.annotations.Inject; +import com.google.cloud.storage.it.runner.annotations.SingleBackend; +import com.google.cloud.storage.it.runner.registry.Generator; +import com.google.common.collect.ImmutableList; +import com.google.common.truth.IterableSubject; +import io.grpc.Metadata; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(StorageITRunner.class) +@SingleBackend(Backend.PROD) +public final class ITGrpcIdempotencyTokenTest { + + private static final Metadata.Key X_GOOG_GCS_IDEMPOTENCY_TOKEN = + Metadata.Key.of("x-goog-gcs-idempotency-token", Metadata.ASCII_STRING_MARSHALLER); + @Inject public BucketInfo bucket; + @Inject public Generator generator; + + private Storage storage; + private GrpcRequestAuditing requestAuditing; + + @Before + public void setUp() throws Exception { + requestAuditing = new GrpcRequestAuditing(); + storage = + StorageOptions.grpc() + .setGrpcInterceptorProvider(() -> ImmutableList.of(requestAuditing)) + .build() + .getService(); + } + + @After + public void tearDown() throws Exception { + if (storage != null) { + storage.close(); + } + } + + @Test + public void simpleUnary() throws Exception { + Bucket gen1 = storage.get(bucket.getName()); + + IterableSubject subject = requestAuditing.assertRequestHeader(X_GOOG_GCS_IDEMPOTENCY_TOKEN); + assertAll(() -> subject.hasSize(1)); + } + + @Test + public void pageObjects() throws Exception { + String baseName = generator.randomObjectName(); + Blob blob1 = storage.create(BlobInfo.newBuilder(bucket, baseName + "1").build()); + Blob blob2 = storage.create(BlobInfo.newBuilder(bucket, baseName + "2").build()); + + requestAuditing.clear(); + ImmutableList expectedNamess = ImmutableList.of(blob1.getName(), blob2.getName()); + Page page = + storage.list(bucket.getName(), BlobListOption.prefix(baseName), BlobListOption.pageSize(1)); + + List collect = page.streamAll().map(BlobInfo::getName).collect(Collectors.toList()); + IterableSubject subject = requestAuditing.assertRequestHeader(X_GOOG_GCS_IDEMPOTENCY_TOKEN); + assertAll( + () -> assertThat(collect).hasSize(2), + () -> assertThat(collect).containsExactlyElementsIn(expectedNamess), + () -> subject.hasSize(2)); + } + + @Test + public void readObject() throws Exception { + byte[] expected = DataGenerator.base64Characters().genBytes(512 * 1024 + 45); + String expectedXxd = xxd(expected); + + Blob gen1 = + storage.create( + BlobInfo.newBuilder(bucket, generator.randomObjectName()).build(), + expected, + BlobTargetOption.doesNotExist()); + + requestAuditing.clear(); + byte[] actual = storage.readAllBytes(gen1.getBlobId(), BlobSourceOption.generationMatch()); + IterableSubject subject = requestAuditing.assertRequestHeader(X_GOOG_GCS_IDEMPOTENCY_TOKEN); + String actualXxd = xxd(actual); + + assertAll(() -> subject.hasSize(1), () -> assertThat(actualXxd).isEqualTo(expectedXxd)); + } + + @Test + public void directUpload() throws Exception { + byte[] expected = DataGenerator.base64Characters().genBytes(512 * 1024 + 45); + String expectedXxd = xxd(expected); + + BlobInfo info = BlobInfo.newBuilder(bucket, generator.randomObjectName()).build(); + requestAuditing.clear(); + Blob gen1 = storage.create(info, expected, BlobTargetOption.doesNotExist()); + IterableSubject subject = requestAuditing.assertRequestHeader(X_GOOG_GCS_IDEMPOTENCY_TOKEN); + + byte[] actual = storage.readAllBytes(gen1.getBlobId(), BlobSourceOption.generationMatch()); + String actualXxd = xxd(actual); + + assertAll(() -> subject.hasSize(1), () -> assertThat(actualXxd).isEqualTo(expectedXxd)); + } + + @Test + public void resumableUpload() throws Exception { + byte[] expected = DataGenerator.base64Characters().genBytes(512 * 1024 + 45); + String expectedXxd = xxd(expected); + + BlobInfo info = BlobInfo.newBuilder(bucket, generator.randomObjectName()).build(); + try (WriteChannel writer = storage.writer(info, BlobWriteOption.doesNotExist())) { + writer.setChunkSize(256 * 1024); + writer.write(ByteBuffer.wrap(Arrays.copyOfRange(expected, 0, 256 * 1024))); + writer.write(ByteBuffer.wrap(Arrays.copyOfRange(expected, 256 * 1024, 512 * 1024))); + writer.write(ByteBuffer.wrap(Arrays.copyOfRange(expected, 512 * 1024, expected.length))); + } + IterableSubject subject = requestAuditing.assertRequestHeader(X_GOOG_GCS_IDEMPOTENCY_TOKEN); + + byte[] actual = storage.readAllBytes(info.getBlobId()); + String actualXxd = xxd(actual); + + // We expect 4 distinct requests: + // 1. start resumable session + // 2. PUT first 256KiB + // 3. PUT second 256KiB + // 4. Finalize session and put final 45B + assertAll(() -> subject.hasSize(4), () -> assertThat(actualXxd).isEqualTo(expectedXxd)); + } +} diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITHttpIdempotencyTokenTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITHttpIdempotencyTokenTest.java new file mode 100644 index 0000000000..7030c52281 --- /dev/null +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITHttpIdempotencyTokenTest.java @@ -0,0 +1,182 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.it; + +import static com.google.cloud.storage.TestUtils.assertAll; +import static com.google.cloud.storage.TestUtils.xxd; +import static com.google.common.truth.Truth.assertThat; + +import com.google.api.gax.paging.Page; +import com.google.cloud.WriteChannel; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Bucket; +import com.google.cloud.storage.BucketInfo; +import com.google.cloud.storage.DataGenerator; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.Storage.BlobListOption; +import com.google.cloud.storage.Storage.BlobSourceOption; +import com.google.cloud.storage.Storage.BlobTargetOption; +import com.google.cloud.storage.Storage.BlobWriteOption; +import com.google.cloud.storage.Storage.BucketListOption; +import com.google.cloud.storage.StorageOptions; +import com.google.cloud.storage.it.runner.StorageITRunner; +import com.google.cloud.storage.it.runner.annotations.Backend; +import com.google.cloud.storage.it.runner.annotations.Inject; +import com.google.cloud.storage.it.runner.annotations.SingleBackend; +import com.google.cloud.storage.it.runner.registry.Generator; +import com.google.common.collect.ImmutableList; +import com.google.common.truth.IterableSubject; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(StorageITRunner.class) +@SingleBackend(Backend.PROD) +public final class ITHttpIdempotencyTokenTest { + + private static final String X_GOOG_GCS_IDEMPOTENCY_TOKEN = "x-goog-gcs-idempotency-token"; + @Inject public BucketInfo bucket; + @Inject public Generator generator; + + private Storage storage; + private RequestAuditing requestAuditing; + + @Before + public void setUp() throws Exception { + requestAuditing = new RequestAuditing(); + storage = StorageOptions.http().setTransportOptions(requestAuditing).build().getService(); + } + + @After + public void tearDown() throws Exception { + if (storage != null) { + storage.close(); + } + } + + @Test + public void simpleUnary() throws Exception { + Bucket gen1 = storage.get(bucket.getName()); + + IterableSubject subject = requestAuditing.assertRequestHeader(X_GOOG_GCS_IDEMPOTENCY_TOKEN); + assertAll(() -> subject.hasSize(1)); + } + + @Test + public void pageObjects() throws Exception { + String baseName = generator.randomObjectName(); + Blob blob1 = storage.create(BlobInfo.newBuilder(bucket, baseName + "1").build()); + Blob blob2 = storage.create(BlobInfo.newBuilder(bucket, baseName + "2").build()); + + requestAuditing.clear(); + ImmutableList expectedNamess = ImmutableList.of(blob1.getName(), blob2.getName()); + Page page = + storage.list(bucket.getName(), BlobListOption.prefix(baseName), BlobListOption.pageSize(1)); + + List collect = page.streamAll().map(BlobInfo::getName).collect(Collectors.toList()); + IterableSubject subject = requestAuditing.assertRequestHeader(X_GOOG_GCS_IDEMPOTENCY_TOKEN); + assertAll( + () -> assertThat(collect).hasSize(2), + () -> assertThat(collect).containsExactlyElementsIn(expectedNamess), + () -> subject.hasSize(2)); + } + + @Test + public void pageBucket() throws Exception { + String baseName = generator.randomBucketName(); + BucketInfo info1 = BucketInfo.of(baseName + "1"); + BucketInfo info2 = BucketInfo.of(baseName + "2"); + try (TemporaryBucket tmp1 = + TemporaryBucket.newBuilder().setBucketInfo(info1).setStorage(storage).build(); + TemporaryBucket tmp2 = + TemporaryBucket.newBuilder().setBucketInfo(info2).setStorage(storage).build()) { + requestAuditing.clear(); + Page page = + storage.list(BucketListOption.prefix(baseName), BucketListOption.pageSize(1)); + + List collect = page.streamAll().collect(Collectors.toList()); + IterableSubject subject = requestAuditing.assertRequestHeader(X_GOOG_GCS_IDEMPOTENCY_TOKEN); + assertAll(() -> assertThat(collect).hasSize(2), () -> subject.hasSize(2)); + } + } + + @Test + public void readObject() throws Exception { + byte[] expected = DataGenerator.base64Characters().genBytes(512 * 1024 + 45); + String expectedXxd = xxd(expected); + + Blob gen1 = + storage.create( + BlobInfo.newBuilder(bucket, generator.randomObjectName()).build(), + expected, + BlobTargetOption.doesNotExist()); + + requestAuditing.clear(); + byte[] actual = storage.readAllBytes(gen1.getBlobId(), BlobSourceOption.generationMatch()); + IterableSubject subject = requestAuditing.assertRequestHeader(X_GOOG_GCS_IDEMPOTENCY_TOKEN); + String actualXxd = xxd(actual); + + assertAll(() -> subject.hasSize(1), () -> assertThat(actualXxd).isEqualTo(expectedXxd)); + } + + @Test + public void directUpload() throws Exception { + byte[] expected = DataGenerator.base64Characters().genBytes(512 * 1024 + 45); + String expectedXxd = xxd(expected); + + BlobInfo info = BlobInfo.newBuilder(bucket, generator.randomObjectName()).build(); + requestAuditing.clear(); + Blob gen1 = storage.create(info, expected, BlobTargetOption.doesNotExist()); + IterableSubject subject = requestAuditing.assertRequestHeader(X_GOOG_GCS_IDEMPOTENCY_TOKEN); + + byte[] actual = storage.readAllBytes(gen1.getBlobId(), BlobSourceOption.generationMatch()); + String actualXxd = xxd(actual); + + assertAll(() -> subject.hasSize(1), () -> assertThat(actualXxd).isEqualTo(expectedXxd)); + } + + @Test + public void resumableUpload() throws Exception { + byte[] expected = DataGenerator.base64Characters().genBytes(512 * 1024 + 45); + String expectedXxd = xxd(expected); + + BlobInfo info = BlobInfo.newBuilder(bucket, generator.randomObjectName()).build(); + try (WriteChannel writer = storage.writer(info, BlobWriteOption.doesNotExist())) { + writer.setChunkSize(256 * 1024); + writer.write(ByteBuffer.wrap(Arrays.copyOfRange(expected, 0, 256 * 1024))); + writer.write(ByteBuffer.wrap(Arrays.copyOfRange(expected, 256 * 1024, 512 * 1024))); + writer.write(ByteBuffer.wrap(Arrays.copyOfRange(expected, 512 * 1024, expected.length))); + } + IterableSubject subject = requestAuditing.assertRequestHeader(X_GOOG_GCS_IDEMPOTENCY_TOKEN); + + byte[] actual = storage.readAllBytes(info.getBlobId()); + String actualXxd = xxd(actual); + + // We expect 4 distinct requests: + // 1. start resumable session + // 2. PUT first 256KiB + // 3. PUT second 256KiB + // 4. Finalize session and put final 45B + assertAll(() -> subject.hasSize(4), () -> assertThat(actualXxd).isEqualTo(expectedXxd)); + } +} diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/RequestAuditing.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/RequestAuditing.java index 1087ad8b2f..bbb1821c9d 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/RequestAuditing.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/RequestAuditing.java @@ -31,6 +31,7 @@ import com.google.cloud.http.HttpTransportOptions; import com.google.cloud.storage.it.CSEKSupport.EncryptionKeyTuple; import com.google.common.collect.ImmutableList; +import com.google.common.truth.IterableSubject; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -194,4 +195,18 @@ void assertMultipartJsonField(String jsonField, Object expectedValue) { .that(collect) .isEqualTo(ImmutableList.of(expectedValue)); } + + IterableSubject assertRequestHeader(String headerName) { + ImmutableList requests = getRequests(); + + List actual = + requests.stream() + .map(HttpRequest::getHeaders) + .map(headers -> headers.get(headerName)) + .filter(Objects::nonNull) + .distinct() + .collect(Collectors.toList()); + + return assertWithMessage(String.format("Headers %s", headerName)).that(actual); + } } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/Generator.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/Generator.java index 383053faa8..e056a3418d 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/Generator.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/Generator.java @@ -45,7 +45,9 @@ public String randomObjectName() { throw new IllegalStateException("No actively running test in registry."); } AtomicInteger counter = counters.computeIfAbsent(currentTest, (d) -> new AtomicInteger(1)); - return String.format("%s-%04d", currentTest.getMethodName(), counter.getAndIncrement()); + return String.format( + "%s.%s-%04d", + currentTest.getClassName(), currentTest.getMethodName(), counter.getAndIncrement()); } @Override From 06a8e71219ce208cebcf691a07e1e1368cb5c482 Mon Sep 17 00:00:00 2001 From: BenWhitehead Date: Tue, 20 Jun 2023 17:03:47 -0400 Subject: [PATCH 10/15] chore: attempt to make TestBench docker image renovateable (#2046) Renovate-bot does not look into java files for docker images. Add a Dockerfile into our test resources that is loaded on TestBench class initialization. --- .../storage/it/runner/SneakyException.java | 2 +- .../storage/it/runner/registry/TestBench.java | 57 +++++++++++++++++-- .../storage/it/runner/registry/Dockerfile | 1 + 3 files changed, 54 insertions(+), 6 deletions(-) create mode 100644 google-cloud-storage/src/test/resources/com/google/cloud/storage/it/runner/registry/Dockerfile diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/SneakyException.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/SneakyException.java index 52790c3ddf..aa6d005995 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/SneakyException.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/SneakyException.java @@ -23,7 +23,7 @@ *

This class provides some utility methods to sneakily (not method declared) throw exceptions * and later unwrap any sneakily wrapped exception if it's needed. */ -final class SneakyException extends RuntimeException { +public final class SneakyException extends RuntimeException { public SneakyException(Throwable cause) { super(cause); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/TestBench.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/TestBench.java index 41c4f49788..03a53f4e2d 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/TestBench.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/TestBench.java @@ -30,9 +30,13 @@ import com.google.api.gax.retrying.BasicResultRetryAlgorithm; import com.google.api.gax.retrying.RetrySettings; import com.google.cloud.RetryHelper.RetryHelperException; +import com.google.cloud.Tuple; import com.google.cloud.conformance.storage.v1.InstructionList; import com.google.cloud.conformance.storage.v1.Method; +import com.google.cloud.storage.it.runner.SneakyException; +import com.google.common.base.Charsets; import com.google.common.collect.ImmutableList; +import com.google.common.io.CharStreams; import com.google.gson.Gson; import com.google.gson.JsonArray; import com.google.gson.JsonElement; @@ -41,15 +45,20 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.net.SocketException; import java.net.URI; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.threeten.bp.Duration; /** @@ -411,9 +420,47 @@ public String toString() { static final class Builder { private static final String DEFAULT_BASE_URI = "http://localhost:9000"; private static final String DEFAULT_GRPC_BASE_URI = "http://localhost:9005"; - private static final String DEFAULT_IMAGE_NAME = - "gcr.io/cloud-devrel-public-resources/storage-testbench"; - private static final String DEFAULT_IMAGE_TAG = "v0.35.0"; + private static final String DEFAULT_IMAGE_NAME; + private static final String DEFAULT_IMAGE_TAG; + + static { + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + Tuple nameAndTag = + SneakyException.unwrap( + () -> { + InputStream dockerfileText = + cl.getResourceAsStream( + "com/google/cloud/storage/it/runner/registry/Dockerfile"); + //noinspection UnstableApiUsage + return Optional.ofNullable(dockerfileText) + .map(is -> new InputStreamReader(is, Charsets.UTF_8)) + .flatMap( + reader -> + SneakyException.sneaky( + () -> + CharStreams.readLines(reader).stream() + .filter(line -> !line.startsWith("#")) + .filter(line -> line.startsWith("FROM")) + .findFirst() + .flatMap( + from -> { + Pattern pattern = + Pattern.compile("FROM (.*?):(.*)$"); + Matcher matcher = pattern.matcher(from); + if (matcher.matches()) { + return Optional.of( + Tuple.of( + matcher.group(1), matcher.group(2))); + } else { + return Optional.empty(); + } + }))); + }) + .orElse(Tuple.of(null, null)); + DEFAULT_IMAGE_NAME = nameAndTag.x(); + DEFAULT_IMAGE_TAG = nameAndTag.y(); + } + private static final String DEFAULT_CONTAINER_NAME = "default"; private boolean ignorePullError; @@ -483,8 +530,8 @@ public TestBench build() { ignorePullError, baseUri, gRPCBaseUri, - dockerImageName, - dockerImageTag, + requireNonNull(dockerImageName, "dockerImageName must be non null"), + requireNonNull(dockerImageTag, "dockerImageTag must be non null"), String.format("storage-testbench_%s", containerName)); } } diff --git a/google-cloud-storage/src/test/resources/com/google/cloud/storage/it/runner/registry/Dockerfile b/google-cloud-storage/src/test/resources/com/google/cloud/storage/it/runner/registry/Dockerfile new file mode 100644 index 0000000000..0314140016 --- /dev/null +++ b/google-cloud-storage/src/test/resources/com/google/cloud/storage/it/runner/registry/Dockerfile @@ -0,0 +1 @@ +FROM gcr.io/cloud-devrel-public-resources/storage-testbench:v0.35.0 From 9f618cddbeb471f7bd0f2332c70e501afbaccc36 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Wed, 21 Jun 2023 00:09:44 +0200 Subject: [PATCH 11/15] deps: update dependency com.google.apis:google-api-services-storage to v1-rev20230617-2.0.0 (#2077) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b744479106..e80701ab27 100644 --- a/pom.xml +++ b/pom.xml @@ -75,7 +75,7 @@ com.google.apis google-api-services-storage - v1-rev20230301-2.0.0 + v1-rev20230617-2.0.0 com.google.cloud From 14a907f664a0d1e46c16a51b5dde164d768b523f Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Wed, 21 Jun 2023 00:10:24 +0200 Subject: [PATCH 12/15] test(deps): update dependency com.google.truth:truth to v1.1.5 (#2076) --- pom.xml | 2 +- samples/install-without-bom/pom.xml | 2 +- samples/native-image-sample/pom.xml | 2 +- samples/snapshot/pom.xml | 2 +- samples/snippets/pom.xml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index e80701ab27..66b97f85da 100644 --- a/pom.xml +++ b/pom.xml @@ -164,7 +164,7 @@ com.google.truth truth - 1.1.4 + 1.1.5 test diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index 6ad8b2bbb0..f4c37b20ec 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -43,7 +43,7 @@ com.google.truth truth - 1.1.4 + 1.1.5 test diff --git a/samples/native-image-sample/pom.xml b/samples/native-image-sample/pom.xml index 14152b6c89..354c81d3cb 100644 --- a/samples/native-image-sample/pom.xml +++ b/samples/native-image-sample/pom.xml @@ -55,7 +55,7 @@ com.google.truth truth - 1.1.4 + 1.1.5 test diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index df4f78504c..423332fc22 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -40,7 +40,7 @@ com.google.truth truth - 1.1.4 + 1.1.5 test diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index 95f581a758..df642f8237 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -54,7 +54,7 @@ com.google.truth truth - 1.1.4 + 1.1.5 test From 9b0154eb2579c2f4816e8347bf486d3de9fea49d Mon Sep 17 00:00:00 2001 From: BenWhitehead Date: Wed, 21 Jun 2023 13:27:45 -0400 Subject: [PATCH 13/15] chore: create 2.22.x branch (#2082) --- .github/release-please.yml | 9 ++++++++- .github/sync-repo-settings.yaml | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/.github/release-please.yml b/.github/release-please.yml index d71d8f4eae..1280ad037a 100644 --- a/.github/release-please.yml +++ b/.github/release-please.yml @@ -1,7 +1,8 @@ bumpMinorPreMajor: true handleGHRelease: true releaseType: java-yoshi -extraFiles: ["README.md"] +extraFiles: + - README.md branches: - bumpMinorPreMajor: true handleGHRelease: true @@ -31,3 +32,9 @@ branches: handleGHRelease: true releaseType: java-backport branch: 2.15.x + - bumpMinorPreMajor: true + handleGHRelease: true + releaseType: java-backport + extraFiles: + - README.md + branch: 2.22.x diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index b7d5dcb3aa..ec449b6a07 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -129,6 +129,23 @@ branchProtectionRules: - cla/google - 'Kokoro - Test: Java GraalVM Native Image' - 'Kokoro - Test: Java 17 GraalVM Native Image' + - pattern: 2.22.x + isAdminEnforced: true + requiredApprovingReviewCount: 1 + requiresCodeOwnerReviews: true + requiresStrictStatusChecks: false + requiredStatusCheckContexts: + - dependencies (8) + - dependencies (11) + - lint + - clirr + - units (8) + - units (11) + - 'Kokoro - Test: Integration' + - cla/google + - OwlBot Post Processor + - 'Kokoro - Test: Java GraalVM Native Image' + - 'Kokoro - Test: Java 17 GraalVM Native Image' permissionRules: - team: yoshi-admins permission: admin From e10bde26416bcf17401516e43949e12246f4831c Mon Sep 17 00:00:00 2001 From: BenWhitehead Date: Wed, 21 Jun 2023 14:10:23 -0400 Subject: [PATCH 14/15] deps: update dependencies io.grpc:* to v1.56.0 (#2072) --- pom.xml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pom.xml b/pom.xml index 66b97f85da..f0003ff23d 100644 --- a/pom.xml +++ b/pom.xml @@ -59,12 +59,25 @@ + + io.grpc + grpc-bom + 1.56.0 + pom + import + com.google.cloud google-cloud-shared-dependencies ${google.cloud.shared-dependencies.version} pom import + + + io.grpc + * + + From 44718bf90b5e47f37ea05ca079d4f6f66cf5b4d2 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 21 Jun 2023 15:11:06 -0400 Subject: [PATCH 15/15] chore(main): release 2.23.0 (#2075) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 14 ++++++++++++++ README.md | 8 ++++---- gapic-google-cloud-storage-v2/pom.xml | 4 ++-- google-cloud-storage-bom/pom.xml | 10 +++++----- google-cloud-storage/pom.xml | 4 ++-- grpc-google-cloud-storage-v2/pom.xml | 4 ++-- pom.xml | 10 +++++----- proto-google-cloud-storage-v2/pom.xml | 4 ++-- samples/snapshot/pom.xml | 2 +- versions.txt | 8 ++++---- 10 files changed, 41 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cbe9b317e5..b4cf5918e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## [2.23.0](https://github.com/googleapis/java-storage/compare/v2.22.4...v2.23.0) (2023-06-21) + + +### Features + +* Add new dedup utility method to Option classes ([#2063](https://github.com/googleapis/java-storage/issues/2063)) ([2ad196c](https://github.com/googleapis/java-storage/commit/2ad196c063e67c7efdac344792b67de3479d789d)) + + +### Dependencies + +* Update dependencies io.grpc:* to v1.56.0 ([#2072](https://github.com/googleapis/java-storage/issues/2072)) ([e10bde2](https://github.com/googleapis/java-storage/commit/e10bde26416bcf17401516e43949e12246f4831c)) +* Update dependency com.google.apis:google-api-services-storage to v1-rev20230617-2.0.0 ([#2077](https://github.com/googleapis/java-storage/issues/2077)) ([9f618cd](https://github.com/googleapis/java-storage/commit/9f618cddbeb471f7bd0f2332c70e501afbaccc36)) +* Update dependency org.graalvm.buildtools:native-maven-plugin to v0.9.23 ([#2074](https://github.com/googleapis/java-storage/issues/2074)) ([427f330](https://github.com/googleapis/java-storage/commit/427f330793a20b0c3da4cbe5e85984a0df508c79)) + ## [2.22.4](https://github.com/googleapis/java-storage/compare/v2.22.3...v2.22.4) (2023-06-07) diff --git a/README.md b/README.md index 1b1d586b02..03d82de001 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ If you are using Maven without the BOM, add this to your dependencies: com.google.cloud google-cloud-storage - 2.22.4 + 2.23.0 ``` @@ -50,20 +50,20 @@ If you are using Maven without the BOM, add this to your dependencies: If you are using Gradle 5.x or later, add this to your dependencies: ```Groovy -implementation platform('com.google.cloud:libraries-bom:26.17.0') +implementation platform('com.google.cloud:libraries-bom:2.23.0') implementation 'com.google.cloud:google-cloud-storage' ``` If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation 'com.google.cloud:google-cloud-storage:2.22.4' +implementation 'com.google.cloud:google-cloud-storage:2.23.0' ``` If you are using SBT, add this to your dependencies: ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-storage" % "2.22.4" +libraryDependencies += "com.google.cloud" % "google-cloud-storage" % "2.23.0" ``` diff --git a/gapic-google-cloud-storage-v2/pom.xml b/gapic-google-cloud-storage-v2/pom.xml index e8aa83e0f1..77a8ee3cbd 100644 --- a/gapic-google-cloud-storage-v2/pom.xml +++ b/gapic-google-cloud-storage-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc gapic-google-cloud-storage-v2 - 2.22.5-alpha-SNAPSHOT + 2.23.0-alpha gapic-google-cloud-storage-v2 GRPC library for gapic-google-cloud-storage-v2 com.google.cloud google-cloud-storage-parent - 2.22.5-SNAPSHOT + 2.23.0 diff --git a/google-cloud-storage-bom/pom.xml b/google-cloud-storage-bom/pom.xml index 356eca8e26..d2a7c7ddbb 100644 --- a/google-cloud-storage-bom/pom.xml +++ b/google-cloud-storage-bom/pom.xml @@ -19,7 +19,7 @@ 4.0.0 com.google.cloud google-cloud-storage-bom - 2.22.5-SNAPSHOT + 2.23.0 pom com.google.cloud @@ -69,22 +69,22 @@ com.google.cloud google-cloud-storage - 2.22.5-SNAPSHOT + 2.23.0 com.google.api.grpc gapic-google-cloud-storage-v2 - 2.22.5-alpha-SNAPSHOT + 2.23.0-alpha com.google.api.grpc grpc-google-cloud-storage-v2 - 2.22.5-alpha-SNAPSHOT + 2.23.0-alpha com.google.api.grpc proto-google-cloud-storage-v2 - 2.22.5-alpha-SNAPSHOT + 2.23.0-alpha diff --git a/google-cloud-storage/pom.xml b/google-cloud-storage/pom.xml index aa16af16d1..f2d89a48e3 100644 --- a/google-cloud-storage/pom.xml +++ b/google-cloud-storage/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-storage - 2.22.5-SNAPSHOT + 2.23.0 jar Google Cloud Storage https://github.com/googleapis/java-storage @@ -12,7 +12,7 @@ com.google.cloud google-cloud-storage-parent - 2.22.5-SNAPSHOT + 2.23.0 google-cloud-storage diff --git a/grpc-google-cloud-storage-v2/pom.xml b/grpc-google-cloud-storage-v2/pom.xml index 1a6e14178f..0812273fba 100644 --- a/grpc-google-cloud-storage-v2/pom.xml +++ b/grpc-google-cloud-storage-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-storage-v2 - 2.22.5-alpha-SNAPSHOT + 2.23.0-alpha grpc-google-cloud-storage-v2 GRPC library for grpc-google-cloud-storage-v2 com.google.cloud google-cloud-storage-parent - 2.22.5-SNAPSHOT + 2.23.0 diff --git a/pom.xml b/pom.xml index f0003ff23d..f0a812e0b7 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-storage-parent pom - 2.22.5-SNAPSHOT + 2.23.0 Storage Parent https://github.com/googleapis/java-storage @@ -83,7 +83,7 @@ com.google.cloud google-cloud-storage - 2.22.5-SNAPSHOT + 2.23.0 com.google.apis @@ -124,17 +124,17 @@ com.google.api.grpc proto-google-cloud-storage-v2 - 2.22.5-alpha-SNAPSHOT + 2.23.0-alpha com.google.api.grpc grpc-google-cloud-storage-v2 - 2.22.5-alpha-SNAPSHOT + 2.23.0-alpha com.google.api.grpc gapic-google-cloud-storage-v2 - 2.22.5-alpha-SNAPSHOT + 2.23.0-alpha com.google.cloud diff --git a/proto-google-cloud-storage-v2/pom.xml b/proto-google-cloud-storage-v2/pom.xml index dc5d1647b8..7ba164b281 100644 --- a/proto-google-cloud-storage-v2/pom.xml +++ b/proto-google-cloud-storage-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-storage-v2 - 2.22.5-alpha-SNAPSHOT + 2.23.0-alpha proto-google-cloud-storage-v2 PROTO library for proto-google-cloud-storage-v2 com.google.cloud google-cloud-storage-parent - 2.22.5-SNAPSHOT + 2.23.0 diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 423332fc22..f9369a6ede 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -28,7 +28,7 @@ com.google.cloud google-cloud-storage - 2.22.5-SNAPSHOT + 2.23.0 diff --git a/versions.txt b/versions.txt index 84a23716b7..8441fe3ade 100644 --- a/versions.txt +++ b/versions.txt @@ -1,7 +1,7 @@ # Format: # module:released-version:current-version -google-cloud-storage:2.22.4:2.22.5-SNAPSHOT -gapic-google-cloud-storage-v2:2.22.4-alpha:2.22.5-alpha-SNAPSHOT -grpc-google-cloud-storage-v2:2.22.4-alpha:2.22.5-alpha-SNAPSHOT -proto-google-cloud-storage-v2:2.22.4-alpha:2.22.5-alpha-SNAPSHOT +google-cloud-storage:2.23.0:2.23.0 +gapic-google-cloud-storage-v2:2.23.0-alpha:2.23.0-alpha +grpc-google-cloud-storage-v2:2.23.0-alpha:2.23.0-alpha +proto-google-cloud-storage-v2:2.23.0-alpha:2.23.0-alpha