Upgrade dependencies in order to avoid backtracking (#27531)

Seems that the new `pip` have become a little more relaxed when it
comes to looking for dependencies, and we need to make sure that
version constraint is specified in every extra otherwise it might
have hard time finding the right set of dependencies while
running "eager upgrade".

Over the last few days all our main builds failed because of that as pip
was backtracking when performing eager upgrade..

This PR updates various dependencies to get rid of the backtracking,
following the process described in the

https://github.com/apache/airflow/blob/main/dev/TRACKING_BACKTRACKING_ISSUES.md

Author: potiuk

Dockerfile

@@ -48,7 +48,7 @@ ARG AIRFLOW_VERSION="2.4.2"
 
 ARG PYTHON_BASE_IMAGE="python:3.7-slim-bullseye"
 
-ARG AIRFLOW_PIP_VERSION=22.3
+ARG AIRFLOW_PIP_VERSION=22.3.1
 ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"
 ARG AIRFLOW_IMAGE_README_URL="https://raw.githubusercontent.com/apache/airflow/main/docs/docker-stack/README.md"
 
@@ -422,7 +422,7 @@ function common::get_airflow_version_specification() {
 function common::override_pip_version_if_needed() {
     if [[ -n ${AIRFLOW_VERSION} ]]; then
         if [[ ${AIRFLOW_VERSION} =~ ^2\.0.* || ${AIRFLOW_VERSION} =~ ^1\.* ]]; then
-            export AIRFLOW_PIP_VERSION="22.3"
+            export AIRFLOW_PIP_VERSION="22.3.1"
         fi
     fi
 }

Dockerfile.ci

@@ -382,7 +382,7 @@ function common::get_airflow_version_specification() {
 function common::override_pip_version_if_needed() {
     if [[ -n ${AIRFLOW_VERSION} ]]; then
         if [[ ${AIRFLOW_VERSION} =~ ^2\.0.* || ${AIRFLOW_VERSION} =~ ^1\.* ]]; then
-            export AIRFLOW_PIP_VERSION="22.3"
+            export AIRFLOW_PIP_VERSION="22.3.1"
         fi
     fi
 }
@@ -1093,7 +1093,7 @@ ARG AIRFLOW_CI_BUILD_EPOCH="3"
 ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true"
 # By default in the image, we are installing all providers when installing from sources
 ARG INSTALL_PROVIDERS_FROM_SOURCES="true"
-ARG AIRFLOW_PIP_VERSION=22.3
+ARG AIRFLOW_PIP_VERSION=22.3.1
 # Setup PIP
 # By default PIP install run without cache to make image smaller
 ARG PIP_NO_CACHE_DIR="true"

IMAGES.rst

@@ -457,7 +457,7 @@ The following build arguments (``--build-arg`` in docker build command) can be u
 | ``ADDITIONAL_DEV_APT_ENV``               |                                          | Additional env variables defined         |
 |                                          |                                          | when installing dev deps                 |
 +------------------------------------------+------------------------------------------+------------------------------------------+
-| ``AIRFLOW_PIP_VERSION``                  | ``22.3``                                 | PIP version used.                        |
+| ``AIRFLOW_PIP_VERSION``                  | ``22.3.1``                               | PIP version used.                        |
 +------------------------------------------+------------------------------------------+------------------------------------------+
 | ``PIP_PROGRESS_BAR``                     | ``on``                                   | Progress bar for PIP installation        |
 +------------------------------------------+------------------------------------------+------------------------------------------+

airflow/providers/amazon/provider.yaml

@@ -48,7 +48,7 @@ versions:
 dependencies:
   - apache-airflow>=2.3.0
   - apache-airflow-providers-common-sql>=1.3.0
-  - boto3>=1.15.0
+  - boto3>=1.24.0
   # watchtower 3 has been released end Jan and introduced breaking change across the board that might
   # change logging behaviour:
   # https://github.com/kislyuk/watchtower/blob/develop/Changes.rst#changes-for-v300-2022-01-26
@@ -58,9 +58,9 @@ dependencies:
   - redshift_connector>=2.0.888
   - sqlalchemy_redshift>=0.8.6
   - pandas>=0.17.1
-  - mypy-boto3-rds>=1.21.0
-  - mypy-boto3-redshift-data>=1.21.0
-  - mypy-boto3-appflow>=1.21.0
+  - mypy-boto3-rds>=1.24.0
+  - mypy-boto3-redshift-data>=1.24.0
+  - mypy-boto3-appflow>=1.24.0
 
 integrations:
   - integration-name: Amazon Athena

airflow/providers/google/provider.yaml

@@ -66,6 +66,7 @@ dependencies:
   # TODO: Upgrade all Google libraries that are limited to <2.0.0
   - PyOpenSSL
   - asgiref>=3.5.2
+  - gcloud_aio_auth>=4.0.0
   - gcloud-aio-bigquery>=6.1.2
   - gcloud-aio-storage
   - google-ads>=15.1.1

airflow/providers/microsoft/azure/provider.yaml

@@ -64,6 +64,7 @@ dependencies:
   # Limited due to https://github.com/Azure/azure-uamqp-python/issues/191
   - azure-servicebus>=7.6.1; platform_machine != "aarch64"
   - azure-synapse-spark
+  - adal>=.1.2.7
 
 integrations:
   - integration-name: Microsoft Azure Batch

dev/breeze/src/airflow_breeze/global_constants.py

@@ -74,7 +74,7 @@
 ALLOWED_MYSQL_VERSIONS = ["5.7", "8"]
 ALLOWED_MSSQL_VERSIONS = ["2017-latest", "2019-latest"]
 
-PIP_VERSION = "22.3"
+PIP_VERSION = "22.3.1"
 
 
 @lru_cache(maxsize=None)

dev/breeze/src/airflow_breeze/utils/find_newer_dependencies.py

@@ -92,7 +92,7 @@ def find_newer_dependencies(
     get_console().print("[info]Use the following pip install command (see the doc above for details)\n")
     get_console().print(
         'pip install ".[devel_all]" --upgrade --upgrade-strategy eager '
-        '"dill<0.3.3" "certifi<2021.0.0"' + constraint_string,
+        '"dill<0.3.3" "pyarrow>=6.0.0" "protobuf<4.21.0"' + constraint_string,
         markup=False,
         soft_wrap=True,
     )

docs/docker-stack/build-arg-ref.rst

@@ -45,7 +45,7 @@ Those are the most common arguments that you use when you want to build a custom
 +------------------------------------------+------------------------------------------+---------------------------------------------+
 | ``AIRFLOW_USER_HOME_DIR``                | ``/home/airflow``                        | Home directory of the Airflow user.         |
 +------------------------------------------+------------------------------------------+---------------------------------------------+
-| ``AIRFLOW_PIP_VERSION``                  | ``22.3``                                 |  PIP version used.                          |
+| ``AIRFLOW_PIP_VERSION``                  | ``22.3.1``                               |  PIP version used.                          |
 +------------------------------------------+------------------------------------------+---------------------------------------------+
 | ``ADDITIONAL_PIP_INSTALL_FLAGS``         |                                          | additional ``pip`` flags passed to the      |
 |                                          |                                          | installation commands (except when          |

generated/provider_dependencies.json

@@ -19,11 +19,11 @@
     "deps": [
       "apache-airflow-providers-common-sql>=1.3.0",
       "apache-airflow>=2.3.0",
-      "boto3>=1.15.0",
+      "boto3>=1.24.0",
       "jsonpath_ng>=1.5.3",
-      "mypy-boto3-appflow>=1.21.0",
-      "mypy-boto3-rds>=1.21.0",
-      "mypy-boto3-redshift-data>=1.21.0",
+      "mypy-boto3-appflow>=1.24.0",
+      "mypy-boto3-rds>=1.24.0",
+      "mypy-boto3-redshift-data>=1.24.0",
       "pandas>=0.17.1",
       "redshift_connector>=2.0.888",
       "sqlalchemy_redshift>=0.8.6",
@@ -305,6 +305,7 @@
       "asgiref>=3.5.2",
       "gcloud-aio-bigquery>=6.1.2",
       "gcloud-aio-storage",
+      "gcloud_aio_auth>=4.0.0",
       "google-ads>=15.1.1",
       "google-api-core>=2.7.0,<3.0.0",
       "google-api-python-client>=1.6.0,<2.0.0",
@@ -426,6 +427,7 @@
   },
   "microsoft.azure": {
     "deps": [
+      "adal>=.1.2.7",
       "apache-airflow>=2.3.0",
       "azure-batch>=8.0.0",
       "azure-cosmos>=4.0.0",