diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index a3da1b0d..ec696b55 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:3e3800bb100af5d7f9e810d48212b37812c1856d20ffeafb99ebe66461b61fc7 -# created: 2023-08-02T10:53:29.114535628Z + digest: sha256:30470597773378105e239b59fce8eb27cc97375580d592699206d17d117143d0 +# created: 2023-11-03T00:57:07.335914631Z diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index e97d89e4..221806ce 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -28,7 +28,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: "3.9" + python-version: "3.10" - name: Install nox run: | python -m pip install --upgrade setuptools pip wheel diff --git a/.gitignore b/.gitignore index 99c3a144..168b201f 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,7 @@ docs.metadata # Virtual environment env/ +venv/ # Test logs coverage.xml diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt index 029bd342..16170d0c 100644 --- a/.kokoro/requirements.txt +++ b/.kokoro/requirements.txt @@ -113,30 +113,30 @@ commonmark==0.9.1 \ --hash=sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60 \ --hash=sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9 # via rich -cryptography==41.0.3 \ - --hash=sha256:0d09fb5356f975974dbcb595ad2d178305e5050656affb7890a1583f5e02a306 \ - --hash=sha256:23c2d778cf829f7d0ae180600b17e9fceea3c2ef8b31a99e3c694cbbf3a24b84 \ - --hash=sha256:3fb248989b6363906827284cd20cca63bb1a757e0a2864d4c1682a985e3dca47 \ - --hash=sha256:41d7aa7cdfded09b3d73a47f429c298e80796c8e825ddfadc84c8a7f12df212d \ - --hash=sha256:42cb413e01a5d36da9929baa9d70ca90d90b969269e5a12d39c1e0d475010116 \ - --hash=sha256:4c2f0d35703d61002a2bbdcf15548ebb701cfdd83cdc12471d2bae80878a4207 \ - --hash=sha256:4fd871184321100fb400d759ad0cddddf284c4b696568204d281c902fc7b0d81 \ - --hash=sha256:5259cb659aa43005eb55a0e4ff2c825ca111a0da1814202c64d28a985d33b087 \ - --hash=sha256:57a51b89f954f216a81c9d057bf1a24e2f36e764a1ca9a501a6964eb4a6800dd \ - --hash=sha256:652627a055cb52a84f8c448185922241dd5217443ca194d5739b44612c5e6507 \ - --hash=sha256:67e120e9a577c64fe1f611e53b30b3e69744e5910ff3b6e97e935aeb96005858 \ - --hash=sha256:6af1c6387c531cd364b72c28daa29232162010d952ceb7e5ca8e2827526aceae \ - --hash=sha256:6d192741113ef5e30d89dcb5b956ef4e1578f304708701b8b73d38e3e1461f34 \ - --hash=sha256:7efe8041897fe7a50863e51b77789b657a133c75c3b094e51b5e4b5cec7bf906 \ - --hash=sha256:84537453d57f55a50a5b6835622ee405816999a7113267739a1b4581f83535bd \ - --hash=sha256:8f09daa483aedea50d249ef98ed500569841d6498aa9c9f4b0531b9964658922 \ - --hash=sha256:95dd7f261bb76948b52a5330ba5202b91a26fbac13ad0e9fc8a3ac04752058c7 \ - --hash=sha256:a74fbcdb2a0d46fe00504f571a2a540532f4c188e6ccf26f1f178480117b33c4 \ - --hash=sha256:a983e441a00a9d57a4d7c91b3116a37ae602907a7618b882c8013b5762e80574 \ - --hash=sha256:ab8de0d091acbf778f74286f4989cf3d1528336af1b59f3e5d2ebca8b5fe49e1 \ - --hash=sha256:aeb57c421b34af8f9fe830e1955bf493a86a7996cc1338fe41b30047d16e962c \ - --hash=sha256:ce785cf81a7bdade534297ef9e490ddff800d956625020ab2ec2780a556c313e \ - --hash=sha256:d0d651aa754ef58d75cec6edfbd21259d93810b73f6ec246436a21b7841908de +cryptography==41.0.4 \ + --hash=sha256:004b6ccc95943f6a9ad3142cfabcc769d7ee38a3f60fb0dddbfb431f818c3a67 \ + --hash=sha256:047c4603aeb4bbd8db2756e38f5b8bd7e94318c047cfe4efeb5d715e08b49311 \ + --hash=sha256:0d9409894f495d465fe6fda92cb70e8323e9648af912d5b9141d616df40a87b8 \ + --hash=sha256:23a25c09dfd0d9f28da2352503b23e086f8e78096b9fd585d1d14eca01613e13 \ + --hash=sha256:2ed09183922d66c4ec5fdaa59b4d14e105c084dd0febd27452de8f6f74704143 \ + --hash=sha256:35c00f637cd0b9d5b6c6bd11b6c3359194a8eba9c46d4e875a3660e3b400005f \ + --hash=sha256:37480760ae08065437e6573d14be973112c9e6dcaf5f11d00147ee74f37a3829 \ + --hash=sha256:3b224890962a2d7b57cf5eeb16ccaafba6083f7b811829f00476309bce2fe0fd \ + --hash=sha256:5a0f09cefded00e648a127048119f77bc2b2ec61e736660b5789e638f43cc397 \ + --hash=sha256:5b72205a360f3b6176485a333256b9bcd48700fc755fef51c8e7e67c4b63e3ac \ + --hash=sha256:7e53db173370dea832190870e975a1e09c86a879b613948f09eb49324218c14d \ + --hash=sha256:7febc3094125fc126a7f6fb1f420d0da639f3f32cb15c8ff0dc3997c4549f51a \ + --hash=sha256:80907d3faa55dc5434a16579952ac6da800935cd98d14dbd62f6f042c7f5e839 \ + --hash=sha256:86defa8d248c3fa029da68ce61fe735432b047e32179883bdb1e79ed9bb8195e \ + --hash=sha256:8ac4f9ead4bbd0bc8ab2d318f97d85147167a488be0e08814a37eb2f439d5cf6 \ + --hash=sha256:93530900d14c37a46ce3d6c9e6fd35dbe5f5601bf6b3a5c325c7bffc030344d9 \ + --hash=sha256:9eeb77214afae972a00dee47382d2591abe77bdae166bda672fb1e24702a3860 \ + --hash=sha256:b5f4dfe950ff0479f1f00eda09c18798d4f49b98f4e2006d644b3301682ebdca \ + --hash=sha256:c3391bd8e6de35f6f1140e50aaeb3e2b3d6a9012536ca23ab0d9c35ec18c8a91 \ + --hash=sha256:c880eba5175f4307129784eca96f4e70b88e57aa3f680aeba3bab0e980b0f37d \ + --hash=sha256:cecfefa17042941f94ab54f769c8ce0fe14beff2694e9ac684176a2535bf9714 \ + --hash=sha256:e40211b4923ba5a6dc9769eab704bdb3fbb58d56c5b336d30996c24fcf12aadb \ + --hash=sha256:efc8ad4e6fc4f1752ebfb58aefece8b4e3c4cae940b0994d43649bdfce8d0d4f # via # gcp-releasetool # secretstorage @@ -382,6 +382,7 @@ protobuf==3.20.3 \ # gcp-docuploader # gcp-releasetool # google-api-core + # googleapis-common-protos pyasn1==0.4.8 \ --hash=sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d \ --hash=sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba @@ -466,9 +467,9 @@ typing-extensions==4.4.0 \ --hash=sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa \ --hash=sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e # via -r requirements.in -urllib3==1.26.12 \ - --hash=sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e \ - --hash=sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997 +urllib3==1.26.18 \ + --hash=sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07 \ + --hash=sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0 # via # requests # twine diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 19409cbd..6a8e1695 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,7 +22,7 @@ repos: - id: end-of-file-fixer - id: check-yaml - repo: https://github.com/psf/black - rev: 22.3.0 + rev: 23.7.0 hooks: - id: black - repo: https://github.com/pycqa/flake8 diff --git a/CHANGELOG.md b/CHANGELOG.md index 93c3848c..7b0dacfe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,20 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.13.0](https://github.com/googleapis/python-api-core/compare/v2.12.0...v2.13.0) (2023-11-03) + + +### Features + +* Add caching to routing header calculation ([#526](https://github.com/googleapis/python-api-core/issues/526)) ([6251eab](https://github.com/googleapis/python-api-core/commit/6251eab3fca5f7e509cb9b6e476ce1184094b711)) + + +### Bug Fixes + +* Add warning to retry target to avoid incorrect usage ([#543](https://github.com/googleapis/python-api-core/issues/543)) ([bfb40e6](https://github.com/googleapis/python-api-core/commit/bfb40e6929ef47be7a6464d2f1e0d06595736b8d)) +* Drop usage of distutils ([#541](https://github.com/googleapis/python-api-core/issues/541)) ([4bd9e10](https://github.com/googleapis/python-api-core/commit/4bd9e10f20eea227c88e3e1496010cca6dd8a270)) +* Ensure exception is available when BackgroundConsumer open stream fails ([#357](https://github.com/googleapis/python-api-core/issues/357)) ([405272c](https://github.com/googleapis/python-api-core/commit/405272c05f8c6d20e242c6172b01f78f0fd3bf32)) + ## [2.12.0](https://github.com/googleapis/python-api-core/compare/v2.11.1...v2.12.0) (2023-09-07) diff --git a/google/api_core/bidi.py b/google/api_core/bidi.py index 57f5f9dd..74abc495 100644 --- a/google/api_core/bidi.py +++ b/google/api_core/bidi.py @@ -92,10 +92,7 @@ def _is_active(self): # Note: there is a possibility that this starts *before* the call # property is set. So we have to check if self.call is set before # seeing if it's active. - if self.call is not None and not self.call.is_active(): - return False - else: - return True + return self.call is not None and self.call.is_active() def __iter__(self): if self._initial_request is not None: @@ -265,6 +262,10 @@ def add_done_callback(self, callback): self._callbacks.append(callback) def _on_call_done(self, future): + # This occurs when the RPC errors or is successfully terminated. + # Note that grpc's "future" here can also be a grpc.RpcError. + # See note in https://github.com/grpc/grpc/issues/10885#issuecomment-302651331 + # that `grpc.RpcError` is also `grpc.call`. for callback in self._callbacks: callback(future) @@ -276,7 +277,13 @@ def open(self): request_generator = _RequestQueueGenerator( self._request_queue, initial_request=self._initial_request ) - call = self._start_rpc(iter(request_generator), metadata=self._rpc_metadata) + try: + call = self._start_rpc(iter(request_generator), metadata=self._rpc_metadata) + except exceptions.GoogleAPICallError as exc: + # The original `grpc.RpcError` (which is usually also a `grpc.Call`) is + # available from the ``response`` property on the mapped exception. + self._on_call_done(exc.response) + raise request_generator.call = call diff --git a/google/api_core/gapic_v1/routing_header.py b/google/api_core/gapic_v1/routing_header.py index 28b13abf..c0c6f648 100644 --- a/google/api_core/gapic_v1/routing_header.py +++ b/google/api_core/gapic_v1/routing_header.py @@ -20,17 +20,22 @@ Generally, these headers are specified as gRPC metadata. """ +import functools from enum import Enum from urllib.parse import urlencode ROUTING_METADATA_KEY = "x-goog-request-params" +# This is the value for the `maxsize` argument of @functools.lru_cache +# https://docs.python.org/3/library/functools.html#functools.lru_cache +# This represents the number of recent function calls to store. +ROUTING_PARAM_CACHE_SIZE = 32 def to_routing_header(params, qualified_enums=True): """Returns a routing header string for the given request parameters. Args: - params (Mapping[str, Any]): A dictionary containing the request + params (Mapping[str, str | bytes | Enum]): A dictionary containing the request parameters used for routing. qualified_enums (bool): Whether to represent enum values as their type-qualified symbol names instead of as their @@ -38,19 +43,11 @@ def to_routing_header(params, qualified_enums=True): Returns: str: The routing header string. - """ + tuples = params.items() if isinstance(params, dict) else params if not qualified_enums: - if isinstance(params, dict): - tuples = params.items() - else: - tuples = params - params = [(x[0], x[1].name) if isinstance(x[1], Enum) else x for x in tuples] - return urlencode( - params, - # Per Google API policy (go/api-url-encoding), / is not encoded. - safe="/", - ) + tuples = [(x[0], x[1].name) if isinstance(x[1], Enum) else x for x in tuples] + return "&".join([_urlencode_param(*t) for t in tuples]) def to_grpc_metadata(params, qualified_enums=True): @@ -58,7 +55,7 @@ def to_grpc_metadata(params, qualified_enums=True): request parameters. Args: - params (Mapping[str, Any]): A dictionary containing the request + params (Mapping[str, str | bytes | Enum]): A dictionary containing the request parameters used for routing. qualified_enums (bool): Whether to represent enum values as their type-qualified symbol names instead of as their @@ -69,3 +66,22 @@ def to_grpc_metadata(params, qualified_enums=True): and value. """ return (ROUTING_METADATA_KEY, to_routing_header(params, qualified_enums)) + + +# use caching to avoid repeated computation +@functools.lru_cache(maxsize=ROUTING_PARAM_CACHE_SIZE) +def _urlencode_param(key, value): + """Cacheable wrapper over urlencode + + Args: + key (str): The key of the parameter to encode. + value (str | bytes | Enum): The value of the parameter to encode. + + Returns: + str: The encoded parameter. + """ + return urlencode( + {key: value}, + # Per Google API policy (go/api-url-encoding), / is not encoded. + safe="/", + ) diff --git a/google/api_core/operations_v1/abstract_operations_client.py b/google/api_core/operations_v1/abstract_operations_client.py index 714c2aae..38f532af 100644 --- a/google/api_core/operations_v1/abstract_operations_client.py +++ b/google/api_core/operations_v1/abstract_operations_client.py @@ -14,7 +14,6 @@ # limitations under the License. # from collections import OrderedDict -from distutils import util import os import re from typing import Dict, Optional, Sequence, Tuple, Type, Union @@ -294,13 +293,16 @@ def __init__( client_options = client_options_lib.ClientOptions() # Create SSL credentials for mutual TLS if needed. - use_client_cert = bool( - util.strtobool(os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false")) - ) - + use_client_cert = os.getenv( + "GOOGLE_API_USE_CLIENT_CERTIFICATE", "false" + ).lower() + if use_client_cert not in ("true", "false"): + raise ValueError( + "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + ) client_cert_source_func = None is_mtls = False - if use_client_cert: + if use_client_cert == "true": if client_options.client_cert_source: is_mtls = True client_cert_source_func = client_options.client_cert_source diff --git a/google/api_core/retry.py b/google/api_core/retry.py index 84b5d0fe..08f8209e 100644 --- a/google/api_core/retry.py +++ b/google/api_core/retry.py @@ -62,6 +62,8 @@ def check_if_exists(): import random import sys import time +import inspect +import warnings from typing import Any, Callable, TypeVar, TYPE_CHECKING import requests.exceptions @@ -84,6 +86,7 @@ def check_if_exists(): _DEFAULT_MAXIMUM_DELAY = 60.0 # seconds _DEFAULT_DELAY_MULTIPLIER = 2.0 _DEFAULT_DEADLINE = 60.0 * 2.0 # seconds +_ASYNC_RETRY_WARNING = "Using the synchronous google.api_core.retry.Retry with asynchronous calls may lead to unexpected results. Please use google.api_core.retry_async.AsyncRetry instead." def if_exception_type( @@ -201,7 +204,10 @@ def retry_target( for sleep in sleep_generator: try: - return target() + result = target() + if inspect.isawaitable(result): + warnings.warn(_ASYNC_RETRY_WARNING) + return result # pylint: disable=broad-except # This function explicitly must deal with broad exceptions. diff --git a/google/api_core/version.py b/google/api_core/version.py index 67e043bd..b6000e20 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.12.0" +__version__ = "2.13.0" diff --git a/noxfile.py b/noxfile.py index 84abb498..82ea2669 100644 --- a/noxfile.py +++ b/noxfile.py @@ -232,7 +232,7 @@ def docs(session): ) -@nox.session(python="3.9") +@nox.session(python="3.10") def docfx(session): """Build the docfx yaml files for this library.""" diff --git a/tests/unit/gapic/test_routing_header.py b/tests/unit/gapic/test_routing_header.py index 9d31eb39..2c8c7546 100644 --- a/tests/unit/gapic/test_routing_header.py +++ b/tests/unit/gapic/test_routing_header.py @@ -70,3 +70,34 @@ def test_to_grpc_metadata(): params = [("name", "meep"), ("book.read", "1")] metadata = routing_header.to_grpc_metadata(params) assert metadata == (routing_header.ROUTING_METADATA_KEY, "name=meep&book.read=1") + + +@pytest.mark.parametrize( + "key,value,expected", + [ + ("book.read", "1", "book.read=1"), + ("name", "me/ep", "name=me/ep"), + ("\\", "=", "%5C=%3D"), + (b"hello", "world", "hello=world"), + ("✔️", "✌️", "%E2%9C%94%EF%B8%8F=%E2%9C%8C%EF%B8%8F"), + ], +) +def test__urlencode_param(key, value, expected): + result = routing_header._urlencode_param(key, value) + assert result == expected + + +def test__urlencode_param_caching_performance(): + import time + + key = "key" * 100 + value = "value" * 100 + # time with empty cache + start_time = time.perf_counter() + routing_header._urlencode_param(key, value) + duration = time.perf_counter() - start_time + second_start_time = time.perf_counter() + routing_header._urlencode_param(key, value) + second_duration = time.perf_counter() - second_start_time + # second call should be approximately 10 times faster + assert second_duration < duration / 10 diff --git a/tests/unit/test_bidi.py b/tests/unit/test_bidi.py index f5e2b72b..84ac9dc5 100644 --- a/tests/unit/test_bidi.py +++ b/tests/unit/test_bidi.py @@ -804,6 +804,21 @@ def test_wake_on_error(self): while consumer.is_active: pass + def test_rpc_callback_fires_when_consumer_start_fails(self): + expected_exception = exceptions.InvalidArgument( + "test", response=grpc.StatusCode.INVALID_ARGUMENT + ) + callback = mock.Mock(spec=["__call__"]) + + rpc, _ = make_rpc() + bidi_rpc = bidi.BidiRpc(rpc) + bidi_rpc.add_done_callback(callback) + bidi_rpc._start_rpc.side_effect = expected_exception + + consumer = bidi.BackgroundConsumer(bidi_rpc, on_response=None) + consumer.start() + assert callback.call_args.args[0] == grpc.StatusCode.INVALID_ARGUMENT + def test_consumer_expected_error(self, caplog): caplog.set_level(logging.DEBUG) diff --git a/tests/unit/test_retry.py b/tests/unit/test_retry.py index ec27056d..d770f7a2 100644 --- a/tests/unit/test_retry.py +++ b/tests/unit/test_retry.py @@ -129,6 +129,26 @@ def test_retry_target_non_retryable_error(utcnow, sleep): sleep.assert_not_called() +@mock.patch("asyncio.sleep", autospec=True) +@mock.patch( + "google.api_core.datetime_helpers.utcnow", + return_value=datetime.datetime.min, + autospec=True, +) +@pytest.mark.asyncio +async def test_retry_target_warning_for_retry(utcnow, sleep): + predicate = retry.if_exception_type(ValueError) + target = mock.AsyncMock(spec=["__call__"]) + + with pytest.warns(Warning) as exc_info: + # Note: predicate is just a filler and doesn't affect the test + retry.retry_target(target, predicate, range(10), None) + + assert len(exc_info) == 2 + assert str(exc_info[0].message) == retry._ASYNC_RETRY_WARNING + sleep.assert_not_called() + + @mock.patch("time.sleep", autospec=True) @mock.patch("google.api_core.datetime_helpers.utcnow", autospec=True) def test_retry_target_deadline_exceeded(utcnow, sleep): pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy