diff --git a/CHANGELOG.md b/CHANGELOG.md index dd09319..b4e96fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added +- `retry_request` and `aretry_request` helpers, which run the retry loop at the client level so that errors raised while reading the response body (such as `ReadTimeout` and `RemoteProtocolError`) are retried. This is something `RetryTransport` cannot do, as transports return before the body is read. They reject a client that already uses `RetryTransport` to avoid retrying every request twice. +- `validate_response` option on `Retry` to retry when a callback rejects an otherwise-successful response (for example, content-level blocks such as a CAPTCHA or authorization wall). +- `request.extensions["retry"]` support, allowing the `Retry` configuration to be overridden per request; the resolved `Retry` is also attached to `response.extensions["retry"]` for introspection. +- `Retry.copy_with` to derive a modified `Retry` (mirroring `httpx.URL.copy_with`), convenient for per-request overrides via `request.extensions["retry"]`. + ## [0.5.0] - 2026-04-20 ### Added @@ -113,7 +121,7 @@ from `httpx.BaseTransport` and `httpx.AsyncBaseTransport`. ### Added - Initial release -[Unreleased]: https://github.com/will-ockmore/httpx-retries/compare/0.3.0...HEAD +[Unreleased]: https://github.com/will-ockmore/httpx-retries/compare/0.5.0...HEAD [0.3.0]: https://github.com/will-ockmore/httpx-retries/releases/tag/0.3.0 [0.2.4]: https://github.com/will-ockmore/httpx-retries/releases/tag/0.2.4 [0.2.3]: https://github.com/will-ockmore/httpx-retries/releases/tag/0.2.3 diff --git a/README.md b/README.md index 678fba8..bbea585 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,11 @@ with httpx.Client(transport=transport) as client: response = client.get("https://example.com") ``` +> **Errors while reading the response body** (such as `ReadTimeout` part-way through a download) happen +> after the transport has returned, so `RetryTransport` can't retry them. This is a niche case, +> but if you read very large or slow bodies and need those retried, see the `retry_request` / `aretry_request` helpers in +> [Why wasn't my `ReadTimeout` retried?](https://will-ockmore.github.io/httpx-retries/faq/#why-wasnt-my-readtimeout-retried). + ## Features HTTPX Retries builds on the patterns users will expect from `urllib` and `requests`. The typical approach has been diff --git a/docs/api.md b/docs/api.md index 7afd832..33b7f99 100644 --- a/docs/api.md +++ b/docs/api.md @@ -2,7 +2,8 @@ options: members: - RetryTransport - - AsyncRetryTransport - Retry + - retry_request + - aretry_request filters: - "!^_" diff --git a/docs/faq.md b/docs/faq.md index 2108b02..1c61682 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -59,22 +59,35 @@ Not retried by [RetryTransport][httpx_retries.RetryTransport]: - Any exception raised during `response.read()`, `response.aread()`, or iteration of a streaming response — including `ReadTimeout` mid-body and `RemoteProtocolError("peer closed connection...")`. -If you need to retry body-phase errors today, do it at the call site: +!!! tip "Most requests don't need this! Keep using `RetryTransport`" + A body-phase error can only occur while the body is in transit, so for the small responses typical of most API traffic that window is tiny and almost every failure is already caught by [RetryTransport][httpx_retries.RetryTransport] at the header or status-code stage. The helpers below earn their keep in a narrower set of cases: reading **large or slow, non-streamed** bodies — big exports and downloads, or payloads proxied from a flaky origin — where a mid-body `ReadTimeout` or a truncated `RemoteProtocolError` is more likely. For ordinary requests, prefer [RetryTransport][httpx_retries.RetryTransport] for simplicity. + +To retry body-phase errors, use [retry_request][httpx_retries.retry_request] (or its async counterpart [aretry_request][httpx_retries.aretry_request]). These helpers drive the retry loop at the *client* level, where the body is read, so the same [Retry][httpx_retries.Retry] configuration covers body-phase errors as well as the header-phase errors and retryable status codes that [RetryTransport][httpx_retries.RetryTransport] already handles: ```python import httpx +from httpx_retries import Retry, retry_request + +with httpx.Client() as client: + response = retry_request(client, "GET", "https://example.com", retry=Retry(total=5, backoff_factor=0.5)) +``` -retryable = (httpx.ReadTimeout, httpx.RemoteProtocolError) +```python +import httpx +from httpx_retries import aretry_request -for attempt in range(5): - try: - response = client.get("https://example.com") - break - except retryable: - if attempt == 4: - raise +async with httpx.AsyncClient() as client: + response = await aretry_request(client, "GET", "https://example.com") ``` +A plain client is all you need — the helpers run the full retry loop themselves, so there's no need to also install [RetryTransport][httpx_retries.RetryTransport]. Passing a client that uses [RetryTransport][httpx_retries.RetryTransport] raises a `ValueError`, because it would retry every request twice. + +!!! warning "These helpers buffer the full response body" + Because the body is read before the helper returns, `retry_request` and `aretry_request` are not suitable for streaming. An error raised while iterating a streaming response (`client.stream(...)`) happens after the body has started arriving and cannot be retried transparently — bytes already handed to your code can't be recalled. For streaming, catch the error and re-issue the request yourself. + +!!! note "Only idempotent methods are retried by default" + Like [RetryTransport][httpx_retries.RetryTransport], the helpers only retry methods in `Retry.allowed_methods` (by default `HEAD`, `GET`, `PUT`, `DELETE`, `OPTIONS`, `TRACE`); other methods are sent once. You can opt a method in with `Retry(allowed_methods=[...])`, but take care: a body-phase retry re-issues the *entire* request, and because the server has already started responding it has most likely processed the original — so only enable non-idempotent methods such as `POST` when duplicate side effects are acceptable. + ## Retrying on response content Sometimes a server returns a valid response but the body or custom headers signals a failure - for example, a block page, a CAPTCHA redirect, or an authorization wall. This commonly occurs if access may be blocked at the content level rather than the HTTP status level. diff --git a/httpx_retries/__init__.py b/httpx_retries/__init__.py index e33aee0..1554442 100644 --- a/httpx_retries/__init__.py +++ b/httpx_retries/__init__.py @@ -1,4 +1,5 @@ +from .helpers import aretry_request, retry_request from .retry import Retry from .transport import RetryTransport -__all__ = ["Retry", "RetryTransport"] +__all__ = ["Retry", "RetryTransport", "aretry_request", "retry_request"] diff --git a/httpx_retries/helpers.py b/httpx_retries/helpers.py new file mode 100644 index 0000000..b16ff92 --- /dev/null +++ b/httpx_retries/helpers.py @@ -0,0 +1,147 @@ +import inspect +from typing import Any + +import httpx + +from .retry import Retry +from .transport import RetryTransport, _retry_operation, _retry_operation_async + +# Arguments accepted by `Client.send` rather than `Client.build_request`; forwarded to send if provided. +_SEND_KWARGS = ("auth", "follow_redirects") + + +def _client_retries(client: httpx.Client | httpx.AsyncClient) -> bool: + """Return True if the client already retries via a [RetryTransport][httpx_retries.RetryTransport]. + + The helpers run the retry loop themselves, so combining them with a retrying transport would retry every + request twice. Detection reaches into httpx's private attributes and degrades to ``False`` if they are absent. + """ + mounts = getattr(client, "_mounts", {}) + transports = [getattr(client, "_transport", None), *mounts.values()] + return any(isinstance(transport, RetryTransport) for transport in transports) + + +def retry_request( + client: httpx.Client, + method: str, + url: httpx.URL | str, + *, + retry: Retry | None = None, + **kwargs: Any, +) -> httpx.Response: + """ + Send a request with retries, including errors raised while reading the response body. + + Unlike [RetryTransport][httpx_retries.RetryTransport], which can only observe what flows through its + `handle_request` method (the response *headers*), this helper drives the retry loop at the client level. + Because `httpx.Client.send` reads the body before returning, body-phase errors such as `httpx.ReadTimeout` + and `httpx.RemoteProtocolError("peer closed connection...")` are caught here and retried. + + ```python + import httpx + from httpx_retries import retry_request + + with httpx.Client() as client: + response = retry_request(client, "GET", "https://example.com") + ``` + + The retry configuration can be customised, just like [RetryTransport][httpx_retries.RetryTransport]: + + ```python + response = retry_request(client, "GET", "https://example.com", retry=Retry(total=5, backoff_factor=0.5)) + ``` + + This helper buffers the full response body, so it is not suitable for streaming. Errors raised while + iterating a streaming response (`client.stream(...)`) cannot be retried. + + Body-phase errors are a niche case; see + [Why wasn't my `ReadTimeout` retried?](faq.md#why-wasnt-my-readtimeout-retried) for when these helpers are + worth using and when to prefer [RetryTransport][httpx_retries.RetryTransport] instead. + + Args: + client: The client used to build and send the request. + method: The HTTP method. + url: The URL to request. + retry: The retry configuration. A per-request `request.extensions["retry"]` takes precedence. + **kwargs: Additional arguments. `auth` and `follow_redirects` are forwarded to `client.send`; all others + (for example `params`, `headers`, `json`, `content`) are passed to `client.build_request`. + + Returns: + The final response. + """ + if _client_retries(client): + raise ValueError( + "retry_request runs the retry loop itself and must be used with a client that does not also retry. " + "The given client uses RetryTransport, which would retry every request twice. Use a plain " + "httpx.Client instead; retry_request already retries header-phase errors and retryable status codes." + ) + + send_kwargs = {key: kwargs.pop(key) for key in _SEND_KWARGS if key in kwargs} + request = client.build_request(method, url, **kwargs) + retry = request.extensions.setdefault("retry", retry or Retry()) + + def send(request: httpx.Request) -> httpx.Response: + return client.send(request, **send_kwargs) + + if not retry.is_retryable_method(request.method): + return send(request) + + if retry.validate_response is not None and inspect.iscoroutinefunction(retry.validate_response): + raise TypeError("validate_response must be a sync function when using a sync client") + + return _retry_operation(request, send, retry) + + +async def aretry_request( + client: httpx.AsyncClient, + method: str, + url: httpx.URL | str, + *, + retry: Retry | None = None, + **kwargs: Any, +) -> httpx.Response: + """ + Send a request asynchronously with retries, including errors raised while reading the response body. + + This is the async counterpart to [retry_request][httpx_retries.retry_request]. Body-phase errors are a niche + case; see [Why wasn't my `ReadTimeout` retried?](faq.md#why-wasnt-my-readtimeout-retried) for when these + helpers are worth using and when to prefer [RetryTransport][httpx_retries.RetryTransport] instead. + + ```python + import httpx + from httpx_retries import aretry_request + + async with httpx.AsyncClient() as client: + response = await aretry_request(client, "GET", "https://example.com") + ``` + + Args: + client: The client used to build and send the request. + method: The HTTP method. + url: The URL to request. + retry: The retry configuration. A per-request `request.extensions["retry"]` takes precedence. + **kwargs: Additional arguments. `auth` and `follow_redirects` are forwarded to `client.send`; all others + (for example `params`, `headers`, `json`, `content`) are passed to `client.build_request`. + + Returns: + The final response. + """ + if _client_retries(client): + raise ValueError( + "aretry_request runs the retry loop itself and must be used with a client that does not also retry. " + "The given client uses RetryTransport, which would retry every request twice. Use a plain " + "httpx.AsyncClient instead; aretry_request already retries header-phase errors and retryable status " + "codes." + ) + + send_kwargs = {key: kwargs.pop(key) for key in _SEND_KWARGS if key in kwargs} + request = client.build_request(method, url, **kwargs) + retry = request.extensions.setdefault("retry", retry or Retry()) + + async def send(request: httpx.Request) -> httpx.Response: + return await client.send(request, **send_kwargs) + + if not retry.is_retryable_method(request.method): + return await send(request) + + return await _retry_operation_async(request, send, retry) diff --git a/httpx_retries/transport.py b/httpx_retries/transport.py index 1990d6d..4abd729 100644 --- a/httpx_retries/transport.py +++ b/httpx_retries/transport.py @@ -11,6 +11,93 @@ logger = logging.getLogger(__name__) +def _retry_operation( + request: httpx.Request, + send_method: Callable[..., httpx.Response], + retry: Retry, +) -> httpx.Response: + response: httpx.Response | Exception | None = None + + while True: + if response is not None: + if isinstance(response, httpx.Response): + response.close() + + logger.debug("_retry_operation retrying request=%s response=%s retry=%s", request, response, retry) + retry = retry.increment() + retry.sleep(response) + try: + response = send_method(request) + except Exception as e: + if retry.is_exhausted() or not retry.is_retryable_exception(e): + raise + + response = e + continue + + if retry.is_exhausted(): + response.extensions["retry"] = retry + return response + + if not retry.is_retryable_status_code(response.status_code): + if retry.validate_response is not None: + # normally set by httpx _after_ calling this function, but we want the request in the validator + response.request = request + try: + retry.validate_response(response) + except Exception as e: + if retry.is_exhausted() or not retry.is_retryable_exception(e): + raise + continue + response.extensions["retry"] = retry + return response + + +async def _retry_operation_async( + request: httpx.Request, + send_method: Callable[..., Coroutine[Any, Any, httpx.Response]], + retry: Retry, +) -> httpx.Response: + response: httpx.Response | Exception | None = None + + while True: + if response is not None: + if isinstance(response, httpx.Response): + await response.aclose() + + logger.debug("_retry_operation_async retrying request=%s response=%s retry=%s", request, response, retry) + retry = retry.increment() + await retry.asleep(response) + try: + response = await send_method(request) + except Exception as e: + if retry.is_exhausted() or not retry.is_retryable_exception(e): + raise + + response = e + continue + + if retry.is_exhausted(): + response.extensions["retry"] = retry + return response + + if not retry.is_retryable_status_code(response.status_code): + if retry.validate_response is not None: + # normally set by httpx _after_ calling this function, but we want the request in the validator + response.request = request + try: + if inspect.iscoroutinefunction(retry.validate_response): + await retry.validate_response(response) + else: + retry.validate_response(response) + except Exception as e: + if retry.is_exhausted() or not retry.is_retryable_exception(e): + raise + continue + response.extensions["retry"] = retry + return response + + class RetryTransport(httpx.BaseTransport, httpx.AsyncBaseTransport): """ A transport that automatically retries requests. @@ -95,7 +182,7 @@ def handle_request(self, request: httpx.Request) -> httpx.Response: raise TypeError("validate_response must be a sync function when using a sync transport") send_method = partial(self._sync_transport.handle_request) - response = self._retry_operation(request, send_method, retry) + response = _retry_operation(request, send_method, retry) else: response = self._sync_transport.handle_request(request) @@ -121,99 +208,10 @@ async def handle_async_request(self, request: httpx.Request) -> httpx.Response: if retry.is_retryable_method(request.method): send_method = partial(self._async_transport.handle_async_request) - response = await self._retry_operation_async(request, send_method, retry) + response = await _retry_operation_async(request, send_method, retry) else: response = await self._async_transport.handle_async_request(request) logger.debug("handle_async_request finished request=%s response=%s", request, response) return response - - def _retry_operation( - self, - request: httpx.Request, - send_method: Callable[..., httpx.Response], - retry: Retry, - ) -> httpx.Response: - response: httpx.Response | Exception | None = None - - while True: - if response is not None: - if isinstance(response, httpx.Response): - response.close() - - logger.debug("_retry_operation retrying request=%s response=%s retry=%s", request, response, retry) - retry = retry.increment() - retry.sleep(response) - try: - response = send_method(request) - except Exception as e: - if retry.is_exhausted() or not retry.is_retryable_exception(e): - raise - - response = e - continue - - if retry.is_exhausted(): - response.extensions["retry"] = retry - return response - - if not retry.is_retryable_status_code(response.status_code): - if retry.validate_response is not None: - # normally set by httpx _after_ calling this function, but we want the request in the validator - response.request = request - try: - retry.validate_response(response) - except Exception as e: - if retry.is_exhausted() or not retry.is_retryable_exception(e): - raise - continue - response.extensions["retry"] = retry - return response - - async def _retry_operation_async( - self, - request: httpx.Request, - send_method: Callable[..., Coroutine[Any, Any, httpx.Response]], - retry: Retry, - ) -> httpx.Response: - response: httpx.Response | Exception | None = None - - while True: - if response is not None: - if isinstance(response, httpx.Response): - await response.aclose() - - logger.debug( - "_retry_operation_async retrying request=%s response=%s retry=%s", request, response, retry - ) - retry = retry.increment() - await retry.asleep(response) - try: - response = await send_method(request) - except Exception as e: - if retry.is_exhausted() or not retry.is_retryable_exception(e): - raise - - response = e - continue - - if retry.is_exhausted(): - response.extensions["retry"] = retry - return response - - if not retry.is_retryable_status_code(response.status_code): - if retry.validate_response is not None: - # normally set by httpx _after_ calling this function, but we want the request in the validator - response.request = request - try: - if inspect.iscoroutinefunction(retry.validate_response): - await retry.validate_response(response) - else: - retry.validate_response(response) - except Exception as e: - if retry.is_exhausted() or not retry.is_retryable_exception(e): - raise - continue - response.extensions["retry"] = retry - return response diff --git a/tests/test_helpers.py b/tests/test_helpers.py new file mode 100644 index 0000000..5f8a78c --- /dev/null +++ b/tests/test_helpers.py @@ -0,0 +1,338 @@ +from collections.abc import AsyncIterator, Generator, Iterator +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest + +from httpx_retries import Retry, RetryTransport, aretry_request, retry_request + + +class _FailingByteStream(httpx.SyncByteStream): + """A response body that raises when read.""" + + def __init__(self, exc: Exception) -> None: + self._exc = exc + + def __iter__(self) -> Iterator[bytes]: + raise self._exc + yield b"" # pragma: no cover + + def close(self) -> None: + pass + + +class _FailingAsyncByteStream(httpx.AsyncByteStream): + """A response body that raises when read.""" + + def __init__(self, exc: Exception) -> None: + self._exc = exc + + async def __aiter__(self) -> AsyncIterator[bytes]: + raise self._exc + yield b"" # pragma: no cover + + async def aclose(self) -> None: + pass + + +class BodyFailTransport(httpx.BaseTransport): + """Returns a response whose body read fails `fail_times` times, then succeeds.""" + + def __init__(self, exc: Exception, fail_times: int) -> None: + self.exc = exc + self.fail_times = fail_times + self.attempts = 0 + + def handle_request(self, request: httpx.Request) -> httpx.Response: + self.attempts += 1 + if self.attempts <= self.fail_times: + return httpx.Response(200, stream=_FailingByteStream(self.exc)) + return httpx.Response(200, content=b"ok") + + +class AsyncBodyFailTransport(httpx.AsyncBaseTransport): + """Returns a response whose body read fails `fail_times` times, then succeeds.""" + + def __init__(self, exc: Exception, fail_times: int) -> None: + self.exc = exc + self.fail_times = fail_times + self.attempts = 0 + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + self.attempts += 1 + if self.attempts <= self.fail_times: + return httpx.Response(200, stream=_FailingAsyncByteStream(self.exc)) + return httpx.Response(200, content=b"ok") + + +class _HeaderAuth(httpx.Auth): + """Auth scheme that stamps a fixed header onto each request.""" + + def auth_flow(self, request: httpx.Request) -> Generator[httpx.Request, httpx.Response, None]: + request.headers["X-Auth"] = "secret" + yield request + + +class RecordingTransport(httpx.BaseTransport): + """Records the requests it receives and returns 200.""" + + def __init__(self) -> None: + self.requests: list[httpx.Request] = [] + + def handle_request(self, request: httpx.Request) -> httpx.Response: + self.requests.append(request) + return httpx.Response(200, content=b"ok") + + +class AsyncRecordingTransport(httpx.AsyncBaseTransport): + """Records the requests it receives and returns 200.""" + + def __init__(self) -> None: + self.requests: list[httpx.Request] = [] + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + self.requests.append(request) + return httpx.Response(200, content=b"ok") + + +class RedirectTransport(httpx.BaseTransport): + """Redirects `/start` to `/final` once, then serves `/final`.""" + + def __init__(self) -> None: + self.paths: list[str] = [] + + def handle_request(self, request: httpx.Request) -> httpx.Response: + self.paths.append(request.url.path) + if request.url.path == "/start": + return httpx.Response(302, headers={"Location": "https://example.com/final"}) + return httpx.Response(200, content=b"final") + + +class AsyncRedirectTransport(httpx.AsyncBaseTransport): + """Redirects `/start` to `/final` once, then serves `/final`.""" + + def __init__(self) -> None: + self.paths: list[str] = [] + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + self.paths.append(request.url.path) + if request.url.path == "/start": + return httpx.Response(302, headers={"Location": "https://example.com/final"}) + return httpx.Response(200, content=b"final") + + +def test_retry_request_success(mock_sleep: MagicMock) -> None: + transport = BodyFailTransport(httpx.ReadTimeout("boom"), fail_times=0) + + with httpx.Client(transport=transport) as client: + response = retry_request(client, "GET", "https://example.com") + + assert response.status_code == 200 + assert response.text == "ok" + assert mock_sleep.call_count == 0 + assert response.extensions["retry"].attempts_made == 0 + + +# Both are in the default retryable set and are the two headline body-phase failures. +BODY_READ_ERRORS = [ + httpx.ReadTimeout("read timed out"), + httpx.RemoteProtocolError("peer closed connection without sending complete message body"), +] + + +@pytest.mark.parametrize("exc", BODY_READ_ERRORS) +def test_retry_request_retries_body_read_error(mock_sleep: MagicMock, exc: Exception) -> None: + transport = BodyFailTransport(exc, fail_times=2) + + with httpx.Client(transport=transport) as client: + response = retry_request(client, "GET", "https://example.com") + + assert response.status_code == 200 + assert response.text == "ok" + assert transport.attempts == 3 + assert mock_sleep.call_count == 2 + assert response.extensions["retry"].attempts_made == 2 + + +def test_retry_request_raises_when_exhausted(mock_sleep: MagicMock) -> None: + transport = BodyFailTransport(httpx.ReadTimeout("boom"), fail_times=100) + retry = Retry(total=3) + + with httpx.Client(transport=transport) as client: + with pytest.raises(httpx.ReadTimeout, match="boom"): + retry_request(client, "GET", "https://example.com", retry=retry) + + assert mock_sleep.call_count == 3 + + +def test_retry_request_non_retryable_method(mock_sleep: MagicMock) -> None: + transport = BodyFailTransport(httpx.ReadTimeout("boom"), fail_times=1) + + with httpx.Client(transport=transport) as client: + with pytest.raises(httpx.ReadTimeout, match="boom"): + retry_request(client, "POST", "https://example.com") + + assert transport.attempts == 1 + assert mock_sleep.call_count == 0 + + +def test_retry_request_extension_overrides_retry_argument(mock_sleep: MagicMock) -> None: + transport = BodyFailTransport(httpx.ReadTimeout("boom"), fail_times=100) + + with httpx.Client(transport=transport) as client: + with pytest.raises(httpx.ReadTimeout, match="boom"): + retry_request( + client, + "GET", + "https://example.com", + retry=Retry(total=10), + extensions={"retry": Retry(total=2)}, + ) + + assert mock_sleep.call_count == 2 + + +def test_retry_request_async_validate_response_raises_for_sync_client() -> None: + async def validate(response: httpx.Response) -> None: # pragma: no cover + pass + + transport = BodyFailTransport(httpx.ReadTimeout("boom"), fail_times=0) + retry = Retry(validate_response=validate) + + with httpx.Client(transport=transport) as client: + with pytest.raises(TypeError, match="validate_response must be a sync function"): + retry_request(client, "GET", "https://example.com", retry=retry) + + +@pytest.mark.asyncio +async def test_aretry_request_success(mock_asleep: AsyncMock) -> None: + transport = AsyncBodyFailTransport(httpx.ReadTimeout("boom"), fail_times=0) + + async with httpx.AsyncClient(transport=transport) as client: + response = await aretry_request(client, "GET", "https://example.com") + + assert response.status_code == 200 + assert response.text == "ok" + assert mock_asleep.call_count == 0 + + +@pytest.mark.parametrize("exc", BODY_READ_ERRORS) +@pytest.mark.asyncio +async def test_aretry_request_retries_body_read_error(mock_asleep: AsyncMock, exc: Exception) -> None: + transport = AsyncBodyFailTransport(exc, fail_times=2) + + async with httpx.AsyncClient(transport=transport) as client: + response = await aretry_request(client, "GET", "https://example.com") + + assert response.status_code == 200 + assert response.text == "ok" + assert transport.attempts == 3 + assert mock_asleep.call_count == 2 + assert response.extensions["retry"].attempts_made == 2 + + +@pytest.mark.asyncio +async def test_aretry_request_non_retryable_method(mock_asleep: AsyncMock) -> None: + transport = AsyncBodyFailTransport(httpx.ReadTimeout("boom"), fail_times=1) + + async with httpx.AsyncClient(transport=transport) as client: + with pytest.raises(httpx.ReadTimeout, match="boom"): + await aretry_request(client, "POST", "https://example.com") + + assert transport.attempts == 1 + assert mock_asleep.call_count == 0 + + +def test_retry_request_forwards_auth(mock_sleep: MagicMock) -> None: + transport = RecordingTransport() + + with httpx.Client(transport=transport) as client: + response = retry_request(client, "GET", "https://example.com", auth=_HeaderAuth()) + + assert response.status_code == 200 + assert transport.requests[0].headers["X-Auth"] == "secret" + + +def test_retry_request_forwards_follow_redirects(mock_sleep: MagicMock) -> None: + transport = RedirectTransport() + + with httpx.Client(transport=transport) as client: + # Without follow_redirects, the client default (off) leaves the 302 in place. + response = retry_request(client, "GET", "https://example.com/start") + assert response.status_code == 302 + + response = retry_request(client, "GET", "https://example.com/start", follow_redirects=True) + + assert response.status_code == 200 + assert response.text == "final" + assert transport.paths == ["/start", "/start", "/final"] + + +@pytest.mark.asyncio +async def test_aretry_request_forwards_auth(mock_asleep: AsyncMock) -> None: + transport = AsyncRecordingTransport() + + async with httpx.AsyncClient(transport=transport) as client: + response = await aretry_request(client, "GET", "https://example.com", auth=_HeaderAuth()) + + assert response.status_code == 200 + assert transport.requests[0].headers["X-Auth"] == "secret" + + +@pytest.mark.asyncio +async def test_aretry_request_forwards_follow_redirects(mock_asleep: AsyncMock) -> None: + transport = AsyncRedirectTransport() + + async with httpx.AsyncClient(transport=transport) as client: + response = await aretry_request(client, "GET", "https://example.com/start", follow_redirects=True) + + assert response.status_code == 200 + assert response.text == "final" + assert transport.paths == ["/start", "/final"] + + +def test_retry_request_rejects_retrying_client() -> None: + with httpx.Client(transport=RetryTransport(transport=RecordingTransport())) as client: + with pytest.raises(ValueError, match="would retry every request twice"): + retry_request(client, "GET", "https://example.com") + + +def test_retry_request_rejects_mounted_retrying_transport() -> None: + mounts = {"https://": RetryTransport(transport=RecordingTransport())} + with httpx.Client(mounts=mounts, trust_env=False) as client: + with pytest.raises(ValueError, match="would retry every request twice"): + retry_request(client, "GET", "https://example.com") + + +@pytest.mark.asyncio +async def test_aretry_request_rejects_retrying_client() -> None: + async with httpx.AsyncClient(transport=RetryTransport(transport=AsyncRecordingTransport())) as client: + with pytest.raises(ValueError, match="would retry every request twice"): + await aretry_request(client, "GET", "https://example.com") + + +@pytest.mark.asyncio +async def test_aretry_request_rejects_mounted_retrying_transport() -> None: + mounts = {"https://": RetryTransport(transport=AsyncRecordingTransport())} + async with httpx.AsyncClient(mounts=mounts, trust_env=False) as client: + with pytest.raises(ValueError, match="would retry every request twice"): + await aretry_request(client, "GET", "https://example.com") + + +def test_retry_request_validate_response_retries(mock_sleep: MagicMock) -> None: + transport = RecordingTransport() + statuses: list[int] = [] + + def validate(response: httpx.Response) -> None: + statuses.append(response.status_code) + if len(statuses) < 3: + raise httpx.TimeoutException("not ready yet") + + retry = Retry(total=5, validate_response=validate) + + with httpx.Client(transport=transport) as client: + response = retry_request(client, "GET", "https://example.com", retry=retry) + + assert response.status_code == 200 + assert statuses == [200, 200, 200] + assert mock_sleep.call_count == 2 diff --git a/tests/test_transport.py b/tests/test_transport.py index 847d0a4..f1998b8 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -9,6 +9,7 @@ from httpx import Request, Response from httpx_retries import Retry, RetryTransport +from httpx_retries.transport import _retry_operation, _retry_operation_async def status_codes( @@ -252,8 +253,6 @@ def test_retryable_exception_custom_exception(mock_responses: MockResponse) -> N @pytest.mark.parametrize("status_code", Retry.RETRYABLE_STATUS_CODES) def test_retry_operation_always_closes_response(status_code: int) -> None: - transport = RetryTransport() - responses = [] def send_method(request: httpx.Request) -> httpx.Response: @@ -266,9 +265,7 @@ def send_method(request: httpx.Request) -> httpx.Response: responses.append(response) return response - transport._retry_operation( - request=httpx.Request("GET", "https://example.com"), send_method=send_method, retry=transport.retry - ) + _retry_operation(request=httpx.Request("GET", "https://example.com"), send_method=send_method, retry=Retry()) assert all(r.close.called for r in responses[:-1]) @@ -615,8 +612,6 @@ async def test_async_retry_after_capped_by_total_timeout( @pytest.mark.parametrize("status_code", Retry.RETRYABLE_STATUS_CODES) @pytest.mark.asyncio async def test_retry_operation_async_always_closes_response(status_code: int) -> None: - transport = RetryTransport() - responses = [] async def send_method(request: httpx.Request) -> httpx.Response: @@ -629,8 +624,8 @@ async def send_method(request: httpx.Request) -> httpx.Response: responses.append(response) return response - await transport._retry_operation_async( - request=httpx.Request("GET", "https://example.com"), send_method=send_method, retry=transport.retry + await _retry_operation_async( + request=httpx.Request("GET", "https://example.com"), send_method=send_method, retry=Retry() ) assert all(r.aclose.called for r in responses[:-1])