Determine status code from python Retry exception

Question

import requests
from requests.adapters import HTTPAdapter
from urllib3 import Retry

DEFAULT_RETRIES = 5
DEFAULT_BACKOFF = 0.3


def session_with_retries(max_retries: int = DEFAULT_RETRIES,
                         backoff_factor: float = DEFAULT_BACKOFF,
                         proxies: dict = None) -> requests.Session:
    new_session = requests.Session()
    retries = Retry(total=max_retries,
                    connect=max_retries,
                    read=max_retries,
                    status=max_retries,
                    allowed_methods=frozenset(['HEAD', 'GET', 'POST']),
                    status_forcelist=frozenset([500, 502, 503, 504]),
                    backoff_factor=backoff_factor,
                    )
    retry_adapter = HTTPAdapter(max_retries=retries)
    new_session.mount('http://', retry_adapter)
    new_session.mount('https://', retry_adapter)

    if proxies is not None:
        new_session.proxies.update(proxies)

    return new_session

This code is for retry logic with python requests. And is working fine too. As expected it will throw exception on 503 status code after max_retries. So collectively it will throw exception for [500, 502, 503, 504].

How I can get the status code on which the exception was thrown from this method. As exception messages i could see something

HTTPSConnectionPool(host='some-host', port=443): Max retries exceeded with url: /services/data/v52.0/connect/communities?pageSize=100 (Caused by ResponseError('too many 503 error responses',))

I traced back the exceptions urllib3.exceptions.MaxRetryError, requests.exceptions.RetryError but could not find about its status code for exception.

Strictly NO for other libraries like tenacity

Answer 1

You can handle this by turning off raising an error on 5xx responses, getting the response, and then either raising the last exception or examining the retry history.

The connect, read, and status are over-ridden when a value for total is passed.

import requests
from requests.exceptions import HTTPError
from requests.adapters import HTTPAdapter
from urllib3 import Retry

DEFAULT_RETRIES = 5
DEFAULT_BACKOFF = 0.3

def session_with_retries(max_retries: int = DEFAULT_RETRIES,
                         backoff_factor: float = DEFAULT_BACKOFF,
                         proxies: dict = None
                         ) -> requests.Session:
    """Creates a new Session with `max_retries` retry attempts at 5xx status
    codes.
    """
    new_session = requests.Session()
    retries = Retry(
        total=max_retries,
        allowed_methods=frozenset(['HEAD', 'GET', 'POST']),
        status_forcelist=frozenset([500, 502, 503, 504]),
        backoff_factor=backoff_factor,
        # ensure a response is returned:
        raise_on_status=False,
    )

    retry_adapter = HTTPAdapter(max_retries=retries)
    new_session.mount('http://', retry_adapter)
    new_session.mount('https://', retry_adapter)

    if proxies is not None:
        new_session.proxies.update(proxies)

    return new_session

Now when you send a request, the response will be returned without raising a MaxRetryError. To get the last response code, you simply call raise_for_status.

sess = session_with_retries()
res = sess.get('https://httpbin.org/status/500')

try:
    res.raise_for_status()
except HTTPError:
    print(f'The request failed with code: {res.status_code}')

# prints:
# The request failed with code: 500

If you want to see the complete retry status code history, you can use:

sess = session_with_retries()
res = sess.get('https://httpbin.org/status/500')

try:
    res.raise_for_status()
except HTTPError:
    print('The request retries failed with codes:')
    print([h.status for h in res.raw.retries.history])

# prints:
# The request retries failed with codes:
# [500, 500, 500]