'Python requests - How to handle different exceptions correctly?

Using requests module I query data from api's and I am trying to figure out how to handle exceptions correctly in order to count statistics on the errors that occured. I have the following piece of code to make the request:

import requests
from urllib3.exceptions import ReadTimeoutError

## class definition and some other code here...

def _make_get_request(self, api_url: str, payload=None) -> json:
    try:
        self._req_stats_dict.update({'api_req_get': self._req_stats_dict.get('api_req_get', 0) + 1})
        if payload:
            response = requests.get(api_url, params=payload, timeout=self._API_TIMEOUT)
        else:
            response = requests.get(api_url, timeout=self._API_TIMEOUT)
        response.raise_for_status()
        clogger.debug(response.json())
        if not response.json():
            self._req_stats_dict.update({'api_no_result': self._req_stats_dict.get('api_no_result', 0) + 1})
        # raise Exception("test api error")
        return response.json()
    except requests.exceptions.HTTPError as e:
        self._req_stats_dict.update({'api_no_response': self._req_stats_dict.get('api_no_response', 0) + 1})
        clogger.warning(f"<{self}>: {e} \n{payload}")
    except TimeoutError as e:
        clogger.warning('Api TimeoutError occured')
        self._req_stats_dict.update({'api_timeout': self._req_stats_dict.get('api_timeout', 0) + 1})
    except ReadTimeoutError as e:
        clogger.warning('Api ReadTimeoutError occured')
        self._req_stats_dict.update({'api_timeout': self._req_stats_dict.get('api_timeout', 0) + 1})
    except requests.exceptions.ReadTimeout as e:
        clogger.warning('Api requests.exceptions.ReadTimeout occured')
        self._req_stats_dict.update({'api_timeout': self._req_stats_dict.get('api_timeout', 0) + 1})
    except Exception as e:
        self._req_stats_dict.update({'api_error': self._req_stats_dict.get('api_error', 0) + 1})
        clogger.exception(f"{self} An unknown api error occured: {e} \n{payload}")

So when getting a timeout error, it looks like this:

WARNING | Api requests.exceptions.ReadTimeout occured
ERROR   | ApiSomehost: HTTPSConnectionPool(host='api.somehost.io', port=443): Read timed out. (read timeout=3)
Traceback (most recent call last):
  File "/opt/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 449, in _make_request
    six.raise_from(e, None)
  File "<string>", line 3, in raise_from
  File "/opt/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 444, in _make_request
    httplib_response = conn.getresponse()
  File "/usr/local/lib/python3.10/http/client.py", line 1374, in getresponse
    response.begin()
  File "/usr/local/lib/python3.10/http/client.py", line 318, in begin
    version, status, reason = self._read_status()
  File "/usr/local/lib/python3.10/http/client.py", line 279, in _read_status
    line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
  File "/usr/local/lib/python3.10/socket.py", line 705, in readinto
    return self._sock.recv_into(b)
  File "/usr/local/lib/python3.10/ssl.py", line 1273, in recv_into
    return self.read(nbytes, buffer)
  File "/usr/local/lib/python3.10/ssl.py", line 1129, in read
    return self._sslobj.read(len, buffer)
TimeoutError: The read operation timed out

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/venv/lib/python3.10/site-packages/requests/adapters.py", line 439, in send
    resp = conn.urlopen(
  File "/opt/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 785, in urlopen
    retries = retries.increment(
  File "/opt/venv/lib/python3.10/site-packages/urllib3/util/retry.py", line 550, in increment
    raise six.reraise(type(error), error, _stacktrace)
  File "/opt/venv/lib/python3.10/site-packages/urllib3/packages/six.py", line 770, in reraise
    raise value
  File "/opt/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 703, in urlopen
    httplib_response = self._make_request(
  File "/opt/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 451, in _make_request
    self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
  File "/opt/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 340, in _raise_timeout
    raise ReadTimeoutError(
urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='api.somehost.io', port=443): Read timed out. (read timeout=3)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/workspace/src/floor/floorapp/api/api_strategy.py", line 39, in _make_get_request
    response = requests.get(api_url, params=payload, timeout=self._API_TIMEOUT)
  File "/opt/venv/lib/python3.10/site-packages/requests/api.py", line 75, in get
    return request('get', url, params=params, **kwargs)
  File "/opt/venv/lib/python3.10/site-packages/requests/api.py", line 61, in request
    return session.request(method=method, url=url, **kwargs)
  File "/opt/venv/lib/python3.10/site-packages/requests/sessions.py", line 542, in request
    resp = self.send(prep, **send_kwargs)
  File "/opt/venv/lib/python3.10/site-packages/requests/sessions.py", line 655, in send
    r = adapter.send(request, **kwargs)
  File "/opt/venv/lib/python3.10/site-packages/requests/adapters.py", line 529, in send
    raise ReadTimeout(e, request=request)
requests.exceptions.ReadTimeout: HTTPSConnectionPool(host='api.somehost.io', port=443): Read timed out. (read timeout=3)

I don't quite understand this chain of tracebacks. It seems only the last exception requests.exceptions.ReadTimeout gets handled correctly, I am not sure what I am doing wrong with the other handlers? Would it be enough to handle just that exception and are the other handlers for TimeoutError and urllib3.exceptions.ReadTimeoutError necessary at all?

Thanks in advance!



Sources

This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.

Source: Stack Overflow

Solution Source