| 1 | import tenacity |
| 2 | import traceback |
| 3 | import logging |
| 4 | |
| 5 | import requests |
| 6 | |
| 7 | def after_func(retry_state: tenacity.RetryCallState) -> None: |
| 8 | if retry_state.outcome.failed: |
| 9 | exc = retry_state.outcome.exception() |
| 10 | logging.warning(f"Retrying {retry_state.fn.__name__} due to {repr(exc)} (Attempt {retry_state.attempt_number})") |
| 11 | logging.debug(traceback.format_exception(type(exc), exc, exc.__traceback__)) |
| 12 | |
| 13 | |
| 14 | def is_retryable_download_error(exc: BaseException) -> bool: |
| 15 | """Network errors and 5xx responses are retryable; other HTTP errors (expired |
| 16 | or invalid URLs, auth failures) will never succeed and must fail fast.""" |
| 17 | if isinstance(exc, requests.HTTPError): |
| 18 | response = exc.response |
| 19 | return response is None or response.status_code >= 500 |
| 20 | return isinstance(exc, requests.RequestException) |
| 21 | |
| 22 | |
| 23 | download_retry = tenacity.retry( |
| 24 | stop=tenacity.stop_after_attempt(3), |
| 25 | wait=tenacity.wait_exponential(multiplier=1, max=10), |
| 26 | retry=tenacity.retry_if_exception(is_retryable_download_error), |
| 27 | after=after_func, |
| 28 | reraise=True, |
| 29 | ) |
| 30 |