summaryrefslogtreecommitdiff
path: root/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/urllib3/util/retry.py
diff options
context:
space:
mode:
Diffstat (limited to 'venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/urllib3/util/retry.py')
-rw-r--r--venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/urllib3/util/retry.py401
1 files changed, 401 insertions, 0 deletions
diff --git a/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/urllib3/util/retry.py b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/urllib3/util/retry.py
new file mode 100644
index 0000000..2a7e8c1
--- /dev/null
+++ b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/urllib3/util/retry.py
@@ -0,0 +1,401 @@
1from __future__ import absolute_import
2import time
3import logging
4from collections import namedtuple
5from itertools import takewhile
6import email
7import re
8
9from ..exceptions import (
10 ConnectTimeoutError,
11 MaxRetryError,
12 ProtocolError,
13 ReadTimeoutError,
14 ResponseError,
15 InvalidHeader,
16)
17from ..packages import six
18
19
20log = logging.getLogger(__name__)
21
22# Data structure for representing the metadata of requests that result in a retry.
23RequestHistory = namedtuple('RequestHistory', ["method", "url", "error",
24 "status", "redirect_location"])
25
26
27class Retry(object):
28 """ Retry configuration.
29
30 Each retry attempt will create a new Retry object with updated values, so
31 they can be safely reused.
32
33 Retries can be defined as a default for a pool::
34
35 retries = Retry(connect=5, read=2, redirect=5)
36 http = PoolManager(retries=retries)
37 response = http.request('GET', 'http://example.com/')
38
39 Or per-request (which overrides the default for the pool)::
40
41 response = http.request('GET', 'http://example.com/', retries=Retry(10))
42
43 Retries can be disabled by passing ``False``::
44
45 response = http.request('GET', 'http://example.com/', retries=False)
46
47 Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless
48 retries are disabled, in which case the causing exception will be raised.
49
50 :param int total:
51 Total number of retries to allow. Takes precedence over other counts.
52
53 Set to ``None`` to remove this constraint and fall back on other
54 counts. It's a good idea to set this to some sensibly-high value to
55 account for unexpected edge cases and avoid infinite retry loops.
56
57 Set to ``0`` to fail on the first retry.
58
59 Set to ``False`` to disable and imply ``raise_on_redirect=False``.
60
61 :param int connect:
62 How many connection-related errors to retry on.
63
64 These are errors raised before the request is sent to the remote server,
65 which we assume has not triggered the server to process the request.
66
67 Set to ``0`` to fail on the first retry of this type.
68
69 :param int read:
70 How many times to retry on read errors.
71
72 These errors are raised after the request was sent to the server, so the
73 request may have side-effects.
74
75 Set to ``0`` to fail on the first retry of this type.
76
77 :param int redirect:
78 How many redirects to perform. Limit this to avoid infinite redirect
79 loops.
80
81 A redirect is a HTTP response with a status code 301, 302, 303, 307 or
82 308.
83
84 Set to ``0`` to fail on the first retry of this type.
85
86 Set to ``False`` to disable and imply ``raise_on_redirect=False``.
87
88 :param int status:
89 How many times to retry on bad status codes.
90
91 These are retries made on responses, where status code matches
92 ``status_forcelist``.
93
94 Set to ``0`` to fail on the first retry of this type.
95
96 :param iterable method_whitelist:
97 Set of uppercased HTTP method verbs that we should retry on.
98
99 By default, we only retry on methods which are considered to be
100 idempotent (multiple requests with the same parameters end with the
101 same state). See :attr:`Retry.DEFAULT_METHOD_WHITELIST`.
102
103 Set to a ``False`` value to retry on any verb.
104
105 :param iterable status_forcelist:
106 A set of integer HTTP status codes that we should force a retry on.
107 A retry is initiated if the request method is in ``method_whitelist``
108 and the response status code is in ``status_forcelist``.
109
110 By default, this is disabled with ``None``.
111
112 :param float backoff_factor:
113 A backoff factor to apply between attempts after the second try
114 (most errors are resolved immediately by a second try without a
115 delay). urllib3 will sleep for::
116
117 {backoff factor} * (2 ^ ({number of total retries} - 1))
118
119 seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep
120 for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer
121 than :attr:`Retry.BACKOFF_MAX`.
122
123 By default, backoff is disabled (set to 0).
124
125 :param bool raise_on_redirect: Whether, if the number of redirects is
126 exhausted, to raise a MaxRetryError, or to return a response with a
127 response code in the 3xx range.
128
129 :param bool raise_on_status: Similar meaning to ``raise_on_redirect``:
130 whether we should raise an exception, or return a response,
131 if status falls in ``status_forcelist`` range and retries have
132 been exhausted.
133
134 :param tuple history: The history of the request encountered during
135 each call to :meth:`~Retry.increment`. The list is in the order
136 the requests occurred. Each list item is of class :class:`RequestHistory`.
137
138 :param bool respect_retry_after_header:
139 Whether to respect Retry-After header on status codes defined as
140 :attr:`Retry.RETRY_AFTER_STATUS_CODES` or not.
141
142 """
143
144 DEFAULT_METHOD_WHITELIST = frozenset([
145 'HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE'])
146
147 RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503])
148
149 #: Maximum backoff time.
150 BACKOFF_MAX = 120
151
152 def __init__(self, total=10, connect=None, read=None, redirect=None, status=None,
153 method_whitelist=DEFAULT_METHOD_WHITELIST, status_forcelist=None,
154 backoff_factor=0, raise_on_redirect=True, raise_on_status=True,
155 history=None, respect_retry_after_header=True):
156
157 self.total = total
158 self.connect = connect
159 self.read = read
160 self.status = status
161
162 if redirect is False or total is False:
163 redirect = 0
164 raise_on_redirect = False
165
166 self.redirect = redirect
167 self.status_forcelist = status_forcelist or set()
168 self.method_whitelist = method_whitelist
169 self.backoff_factor = backoff_factor
170 self.raise_on_redirect = raise_on_redirect
171 self.raise_on_status = raise_on_status
172 self.history = history or tuple()
173 self.respect_retry_after_header = respect_retry_after_header
174
175 def new(self, **kw):
176 params = dict(
177 total=self.total,
178 connect=self.connect, read=self.read, redirect=self.redirect, status=self.status,
179 method_whitelist=self.method_whitelist,
180 status_forcelist=self.status_forcelist,
181 backoff_factor=self.backoff_factor,
182 raise_on_redirect=self.raise_on_redirect,
183 raise_on_status=self.raise_on_status,
184 history=self.history,
185 )
186 params.update(kw)
187 return type(self)(**params)
188
189 @classmethod
190 def from_int(cls, retries, redirect=True, default=None):
191 """ Backwards-compatibility for the old retries format."""
192 if retries is None:
193 retries = default if default is not None else cls.DEFAULT
194
195 if isinstance(retries, Retry):
196 return retries
197
198 redirect = bool(redirect) and None
199 new_retries = cls(retries, redirect=redirect)
200 log.debug("Converted retries value: %r -> %r", retries, new_retries)
201 return new_retries
202
203 def get_backoff_time(self):
204 """ Formula for computing the current backoff
205
206 :rtype: float
207 """
208 # We want to consider only the last consecutive errors sequence (Ignore redirects).
209 consecutive_errors_len = len(list(takewhile(lambda x: x.redirect_location is None,
210 reversed(self.history))))
211 if consecutive_errors_len <= 1:
212 return 0
213
214 backoff_value = self.backoff_factor * (2 ** (consecutive_errors_len - 1))
215 return min(self.BACKOFF_MAX, backoff_value)
216
217 def parse_retry_after(self, retry_after):
218 # Whitespace: https://tools.ietf.org/html/rfc7230#section-3.2.4
219 if re.match(r"^\s*[0-9]+\s*$", retry_after):
220 seconds = int(retry_after)
221 else:
222 retry_date_tuple = email.utils.parsedate(retry_after)
223 if retry_date_tuple is None:
224 raise InvalidHeader("Invalid Retry-After header: %s" % retry_after)
225 retry_date = time.mktime(retry_date_tuple)
226 seconds = retry_date - time.time()
227
228 if seconds < 0:
229 seconds = 0
230
231 return seconds
232
233 def get_retry_after(self, response):
234 """ Get the value of Retry-After in seconds. """
235
236 retry_after = response.getheader("Retry-After")
237
238 if retry_after is None:
239 return None
240
241 return self.parse_retry_after(retry_after)
242
243 def sleep_for_retry(self, response=None):
244 retry_after = self.get_retry_after(response)
245 if retry_after:
246 time.sleep(retry_after)
247 return True
248
249 return False
250
251 def _sleep_backoff(self):
252 backoff = self.get_backoff_time()
253 if backoff <= 0:
254 return
255 time.sleep(backoff)
256
257 def sleep(self, response=None):
258 """ Sleep between retry attempts.
259
260 This method will respect a server's ``Retry-After`` response header
261 and sleep the duration of the time requested. If that is not present, it
262 will use an exponential backoff. By default, the backoff factor is 0 and
263 this method will return immediately.
264 """
265
266 if response:
267 slept = self.sleep_for_retry(response)
268 if slept:
269 return
270
271 self._sleep_backoff()
272
273 def _is_connection_error(self, err):
274 """ Errors when we're fairly sure that the server did not receive the
275 request, so it should be safe to retry.
276 """
277 return isinstance(err, ConnectTimeoutError)
278
279 def _is_read_error(self, err):
280 """ Errors that occur after the request has been started, so we should
281 assume that the server began processing it.
282 """
283 return isinstance(err, (ReadTimeoutError, ProtocolError))
284
285 def _is_method_retryable(self, method):
286 """ Checks if a given HTTP method should be retried upon, depending if
287 it is included on the method whitelist.
288 """
289 if self.method_whitelist and method.upper() not in self.method_whitelist:
290 return False
291
292 return True
293
294 def is_retry(self, method, status_code, has_retry_after=False):
295 """ Is this method/status code retryable? (Based on whitelists and control
296 variables such as the number of total retries to allow, whether to
297 respect the Retry-After header, whether this header is present, and
298 whether the returned status code is on the list of status codes to
299 be retried upon on the presence of the aforementioned header)
300 """
301 if not self._is_method_retryable(method):
302 return False
303
304 if self.status_forcelist and status_code in self.status_forcelist:
305 return True
306
307 return (self.total and self.respect_retry_after_header and
308 has_retry_after and (status_code in self.RETRY_AFTER_STATUS_CODES))
309
310 def is_exhausted(self):
311 """ Are we out of retries? """
312 retry_counts = (self.total, self.connect, self.read, self.redirect, self.status)
313 retry_counts = list(filter(None, retry_counts))
314 if not retry_counts:
315 return False
316
317 return min(retry_counts) < 0
318
319 def increment(self, method=None, url=None, response=None, error=None,
320 _pool=None, _stacktrace=None):
321 """ Return a new Retry object with incremented retry counters.
322
323 :param response: A response object, or None, if the server did not
324 return a response.
325 :type response: :class:`~urllib3.response.HTTPResponse`
326 :param Exception error: An error encountered during the request, or
327 None if the response was received successfully.
328
329 :return: A new ``Retry`` object.
330 """
331 if self.total is False and error:
332 # Disabled, indicate to re-raise the error.
333 raise six.reraise(type(error), error, _stacktrace)
334
335 total = self.total
336 if total is not None:
337 total -= 1
338
339 connect = self.connect
340 read = self.read
341 redirect = self.redirect
342 status_count = self.status
343 cause = 'unknown'
344 status = None
345 redirect_location = None
346
347 if error and self._is_connection_error(error):
348 # Connect retry?
349 if connect is False:
350 raise six.reraise(type(error), error, _stacktrace)
351 elif connect is not None:
352 connect -= 1
353
354 elif error and self._is_read_error(error):
355 # Read retry?
356 if read is False or not self._is_method_retryable(method):
357 raise six.reraise(type(error), error, _stacktrace)
358 elif read is not None:
359 read -= 1
360
361 elif response and response.get_redirect_location():
362 # Redirect retry?
363 if redirect is not None:
364 redirect -= 1
365 cause = 'too many redirects'
366 redirect_location = response.get_redirect_location()
367 status = response.status
368
369 else:
370 # Incrementing because of a server error like a 500 in
371 # status_forcelist and a the given method is in the whitelist
372 cause = ResponseError.GENERIC_ERROR
373 if response and response.status:
374 if status_count is not None:
375 status_count -= 1
376 cause = ResponseError.SPECIFIC_ERROR.format(
377 status_code=response.status)
378 status = response.status
379
380 history = self.history + (RequestHistory(method, url, error, status, redirect_location),)
381
382 new_retry = self.new(
383 total=total,
384 connect=connect, read=read, redirect=redirect, status=status_count,
385 history=history)
386
387 if new_retry.is_exhausted():
388 raise MaxRetryError(_pool, url, error or ResponseError(cause))
389
390 log.debug("Incremented Retry for (url='%s'): %r", url, new_retry)
391
392 return new_retry
393
394 def __repr__(self):
395 return ('{cls.__name__}(total={self.total}, connect={self.connect}, '
396 'read={self.read}, redirect={self.redirect}, status={self.status})').format(
397 cls=type(self), self=self)
398
399
400# For backwards compatibility (equivalent to pre-v1.9):
401Retry.DEFAULT = Retry(3)