diff options
author | Shubham Saini <shubham6405@gmail.com> | 2018-12-11 10:01:23 +0000 |
---|---|---|
committer | Shubham Saini <shubham6405@gmail.com> | 2018-12-11 10:01:23 +0000 |
commit | 68df54d6629ec019142eb149dd037774f2d11e7c (patch) | |
tree | 345bc22d46b4e01a4ba8303b94278952a4ed2b9e /venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/urllib3/util/retry.py |
First commit
Diffstat (limited to 'venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/urllib3/util/retry.py')
-rw-r--r-- | venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/urllib3/util/retry.py | 401 |
1 files changed, 401 insertions, 0 deletions
diff --git a/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/urllib3/util/retry.py b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/urllib3/util/retry.py new file mode 100644 index 0000000..2a7e8c1 --- /dev/null +++ b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/urllib3/util/retry.py | |||
@@ -0,0 +1,401 @@ | |||
1 | from __future__ import absolute_import | ||
2 | import time | ||
3 | import logging | ||
4 | from collections import namedtuple | ||
5 | from itertools import takewhile | ||
6 | import email | ||
7 | import re | ||
8 | |||
9 | from ..exceptions import ( | ||
10 | ConnectTimeoutError, | ||
11 | MaxRetryError, | ||
12 | ProtocolError, | ||
13 | ReadTimeoutError, | ||
14 | ResponseError, | ||
15 | InvalidHeader, | ||
16 | ) | ||
17 | from ..packages import six | ||
18 | |||
19 | |||
20 | log = logging.getLogger(__name__) | ||
21 | |||
22 | # Data structure for representing the metadata of requests that result in a retry. | ||
23 | RequestHistory = namedtuple('RequestHistory', ["method", "url", "error", | ||
24 | "status", "redirect_location"]) | ||
25 | |||
26 | |||
27 | class Retry(object): | ||
28 | """ Retry configuration. | ||
29 | |||
30 | Each retry attempt will create a new Retry object with updated values, so | ||
31 | they can be safely reused. | ||
32 | |||
33 | Retries can be defined as a default for a pool:: | ||
34 | |||
35 | retries = Retry(connect=5, read=2, redirect=5) | ||
36 | http = PoolManager(retries=retries) | ||
37 | response = http.request('GET', 'http://example.com/') | ||
38 | |||
39 | Or per-request (which overrides the default for the pool):: | ||
40 | |||
41 | response = http.request('GET', 'http://example.com/', retries=Retry(10)) | ||
42 | |||
43 | Retries can be disabled by passing ``False``:: | ||
44 | |||
45 | response = http.request('GET', 'http://example.com/', retries=False) | ||
46 | |||
47 | Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless | ||
48 | retries are disabled, in which case the causing exception will be raised. | ||
49 | |||
50 | :param int total: | ||
51 | Total number of retries to allow. Takes precedence over other counts. | ||
52 | |||
53 | Set to ``None`` to remove this constraint and fall back on other | ||
54 | counts. It's a good idea to set this to some sensibly-high value to | ||
55 | account for unexpected edge cases and avoid infinite retry loops. | ||
56 | |||
57 | Set to ``0`` to fail on the first retry. | ||
58 | |||
59 | Set to ``False`` to disable and imply ``raise_on_redirect=False``. | ||
60 | |||
61 | :param int connect: | ||
62 | How many connection-related errors to retry on. | ||
63 | |||
64 | These are errors raised before the request is sent to the remote server, | ||
65 | which we assume has not triggered the server to process the request. | ||
66 | |||
67 | Set to ``0`` to fail on the first retry of this type. | ||
68 | |||
69 | :param int read: | ||
70 | How many times to retry on read errors. | ||
71 | |||
72 | These errors are raised after the request was sent to the server, so the | ||
73 | request may have side-effects. | ||
74 | |||
75 | Set to ``0`` to fail on the first retry of this type. | ||
76 | |||
77 | :param int redirect: | ||
78 | How many redirects to perform. Limit this to avoid infinite redirect | ||
79 | loops. | ||
80 | |||
81 | A redirect is a HTTP response with a status code 301, 302, 303, 307 or | ||
82 | 308. | ||
83 | |||
84 | Set to ``0`` to fail on the first retry of this type. | ||
85 | |||
86 | Set to ``False`` to disable and imply ``raise_on_redirect=False``. | ||
87 | |||
88 | :param int status: | ||
89 | How many times to retry on bad status codes. | ||
90 | |||
91 | These are retries made on responses, where status code matches | ||
92 | ``status_forcelist``. | ||
93 | |||
94 | Set to ``0`` to fail on the first retry of this type. | ||
95 | |||
96 | :param iterable method_whitelist: | ||
97 | Set of uppercased HTTP method verbs that we should retry on. | ||
98 | |||
99 | By default, we only retry on methods which are considered to be | ||
100 | idempotent (multiple requests with the same parameters end with the | ||
101 | same state). See :attr:`Retry.DEFAULT_METHOD_WHITELIST`. | ||
102 | |||
103 | Set to a ``False`` value to retry on any verb. | ||
104 | |||
105 | :param iterable status_forcelist: | ||
106 | A set of integer HTTP status codes that we should force a retry on. | ||
107 | A retry is initiated if the request method is in ``method_whitelist`` | ||
108 | and the response status code is in ``status_forcelist``. | ||
109 | |||
110 | By default, this is disabled with ``None``. | ||
111 | |||
112 | :param float backoff_factor: | ||
113 | A backoff factor to apply between attempts after the second try | ||
114 | (most errors are resolved immediately by a second try without a | ||
115 | delay). urllib3 will sleep for:: | ||
116 | |||
117 | {backoff factor} * (2 ^ ({number of total retries} - 1)) | ||
118 | |||
119 | seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep | ||
120 | for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer | ||
121 | than :attr:`Retry.BACKOFF_MAX`. | ||
122 | |||
123 | By default, backoff is disabled (set to 0). | ||
124 | |||
125 | :param bool raise_on_redirect: Whether, if the number of redirects is | ||
126 | exhausted, to raise a MaxRetryError, or to return a response with a | ||
127 | response code in the 3xx range. | ||
128 | |||
129 | :param bool raise_on_status: Similar meaning to ``raise_on_redirect``: | ||
130 | whether we should raise an exception, or return a response, | ||
131 | if status falls in ``status_forcelist`` range and retries have | ||
132 | been exhausted. | ||
133 | |||
134 | :param tuple history: The history of the request encountered during | ||
135 | each call to :meth:`~Retry.increment`. The list is in the order | ||
136 | the requests occurred. Each list item is of class :class:`RequestHistory`. | ||
137 | |||
138 | :param bool respect_retry_after_header: | ||
139 | Whether to respect Retry-After header on status codes defined as | ||
140 | :attr:`Retry.RETRY_AFTER_STATUS_CODES` or not. | ||
141 | |||
142 | """ | ||
143 | |||
144 | DEFAULT_METHOD_WHITELIST = frozenset([ | ||
145 | 'HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE']) | ||
146 | |||
147 | RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503]) | ||
148 | |||
149 | #: Maximum backoff time. | ||
150 | BACKOFF_MAX = 120 | ||
151 | |||
152 | def __init__(self, total=10, connect=None, read=None, redirect=None, status=None, | ||
153 | method_whitelist=DEFAULT_METHOD_WHITELIST, status_forcelist=None, | ||
154 | backoff_factor=0, raise_on_redirect=True, raise_on_status=True, | ||
155 | history=None, respect_retry_after_header=True): | ||
156 | |||
157 | self.total = total | ||
158 | self.connect = connect | ||
159 | self.read = read | ||
160 | self.status = status | ||
161 | |||
162 | if redirect is False or total is False: | ||
163 | redirect = 0 | ||
164 | raise_on_redirect = False | ||
165 | |||
166 | self.redirect = redirect | ||
167 | self.status_forcelist = status_forcelist or set() | ||
168 | self.method_whitelist = method_whitelist | ||
169 | self.backoff_factor = backoff_factor | ||
170 | self.raise_on_redirect = raise_on_redirect | ||
171 | self.raise_on_status = raise_on_status | ||
172 | self.history = history or tuple() | ||
173 | self.respect_retry_after_header = respect_retry_after_header | ||
174 | |||
175 | def new(self, **kw): | ||
176 | params = dict( | ||
177 | total=self.total, | ||
178 | connect=self.connect, read=self.read, redirect=self.redirect, status=self.status, | ||
179 | method_whitelist=self.method_whitelist, | ||
180 | status_forcelist=self.status_forcelist, | ||
181 | backoff_factor=self.backoff_factor, | ||
182 | raise_on_redirect=self.raise_on_redirect, | ||
183 | raise_on_status=self.raise_on_status, | ||
184 | history=self.history, | ||
185 | ) | ||
186 | params.update(kw) | ||
187 | return type(self)(**params) | ||
188 | |||
189 | @classmethod | ||
190 | def from_int(cls, retries, redirect=True, default=None): | ||
191 | """ Backwards-compatibility for the old retries format.""" | ||
192 | if retries is None: | ||
193 | retries = default if default is not None else cls.DEFAULT | ||
194 | |||
195 | if isinstance(retries, Retry): | ||
196 | return retries | ||
197 | |||
198 | redirect = bool(redirect) and None | ||
199 | new_retries = cls(retries, redirect=redirect) | ||
200 | log.debug("Converted retries value: %r -> %r", retries, new_retries) | ||
201 | return new_retries | ||
202 | |||
203 | def get_backoff_time(self): | ||
204 | """ Formula for computing the current backoff | ||
205 | |||
206 | :rtype: float | ||
207 | """ | ||
208 | # We want to consider only the last consecutive errors sequence (Ignore redirects). | ||
209 | consecutive_errors_len = len(list(takewhile(lambda x: x.redirect_location is None, | ||
210 | reversed(self.history)))) | ||
211 | if consecutive_errors_len <= 1: | ||
212 | return 0 | ||
213 | |||
214 | backoff_value = self.backoff_factor * (2 ** (consecutive_errors_len - 1)) | ||
215 | return min(self.BACKOFF_MAX, backoff_value) | ||
216 | |||
217 | def parse_retry_after(self, retry_after): | ||
218 | # Whitespace: https://tools.ietf.org/html/rfc7230#section-3.2.4 | ||
219 | if re.match(r"^\s*[0-9]+\s*$", retry_after): | ||
220 | seconds = int(retry_after) | ||
221 | else: | ||
222 | retry_date_tuple = email.utils.parsedate(retry_after) | ||
223 | if retry_date_tuple is None: | ||
224 | raise InvalidHeader("Invalid Retry-After header: %s" % retry_after) | ||
225 | retry_date = time.mktime(retry_date_tuple) | ||
226 | seconds = retry_date - time.time() | ||
227 | |||
228 | if seconds < 0: | ||
229 | seconds = 0 | ||
230 | |||
231 | return seconds | ||
232 | |||
233 | def get_retry_after(self, response): | ||
234 | """ Get the value of Retry-After in seconds. """ | ||
235 | |||
236 | retry_after = response.getheader("Retry-After") | ||
237 | |||
238 | if retry_after is None: | ||
239 | return None | ||
240 | |||
241 | return self.parse_retry_after(retry_after) | ||
242 | |||
243 | def sleep_for_retry(self, response=None): | ||
244 | retry_after = self.get_retry_after(response) | ||
245 | if retry_after: | ||
246 | time.sleep(retry_after) | ||
247 | return True | ||
248 | |||
249 | return False | ||
250 | |||
251 | def _sleep_backoff(self): | ||
252 | backoff = self.get_backoff_time() | ||
253 | if backoff <= 0: | ||
254 | return | ||
255 | time.sleep(backoff) | ||
256 | |||
257 | def sleep(self, response=None): | ||
258 | """ Sleep between retry attempts. | ||
259 | |||
260 | This method will respect a server's ``Retry-After`` response header | ||
261 | and sleep the duration of the time requested. If that is not present, it | ||
262 | will use an exponential backoff. By default, the backoff factor is 0 and | ||
263 | this method will return immediately. | ||
264 | """ | ||
265 | |||
266 | if response: | ||
267 | slept = self.sleep_for_retry(response) | ||
268 | if slept: | ||
269 | return | ||
270 | |||
271 | self._sleep_backoff() | ||
272 | |||
273 | def _is_connection_error(self, err): | ||
274 | """ Errors when we're fairly sure that the server did not receive the | ||
275 | request, so it should be safe to retry. | ||
276 | """ | ||
277 | return isinstance(err, ConnectTimeoutError) | ||
278 | |||
279 | def _is_read_error(self, err): | ||
280 | """ Errors that occur after the request has been started, so we should | ||
281 | assume that the server began processing it. | ||
282 | """ | ||
283 | return isinstance(err, (ReadTimeoutError, ProtocolError)) | ||
284 | |||
285 | def _is_method_retryable(self, method): | ||
286 | """ Checks if a given HTTP method should be retried upon, depending if | ||
287 | it is included on the method whitelist. | ||
288 | """ | ||
289 | if self.method_whitelist and method.upper() not in self.method_whitelist: | ||
290 | return False | ||
291 | |||
292 | return True | ||
293 | |||
294 | def is_retry(self, method, status_code, has_retry_after=False): | ||
295 | """ Is this method/status code retryable? (Based on whitelists and control | ||
296 | variables such as the number of total retries to allow, whether to | ||
297 | respect the Retry-After header, whether this header is present, and | ||
298 | whether the returned status code is on the list of status codes to | ||
299 | be retried upon on the presence of the aforementioned header) | ||
300 | """ | ||
301 | if not self._is_method_retryable(method): | ||
302 | return False | ||
303 | |||
304 | if self.status_forcelist and status_code in self.status_forcelist: | ||
305 | return True | ||
306 | |||
307 | return (self.total and self.respect_retry_after_header and | ||
308 | has_retry_after and (status_code in self.RETRY_AFTER_STATUS_CODES)) | ||
309 | |||
310 | def is_exhausted(self): | ||
311 | """ Are we out of retries? """ | ||
312 | retry_counts = (self.total, self.connect, self.read, self.redirect, self.status) | ||
313 | retry_counts = list(filter(None, retry_counts)) | ||
314 | if not retry_counts: | ||
315 | return False | ||
316 | |||
317 | return min(retry_counts) < 0 | ||
318 | |||
319 | def increment(self, method=None, url=None, response=None, error=None, | ||
320 | _pool=None, _stacktrace=None): | ||
321 | """ Return a new Retry object with incremented retry counters. | ||
322 | |||
323 | :param response: A response object, or None, if the server did not | ||
324 | return a response. | ||
325 | :type response: :class:`~urllib3.response.HTTPResponse` | ||
326 | :param Exception error: An error encountered during the request, or | ||
327 | None if the response was received successfully. | ||
328 | |||
329 | :return: A new ``Retry`` object. | ||
330 | """ | ||
331 | if self.total is False and error: | ||
332 | # Disabled, indicate to re-raise the error. | ||
333 | raise six.reraise(type(error), error, _stacktrace) | ||
334 | |||
335 | total = self.total | ||
336 | if total is not None: | ||
337 | total -= 1 | ||
338 | |||
339 | connect = self.connect | ||
340 | read = self.read | ||
341 | redirect = self.redirect | ||
342 | status_count = self.status | ||
343 | cause = 'unknown' | ||
344 | status = None | ||
345 | redirect_location = None | ||
346 | |||
347 | if error and self._is_connection_error(error): | ||
348 | # Connect retry? | ||
349 | if connect is False: | ||
350 | raise six.reraise(type(error), error, _stacktrace) | ||
351 | elif connect is not None: | ||
352 | connect -= 1 | ||
353 | |||
354 | elif error and self._is_read_error(error): | ||
355 | # Read retry? | ||
356 | if read is False or not self._is_method_retryable(method): | ||
357 | raise six.reraise(type(error), error, _stacktrace) | ||
358 | elif read is not None: | ||
359 | read -= 1 | ||
360 | |||
361 | elif response and response.get_redirect_location(): | ||
362 | # Redirect retry? | ||
363 | if redirect is not None: | ||
364 | redirect -= 1 | ||
365 | cause = 'too many redirects' | ||
366 | redirect_location = response.get_redirect_location() | ||
367 | status = response.status | ||
368 | |||
369 | else: | ||
370 | # Incrementing because of a server error like a 500 in | ||
371 | # status_forcelist and a the given method is in the whitelist | ||
372 | cause = ResponseError.GENERIC_ERROR | ||
373 | if response and response.status: | ||
374 | if status_count is not None: | ||
375 | status_count -= 1 | ||
376 | cause = ResponseError.SPECIFIC_ERROR.format( | ||
377 | status_code=response.status) | ||
378 | status = response.status | ||
379 | |||
380 | history = self.history + (RequestHistory(method, url, error, status, redirect_location),) | ||
381 | |||
382 | new_retry = self.new( | ||
383 | total=total, | ||
384 | connect=connect, read=read, redirect=redirect, status=status_count, | ||
385 | history=history) | ||
386 | |||
387 | if new_retry.is_exhausted(): | ||
388 | raise MaxRetryError(_pool, url, error or ResponseError(cause)) | ||
389 | |||
390 | log.debug("Incremented Retry for (url='%s'): %r", url, new_retry) | ||
391 | |||
392 | return new_retry | ||
393 | |||
394 | def __repr__(self): | ||
395 | return ('{cls.__name__}(total={self.total}, connect={self.connect}, ' | ||
396 | 'read={self.read}, redirect={self.redirect}, status={self.status})').format( | ||
397 | cls=type(self), self=self) | ||
398 | |||
399 | |||
400 | # For backwards compatibility (equivalent to pre-v1.9): | ||
401 | Retry.DEFAULT = Retry(3) | ||