diff options
author | Shubham Saini <shubham6405@gmail.com> | 2018-12-11 10:01:23 +0000 |
---|---|---|
committer | Shubham Saini <shubham6405@gmail.com> | 2018-12-11 10:01:23 +0000 |
commit | 68df54d6629ec019142eb149dd037774f2d11e7c (patch) | |
tree | 345bc22d46b4e01a4ba8303b94278952a4ed2b9e /venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/urllib3/poolmanager.py |
First commit
Diffstat (limited to 'venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/urllib3/poolmanager.py')
-rw-r--r-- | venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/urllib3/poolmanager.py | 440 |
1 files changed, 440 insertions, 0 deletions
diff --git a/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/urllib3/poolmanager.py b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/urllib3/poolmanager.py new file mode 100644 index 0000000..607ae0f --- /dev/null +++ b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/urllib3/poolmanager.py | |||
@@ -0,0 +1,440 @@ | |||
1 | from __future__ import absolute_import | ||
2 | import collections | ||
3 | import functools | ||
4 | import logging | ||
5 | |||
6 | from ._collections import RecentlyUsedContainer | ||
7 | from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool | ||
8 | from .connectionpool import port_by_scheme | ||
9 | from .exceptions import LocationValueError, MaxRetryError, ProxySchemeUnknown | ||
10 | from .packages.six.moves.urllib.parse import urljoin | ||
11 | from .request import RequestMethods | ||
12 | from .util.url import parse_url | ||
13 | from .util.retry import Retry | ||
14 | |||
15 | |||
16 | __all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url'] | ||
17 | |||
18 | |||
19 | log = logging.getLogger(__name__) | ||
20 | |||
21 | SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs', | ||
22 | 'ssl_version', 'ca_cert_dir', 'ssl_context') | ||
23 | |||
24 | # All known keyword arguments that could be provided to the pool manager, its | ||
25 | # pools, or the underlying connections. This is used to construct a pool key. | ||
26 | _key_fields = ( | ||
27 | 'key_scheme', # str | ||
28 | 'key_host', # str | ||
29 | 'key_port', # int | ||
30 | 'key_timeout', # int or float or Timeout | ||
31 | 'key_retries', # int or Retry | ||
32 | 'key_strict', # bool | ||
33 | 'key_block', # bool | ||
34 | 'key_source_address', # str | ||
35 | 'key_key_file', # str | ||
36 | 'key_cert_file', # str | ||
37 | 'key_cert_reqs', # str | ||
38 | 'key_ca_certs', # str | ||
39 | 'key_ssl_version', # str | ||
40 | 'key_ca_cert_dir', # str | ||
41 | 'key_ssl_context', # instance of ssl.SSLContext or urllib3.util.ssl_.SSLContext | ||
42 | 'key_maxsize', # int | ||
43 | 'key_headers', # dict | ||
44 | 'key__proxy', # parsed proxy url | ||
45 | 'key__proxy_headers', # dict | ||
46 | 'key_socket_options', # list of (level (int), optname (int), value (int or str)) tuples | ||
47 | 'key__socks_options', # dict | ||
48 | 'key_assert_hostname', # bool or string | ||
49 | 'key_assert_fingerprint', # str | ||
50 | ) | ||
51 | |||
52 | #: The namedtuple class used to construct keys for the connection pool. | ||
53 | #: All custom key schemes should include the fields in this key at a minimum. | ||
54 | PoolKey = collections.namedtuple('PoolKey', _key_fields) | ||
55 | |||
56 | |||
57 | def _default_key_normalizer(key_class, request_context): | ||
58 | """ | ||
59 | Create a pool key out of a request context dictionary. | ||
60 | |||
61 | According to RFC 3986, both the scheme and host are case-insensitive. | ||
62 | Therefore, this function normalizes both before constructing the pool | ||
63 | key for an HTTPS request. If you wish to change this behaviour, provide | ||
64 | alternate callables to ``key_fn_by_scheme``. | ||
65 | |||
66 | :param key_class: | ||
67 | The class to use when constructing the key. This should be a namedtuple | ||
68 | with the ``scheme`` and ``host`` keys at a minimum. | ||
69 | :type key_class: namedtuple | ||
70 | :param request_context: | ||
71 | A dictionary-like object that contain the context for a request. | ||
72 | :type request_context: dict | ||
73 | |||
74 | :return: A namedtuple that can be used as a connection pool key. | ||
75 | :rtype: PoolKey | ||
76 | """ | ||
77 | # Since we mutate the dictionary, make a copy first | ||
78 | context = request_context.copy() | ||
79 | context['scheme'] = context['scheme'].lower() | ||
80 | context['host'] = context['host'].lower() | ||
81 | |||
82 | # These are both dictionaries and need to be transformed into frozensets | ||
83 | for key in ('headers', '_proxy_headers', '_socks_options'): | ||
84 | if key in context and context[key] is not None: | ||
85 | context[key] = frozenset(context[key].items()) | ||
86 | |||
87 | # The socket_options key may be a list and needs to be transformed into a | ||
88 | # tuple. | ||
89 | socket_opts = context.get('socket_options') | ||
90 | if socket_opts is not None: | ||
91 | context['socket_options'] = tuple(socket_opts) | ||
92 | |||
93 | # Map the kwargs to the names in the namedtuple - this is necessary since | ||
94 | # namedtuples can't have fields starting with '_'. | ||
95 | for key in list(context.keys()): | ||
96 | context['key_' + key] = context.pop(key) | ||
97 | |||
98 | # Default to ``None`` for keys missing from the context | ||
99 | for field in key_class._fields: | ||
100 | if field not in context: | ||
101 | context[field] = None | ||
102 | |||
103 | return key_class(**context) | ||
104 | |||
105 | |||
106 | #: A dictionary that maps a scheme to a callable that creates a pool key. | ||
107 | #: This can be used to alter the way pool keys are constructed, if desired. | ||
108 | #: Each PoolManager makes a copy of this dictionary so they can be configured | ||
109 | #: globally here, or individually on the instance. | ||
110 | key_fn_by_scheme = { | ||
111 | 'http': functools.partial(_default_key_normalizer, PoolKey), | ||
112 | 'https': functools.partial(_default_key_normalizer, PoolKey), | ||
113 | } | ||
114 | |||
115 | pool_classes_by_scheme = { | ||
116 | 'http': HTTPConnectionPool, | ||
117 | 'https': HTTPSConnectionPool, | ||
118 | } | ||
119 | |||
120 | |||
121 | class PoolManager(RequestMethods): | ||
122 | """ | ||
123 | Allows for arbitrary requests while transparently keeping track of | ||
124 | necessary connection pools for you. | ||
125 | |||
126 | :param num_pools: | ||
127 | Number of connection pools to cache before discarding the least | ||
128 | recently used pool. | ||
129 | |||
130 | :param headers: | ||
131 | Headers to include with all requests, unless other headers are given | ||
132 | explicitly. | ||
133 | |||
134 | :param \\**connection_pool_kw: | ||
135 | Additional parameters are used to create fresh | ||
136 | :class:`urllib3.connectionpool.ConnectionPool` instances. | ||
137 | |||
138 | Example:: | ||
139 | |||
140 | >>> manager = PoolManager(num_pools=2) | ||
141 | >>> r = manager.request('GET', 'http://google.com/') | ||
142 | >>> r = manager.request('GET', 'http://google.com/mail') | ||
143 | >>> r = manager.request('GET', 'http://yahoo.com/') | ||
144 | >>> len(manager.pools) | ||
145 | 2 | ||
146 | |||
147 | """ | ||
148 | |||
149 | proxy = None | ||
150 | |||
151 | def __init__(self, num_pools=10, headers=None, **connection_pool_kw): | ||
152 | RequestMethods.__init__(self, headers) | ||
153 | self.connection_pool_kw = connection_pool_kw | ||
154 | self.pools = RecentlyUsedContainer(num_pools, | ||
155 | dispose_func=lambda p: p.close()) | ||
156 | |||
157 | # Locally set the pool classes and keys so other PoolManagers can | ||
158 | # override them. | ||
159 | self.pool_classes_by_scheme = pool_classes_by_scheme | ||
160 | self.key_fn_by_scheme = key_fn_by_scheme.copy() | ||
161 | |||
162 | def __enter__(self): | ||
163 | return self | ||
164 | |||
165 | def __exit__(self, exc_type, exc_val, exc_tb): | ||
166 | self.clear() | ||
167 | # Return False to re-raise any potential exceptions | ||
168 | return False | ||
169 | |||
170 | def _new_pool(self, scheme, host, port, request_context=None): | ||
171 | """ | ||
172 | Create a new :class:`ConnectionPool` based on host, port, scheme, and | ||
173 | any additional pool keyword arguments. | ||
174 | |||
175 | If ``request_context`` is provided, it is provided as keyword arguments | ||
176 | to the pool class used. This method is used to actually create the | ||
177 | connection pools handed out by :meth:`connection_from_url` and | ||
178 | companion methods. It is intended to be overridden for customization. | ||
179 | """ | ||
180 | pool_cls = self.pool_classes_by_scheme[scheme] | ||
181 | if request_context is None: | ||
182 | request_context = self.connection_pool_kw.copy() | ||
183 | |||
184 | # Although the context has everything necessary to create the pool, | ||
185 | # this function has historically only used the scheme, host, and port | ||
186 | # in the positional args. When an API change is acceptable these can | ||
187 | # be removed. | ||
188 | for key in ('scheme', 'host', 'port'): | ||
189 | request_context.pop(key, None) | ||
190 | |||
191 | if scheme == 'http': | ||
192 | for kw in SSL_KEYWORDS: | ||
193 | request_context.pop(kw, None) | ||
194 | |||
195 | return pool_cls(host, port, **request_context) | ||
196 | |||
197 | def clear(self): | ||
198 | """ | ||
199 | Empty our store of pools and direct them all to close. | ||
200 | |||
201 | This will not affect in-flight connections, but they will not be | ||
202 | re-used after completion. | ||
203 | """ | ||
204 | self.pools.clear() | ||
205 | |||
206 | def connection_from_host(self, host, port=None, scheme='http', pool_kwargs=None): | ||
207 | """ | ||
208 | Get a :class:`ConnectionPool` based on the host, port, and scheme. | ||
209 | |||
210 | If ``port`` isn't given, it will be derived from the ``scheme`` using | ||
211 | ``urllib3.connectionpool.port_by_scheme``. If ``pool_kwargs`` is | ||
212 | provided, it is merged with the instance's ``connection_pool_kw`` | ||
213 | variable and used to create the new connection pool, if one is | ||
214 | needed. | ||
215 | """ | ||
216 | |||
217 | if not host: | ||
218 | raise LocationValueError("No host specified.") | ||
219 | |||
220 | request_context = self._merge_pool_kwargs(pool_kwargs) | ||
221 | request_context['scheme'] = scheme or 'http' | ||
222 | if not port: | ||
223 | port = port_by_scheme.get(request_context['scheme'].lower(), 80) | ||
224 | request_context['port'] = port | ||
225 | request_context['host'] = host | ||
226 | |||
227 | return self.connection_from_context(request_context) | ||
228 | |||
229 | def connection_from_context(self, request_context): | ||
230 | """ | ||
231 | Get a :class:`ConnectionPool` based on the request context. | ||
232 | |||
233 | ``request_context`` must at least contain the ``scheme`` key and its | ||
234 | value must be a key in ``key_fn_by_scheme`` instance variable. | ||
235 | """ | ||
236 | scheme = request_context['scheme'].lower() | ||
237 | pool_key_constructor = self.key_fn_by_scheme[scheme] | ||
238 | pool_key = pool_key_constructor(request_context) | ||
239 | |||
240 | return self.connection_from_pool_key(pool_key, request_context=request_context) | ||
241 | |||
242 | def connection_from_pool_key(self, pool_key, request_context=None): | ||
243 | """ | ||
244 | Get a :class:`ConnectionPool` based on the provided pool key. | ||
245 | |||
246 | ``pool_key`` should be a namedtuple that only contains immutable | ||
247 | objects. At a minimum it must have the ``scheme``, ``host``, and | ||
248 | ``port`` fields. | ||
249 | """ | ||
250 | with self.pools.lock: | ||
251 | # If the scheme, host, or port doesn't match existing open | ||
252 | # connections, open a new ConnectionPool. | ||
253 | pool = self.pools.get(pool_key) | ||
254 | if pool: | ||
255 | return pool | ||
256 | |||
257 | # Make a fresh ConnectionPool of the desired type | ||
258 | scheme = request_context['scheme'] | ||
259 | host = request_context['host'] | ||
260 | port = request_context['port'] | ||
261 | pool = self._new_pool(scheme, host, port, request_context=request_context) | ||
262 | self.pools[pool_key] = pool | ||
263 | |||
264 | return pool | ||
265 | |||
266 | def connection_from_url(self, url, pool_kwargs=None): | ||
267 | """ | ||
268 | Similar to :func:`urllib3.connectionpool.connection_from_url`. | ||
269 | |||
270 | If ``pool_kwargs`` is not provided and a new pool needs to be | ||
271 | constructed, ``self.connection_pool_kw`` is used to initialize | ||
272 | the :class:`urllib3.connectionpool.ConnectionPool`. If ``pool_kwargs`` | ||
273 | is provided, it is used instead. Note that if a new pool does not | ||
274 | need to be created for the request, the provided ``pool_kwargs`` are | ||
275 | not used. | ||
276 | """ | ||
277 | u = parse_url(url) | ||
278 | return self.connection_from_host(u.host, port=u.port, scheme=u.scheme, | ||
279 | pool_kwargs=pool_kwargs) | ||
280 | |||
281 | def _merge_pool_kwargs(self, override): | ||
282 | """ | ||
283 | Merge a dictionary of override values for self.connection_pool_kw. | ||
284 | |||
285 | This does not modify self.connection_pool_kw and returns a new dict. | ||
286 | Any keys in the override dictionary with a value of ``None`` are | ||
287 | removed from the merged dictionary. | ||
288 | """ | ||
289 | base_pool_kwargs = self.connection_pool_kw.copy() | ||
290 | if override: | ||
291 | for key, value in override.items(): | ||
292 | if value is None: | ||
293 | try: | ||
294 | del base_pool_kwargs[key] | ||
295 | except KeyError: | ||
296 | pass | ||
297 | else: | ||
298 | base_pool_kwargs[key] = value | ||
299 | return base_pool_kwargs | ||
300 | |||
301 | def urlopen(self, method, url, redirect=True, **kw): | ||
302 | """ | ||
303 | Same as :meth:`urllib3.connectionpool.HTTPConnectionPool.urlopen` | ||
304 | with custom cross-host redirect logic and only sends the request-uri | ||
305 | portion of the ``url``. | ||
306 | |||
307 | The given ``url`` parameter must be absolute, such that an appropriate | ||
308 | :class:`urllib3.connectionpool.ConnectionPool` can be chosen for it. | ||
309 | """ | ||
310 | u = parse_url(url) | ||
311 | conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme) | ||
312 | |||
313 | kw['assert_same_host'] = False | ||
314 | kw['redirect'] = False | ||
315 | if 'headers' not in kw: | ||
316 | kw['headers'] = self.headers | ||
317 | |||
318 | if self.proxy is not None and u.scheme == "http": | ||
319 | response = conn.urlopen(method, url, **kw) | ||
320 | else: | ||
321 | response = conn.urlopen(method, u.request_uri, **kw) | ||
322 | |||
323 | redirect_location = redirect and response.get_redirect_location() | ||
324 | if not redirect_location: | ||
325 | return response | ||
326 | |||
327 | # Support relative URLs for redirecting. | ||
328 | redirect_location = urljoin(url, redirect_location) | ||
329 | |||
330 | # RFC 7231, Section 6.4.4 | ||
331 | if response.status == 303: | ||
332 | method = 'GET' | ||
333 | |||
334 | retries = kw.get('retries') | ||
335 | if not isinstance(retries, Retry): | ||
336 | retries = Retry.from_int(retries, redirect=redirect) | ||
337 | |||
338 | try: | ||
339 | retries = retries.increment(method, url, response=response, _pool=conn) | ||
340 | except MaxRetryError: | ||
341 | if retries.raise_on_redirect: | ||
342 | raise | ||
343 | return response | ||
344 | |||
345 | kw['retries'] = retries | ||
346 | kw['redirect'] = redirect | ||
347 | |||
348 | log.info("Redirecting %s -> %s", url, redirect_location) | ||
349 | return self.urlopen(method, redirect_location, **kw) | ||
350 | |||
351 | |||
352 | class ProxyManager(PoolManager): | ||
353 | """ | ||
354 | Behaves just like :class:`PoolManager`, but sends all requests through | ||
355 | the defined proxy, using the CONNECT method for HTTPS URLs. | ||
356 | |||
357 | :param proxy_url: | ||
358 | The URL of the proxy to be used. | ||
359 | |||
360 | :param proxy_headers: | ||
361 | A dictionary contaning headers that will be sent to the proxy. In case | ||
362 | of HTTP they are being sent with each request, while in the | ||
363 | HTTPS/CONNECT case they are sent only once. Could be used for proxy | ||
364 | authentication. | ||
365 | |||
366 | Example: | ||
367 | >>> proxy = urllib3.ProxyManager('http://localhost:3128/') | ||
368 | >>> r1 = proxy.request('GET', 'http://google.com/') | ||
369 | >>> r2 = proxy.request('GET', 'http://httpbin.org/') | ||
370 | >>> len(proxy.pools) | ||
371 | 1 | ||
372 | >>> r3 = proxy.request('GET', 'https://httpbin.org/') | ||
373 | >>> r4 = proxy.request('GET', 'https://twitter.com/') | ||
374 | >>> len(proxy.pools) | ||
375 | 3 | ||
376 | |||
377 | """ | ||
378 | |||
379 | def __init__(self, proxy_url, num_pools=10, headers=None, | ||
380 | proxy_headers=None, **connection_pool_kw): | ||
381 | |||
382 | if isinstance(proxy_url, HTTPConnectionPool): | ||
383 | proxy_url = '%s://%s:%i' % (proxy_url.scheme, proxy_url.host, | ||
384 | proxy_url.port) | ||
385 | proxy = parse_url(proxy_url) | ||
386 | if not proxy.port: | ||
387 | port = port_by_scheme.get(proxy.scheme, 80) | ||
388 | proxy = proxy._replace(port=port) | ||
389 | |||
390 | if proxy.scheme not in ("http", "https"): | ||
391 | raise ProxySchemeUnknown(proxy.scheme) | ||
392 | |||
393 | self.proxy = proxy | ||
394 | self.proxy_headers = proxy_headers or {} | ||
395 | |||
396 | connection_pool_kw['_proxy'] = self.proxy | ||
397 | connection_pool_kw['_proxy_headers'] = self.proxy_headers | ||
398 | |||
399 | super(ProxyManager, self).__init__( | ||
400 | num_pools, headers, **connection_pool_kw) | ||
401 | |||
402 | def connection_from_host(self, host, port=None, scheme='http', pool_kwargs=None): | ||
403 | if scheme == "https": | ||
404 | return super(ProxyManager, self).connection_from_host( | ||
405 | host, port, scheme, pool_kwargs=pool_kwargs) | ||
406 | |||
407 | return super(ProxyManager, self).connection_from_host( | ||
408 | self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs) | ||
409 | |||
410 | def _set_proxy_headers(self, url, headers=None): | ||
411 | """ | ||
412 | Sets headers needed by proxies: specifically, the Accept and Host | ||
413 | headers. Only sets headers not provided by the user. | ||
414 | """ | ||
415 | headers_ = {'Accept': '*/*'} | ||
416 | |||
417 | netloc = parse_url(url).netloc | ||
418 | if netloc: | ||
419 | headers_['Host'] = netloc | ||
420 | |||
421 | if headers: | ||
422 | headers_.update(headers) | ||
423 | return headers_ | ||
424 | |||
425 | def urlopen(self, method, url, redirect=True, **kw): | ||
426 | "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute." | ||
427 | u = parse_url(url) | ||
428 | |||
429 | if u.scheme == "http": | ||
430 | # For proxied HTTPS requests, httplib sets the necessary headers | ||
431 | # on the CONNECT to the proxy. For HTTP, we'll definitely | ||
432 | # need to set 'Host' at the very least. | ||
433 | headers = kw.get('headers', self.headers) | ||
434 | kw['headers'] = self._set_proxy_headers(url, headers) | ||
435 | |||
436 | return super(ProxyManager, self).urlopen(method, url, redirect=redirect, **kw) | ||
437 | |||
438 | |||
439 | def proxy_from_url(url, **kw): | ||
440 | return ProxyManager(proxy_url=url, **kw) | ||