diff options
author | Shubham Saini <shubham6405@gmail.com> | 2019-08-05 08:32:33 +0000 |
---|---|---|
committer | Shubham Saini <shubham6405@gmail.com> | 2019-08-05 08:32:33 +0000 |
commit | 227b2d30a8675b44918f9d9ca89b24144a938215 (patch) | |
tree | 9f8e6a28724514b6fdf463a9ab2067a7ef309b72 /venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/download.py | |
parent | 842a8cfbbbdb1f92889d892e4859dbd5d40c5be8 (diff) |
removing venv files
Diffstat (limited to 'venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/download.py')
-rw-r--r-- | venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/download.py | 922 |
1 files changed, 0 insertions, 922 deletions
diff --git a/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/download.py b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/download.py deleted file mode 100644 index e0e2d24..0000000 --- a/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/download.py +++ /dev/null | |||
@@ -1,922 +0,0 @@ | |||
1 | from __future__ import absolute_import | ||
2 | |||
3 | import cgi | ||
4 | import email.utils | ||
5 | import getpass | ||
6 | import json | ||
7 | import logging | ||
8 | import mimetypes | ||
9 | import os | ||
10 | import platform | ||
11 | import re | ||
12 | import shutil | ||
13 | import sys | ||
14 | |||
15 | from pip._vendor import requests, six, urllib3 | ||
16 | from pip._vendor.cachecontrol import CacheControlAdapter | ||
17 | from pip._vendor.cachecontrol.caches import FileCache | ||
18 | from pip._vendor.lockfile import LockError | ||
19 | from pip._vendor.requests.adapters import BaseAdapter, HTTPAdapter | ||
20 | from pip._vendor.requests.auth import AuthBase, HTTPBasicAuth | ||
21 | from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response | ||
22 | from pip._vendor.requests.structures import CaseInsensitiveDict | ||
23 | from pip._vendor.requests.utils import get_netrc_auth | ||
24 | # NOTE: XMLRPC Client is not annotated in typeshed as on 2017-07-17, which is | ||
25 | # why we ignore the type on this import | ||
26 | from pip._vendor.six.moves import xmlrpc_client # type: ignore | ||
27 | from pip._vendor.six.moves.urllib import parse as urllib_parse | ||
28 | from pip._vendor.six.moves.urllib import request as urllib_request | ||
29 | from pip._vendor.six.moves.urllib.parse import unquote as urllib_unquote | ||
30 | from pip._vendor.urllib3.util import IS_PYOPENSSL | ||
31 | |||
32 | import pip | ||
33 | from pip._internal.compat import WINDOWS | ||
34 | from pip._internal.exceptions import HashMismatch, InstallationError | ||
35 | from pip._internal.locations import write_delete_marker_file | ||
36 | from pip._internal.models import PyPI | ||
37 | from pip._internal.utils.encoding import auto_decode | ||
38 | from pip._internal.utils.filesystem import check_path_owner | ||
39 | from pip._internal.utils.glibc import libc_ver | ||
40 | from pip._internal.utils.logging import indent_log | ||
41 | from pip._internal.utils.misc import ( | ||
42 | ARCHIVE_EXTENSIONS, ask_path_exists, backup_dir, call_subprocess, consume, | ||
43 | display_path, format_size, get_installed_version, rmtree, splitext, | ||
44 | unpack_file, | ||
45 | ) | ||
46 | from pip._internal.utils.setuptools_build import SETUPTOOLS_SHIM | ||
47 | from pip._internal.utils.temp_dir import TempDirectory | ||
48 | from pip._internal.utils.ui import DownloadProgressProvider | ||
49 | from pip._internal.vcs import vcs | ||
50 | |||
51 | try: | ||
52 | import ssl # noqa | ||
53 | except ImportError: | ||
54 | ssl = None | ||
55 | |||
56 | HAS_TLS = (ssl is not None) or IS_PYOPENSSL | ||
57 | |||
58 | __all__ = ['get_file_content', | ||
59 | 'is_url', 'url_to_path', 'path_to_url', | ||
60 | 'is_archive_file', 'unpack_vcs_link', | ||
61 | 'unpack_file_url', 'is_vcs_url', 'is_file_url', | ||
62 | 'unpack_http_url', 'unpack_url'] | ||
63 | |||
64 | |||
65 | logger = logging.getLogger(__name__) | ||
66 | |||
67 | |||
68 | def user_agent(): | ||
69 | """ | ||
70 | Return a string representing the user agent. | ||
71 | """ | ||
72 | data = { | ||
73 | "installer": {"name": "pip", "version": pip.__version__}, | ||
74 | "python": platform.python_version(), | ||
75 | "implementation": { | ||
76 | "name": platform.python_implementation(), | ||
77 | }, | ||
78 | } | ||
79 | |||
80 | if data["implementation"]["name"] == 'CPython': | ||
81 | data["implementation"]["version"] = platform.python_version() | ||
82 | elif data["implementation"]["name"] == 'PyPy': | ||
83 | if sys.pypy_version_info.releaselevel == 'final': | ||
84 | pypy_version_info = sys.pypy_version_info[:3] | ||
85 | else: | ||
86 | pypy_version_info = sys.pypy_version_info | ||
87 | data["implementation"]["version"] = ".".join( | ||
88 | [str(x) for x in pypy_version_info] | ||
89 | ) | ||
90 | elif data["implementation"]["name"] == 'Jython': | ||
91 | # Complete Guess | ||
92 | data["implementation"]["version"] = platform.python_version() | ||
93 | elif data["implementation"]["name"] == 'IronPython': | ||
94 | # Complete Guess | ||
95 | data["implementation"]["version"] = platform.python_version() | ||
96 | |||
97 | if sys.platform.startswith("linux"): | ||
98 | from pip._vendor import distro | ||
99 | distro_infos = dict(filter( | ||
100 | lambda x: x[1], | ||
101 | zip(["name", "version", "id"], distro.linux_distribution()), | ||
102 | )) | ||
103 | libc = dict(filter( | ||
104 | lambda x: x[1], | ||
105 | zip(["lib", "version"], libc_ver()), | ||
106 | )) | ||
107 | if libc: | ||
108 | distro_infos["libc"] = libc | ||
109 | if distro_infos: | ||
110 | data["distro"] = distro_infos | ||
111 | |||
112 | if sys.platform.startswith("darwin") and platform.mac_ver()[0]: | ||
113 | data["distro"] = {"name": "macOS", "version": platform.mac_ver()[0]} | ||
114 | |||
115 | if platform.system(): | ||
116 | data.setdefault("system", {})["name"] = platform.system() | ||
117 | |||
118 | if platform.release(): | ||
119 | data.setdefault("system", {})["release"] = platform.release() | ||
120 | |||
121 | if platform.machine(): | ||
122 | data["cpu"] = platform.machine() | ||
123 | |||
124 | if HAS_TLS: | ||
125 | data["openssl_version"] = ssl.OPENSSL_VERSION | ||
126 | |||
127 | setuptools_version = get_installed_version("setuptools") | ||
128 | if setuptools_version is not None: | ||
129 | data["setuptools_version"] = setuptools_version | ||
130 | |||
131 | return "{data[installer][name]}/{data[installer][version]} {json}".format( | ||
132 | data=data, | ||
133 | json=json.dumps(data, separators=(",", ":"), sort_keys=True), | ||
134 | ) | ||
135 | |||
136 | |||
137 | class MultiDomainBasicAuth(AuthBase): | ||
138 | |||
139 | def __init__(self, prompting=True): | ||
140 | self.prompting = prompting | ||
141 | self.passwords = {} | ||
142 | |||
143 | def __call__(self, req): | ||
144 | parsed = urllib_parse.urlparse(req.url) | ||
145 | |||
146 | # Get the netloc without any embedded credentials | ||
147 | netloc = parsed.netloc.rsplit("@", 1)[-1] | ||
148 | |||
149 | # Set the url of the request to the url without any credentials | ||
150 | req.url = urllib_parse.urlunparse(parsed[:1] + (netloc,) + parsed[2:]) | ||
151 | |||
152 | # Use any stored credentials that we have for this netloc | ||
153 | username, password = self.passwords.get(netloc, (None, None)) | ||
154 | |||
155 | # Extract credentials embedded in the url if we have none stored | ||
156 | if username is None: | ||
157 | username, password = self.parse_credentials(parsed.netloc) | ||
158 | |||
159 | # Get creds from netrc if we still don't have them | ||
160 | if username is None and password is None: | ||
161 | netrc_auth = get_netrc_auth(req.url) | ||
162 | username, password = netrc_auth if netrc_auth else (None, None) | ||
163 | |||
164 | if username or password: | ||
165 | # Store the username and password | ||
166 | self.passwords[netloc] = (username, password) | ||
167 | |||
168 | # Send the basic auth with this request | ||
169 | req = HTTPBasicAuth(username or "", password or "")(req) | ||
170 | |||
171 | # Attach a hook to handle 401 responses | ||
172 | req.register_hook("response", self.handle_401) | ||
173 | |||
174 | return req | ||
175 | |||
176 | def handle_401(self, resp, **kwargs): | ||
177 | # We only care about 401 responses, anything else we want to just | ||
178 | # pass through the actual response | ||
179 | if resp.status_code != 401: | ||
180 | return resp | ||
181 | |||
182 | # We are not able to prompt the user so simply return the response | ||
183 | if not self.prompting: | ||
184 | return resp | ||
185 | |||
186 | parsed = urllib_parse.urlparse(resp.url) | ||
187 | |||
188 | # Prompt the user for a new username and password | ||
189 | username = six.moves.input("User for %s: " % parsed.netloc) | ||
190 | password = getpass.getpass("Password: ") | ||
191 | |||
192 | # Store the new username and password to use for future requests | ||
193 | if username or password: | ||
194 | self.passwords[parsed.netloc] = (username, password) | ||
195 | |||
196 | # Consume content and release the original connection to allow our new | ||
197 | # request to reuse the same one. | ||
198 | resp.content | ||
199 | resp.raw.release_conn() | ||
200 | |||
201 | # Add our new username and password to the request | ||
202 | req = HTTPBasicAuth(username or "", password or "")(resp.request) | ||
203 | |||
204 | # Send our new request | ||
205 | new_resp = resp.connection.send(req, **kwargs) | ||
206 | new_resp.history.append(resp) | ||
207 | |||
208 | return new_resp | ||
209 | |||
210 | def parse_credentials(self, netloc): | ||
211 | if "@" in netloc: | ||
212 | userinfo = netloc.rsplit("@", 1)[0] | ||
213 | if ":" in userinfo: | ||
214 | user, pwd = userinfo.split(":", 1) | ||
215 | return (urllib_unquote(user), urllib_unquote(pwd)) | ||
216 | return urllib_unquote(userinfo), None | ||
217 | return None, None | ||
218 | |||
219 | |||
220 | class LocalFSAdapter(BaseAdapter): | ||
221 | |||
222 | def send(self, request, stream=None, timeout=None, verify=None, cert=None, | ||
223 | proxies=None): | ||
224 | pathname = url_to_path(request.url) | ||
225 | |||
226 | resp = Response() | ||
227 | resp.status_code = 200 | ||
228 | resp.url = request.url | ||
229 | |||
230 | try: | ||
231 | stats = os.stat(pathname) | ||
232 | except OSError as exc: | ||
233 | resp.status_code = 404 | ||
234 | resp.raw = exc | ||
235 | else: | ||
236 | modified = email.utils.formatdate(stats.st_mtime, usegmt=True) | ||
237 | content_type = mimetypes.guess_type(pathname)[0] or "text/plain" | ||
238 | resp.headers = CaseInsensitiveDict({ | ||
239 | "Content-Type": content_type, | ||
240 | "Content-Length": stats.st_size, | ||
241 | "Last-Modified": modified, | ||
242 | }) | ||
243 | |||
244 | resp.raw = open(pathname, "rb") | ||
245 | resp.close = resp.raw.close | ||
246 | |||
247 | return resp | ||
248 | |||
249 | def close(self): | ||
250 | pass | ||
251 | |||
252 | |||
253 | class SafeFileCache(FileCache): | ||
254 | """ | ||
255 | A file based cache which is safe to use even when the target directory may | ||
256 | not be accessible or writable. | ||
257 | """ | ||
258 | |||
259 | def __init__(self, *args, **kwargs): | ||
260 | super(SafeFileCache, self).__init__(*args, **kwargs) | ||
261 | |||
262 | # Check to ensure that the directory containing our cache directory | ||
263 | # is owned by the user current executing pip. If it does not exist | ||
264 | # we will check the parent directory until we find one that does exist. | ||
265 | # If it is not owned by the user executing pip then we will disable | ||
266 | # the cache and log a warning. | ||
267 | if not check_path_owner(self.directory): | ||
268 | logger.warning( | ||
269 | "The directory '%s' or its parent directory is not owned by " | ||
270 | "the current user and the cache has been disabled. Please " | ||
271 | "check the permissions and owner of that directory. If " | ||
272 | "executing pip with sudo, you may want sudo's -H flag.", | ||
273 | self.directory, | ||
274 | ) | ||
275 | |||
276 | # Set our directory to None to disable the Cache | ||
277 | self.directory = None | ||
278 | |||
279 | def get(self, *args, **kwargs): | ||
280 | # If we don't have a directory, then the cache should be a no-op. | ||
281 | if self.directory is None: | ||
282 | return | ||
283 | |||
284 | try: | ||
285 | return super(SafeFileCache, self).get(*args, **kwargs) | ||
286 | except (LockError, OSError, IOError): | ||
287 | # We intentionally silence this error, if we can't access the cache | ||
288 | # then we can just skip caching and process the request as if | ||
289 | # caching wasn't enabled. | ||
290 | pass | ||
291 | |||
292 | def set(self, *args, **kwargs): | ||
293 | # If we don't have a directory, then the cache should be a no-op. | ||
294 | if self.directory is None: | ||
295 | return | ||
296 | |||
297 | try: | ||
298 | return super(SafeFileCache, self).set(*args, **kwargs) | ||
299 | except (LockError, OSError, IOError): | ||
300 | # We intentionally silence this error, if we can't access the cache | ||
301 | # then we can just skip caching and process the request as if | ||
302 | # caching wasn't enabled. | ||
303 | pass | ||
304 | |||
305 | def delete(self, *args, **kwargs): | ||
306 | # If we don't have a directory, then the cache should be a no-op. | ||
307 | if self.directory is None: | ||
308 | return | ||
309 | |||
310 | try: | ||
311 | return super(SafeFileCache, self).delete(*args, **kwargs) | ||
312 | except (LockError, OSError, IOError): | ||
313 | # We intentionally silence this error, if we can't access the cache | ||
314 | # then we can just skip caching and process the request as if | ||
315 | # caching wasn't enabled. | ||
316 | pass | ||
317 | |||
318 | |||
319 | class InsecureHTTPAdapter(HTTPAdapter): | ||
320 | |||
321 | def cert_verify(self, conn, url, verify, cert): | ||
322 | conn.cert_reqs = 'CERT_NONE' | ||
323 | conn.ca_certs = None | ||
324 | |||
325 | |||
326 | class PipSession(requests.Session): | ||
327 | |||
328 | timeout = None | ||
329 | |||
330 | def __init__(self, *args, **kwargs): | ||
331 | retries = kwargs.pop("retries", 0) | ||
332 | cache = kwargs.pop("cache", None) | ||
333 | insecure_hosts = kwargs.pop("insecure_hosts", []) | ||
334 | |||
335 | super(PipSession, self).__init__(*args, **kwargs) | ||
336 | |||
337 | # Attach our User Agent to the request | ||
338 | self.headers["User-Agent"] = user_agent() | ||
339 | |||
340 | # Attach our Authentication handler to the session | ||
341 | self.auth = MultiDomainBasicAuth() | ||
342 | |||
343 | # Create our urllib3.Retry instance which will allow us to customize | ||
344 | # how we handle retries. | ||
345 | retries = urllib3.Retry( | ||
346 | # Set the total number of retries that a particular request can | ||
347 | # have. | ||
348 | total=retries, | ||
349 | |||
350 | # A 503 error from PyPI typically means that the Fastly -> Origin | ||
351 | # connection got interrupted in some way. A 503 error in general | ||
352 | # is typically considered a transient error so we'll go ahead and | ||
353 | # retry it. | ||
354 | # A 500 may indicate transient error in Amazon S3 | ||
355 | # A 520 or 527 - may indicate transient error in CloudFlare | ||
356 | status_forcelist=[500, 503, 520, 527], | ||
357 | |||
358 | # Add a small amount of back off between failed requests in | ||
359 | # order to prevent hammering the service. | ||
360 | backoff_factor=0.25, | ||
361 | ) | ||
362 | |||
363 | # We want to _only_ cache responses on securely fetched origins. We do | ||
364 | # this because we can't validate the response of an insecurely fetched | ||
365 | # origin, and we don't want someone to be able to poison the cache and | ||
366 | # require manual eviction from the cache to fix it. | ||
367 | if cache: | ||
368 | secure_adapter = CacheControlAdapter( | ||
369 | cache=SafeFileCache(cache, use_dir_lock=True), | ||
370 | max_retries=retries, | ||
371 | ) | ||
372 | else: | ||
373 | secure_adapter = HTTPAdapter(max_retries=retries) | ||
374 | |||
375 | # Our Insecure HTTPAdapter disables HTTPS validation. It does not | ||
376 | # support caching (see above) so we'll use it for all http:// URLs as | ||
377 | # well as any https:// host that we've marked as ignoring TLS errors | ||
378 | # for. | ||
379 | insecure_adapter = InsecureHTTPAdapter(max_retries=retries) | ||
380 | |||
381 | self.mount("https://", secure_adapter) | ||
382 | self.mount("http://", insecure_adapter) | ||
383 | |||
384 | # Enable file:// urls | ||
385 | self.mount("file://", LocalFSAdapter()) | ||
386 | |||
387 | # We want to use a non-validating adapter for any requests which are | ||
388 | # deemed insecure. | ||
389 | for host in insecure_hosts: | ||
390 | self.mount("https://{}/".format(host), insecure_adapter) | ||
391 | |||
392 | def request(self, method, url, *args, **kwargs): | ||
393 | # Allow setting a default timeout on a session | ||
394 | kwargs.setdefault("timeout", self.timeout) | ||
395 | |||
396 | # Dispatch the actual request | ||
397 | return super(PipSession, self).request(method, url, *args, **kwargs) | ||
398 | |||
399 | |||
400 | def get_file_content(url, comes_from=None, session=None): | ||
401 | """Gets the content of a file; it may be a filename, file: URL, or | ||
402 | http: URL. Returns (location, content). Content is unicode. | ||
403 | |||
404 | :param url: File path or url. | ||
405 | :param comes_from: Origin description of requirements. | ||
406 | :param session: Instance of pip.download.PipSession. | ||
407 | """ | ||
408 | if session is None: | ||
409 | raise TypeError( | ||
410 | "get_file_content() missing 1 required keyword argument: 'session'" | ||
411 | ) | ||
412 | |||
413 | match = _scheme_re.search(url) | ||
414 | if match: | ||
415 | scheme = match.group(1).lower() | ||
416 | if (scheme == 'file' and comes_from and | ||
417 | comes_from.startswith('http')): | ||
418 | raise InstallationError( | ||
419 | 'Requirements file %s references URL %s, which is local' | ||
420 | % (comes_from, url)) | ||
421 | if scheme == 'file': | ||
422 | path = url.split(':', 1)[1] | ||
423 | path = path.replace('\\', '/') | ||
424 | match = _url_slash_drive_re.match(path) | ||
425 | if match: | ||
426 | path = match.group(1) + ':' + path.split('|', 1)[1] | ||
427 | path = urllib_parse.unquote(path) | ||
428 | if path.startswith('/'): | ||
429 | path = '/' + path.lstrip('/') | ||
430 | url = path | ||
431 | else: | ||
432 | # FIXME: catch some errors | ||
433 | resp = session.get(url) | ||
434 | resp.raise_for_status() | ||
435 | return resp.url, resp.text | ||
436 | try: | ||
437 | with open(url, 'rb') as f: | ||
438 | content = auto_decode(f.read()) | ||
439 | except IOError as exc: | ||
440 | raise InstallationError( | ||
441 | 'Could not open requirements file: %s' % str(exc) | ||
442 | ) | ||
443 | return url, content | ||
444 | |||
445 | |||
446 | _scheme_re = re.compile(r'^(http|https|file):', re.I) | ||
447 | _url_slash_drive_re = re.compile(r'/*([a-z])\|', re.I) | ||
448 | |||
449 | |||
450 | def is_url(name): | ||
451 | """Returns true if the name looks like a URL""" | ||
452 | if ':' not in name: | ||
453 | return False | ||
454 | scheme = name.split(':', 1)[0].lower() | ||
455 | return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes | ||
456 | |||
457 | |||
458 | def url_to_path(url): | ||
459 | """ | ||
460 | Convert a file: URL to a path. | ||
461 | """ | ||
462 | assert url.startswith('file:'), ( | ||
463 | "You can only turn file: urls into filenames (not %r)" % url) | ||
464 | |||
465 | _, netloc, path, _, _ = urllib_parse.urlsplit(url) | ||
466 | |||
467 | # if we have a UNC path, prepend UNC share notation | ||
468 | if netloc: | ||
469 | netloc = '\\\\' + netloc | ||
470 | |||
471 | path = urllib_request.url2pathname(netloc + path) | ||
472 | return path | ||
473 | |||
474 | |||
475 | def path_to_url(path): | ||
476 | """ | ||
477 | Convert a path to a file: URL. The path will be made absolute and have | ||
478 | quoted path parts. | ||
479 | """ | ||
480 | path = os.path.normpath(os.path.abspath(path)) | ||
481 | url = urllib_parse.urljoin('file:', urllib_request.pathname2url(path)) | ||
482 | return url | ||
483 | |||
484 | |||
485 | def is_archive_file(name): | ||
486 | """Return True if `name` is a considered as an archive file.""" | ||
487 | ext = splitext(name)[1].lower() | ||
488 | if ext in ARCHIVE_EXTENSIONS: | ||
489 | return True | ||
490 | return False | ||
491 | |||
492 | |||
493 | def unpack_vcs_link(link, location): | ||
494 | vcs_backend = _get_used_vcs_backend(link) | ||
495 | vcs_backend.unpack(location) | ||
496 | |||
497 | |||
498 | def _get_used_vcs_backend(link): | ||
499 | for backend in vcs.backends: | ||
500 | if link.scheme in backend.schemes: | ||
501 | vcs_backend = backend(link.url) | ||
502 | return vcs_backend | ||
503 | |||
504 | |||
505 | def is_vcs_url(link): | ||
506 | return bool(_get_used_vcs_backend(link)) | ||
507 | |||
508 | |||
509 | def is_file_url(link): | ||
510 | return link.url.lower().startswith('file:') | ||
511 | |||
512 | |||
513 | def is_dir_url(link): | ||
514 | """Return whether a file:// Link points to a directory. | ||
515 | |||
516 | ``link`` must not have any other scheme but file://. Call is_file_url() | ||
517 | first. | ||
518 | |||
519 | """ | ||
520 | link_path = url_to_path(link.url_without_fragment) | ||
521 | return os.path.isdir(link_path) | ||
522 | |||
523 | |||
524 | def _progress_indicator(iterable, *args, **kwargs): | ||
525 | return iterable | ||
526 | |||
527 | |||
528 | def _download_url(resp, link, content_file, hashes, progress_bar): | ||
529 | try: | ||
530 | total_length = int(resp.headers['content-length']) | ||
531 | except (ValueError, KeyError, TypeError): | ||
532 | total_length = 0 | ||
533 | |||
534 | cached_resp = getattr(resp, "from_cache", False) | ||
535 | if logger.getEffectiveLevel() > logging.INFO: | ||
536 | show_progress = False | ||
537 | elif cached_resp: | ||
538 | show_progress = False | ||
539 | elif total_length > (40 * 1000): | ||
540 | show_progress = True | ||
541 | elif not total_length: | ||
542 | show_progress = True | ||
543 | else: | ||
544 | show_progress = False | ||
545 | |||
546 | show_url = link.show_url | ||
547 | |||
548 | def resp_read(chunk_size): | ||
549 | try: | ||
550 | # Special case for urllib3. | ||
551 | for chunk in resp.raw.stream( | ||
552 | chunk_size, | ||
553 | # We use decode_content=False here because we don't | ||
554 | # want urllib3 to mess with the raw bytes we get | ||
555 | # from the server. If we decompress inside of | ||
556 | # urllib3 then we cannot verify the checksum | ||
557 | # because the checksum will be of the compressed | ||
558 | # file. This breakage will only occur if the | ||
559 | # server adds a Content-Encoding header, which | ||
560 | # depends on how the server was configured: | ||
561 | # - Some servers will notice that the file isn't a | ||
562 | # compressible file and will leave the file alone | ||
563 | # and with an empty Content-Encoding | ||
564 | # - Some servers will notice that the file is | ||
565 | # already compressed and will leave the file | ||
566 | # alone and will add a Content-Encoding: gzip | ||
567 | # header | ||
568 | # - Some servers won't notice anything at all and | ||
569 | # will take a file that's already been compressed | ||
570 | # and compress it again and set the | ||
571 | # Content-Encoding: gzip header | ||
572 | # | ||
573 | # By setting this not to decode automatically we | ||
574 | # hope to eliminate problems with the second case. | ||
575 | decode_content=False): | ||
576 | yield chunk | ||
577 | except AttributeError: | ||
578 | # Standard file-like object. | ||
579 | while True: | ||
580 | chunk = resp.raw.read(chunk_size) | ||
581 | if not chunk: | ||
582 | break | ||
583 | yield chunk | ||
584 | |||
585 | def written_chunks(chunks): | ||
586 | for chunk in chunks: | ||
587 | content_file.write(chunk) | ||
588 | yield chunk | ||
589 | |||
590 | progress_indicator = _progress_indicator | ||
591 | |||
592 | if link.netloc == PyPI.netloc: | ||
593 | url = show_url | ||
594 | else: | ||
595 | url = link.url_without_fragment | ||
596 | |||
597 | if show_progress: # We don't show progress on cached responses | ||
598 | progress_indicator = DownloadProgressProvider(progress_bar, | ||
599 | max=total_length) | ||
600 | if total_length: | ||
601 | logger.info("Downloading %s (%s)", url, format_size(total_length)) | ||
602 | else: | ||
603 | logger.info("Downloading %s", url) | ||
604 | elif cached_resp: | ||
605 | logger.info("Using cached %s", url) | ||
606 | else: | ||
607 | logger.info("Downloading %s", url) | ||
608 | |||
609 | logger.debug('Downloading from URL %s', link) | ||
610 | |||
611 | downloaded_chunks = written_chunks( | ||
612 | progress_indicator( | ||
613 | resp_read(CONTENT_CHUNK_SIZE), | ||
614 | CONTENT_CHUNK_SIZE | ||
615 | ) | ||
616 | ) | ||
617 | if hashes: | ||
618 | hashes.check_against_chunks(downloaded_chunks) | ||
619 | else: | ||
620 | consume(downloaded_chunks) | ||
621 | |||
622 | |||
623 | def _copy_file(filename, location, link): | ||
624 | copy = True | ||
625 | download_location = os.path.join(location, link.filename) | ||
626 | if os.path.exists(download_location): | ||
627 | response = ask_path_exists( | ||
628 | 'The file %s exists. (i)gnore, (w)ipe, (b)ackup, (a)abort' % | ||
629 | display_path(download_location), ('i', 'w', 'b', 'a')) | ||
630 | if response == 'i': | ||
631 | copy = False | ||
632 | elif response == 'w': | ||
633 | logger.warning('Deleting %s', display_path(download_location)) | ||
634 | os.remove(download_location) | ||
635 | elif response == 'b': | ||
636 | dest_file = backup_dir(download_location) | ||
637 | logger.warning( | ||
638 | 'Backing up %s to %s', | ||
639 | display_path(download_location), | ||
640 | display_path(dest_file), | ||
641 | ) | ||
642 | shutil.move(download_location, dest_file) | ||
643 | elif response == 'a': | ||
644 | sys.exit(-1) | ||
645 | if copy: | ||
646 | shutil.copy(filename, download_location) | ||
647 | logger.info('Saved %s', display_path(download_location)) | ||
648 | |||
649 | |||
650 | def unpack_http_url(link, location, download_dir=None, | ||
651 | session=None, hashes=None, progress_bar="on"): | ||
652 | if session is None: | ||
653 | raise TypeError( | ||
654 | "unpack_http_url() missing 1 required keyword argument: 'session'" | ||
655 | ) | ||
656 | |||
657 | with TempDirectory(kind="unpack") as temp_dir: | ||
658 | # If a download dir is specified, is the file already downloaded there? | ||
659 | already_downloaded_path = None | ||
660 | if download_dir: | ||
661 | already_downloaded_path = _check_download_dir(link, | ||
662 | download_dir, | ||
663 | hashes) | ||
664 | |||
665 | if already_downloaded_path: | ||
666 | from_path = already_downloaded_path | ||
667 | content_type = mimetypes.guess_type(from_path)[0] | ||
668 | else: | ||
669 | # let's download to a tmp dir | ||
670 | from_path, content_type = _download_http_url(link, | ||
671 | session, | ||
672 | temp_dir.path, | ||
673 | hashes, | ||
674 | progress_bar) | ||
675 | |||
676 | # unpack the archive to the build dir location. even when only | ||
677 | # downloading archives, they have to be unpacked to parse dependencies | ||
678 | unpack_file(from_path, location, content_type, link) | ||
679 | |||
680 | # a download dir is specified; let's copy the archive there | ||
681 | if download_dir and not already_downloaded_path: | ||
682 | _copy_file(from_path, download_dir, link) | ||
683 | |||
684 | if not already_downloaded_path: | ||
685 | os.unlink(from_path) | ||
686 | |||
687 | |||
688 | def unpack_file_url(link, location, download_dir=None, hashes=None): | ||
689 | """Unpack link into location. | ||
690 | |||
691 | If download_dir is provided and link points to a file, make a copy | ||
692 | of the link file inside download_dir. | ||
693 | """ | ||
694 | link_path = url_to_path(link.url_without_fragment) | ||
695 | |||
696 | # If it's a url to a local directory | ||
697 | if is_dir_url(link): | ||
698 | if os.path.isdir(location): | ||
699 | rmtree(location) | ||
700 | shutil.copytree(link_path, location, symlinks=True) | ||
701 | if download_dir: | ||
702 | logger.info('Link is a directory, ignoring download_dir') | ||
703 | return | ||
704 | |||
705 | # If --require-hashes is off, `hashes` is either empty, the | ||
706 | # link's embedded hash, or MissingHashes; it is required to | ||
707 | # match. If --require-hashes is on, we are satisfied by any | ||
708 | # hash in `hashes` matching: a URL-based or an option-based | ||
709 | # one; no internet-sourced hash will be in `hashes`. | ||
710 | if hashes: | ||
711 | hashes.check_against_path(link_path) | ||
712 | |||
713 | # If a download dir is specified, is the file already there and valid? | ||
714 | already_downloaded_path = None | ||
715 | if download_dir: | ||
716 | already_downloaded_path = _check_download_dir(link, | ||
717 | download_dir, | ||
718 | hashes) | ||
719 | |||
720 | if already_downloaded_path: | ||
721 | from_path = already_downloaded_path | ||
722 | else: | ||
723 | from_path = link_path | ||
724 | |||
725 | content_type = mimetypes.guess_type(from_path)[0] | ||
726 | |||
727 | # unpack the archive to the build dir location. even when only downloading | ||
728 | # archives, they have to be unpacked to parse dependencies | ||
729 | unpack_file(from_path, location, content_type, link) | ||
730 | |||
731 | # a download dir is specified and not already downloaded | ||
732 | if download_dir and not already_downloaded_path: | ||
733 | _copy_file(from_path, download_dir, link) | ||
734 | |||
735 | |||
736 | def _copy_dist_from_dir(link_path, location): | ||
737 | """Copy distribution files in `link_path` to `location`. | ||
738 | |||
739 | Invoked when user requests to install a local directory. E.g.: | ||
740 | |||
741 | pip install . | ||
742 | pip install ~/dev/git-repos/python-prompt-toolkit | ||
743 | |||
744 | """ | ||
745 | |||
746 | # Note: This is currently VERY SLOW if you have a lot of data in the | ||
747 | # directory, because it copies everything with `shutil.copytree`. | ||
748 | # What it should really do is build an sdist and install that. | ||
749 | # See https://github.com/pypa/pip/issues/2195 | ||
750 | |||
751 | if os.path.isdir(location): | ||
752 | rmtree(location) | ||
753 | |||
754 | # build an sdist | ||
755 | setup_py = 'setup.py' | ||
756 | sdist_args = [sys.executable] | ||
757 | sdist_args.append('-c') | ||
758 | sdist_args.append(SETUPTOOLS_SHIM % setup_py) | ||
759 | sdist_args.append('sdist') | ||
760 | sdist_args += ['--dist-dir', location] | ||
761 | logger.info('Running setup.py sdist for %s', link_path) | ||
762 | |||
763 | with indent_log(): | ||
764 | call_subprocess(sdist_args, cwd=link_path, show_stdout=False) | ||
765 | |||
766 | # unpack sdist into `location` | ||
767 | sdist = os.path.join(location, os.listdir(location)[0]) | ||
768 | logger.info('Unpacking sdist %s into %s', sdist, location) | ||
769 | unpack_file(sdist, location, content_type=None, link=None) | ||
770 | |||
771 | |||
772 | class PipXmlrpcTransport(xmlrpc_client.Transport): | ||
773 | """Provide a `xmlrpclib.Transport` implementation via a `PipSession` | ||
774 | object. | ||
775 | """ | ||
776 | |||
777 | def __init__(self, index_url, session, use_datetime=False): | ||
778 | xmlrpc_client.Transport.__init__(self, use_datetime) | ||
779 | index_parts = urllib_parse.urlparse(index_url) | ||
780 | self._scheme = index_parts.scheme | ||
781 | self._session = session | ||
782 | |||
783 | def request(self, host, handler, request_body, verbose=False): | ||
784 | parts = (self._scheme, host, handler, None, None, None) | ||
785 | url = urllib_parse.urlunparse(parts) | ||
786 | try: | ||
787 | headers = {'Content-Type': 'text/xml'} | ||
788 | response = self._session.post(url, data=request_body, | ||
789 | headers=headers, stream=True) | ||
790 | response.raise_for_status() | ||
791 | self.verbose = verbose | ||
792 | return self.parse_response(response.raw) | ||
793 | except requests.HTTPError as exc: | ||
794 | logger.critical( | ||
795 | "HTTP error %s while getting %s", | ||
796 | exc.response.status_code, url, | ||
797 | ) | ||
798 | raise | ||
799 | |||
800 | |||
801 | def unpack_url(link, location, download_dir=None, | ||
802 | only_download=False, session=None, hashes=None, | ||
803 | progress_bar="on"): | ||
804 | """Unpack link. | ||
805 | If link is a VCS link: | ||
806 | if only_download, export into download_dir and ignore location | ||
807 | else unpack into location | ||
808 | for other types of link: | ||
809 | - unpack into location | ||
810 | - if download_dir, copy the file into download_dir | ||
811 | - if only_download, mark location for deletion | ||
812 | |||
813 | :param hashes: A Hashes object, one of whose embedded hashes must match, | ||
814 | or HashMismatch will be raised. If the Hashes is empty, no matches are | ||
815 | required, and unhashable types of requirements (like VCS ones, which | ||
816 | would ordinarily raise HashUnsupported) are allowed. | ||
817 | """ | ||
818 | # non-editable vcs urls | ||
819 | if is_vcs_url(link): | ||
820 | unpack_vcs_link(link, location) | ||
821 | |||
822 | # file urls | ||
823 | elif is_file_url(link): | ||
824 | unpack_file_url(link, location, download_dir, hashes=hashes) | ||
825 | |||
826 | # http urls | ||
827 | else: | ||
828 | if session is None: | ||
829 | session = PipSession() | ||
830 | |||
831 | unpack_http_url( | ||
832 | link, | ||
833 | location, | ||
834 | download_dir, | ||
835 | session, | ||
836 | hashes=hashes, | ||
837 | progress_bar=progress_bar | ||
838 | ) | ||
839 | if only_download: | ||
840 | write_delete_marker_file(location) | ||
841 | |||
842 | |||
843 | def _download_http_url(link, session, temp_dir, hashes, progress_bar): | ||
844 | """Download link url into temp_dir using provided session""" | ||
845 | target_url = link.url.split('#', 1)[0] | ||
846 | try: | ||
847 | resp = session.get( | ||
848 | target_url, | ||
849 | # We use Accept-Encoding: identity here because requests | ||
850 | # defaults to accepting compressed responses. This breaks in | ||
851 | # a variety of ways depending on how the server is configured. | ||
852 | # - Some servers will notice that the file isn't a compressible | ||
853 | # file and will leave the file alone and with an empty | ||
854 | # Content-Encoding | ||
855 | # - Some servers will notice that the file is already | ||
856 | # compressed and will leave the file alone and will add a | ||
857 | # Content-Encoding: gzip header | ||
858 | # - Some servers won't notice anything at all and will take | ||
859 | # a file that's already been compressed and compress it again | ||
860 | # and set the Content-Encoding: gzip header | ||
861 | # By setting this to request only the identity encoding We're | ||
862 | # hoping to eliminate the third case. Hopefully there does not | ||
863 | # exist a server which when given a file will notice it is | ||
864 | # already compressed and that you're not asking for a | ||
865 | # compressed file and will then decompress it before sending | ||
866 | # because if that's the case I don't think it'll ever be | ||
867 | # possible to make this work. | ||
868 | headers={"Accept-Encoding": "identity"}, | ||
869 | stream=True, | ||
870 | ) | ||
871 | resp.raise_for_status() | ||
872 | except requests.HTTPError as exc: | ||
873 | logger.critical( | ||
874 | "HTTP error %s while getting %s", exc.response.status_code, link, | ||
875 | ) | ||
876 | raise | ||
877 | |||
878 | content_type = resp.headers.get('content-type', '') | ||
879 | filename = link.filename # fallback | ||
880 | # Have a look at the Content-Disposition header for a better guess | ||
881 | content_disposition = resp.headers.get('content-disposition') | ||
882 | if content_disposition: | ||
883 | type, params = cgi.parse_header(content_disposition) | ||
884 | # We use ``or`` here because we don't want to use an "empty" value | ||
885 | # from the filename param. | ||
886 | filename = params.get('filename') or filename | ||
887 | ext = splitext(filename)[1] | ||
888 | if not ext: | ||
889 | ext = mimetypes.guess_extension(content_type) | ||
890 | if ext: | ||
891 | filename += ext | ||
892 | if not ext and link.url != resp.url: | ||
893 | ext = os.path.splitext(resp.url)[1] | ||
894 | if ext: | ||
895 | filename += ext | ||
896 | file_path = os.path.join(temp_dir, filename) | ||
897 | with open(file_path, 'wb') as content_file: | ||
898 | _download_url(resp, link, content_file, hashes, progress_bar) | ||
899 | return file_path, content_type | ||
900 | |||
901 | |||
902 | def _check_download_dir(link, download_dir, hashes): | ||
903 | """ Check download_dir for previously downloaded file with correct hash | ||
904 | If a correct file is found return its path else None | ||
905 | """ | ||
906 | download_path = os.path.join(download_dir, link.filename) | ||
907 | if os.path.exists(download_path): | ||
908 | # If already downloaded, does its hash match? | ||
909 | logger.info('File was already downloaded %s', download_path) | ||
910 | if hashes: | ||
911 | try: | ||
912 | hashes.check_against_path(download_path) | ||
913 | except HashMismatch: | ||
914 | logger.warning( | ||
915 | 'Previously-downloaded file %s has bad hash. ' | ||
916 | 'Re-downloading.', | ||
917 | download_path | ||
918 | ) | ||
919 | os.unlink(download_path) | ||
920 | return None | ||
921 | return download_path | ||
922 | return None | ||