diff options
Diffstat (limited to 'venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/download.py')
| -rw-r--r-- | venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/download.py | 922 |
1 files changed, 922 insertions, 0 deletions
diff --git a/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/download.py b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/download.py new file mode 100644 index 0000000..e0e2d24 --- /dev/null +++ b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/download.py | |||
| @@ -0,0 +1,922 @@ | |||
| 1 | from __future__ import absolute_import | ||
| 2 | |||
| 3 | import cgi | ||
| 4 | import email.utils | ||
| 5 | import getpass | ||
| 6 | import json | ||
| 7 | import logging | ||
| 8 | import mimetypes | ||
| 9 | import os | ||
| 10 | import platform | ||
| 11 | import re | ||
| 12 | import shutil | ||
| 13 | import sys | ||
| 14 | |||
| 15 | from pip._vendor import requests, six, urllib3 | ||
| 16 | from pip._vendor.cachecontrol import CacheControlAdapter | ||
| 17 | from pip._vendor.cachecontrol.caches import FileCache | ||
| 18 | from pip._vendor.lockfile import LockError | ||
| 19 | from pip._vendor.requests.adapters import BaseAdapter, HTTPAdapter | ||
| 20 | from pip._vendor.requests.auth import AuthBase, HTTPBasicAuth | ||
| 21 | from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response | ||
| 22 | from pip._vendor.requests.structures import CaseInsensitiveDict | ||
| 23 | from pip._vendor.requests.utils import get_netrc_auth | ||
| 24 | # NOTE: XMLRPC Client is not annotated in typeshed as on 2017-07-17, which is | ||
| 25 | # why we ignore the type on this import | ||
| 26 | from pip._vendor.six.moves import xmlrpc_client # type: ignore | ||
| 27 | from pip._vendor.six.moves.urllib import parse as urllib_parse | ||
| 28 | from pip._vendor.six.moves.urllib import request as urllib_request | ||
| 29 | from pip._vendor.six.moves.urllib.parse import unquote as urllib_unquote | ||
| 30 | from pip._vendor.urllib3.util import IS_PYOPENSSL | ||
| 31 | |||
| 32 | import pip | ||
| 33 | from pip._internal.compat import WINDOWS | ||
| 34 | from pip._internal.exceptions import HashMismatch, InstallationError | ||
| 35 | from pip._internal.locations import write_delete_marker_file | ||
| 36 | from pip._internal.models import PyPI | ||
| 37 | from pip._internal.utils.encoding import auto_decode | ||
| 38 | from pip._internal.utils.filesystem import check_path_owner | ||
| 39 | from pip._internal.utils.glibc import libc_ver | ||
| 40 | from pip._internal.utils.logging import indent_log | ||
| 41 | from pip._internal.utils.misc import ( | ||
| 42 | ARCHIVE_EXTENSIONS, ask_path_exists, backup_dir, call_subprocess, consume, | ||
| 43 | display_path, format_size, get_installed_version, rmtree, splitext, | ||
| 44 | unpack_file, | ||
| 45 | ) | ||
| 46 | from pip._internal.utils.setuptools_build import SETUPTOOLS_SHIM | ||
| 47 | from pip._internal.utils.temp_dir import TempDirectory | ||
| 48 | from pip._internal.utils.ui import DownloadProgressProvider | ||
| 49 | from pip._internal.vcs import vcs | ||
| 50 | |||
| 51 | try: | ||
| 52 | import ssl # noqa | ||
| 53 | except ImportError: | ||
| 54 | ssl = None | ||
| 55 | |||
| 56 | HAS_TLS = (ssl is not None) or IS_PYOPENSSL | ||
| 57 | |||
| 58 | __all__ = ['get_file_content', | ||
| 59 | 'is_url', 'url_to_path', 'path_to_url', | ||
| 60 | 'is_archive_file', 'unpack_vcs_link', | ||
| 61 | 'unpack_file_url', 'is_vcs_url', 'is_file_url', | ||
| 62 | 'unpack_http_url', 'unpack_url'] | ||
| 63 | |||
| 64 | |||
| 65 | logger = logging.getLogger(__name__) | ||
| 66 | |||
| 67 | |||
| 68 | def user_agent(): | ||
| 69 | """ | ||
| 70 | Return a string representing the user agent. | ||
| 71 | """ | ||
| 72 | data = { | ||
| 73 | "installer": {"name": "pip", "version": pip.__version__}, | ||
| 74 | "python": platform.python_version(), | ||
| 75 | "implementation": { | ||
| 76 | "name": platform.python_implementation(), | ||
| 77 | }, | ||
| 78 | } | ||
| 79 | |||
| 80 | if data["implementation"]["name"] == 'CPython': | ||
| 81 | data["implementation"]["version"] = platform.python_version() | ||
| 82 | elif data["implementation"]["name"] == 'PyPy': | ||
| 83 | if sys.pypy_version_info.releaselevel == 'final': | ||
| 84 | pypy_version_info = sys.pypy_version_info[:3] | ||
| 85 | else: | ||
| 86 | pypy_version_info = sys.pypy_version_info | ||
| 87 | data["implementation"]["version"] = ".".join( | ||
| 88 | [str(x) for x in pypy_version_info] | ||
| 89 | ) | ||
| 90 | elif data["implementation"]["name"] == 'Jython': | ||
| 91 | # Complete Guess | ||
| 92 | data["implementation"]["version"] = platform.python_version() | ||
| 93 | elif data["implementation"]["name"] == 'IronPython': | ||
| 94 | # Complete Guess | ||
| 95 | data["implementation"]["version"] = platform.python_version() | ||
| 96 | |||
| 97 | if sys.platform.startswith("linux"): | ||
| 98 | from pip._vendor import distro | ||
| 99 | distro_infos = dict(filter( | ||
| 100 | lambda x: x[1], | ||
| 101 | zip(["name", "version", "id"], distro.linux_distribution()), | ||
| 102 | )) | ||
| 103 | libc = dict(filter( | ||
| 104 | lambda x: x[1], | ||
| 105 | zip(["lib", "version"], libc_ver()), | ||
| 106 | )) | ||
| 107 | if libc: | ||
| 108 | distro_infos["libc"] = libc | ||
| 109 | if distro_infos: | ||
| 110 | data["distro"] = distro_infos | ||
| 111 | |||
| 112 | if sys.platform.startswith("darwin") and platform.mac_ver()[0]: | ||
| 113 | data["distro"] = {"name": "macOS", "version": platform.mac_ver()[0]} | ||
| 114 | |||
| 115 | if platform.system(): | ||
| 116 | data.setdefault("system", {})["name"] = platform.system() | ||
| 117 | |||
| 118 | if platform.release(): | ||
| 119 | data.setdefault("system", {})["release"] = platform.release() | ||
| 120 | |||
| 121 | if platform.machine(): | ||
| 122 | data["cpu"] = platform.machine() | ||
| 123 | |||
| 124 | if HAS_TLS: | ||
| 125 | data["openssl_version"] = ssl.OPENSSL_VERSION | ||
| 126 | |||
| 127 | setuptools_version = get_installed_version("setuptools") | ||
| 128 | if setuptools_version is not None: | ||
| 129 | data["setuptools_version"] = setuptools_version | ||
| 130 | |||
| 131 | return "{data[installer][name]}/{data[installer][version]} {json}".format( | ||
| 132 | data=data, | ||
| 133 | json=json.dumps(data, separators=(",", ":"), sort_keys=True), | ||
| 134 | ) | ||
| 135 | |||
| 136 | |||
| 137 | class MultiDomainBasicAuth(AuthBase): | ||
| 138 | |||
| 139 | def __init__(self, prompting=True): | ||
| 140 | self.prompting = prompting | ||
| 141 | self.passwords = {} | ||
| 142 | |||
| 143 | def __call__(self, req): | ||
| 144 | parsed = urllib_parse.urlparse(req.url) | ||
| 145 | |||
| 146 | # Get the netloc without any embedded credentials | ||
| 147 | netloc = parsed.netloc.rsplit("@", 1)[-1] | ||
| 148 | |||
| 149 | # Set the url of the request to the url without any credentials | ||
| 150 | req.url = urllib_parse.urlunparse(parsed[:1] + (netloc,) + parsed[2:]) | ||
| 151 | |||
| 152 | # Use any stored credentials that we have for this netloc | ||
| 153 | username, password = self.passwords.get(netloc, (None, None)) | ||
| 154 | |||
| 155 | # Extract credentials embedded in the url if we have none stored | ||
| 156 | if username is None: | ||
| 157 | username, password = self.parse_credentials(parsed.netloc) | ||
| 158 | |||
| 159 | # Get creds from netrc if we still don't have them | ||
| 160 | if username is None and password is None: | ||
| 161 | netrc_auth = get_netrc_auth(req.url) | ||
| 162 | username, password = netrc_auth if netrc_auth else (None, None) | ||
| 163 | |||
| 164 | if username or password: | ||
| 165 | # Store the username and password | ||
| 166 | self.passwords[netloc] = (username, password) | ||
| 167 | |||
| 168 | # Send the basic auth with this request | ||
| 169 | req = HTTPBasicAuth(username or "", password or "")(req) | ||
| 170 | |||
| 171 | # Attach a hook to handle 401 responses | ||
| 172 | req.register_hook("response", self.handle_401) | ||
| 173 | |||
| 174 | return req | ||
| 175 | |||
| 176 | def handle_401(self, resp, **kwargs): | ||
| 177 | # We only care about 401 responses, anything else we want to just | ||
| 178 | # pass through the actual response | ||
| 179 | if resp.status_code != 401: | ||
| 180 | return resp | ||
| 181 | |||
| 182 | # We are not able to prompt the user so simply return the response | ||
| 183 | if not self.prompting: | ||
| 184 | return resp | ||
| 185 | |||
| 186 | parsed = urllib_parse.urlparse(resp.url) | ||
| 187 | |||
| 188 | # Prompt the user for a new username and password | ||
| 189 | username = six.moves.input("User for %s: " % parsed.netloc) | ||
| 190 | password = getpass.getpass("Password: ") | ||
| 191 | |||
| 192 | # Store the new username and password to use for future requests | ||
| 193 | if username or password: | ||
| 194 | self.passwords[parsed.netloc] = (username, password) | ||
| 195 | |||
| 196 | # Consume content and release the original connection to allow our new | ||
| 197 | # request to reuse the same one. | ||
| 198 | resp.content | ||
| 199 | resp.raw.release_conn() | ||
| 200 | |||
| 201 | # Add our new username and password to the request | ||
| 202 | req = HTTPBasicAuth(username or "", password or "")(resp.request) | ||
| 203 | |||
| 204 | # Send our new request | ||
| 205 | new_resp = resp.connection.send(req, **kwargs) | ||
| 206 | new_resp.history.append(resp) | ||
| 207 | |||
| 208 | return new_resp | ||
| 209 | |||
| 210 | def parse_credentials(self, netloc): | ||
| 211 | if "@" in netloc: | ||
| 212 | userinfo = netloc.rsplit("@", 1)[0] | ||
| 213 | if ":" in userinfo: | ||
| 214 | user, pwd = userinfo.split(":", 1) | ||
| 215 | return (urllib_unquote(user), urllib_unquote(pwd)) | ||
| 216 | return urllib_unquote(userinfo), None | ||
| 217 | return None, None | ||
| 218 | |||
| 219 | |||
| 220 | class LocalFSAdapter(BaseAdapter): | ||
| 221 | |||
| 222 | def send(self, request, stream=None, timeout=None, verify=None, cert=None, | ||
| 223 | proxies=None): | ||
| 224 | pathname = url_to_path(request.url) | ||
| 225 | |||
| 226 | resp = Response() | ||
| 227 | resp.status_code = 200 | ||
| 228 | resp.url = request.url | ||
| 229 | |||
| 230 | try: | ||
| 231 | stats = os.stat(pathname) | ||
| 232 | except OSError as exc: | ||
| 233 | resp.status_code = 404 | ||
| 234 | resp.raw = exc | ||
| 235 | else: | ||
| 236 | modified = email.utils.formatdate(stats.st_mtime, usegmt=True) | ||
| 237 | content_type = mimetypes.guess_type(pathname)[0] or "text/plain" | ||
| 238 | resp.headers = CaseInsensitiveDict({ | ||
| 239 | "Content-Type": content_type, | ||
| 240 | "Content-Length": stats.st_size, | ||
| 241 | "Last-Modified": modified, | ||
| 242 | }) | ||
| 243 | |||
| 244 | resp.raw = open(pathname, "rb") | ||
| 245 | resp.close = resp.raw.close | ||
| 246 | |||
| 247 | return resp | ||
| 248 | |||
| 249 | def close(self): | ||
| 250 | pass | ||
| 251 | |||
| 252 | |||
| 253 | class SafeFileCache(FileCache): | ||
| 254 | """ | ||
| 255 | A file based cache which is safe to use even when the target directory may | ||
| 256 | not be accessible or writable. | ||
| 257 | """ | ||
| 258 | |||
| 259 | def __init__(self, *args, **kwargs): | ||
| 260 | super(SafeFileCache, self).__init__(*args, **kwargs) | ||
| 261 | |||
| 262 | # Check to ensure that the directory containing our cache directory | ||
| 263 | # is owned by the user current executing pip. If it does not exist | ||
| 264 | # we will check the parent directory until we find one that does exist. | ||
| 265 | # If it is not owned by the user executing pip then we will disable | ||
| 266 | # the cache and log a warning. | ||
| 267 | if not check_path_owner(self.directory): | ||
| 268 | logger.warning( | ||
| 269 | "The directory '%s' or its parent directory is not owned by " | ||
| 270 | "the current user and the cache has been disabled. Please " | ||
| 271 | "check the permissions and owner of that directory. If " | ||
| 272 | "executing pip with sudo, you may want sudo's -H flag.", | ||
| 273 | self.directory, | ||
| 274 | ) | ||
| 275 | |||
| 276 | # Set our directory to None to disable the Cache | ||
| 277 | self.directory = None | ||
| 278 | |||
| 279 | def get(self, *args, **kwargs): | ||
| 280 | # If we don't have a directory, then the cache should be a no-op. | ||
| 281 | if self.directory is None: | ||
| 282 | return | ||
| 283 | |||
| 284 | try: | ||
| 285 | return super(SafeFileCache, self).get(*args, **kwargs) | ||
| 286 | except (LockError, OSError, IOError): | ||
| 287 | # We intentionally silence this error, if we can't access the cache | ||
| 288 | # then we can just skip caching and process the request as if | ||
| 289 | # caching wasn't enabled. | ||
| 290 | pass | ||
| 291 | |||
| 292 | def set(self, *args, **kwargs): | ||
| 293 | # If we don't have a directory, then the cache should be a no-op. | ||
| 294 | if self.directory is None: | ||
| 295 | return | ||
| 296 | |||
| 297 | try: | ||
| 298 | return super(SafeFileCache, self).set(*args, **kwargs) | ||
| 299 | except (LockError, OSError, IOError): | ||
| 300 | # We intentionally silence this error, if we can't access the cache | ||
| 301 | # then we can just skip caching and process the request as if | ||
| 302 | # caching wasn't enabled. | ||
| 303 | pass | ||
| 304 | |||
| 305 | def delete(self, *args, **kwargs): | ||
| 306 | # If we don't have a directory, then the cache should be a no-op. | ||
| 307 | if self.directory is None: | ||
| 308 | return | ||
| 309 | |||
| 310 | try: | ||
| 311 | return super(SafeFileCache, self).delete(*args, **kwargs) | ||
| 312 | except (LockError, OSError, IOError): | ||
| 313 | # We intentionally silence this error, if we can't access the cache | ||
| 314 | # then we can just skip caching and process the request as if | ||
| 315 | # caching wasn't enabled. | ||
| 316 | pass | ||
| 317 | |||
| 318 | |||
| 319 | class InsecureHTTPAdapter(HTTPAdapter): | ||
| 320 | |||
| 321 | def cert_verify(self, conn, url, verify, cert): | ||
| 322 | conn.cert_reqs = 'CERT_NONE' | ||
| 323 | conn.ca_certs = None | ||
| 324 | |||
| 325 | |||
| 326 | class PipSession(requests.Session): | ||
| 327 | |||
| 328 | timeout = None | ||
| 329 | |||
| 330 | def __init__(self, *args, **kwargs): | ||
| 331 | retries = kwargs.pop("retries", 0) | ||
| 332 | cache = kwargs.pop("cache", None) | ||
| 333 | insecure_hosts = kwargs.pop("insecure_hosts", []) | ||
| 334 | |||
| 335 | super(PipSession, self).__init__(*args, **kwargs) | ||
| 336 | |||
| 337 | # Attach our User Agent to the request | ||
| 338 | self.headers["User-Agent"] = user_agent() | ||
| 339 | |||
| 340 | # Attach our Authentication handler to the session | ||
| 341 | self.auth = MultiDomainBasicAuth() | ||
| 342 | |||
| 343 | # Create our urllib3.Retry instance which will allow us to customize | ||
| 344 | # how we handle retries. | ||
| 345 | retries = urllib3.Retry( | ||
| 346 | # Set the total number of retries that a particular request can | ||
| 347 | # have. | ||
| 348 | total=retries, | ||
| 349 | |||
| 350 | # A 503 error from PyPI typically means that the Fastly -> Origin | ||
| 351 | # connection got interrupted in some way. A 503 error in general | ||
| 352 | # is typically considered a transient error so we'll go ahead and | ||
| 353 | # retry it. | ||
| 354 | # A 500 may indicate transient error in Amazon S3 | ||
| 355 | # A 520 or 527 - may indicate transient error in CloudFlare | ||
| 356 | status_forcelist=[500, 503, 520, 527], | ||
| 357 | |||
| 358 | # Add a small amount of back off between failed requests in | ||
| 359 | # order to prevent hammering the service. | ||
| 360 | backoff_factor=0.25, | ||
| 361 | ) | ||
| 362 | |||
| 363 | # We want to _only_ cache responses on securely fetched origins. We do | ||
| 364 | # this because we can't validate the response of an insecurely fetched | ||
| 365 | # origin, and we don't want someone to be able to poison the cache and | ||
| 366 | # require manual eviction from the cache to fix it. | ||
| 367 | if cache: | ||
| 368 | secure_adapter = CacheControlAdapter( | ||
| 369 | cache=SafeFileCache(cache, use_dir_lock=True), | ||
| 370 | max_retries=retries, | ||
| 371 | ) | ||
| 372 | else: | ||
| 373 | secure_adapter = HTTPAdapter(max_retries=retries) | ||
| 374 | |||
| 375 | # Our Insecure HTTPAdapter disables HTTPS validation. It does not | ||
| 376 | # support caching (see above) so we'll use it for all http:// URLs as | ||
| 377 | # well as any https:// host that we've marked as ignoring TLS errors | ||
| 378 | # for. | ||
| 379 | insecure_adapter = InsecureHTTPAdapter(max_retries=retries) | ||
| 380 | |||
| 381 | self.mount("https://", secure_adapter) | ||
| 382 | self.mount("http://", insecure_adapter) | ||
| 383 | |||
| 384 | # Enable file:// urls | ||
| 385 | self.mount("file://", LocalFSAdapter()) | ||
| 386 | |||
| 387 | # We want to use a non-validating adapter for any requests which are | ||
| 388 | # deemed insecure. | ||
| 389 | for host in insecure_hosts: | ||
| 390 | self.mount("https://{}/".format(host), insecure_adapter) | ||
| 391 | |||
| 392 | def request(self, method, url, *args, **kwargs): | ||
| 393 | # Allow setting a default timeout on a session | ||
| 394 | kwargs.setdefault("timeout", self.timeout) | ||
| 395 | |||
| 396 | # Dispatch the actual request | ||
| 397 | return super(PipSession, self).request(method, url, *args, **kwargs) | ||
| 398 | |||
| 399 | |||
| 400 | def get_file_content(url, comes_from=None, session=None): | ||
| 401 | """Gets the content of a file; it may be a filename, file: URL, or | ||
| 402 | http: URL. Returns (location, content). Content is unicode. | ||
| 403 | |||
| 404 | :param url: File path or url. | ||
| 405 | :param comes_from: Origin description of requirements. | ||
| 406 | :param session: Instance of pip.download.PipSession. | ||
| 407 | """ | ||
| 408 | if session is None: | ||
| 409 | raise TypeError( | ||
| 410 | "get_file_content() missing 1 required keyword argument: 'session'" | ||
| 411 | ) | ||
| 412 | |||
| 413 | match = _scheme_re.search(url) | ||
| 414 | if match: | ||
| 415 | scheme = match.group(1).lower() | ||
| 416 | if (scheme == 'file' and comes_from and | ||
| 417 | comes_from.startswith('http')): | ||
| 418 | raise InstallationError( | ||
| 419 | 'Requirements file %s references URL %s, which is local' | ||
| 420 | % (comes_from, url)) | ||
| 421 | if scheme == 'file': | ||
| 422 | path = url.split(':', 1)[1] | ||
| 423 | path = path.replace('\\', '/') | ||
| 424 | match = _url_slash_drive_re.match(path) | ||
| 425 | if match: | ||
| 426 | path = match.group(1) + ':' + path.split('|', 1)[1] | ||
| 427 | path = urllib_parse.unquote(path) | ||
| 428 | if path.startswith('/'): | ||
| 429 | path = '/' + path.lstrip('/') | ||
| 430 | url = path | ||
| 431 | else: | ||
| 432 | # FIXME: catch some errors | ||
| 433 | resp = session.get(url) | ||
| 434 | resp.raise_for_status() | ||
| 435 | return resp.url, resp.text | ||
| 436 | try: | ||
| 437 | with open(url, 'rb') as f: | ||
| 438 | content = auto_decode(f.read()) | ||
| 439 | except IOError as exc: | ||
| 440 | raise InstallationError( | ||
| 441 | 'Could not open requirements file: %s' % str(exc) | ||
| 442 | ) | ||
| 443 | return url, content | ||
| 444 | |||
| 445 | |||
| 446 | _scheme_re = re.compile(r'^(http|https|file):', re.I) | ||
| 447 | _url_slash_drive_re = re.compile(r'/*([a-z])\|', re.I) | ||
| 448 | |||
| 449 | |||
| 450 | def is_url(name): | ||
| 451 | """Returns true if the name looks like a URL""" | ||
| 452 | if ':' not in name: | ||
| 453 | return False | ||
| 454 | scheme = name.split(':', 1)[0].lower() | ||
| 455 | return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes | ||
| 456 | |||
| 457 | |||
| 458 | def url_to_path(url): | ||
| 459 | """ | ||
| 460 | Convert a file: URL to a path. | ||
| 461 | """ | ||
| 462 | assert url.startswith('file:'), ( | ||
| 463 | "You can only turn file: urls into filenames (not %r)" % url) | ||
| 464 | |||
| 465 | _, netloc, path, _, _ = urllib_parse.urlsplit(url) | ||
| 466 | |||
| 467 | # if we have a UNC path, prepend UNC share notation | ||
| 468 | if netloc: | ||
| 469 | netloc = '\\\\' + netloc | ||
| 470 | |||
| 471 | path = urllib_request.url2pathname(netloc + path) | ||
| 472 | return path | ||
| 473 | |||
| 474 | |||
| 475 | def path_to_url(path): | ||
| 476 | """ | ||
| 477 | Convert a path to a file: URL. The path will be made absolute and have | ||
| 478 | quoted path parts. | ||
| 479 | """ | ||
| 480 | path = os.path.normpath(os.path.abspath(path)) | ||
| 481 | url = urllib_parse.urljoin('file:', urllib_request.pathname2url(path)) | ||
| 482 | return url | ||
| 483 | |||
| 484 | |||
| 485 | def is_archive_file(name): | ||
| 486 | """Return True if `name` is a considered as an archive file.""" | ||
| 487 | ext = splitext(name)[1].lower() | ||
| 488 | if ext in ARCHIVE_EXTENSIONS: | ||
| 489 | return True | ||
| 490 | return False | ||
| 491 | |||
| 492 | |||
| 493 | def unpack_vcs_link(link, location): | ||
| 494 | vcs_backend = _get_used_vcs_backend(link) | ||
| 495 | vcs_backend.unpack(location) | ||
| 496 | |||
| 497 | |||
| 498 | def _get_used_vcs_backend(link): | ||
| 499 | for backend in vcs.backends: | ||
| 500 | if link.scheme in backend.schemes: | ||
| 501 | vcs_backend = backend(link.url) | ||
| 502 | return vcs_backend | ||
| 503 | |||
| 504 | |||
| 505 | def is_vcs_url(link): | ||
| 506 | return bool(_get_used_vcs_backend(link)) | ||
| 507 | |||
| 508 | |||
| 509 | def is_file_url(link): | ||
| 510 | return link.url.lower().startswith('file:') | ||
| 511 | |||
| 512 | |||
| 513 | def is_dir_url(link): | ||
| 514 | """Return whether a file:// Link points to a directory. | ||
| 515 | |||
| 516 | ``link`` must not have any other scheme but file://. Call is_file_url() | ||
| 517 | first. | ||
| 518 | |||
| 519 | """ | ||
| 520 | link_path = url_to_path(link.url_without_fragment) | ||
| 521 | return os.path.isdir(link_path) | ||
| 522 | |||
| 523 | |||
| 524 | def _progress_indicator(iterable, *args, **kwargs): | ||
| 525 | return iterable | ||
| 526 | |||
| 527 | |||
| 528 | def _download_url(resp, link, content_file, hashes, progress_bar): | ||
| 529 | try: | ||
| 530 | total_length = int(resp.headers['content-length']) | ||
| 531 | except (ValueError, KeyError, TypeError): | ||
| 532 | total_length = 0 | ||
| 533 | |||
| 534 | cached_resp = getattr(resp, "from_cache", False) | ||
| 535 | if logger.getEffectiveLevel() > logging.INFO: | ||
| 536 | show_progress = False | ||
| 537 | elif cached_resp: | ||
| 538 | show_progress = False | ||
| 539 | elif total_length > (40 * 1000): | ||
| 540 | show_progress = True | ||
| 541 | elif not total_length: | ||
| 542 | show_progress = True | ||
| 543 | else: | ||
| 544 | show_progress = False | ||
| 545 | |||
| 546 | show_url = link.show_url | ||
| 547 | |||
| 548 | def resp_read(chunk_size): | ||
| 549 | try: | ||
| 550 | # Special case for urllib3. | ||
| 551 | for chunk in resp.raw.stream( | ||
| 552 | chunk_size, | ||
| 553 | # We use decode_content=False here because we don't | ||
| 554 | # want urllib3 to mess with the raw bytes we get | ||
| 555 | # from the server. If we decompress inside of | ||
| 556 | # urllib3 then we cannot verify the checksum | ||
| 557 | # because the checksum will be of the compressed | ||
| 558 | # file. This breakage will only occur if the | ||
| 559 | # server adds a Content-Encoding header, which | ||
| 560 | # depends on how the server was configured: | ||
| 561 | # - Some servers will notice that the file isn't a | ||
| 562 | # compressible file and will leave the file alone | ||
| 563 | # and with an empty Content-Encoding | ||
| 564 | # - Some servers will notice that the file is | ||
| 565 | # already compressed and will leave the file | ||
| 566 | # alone and will add a Content-Encoding: gzip | ||
| 567 | # header | ||
| 568 | # - Some servers won't notice anything at all and | ||
| 569 | # will take a file that's already been compressed | ||
| 570 | # and compress it again and set the | ||
| 571 | # Content-Encoding: gzip header | ||
| 572 | # | ||
| 573 | # By setting this not to decode automatically we | ||
| 574 | # hope to eliminate problems with the second case. | ||
| 575 | decode_content=False): | ||
| 576 | yield chunk | ||
| 577 | except AttributeError: | ||
| 578 | # Standard file-like object. | ||
| 579 | while True: | ||
| 580 | chunk = resp.raw.read(chunk_size) | ||
| 581 | if not chunk: | ||
| 582 | break | ||
| 583 | yield chunk | ||
| 584 | |||
| 585 | def written_chunks(chunks): | ||
| 586 | for chunk in chunks: | ||
| 587 | content_file.write(chunk) | ||
| 588 | yield chunk | ||
| 589 | |||
| 590 | progress_indicator = _progress_indicator | ||
| 591 | |||
| 592 | if link.netloc == PyPI.netloc: | ||
| 593 | url = show_url | ||
| 594 | else: | ||
| 595 | url = link.url_without_fragment | ||
| 596 | |||
| 597 | if show_progress: # We don't show progress on cached responses | ||
| 598 | progress_indicator = DownloadProgressProvider(progress_bar, | ||
| 599 | max=total_length) | ||
| 600 | if total_length: | ||
| 601 | logger.info("Downloading %s (%s)", url, format_size(total_length)) | ||
| 602 | else: | ||
| 603 | logger.info("Downloading %s", url) | ||
| 604 | elif cached_resp: | ||
| 605 | logger.info("Using cached %s", url) | ||
| 606 | else: | ||
| 607 | logger.info("Downloading %s", url) | ||
| 608 | |||
| 609 | logger.debug('Downloading from URL %s', link) | ||
| 610 | |||
| 611 | downloaded_chunks = written_chunks( | ||
| 612 | progress_indicator( | ||
| 613 | resp_read(CONTENT_CHUNK_SIZE), | ||
| 614 | CONTENT_CHUNK_SIZE | ||
| 615 | ) | ||
| 616 | ) | ||
| 617 | if hashes: | ||
| 618 | hashes.check_against_chunks(downloaded_chunks) | ||
| 619 | else: | ||
| 620 | consume(downloaded_chunks) | ||
| 621 | |||
| 622 | |||
| 623 | def _copy_file(filename, location, link): | ||
| 624 | copy = True | ||
| 625 | download_location = os.path.join(location, link.filename) | ||
| 626 | if os.path.exists(download_location): | ||
| 627 | response = ask_path_exists( | ||
| 628 | 'The file %s exists. (i)gnore, (w)ipe, (b)ackup, (a)abort' % | ||
| 629 | display_path(download_location), ('i', 'w', 'b', 'a')) | ||
| 630 | if response == 'i': | ||
| 631 | copy = False | ||
| 632 | elif response == 'w': | ||
| 633 | logger.warning('Deleting %s', display_path(download_location)) | ||
| 634 | os.remove(download_location) | ||
| 635 | elif response == 'b': | ||
| 636 | dest_file = backup_dir(download_location) | ||
| 637 | logger.warning( | ||
| 638 | 'Backing up %s to %s', | ||
| 639 | display_path(download_location), | ||
| 640 | display_path(dest_file), | ||
| 641 | ) | ||
| 642 | shutil.move(download_location, dest_file) | ||
| 643 | elif response == 'a': | ||
| 644 | sys.exit(-1) | ||
| 645 | if copy: | ||
| 646 | shutil.copy(filename, download_location) | ||
| 647 | logger.info('Saved %s', display_path(download_location)) | ||
| 648 | |||
| 649 | |||
| 650 | def unpack_http_url(link, location, download_dir=None, | ||
| 651 | session=None, hashes=None, progress_bar="on"): | ||
| 652 | if session is None: | ||
| 653 | raise TypeError( | ||
| 654 | "unpack_http_url() missing 1 required keyword argument: 'session'" | ||
| 655 | ) | ||
| 656 | |||
| 657 | with TempDirectory(kind="unpack") as temp_dir: | ||
| 658 | # If a download dir is specified, is the file already downloaded there? | ||
| 659 | already_downloaded_path = None | ||
| 660 | if download_dir: | ||
| 661 | already_downloaded_path = _check_download_dir(link, | ||
| 662 | download_dir, | ||
| 663 | hashes) | ||
| 664 | |||
| 665 | if already_downloaded_path: | ||
| 666 | from_path = already_downloaded_path | ||
| 667 | content_type = mimetypes.guess_type(from_path)[0] | ||
| 668 | else: | ||
| 669 | # let's download to a tmp dir | ||
| 670 | from_path, content_type = _download_http_url(link, | ||
| 671 | session, | ||
| 672 | temp_dir.path, | ||
| 673 | hashes, | ||
| 674 | progress_bar) | ||
| 675 | |||
| 676 | # unpack the archive to the build dir location. even when only | ||
| 677 | # downloading archives, they have to be unpacked to parse dependencies | ||
| 678 | unpack_file(from_path, location, content_type, link) | ||
| 679 | |||
| 680 | # a download dir is specified; let's copy the archive there | ||
| 681 | if download_dir and not already_downloaded_path: | ||
| 682 | _copy_file(from_path, download_dir, link) | ||
| 683 | |||
| 684 | if not already_downloaded_path: | ||
| 685 | os.unlink(from_path) | ||
| 686 | |||
| 687 | |||
| 688 | def unpack_file_url(link, location, download_dir=None, hashes=None): | ||
| 689 | """Unpack link into location. | ||
| 690 | |||
| 691 | If download_dir is provided and link points to a file, make a copy | ||
| 692 | of the link file inside download_dir. | ||
| 693 | """ | ||
| 694 | link_path = url_to_path(link.url_without_fragment) | ||
| 695 | |||
| 696 | # If it's a url to a local directory | ||
| 697 | if is_dir_url(link): | ||
| 698 | if os.path.isdir(location): | ||
| 699 | rmtree(location) | ||
| 700 | shutil.copytree(link_path, location, symlinks=True) | ||
| 701 | if download_dir: | ||
| 702 | logger.info('Link is a directory, ignoring download_dir') | ||
| 703 | return | ||
| 704 | |||
| 705 | # If --require-hashes is off, `hashes` is either empty, the | ||
| 706 | # link's embedded hash, or MissingHashes; it is required to | ||
| 707 | # match. If --require-hashes is on, we are satisfied by any | ||
| 708 | # hash in `hashes` matching: a URL-based or an option-based | ||
| 709 | # one; no internet-sourced hash will be in `hashes`. | ||
| 710 | if hashes: | ||
| 711 | hashes.check_against_path(link_path) | ||
| 712 | |||
| 713 | # If a download dir is specified, is the file already there and valid? | ||
| 714 | already_downloaded_path = None | ||
| 715 | if download_dir: | ||
| 716 | already_downloaded_path = _check_download_dir(link, | ||
| 717 | download_dir, | ||
| 718 | hashes) | ||
| 719 | |||
| 720 | if already_downloaded_path: | ||
| 721 | from_path = already_downloaded_path | ||
| 722 | else: | ||
| 723 | from_path = link_path | ||
| 724 | |||
| 725 | content_type = mimetypes.guess_type(from_path)[0] | ||
| 726 | |||
| 727 | # unpack the archive to the build dir location. even when only downloading | ||
| 728 | # archives, they have to be unpacked to parse dependencies | ||
| 729 | unpack_file(from_path, location, content_type, link) | ||
| 730 | |||
| 731 | # a download dir is specified and not already downloaded | ||
| 732 | if download_dir and not already_downloaded_path: | ||
| 733 | _copy_file(from_path, download_dir, link) | ||
| 734 | |||
| 735 | |||
| 736 | def _copy_dist_from_dir(link_path, location): | ||
| 737 | """Copy distribution files in `link_path` to `location`. | ||
| 738 | |||
| 739 | Invoked when user requests to install a local directory. E.g.: | ||
| 740 | |||
| 741 | pip install . | ||
| 742 | pip install ~/dev/git-repos/python-prompt-toolkit | ||
| 743 | |||
| 744 | """ | ||
| 745 | |||
| 746 | # Note: This is currently VERY SLOW if you have a lot of data in the | ||
| 747 | # directory, because it copies everything with `shutil.copytree`. | ||
| 748 | # What it should really do is build an sdist and install that. | ||
| 749 | # See https://github.com/pypa/pip/issues/2195 | ||
| 750 | |||
| 751 | if os.path.isdir(location): | ||
| 752 | rmtree(location) | ||
| 753 | |||
| 754 | # build an sdist | ||
| 755 | setup_py = 'setup.py' | ||
| 756 | sdist_args = [sys.executable] | ||
| 757 | sdist_args.append('-c') | ||
| 758 | sdist_args.append(SETUPTOOLS_SHIM % setup_py) | ||
| 759 | sdist_args.append('sdist') | ||
| 760 | sdist_args += ['--dist-dir', location] | ||
| 761 | logger.info('Running setup.py sdist for %s', link_path) | ||
| 762 | |||
| 763 | with indent_log(): | ||
| 764 | call_subprocess(sdist_args, cwd=link_path, show_stdout=False) | ||
| 765 | |||
| 766 | # unpack sdist into `location` | ||
| 767 | sdist = os.path.join(location, os.listdir(location)[0]) | ||
| 768 | logger.info('Unpacking sdist %s into %s', sdist, location) | ||
| 769 | unpack_file(sdist, location, content_type=None, link=None) | ||
| 770 | |||
| 771 | |||
| 772 | class PipXmlrpcTransport(xmlrpc_client.Transport): | ||
| 773 | """Provide a `xmlrpclib.Transport` implementation via a `PipSession` | ||
| 774 | object. | ||
| 775 | """ | ||
| 776 | |||
| 777 | def __init__(self, index_url, session, use_datetime=False): | ||
| 778 | xmlrpc_client.Transport.__init__(self, use_datetime) | ||
| 779 | index_parts = urllib_parse.urlparse(index_url) | ||
| 780 | self._scheme = index_parts.scheme | ||
| 781 | self._session = session | ||
| 782 | |||
| 783 | def request(self, host, handler, request_body, verbose=False): | ||
| 784 | parts = (self._scheme, host, handler, None, None, None) | ||
| 785 | url = urllib_parse.urlunparse(parts) | ||
| 786 | try: | ||
| 787 | headers = {'Content-Type': 'text/xml'} | ||
| 788 | response = self._session.post(url, data=request_body, | ||
| 789 | headers=headers, stream=True) | ||
| 790 | response.raise_for_status() | ||
| 791 | self.verbose = verbose | ||
| 792 | return self.parse_response(response.raw) | ||
| 793 | except requests.HTTPError as exc: | ||
| 794 | logger.critical( | ||
| 795 | "HTTP error %s while getting %s", | ||
| 796 | exc.response.status_code, url, | ||
| 797 | ) | ||
| 798 | raise | ||
| 799 | |||
| 800 | |||
| 801 | def unpack_url(link, location, download_dir=None, | ||
| 802 | only_download=False, session=None, hashes=None, | ||
| 803 | progress_bar="on"): | ||
| 804 | """Unpack link. | ||
| 805 | If link is a VCS link: | ||
| 806 | if only_download, export into download_dir and ignore location | ||
| 807 | else unpack into location | ||
| 808 | for other types of link: | ||
| 809 | - unpack into location | ||
| 810 | - if download_dir, copy the file into download_dir | ||
| 811 | - if only_download, mark location for deletion | ||
| 812 | |||
| 813 | :param hashes: A Hashes object, one of whose embedded hashes must match, | ||
| 814 | or HashMismatch will be raised. If the Hashes is empty, no matches are | ||
| 815 | required, and unhashable types of requirements (like VCS ones, which | ||
| 816 | would ordinarily raise HashUnsupported) are allowed. | ||
| 817 | """ | ||
| 818 | # non-editable vcs urls | ||
| 819 | if is_vcs_url(link): | ||
| 820 | unpack_vcs_link(link, location) | ||
| 821 | |||
| 822 | # file urls | ||
| 823 | elif is_file_url(link): | ||
| 824 | unpack_file_url(link, location, download_dir, hashes=hashes) | ||
| 825 | |||
| 826 | # http urls | ||
| 827 | else: | ||
| 828 | if session is None: | ||
| 829 | session = PipSession() | ||
| 830 | |||
| 831 | unpack_http_url( | ||
| 832 | link, | ||
| 833 | location, | ||
| 834 | download_dir, | ||
| 835 | session, | ||
| 836 | hashes=hashes, | ||
| 837 | progress_bar=progress_bar | ||
| 838 | ) | ||
| 839 | if only_download: | ||
| 840 | write_delete_marker_file(location) | ||
| 841 | |||
| 842 | |||
| 843 | def _download_http_url(link, session, temp_dir, hashes, progress_bar): | ||
| 844 | """Download link url into temp_dir using provided session""" | ||
| 845 | target_url = link.url.split('#', 1)[0] | ||
| 846 | try: | ||
| 847 | resp = session.get( | ||
| 848 | target_url, | ||
| 849 | # We use Accept-Encoding: identity here because requests | ||
| 850 | # defaults to accepting compressed responses. This breaks in | ||
| 851 | # a variety of ways depending on how the server is configured. | ||
| 852 | # - Some servers will notice that the file isn't a compressible | ||
| 853 | # file and will leave the file alone and with an empty | ||
| 854 | # Content-Encoding | ||
| 855 | # - Some servers will notice that the file is already | ||
| 856 | # compressed and will leave the file alone and will add a | ||
| 857 | # Content-Encoding: gzip header | ||
| 858 | # - Some servers won't notice anything at all and will take | ||
| 859 | # a file that's already been compressed and compress it again | ||
| 860 | # and set the Content-Encoding: gzip header | ||
| 861 | # By setting this to request only the identity encoding We're | ||
| 862 | # hoping to eliminate the third case. Hopefully there does not | ||
| 863 | # exist a server which when given a file will notice it is | ||
| 864 | # already compressed and that you're not asking for a | ||
| 865 | # compressed file and will then decompress it before sending | ||
| 866 | # because if that's the case I don't think it'll ever be | ||
| 867 | # possible to make this work. | ||
| 868 | headers={"Accept-Encoding": "identity"}, | ||
| 869 | stream=True, | ||
| 870 | ) | ||
| 871 | resp.raise_for_status() | ||
| 872 | except requests.HTTPError as exc: | ||
| 873 | logger.critical( | ||
| 874 | "HTTP error %s while getting %s", exc.response.status_code, link, | ||
| 875 | ) | ||
| 876 | raise | ||
| 877 | |||
| 878 | content_type = resp.headers.get('content-type', '') | ||
| 879 | filename = link.filename # fallback | ||
| 880 | # Have a look at the Content-Disposition header for a better guess | ||
| 881 | content_disposition = resp.headers.get('content-disposition') | ||
| 882 | if content_disposition: | ||
| 883 | type, params = cgi.parse_header(content_disposition) | ||
| 884 | # We use ``or`` here because we don't want to use an "empty" value | ||
| 885 | # from the filename param. | ||
| 886 | filename = params.get('filename') or filename | ||
| 887 | ext = splitext(filename)[1] | ||
| 888 | if not ext: | ||
| 889 | ext = mimetypes.guess_extension(content_type) | ||
| 890 | if ext: | ||
| 891 | filename += ext | ||
| 892 | if not ext and link.url != resp.url: | ||
| 893 | ext = os.path.splitext(resp.url)[1] | ||
| 894 | if ext: | ||
| 895 | filename += ext | ||
| 896 | file_path = os.path.join(temp_dir, filename) | ||
| 897 | with open(file_path, 'wb') as content_file: | ||
| 898 | _download_url(resp, link, content_file, hashes, progress_bar) | ||
| 899 | return file_path, content_type | ||
| 900 | |||
| 901 | |||
| 902 | def _check_download_dir(link, download_dir, hashes): | ||
| 903 | """ Check download_dir for previously downloaded file with correct hash | ||
| 904 | If a correct file is found return its path else None | ||
| 905 | """ | ||
| 906 | download_path = os.path.join(download_dir, link.filename) | ||
| 907 | if os.path.exists(download_path): | ||
| 908 | # If already downloaded, does its hash match? | ||
| 909 | logger.info('File was already downloaded %s', download_path) | ||
| 910 | if hashes: | ||
| 911 | try: | ||
| 912 | hashes.check_against_path(download_path) | ||
| 913 | except HashMismatch: | ||
| 914 | logger.warning( | ||
| 915 | 'Previously-downloaded file %s has bad hash. ' | ||
| 916 | 'Re-downloading.', | ||
| 917 | download_path | ||
| 918 | ) | ||
| 919 | os.unlink(download_path) | ||
| 920 | return None | ||
| 921 | return download_path | ||
| 922 | return None | ||
