summaryrefslogtreecommitdiff
path: root/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/download.py
diff options
context:
space:
mode:
Diffstat (limited to 'venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/download.py')
-rw-r--r--venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/download.py922
1 files changed, 922 insertions, 0 deletions
diff --git a/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/download.py b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/download.py
new file mode 100644
index 0000000..e0e2d24
--- /dev/null
+++ b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/download.py
@@ -0,0 +1,922 @@
1from __future__ import absolute_import
2
3import cgi
4import email.utils
5import getpass
6import json
7import logging
8import mimetypes
9import os
10import platform
11import re
12import shutil
13import sys
14
15from pip._vendor import requests, six, urllib3
16from pip._vendor.cachecontrol import CacheControlAdapter
17from pip._vendor.cachecontrol.caches import FileCache
18from pip._vendor.lockfile import LockError
19from pip._vendor.requests.adapters import BaseAdapter, HTTPAdapter
20from pip._vendor.requests.auth import AuthBase, HTTPBasicAuth
21from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response
22from pip._vendor.requests.structures import CaseInsensitiveDict
23from pip._vendor.requests.utils import get_netrc_auth
24# NOTE: XMLRPC Client is not annotated in typeshed as on 2017-07-17, which is
25# why we ignore the type on this import
26from pip._vendor.six.moves import xmlrpc_client # type: ignore
27from pip._vendor.six.moves.urllib import parse as urllib_parse
28from pip._vendor.six.moves.urllib import request as urllib_request
29from pip._vendor.six.moves.urllib.parse import unquote as urllib_unquote
30from pip._vendor.urllib3.util import IS_PYOPENSSL
31
32import pip
33from pip._internal.compat import WINDOWS
34from pip._internal.exceptions import HashMismatch, InstallationError
35from pip._internal.locations import write_delete_marker_file
36from pip._internal.models import PyPI
37from pip._internal.utils.encoding import auto_decode
38from pip._internal.utils.filesystem import check_path_owner
39from pip._internal.utils.glibc import libc_ver
40from pip._internal.utils.logging import indent_log
41from pip._internal.utils.misc import (
42 ARCHIVE_EXTENSIONS, ask_path_exists, backup_dir, call_subprocess, consume,
43 display_path, format_size, get_installed_version, rmtree, splitext,
44 unpack_file,
45)
46from pip._internal.utils.setuptools_build import SETUPTOOLS_SHIM
47from pip._internal.utils.temp_dir import TempDirectory
48from pip._internal.utils.ui import DownloadProgressProvider
49from pip._internal.vcs import vcs
50
51try:
52 import ssl # noqa
53except ImportError:
54 ssl = None
55
56HAS_TLS = (ssl is not None) or IS_PYOPENSSL
57
58__all__ = ['get_file_content',
59 'is_url', 'url_to_path', 'path_to_url',
60 'is_archive_file', 'unpack_vcs_link',
61 'unpack_file_url', 'is_vcs_url', 'is_file_url',
62 'unpack_http_url', 'unpack_url']
63
64
65logger = logging.getLogger(__name__)
66
67
68def user_agent():
69 """
70 Return a string representing the user agent.
71 """
72 data = {
73 "installer": {"name": "pip", "version": pip.__version__},
74 "python": platform.python_version(),
75 "implementation": {
76 "name": platform.python_implementation(),
77 },
78 }
79
80 if data["implementation"]["name"] == 'CPython':
81 data["implementation"]["version"] = platform.python_version()
82 elif data["implementation"]["name"] == 'PyPy':
83 if sys.pypy_version_info.releaselevel == 'final':
84 pypy_version_info = sys.pypy_version_info[:3]
85 else:
86 pypy_version_info = sys.pypy_version_info
87 data["implementation"]["version"] = ".".join(
88 [str(x) for x in pypy_version_info]
89 )
90 elif data["implementation"]["name"] == 'Jython':
91 # Complete Guess
92 data["implementation"]["version"] = platform.python_version()
93 elif data["implementation"]["name"] == 'IronPython':
94 # Complete Guess
95 data["implementation"]["version"] = platform.python_version()
96
97 if sys.platform.startswith("linux"):
98 from pip._vendor import distro
99 distro_infos = dict(filter(
100 lambda x: x[1],
101 zip(["name", "version", "id"], distro.linux_distribution()),
102 ))
103 libc = dict(filter(
104 lambda x: x[1],
105 zip(["lib", "version"], libc_ver()),
106 ))
107 if libc:
108 distro_infos["libc"] = libc
109 if distro_infos:
110 data["distro"] = distro_infos
111
112 if sys.platform.startswith("darwin") and platform.mac_ver()[0]:
113 data["distro"] = {"name": "macOS", "version": platform.mac_ver()[0]}
114
115 if platform.system():
116 data.setdefault("system", {})["name"] = platform.system()
117
118 if platform.release():
119 data.setdefault("system", {})["release"] = platform.release()
120
121 if platform.machine():
122 data["cpu"] = platform.machine()
123
124 if HAS_TLS:
125 data["openssl_version"] = ssl.OPENSSL_VERSION
126
127 setuptools_version = get_installed_version("setuptools")
128 if setuptools_version is not None:
129 data["setuptools_version"] = setuptools_version
130
131 return "{data[installer][name]}/{data[installer][version]} {json}".format(
132 data=data,
133 json=json.dumps(data, separators=(",", ":"), sort_keys=True),
134 )
135
136
137class MultiDomainBasicAuth(AuthBase):
138
139 def __init__(self, prompting=True):
140 self.prompting = prompting
141 self.passwords = {}
142
143 def __call__(self, req):
144 parsed = urllib_parse.urlparse(req.url)
145
146 # Get the netloc without any embedded credentials
147 netloc = parsed.netloc.rsplit("@", 1)[-1]
148
149 # Set the url of the request to the url without any credentials
150 req.url = urllib_parse.urlunparse(parsed[:1] + (netloc,) + parsed[2:])
151
152 # Use any stored credentials that we have for this netloc
153 username, password = self.passwords.get(netloc, (None, None))
154
155 # Extract credentials embedded in the url if we have none stored
156 if username is None:
157 username, password = self.parse_credentials(parsed.netloc)
158
159 # Get creds from netrc if we still don't have them
160 if username is None and password is None:
161 netrc_auth = get_netrc_auth(req.url)
162 username, password = netrc_auth if netrc_auth else (None, None)
163
164 if username or password:
165 # Store the username and password
166 self.passwords[netloc] = (username, password)
167
168 # Send the basic auth with this request
169 req = HTTPBasicAuth(username or "", password or "")(req)
170
171 # Attach a hook to handle 401 responses
172 req.register_hook("response", self.handle_401)
173
174 return req
175
176 def handle_401(self, resp, **kwargs):
177 # We only care about 401 responses, anything else we want to just
178 # pass through the actual response
179 if resp.status_code != 401:
180 return resp
181
182 # We are not able to prompt the user so simply return the response
183 if not self.prompting:
184 return resp
185
186 parsed = urllib_parse.urlparse(resp.url)
187
188 # Prompt the user for a new username and password
189 username = six.moves.input("User for %s: " % parsed.netloc)
190 password = getpass.getpass("Password: ")
191
192 # Store the new username and password to use for future requests
193 if username or password:
194 self.passwords[parsed.netloc] = (username, password)
195
196 # Consume content and release the original connection to allow our new
197 # request to reuse the same one.
198 resp.content
199 resp.raw.release_conn()
200
201 # Add our new username and password to the request
202 req = HTTPBasicAuth(username or "", password or "")(resp.request)
203
204 # Send our new request
205 new_resp = resp.connection.send(req, **kwargs)
206 new_resp.history.append(resp)
207
208 return new_resp
209
210 def parse_credentials(self, netloc):
211 if "@" in netloc:
212 userinfo = netloc.rsplit("@", 1)[0]
213 if ":" in userinfo:
214 user, pwd = userinfo.split(":", 1)
215 return (urllib_unquote(user), urllib_unquote(pwd))
216 return urllib_unquote(userinfo), None
217 return None, None
218
219
220class LocalFSAdapter(BaseAdapter):
221
222 def send(self, request, stream=None, timeout=None, verify=None, cert=None,
223 proxies=None):
224 pathname = url_to_path(request.url)
225
226 resp = Response()
227 resp.status_code = 200
228 resp.url = request.url
229
230 try:
231 stats = os.stat(pathname)
232 except OSError as exc:
233 resp.status_code = 404
234 resp.raw = exc
235 else:
236 modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
237 content_type = mimetypes.guess_type(pathname)[0] or "text/plain"
238 resp.headers = CaseInsensitiveDict({
239 "Content-Type": content_type,
240 "Content-Length": stats.st_size,
241 "Last-Modified": modified,
242 })
243
244 resp.raw = open(pathname, "rb")
245 resp.close = resp.raw.close
246
247 return resp
248
249 def close(self):
250 pass
251
252
253class SafeFileCache(FileCache):
254 """
255 A file based cache which is safe to use even when the target directory may
256 not be accessible or writable.
257 """
258
259 def __init__(self, *args, **kwargs):
260 super(SafeFileCache, self).__init__(*args, **kwargs)
261
262 # Check to ensure that the directory containing our cache directory
263 # is owned by the user current executing pip. If it does not exist
264 # we will check the parent directory until we find one that does exist.
265 # If it is not owned by the user executing pip then we will disable
266 # the cache and log a warning.
267 if not check_path_owner(self.directory):
268 logger.warning(
269 "The directory '%s' or its parent directory is not owned by "
270 "the current user and the cache has been disabled. Please "
271 "check the permissions and owner of that directory. If "
272 "executing pip with sudo, you may want sudo's -H flag.",
273 self.directory,
274 )
275
276 # Set our directory to None to disable the Cache
277 self.directory = None
278
279 def get(self, *args, **kwargs):
280 # If we don't have a directory, then the cache should be a no-op.
281 if self.directory is None:
282 return
283
284 try:
285 return super(SafeFileCache, self).get(*args, **kwargs)
286 except (LockError, OSError, IOError):
287 # We intentionally silence this error, if we can't access the cache
288 # then we can just skip caching and process the request as if
289 # caching wasn't enabled.
290 pass
291
292 def set(self, *args, **kwargs):
293 # If we don't have a directory, then the cache should be a no-op.
294 if self.directory is None:
295 return
296
297 try:
298 return super(SafeFileCache, self).set(*args, **kwargs)
299 except (LockError, OSError, IOError):
300 # We intentionally silence this error, if we can't access the cache
301 # then we can just skip caching and process the request as if
302 # caching wasn't enabled.
303 pass
304
305 def delete(self, *args, **kwargs):
306 # If we don't have a directory, then the cache should be a no-op.
307 if self.directory is None:
308 return
309
310 try:
311 return super(SafeFileCache, self).delete(*args, **kwargs)
312 except (LockError, OSError, IOError):
313 # We intentionally silence this error, if we can't access the cache
314 # then we can just skip caching and process the request as if
315 # caching wasn't enabled.
316 pass
317
318
319class InsecureHTTPAdapter(HTTPAdapter):
320
321 def cert_verify(self, conn, url, verify, cert):
322 conn.cert_reqs = 'CERT_NONE'
323 conn.ca_certs = None
324
325
326class PipSession(requests.Session):
327
328 timeout = None
329
330 def __init__(self, *args, **kwargs):
331 retries = kwargs.pop("retries", 0)
332 cache = kwargs.pop("cache", None)
333 insecure_hosts = kwargs.pop("insecure_hosts", [])
334
335 super(PipSession, self).__init__(*args, **kwargs)
336
337 # Attach our User Agent to the request
338 self.headers["User-Agent"] = user_agent()
339
340 # Attach our Authentication handler to the session
341 self.auth = MultiDomainBasicAuth()
342
343 # Create our urllib3.Retry instance which will allow us to customize
344 # how we handle retries.
345 retries = urllib3.Retry(
346 # Set the total number of retries that a particular request can
347 # have.
348 total=retries,
349
350 # A 503 error from PyPI typically means that the Fastly -> Origin
351 # connection got interrupted in some way. A 503 error in general
352 # is typically considered a transient error so we'll go ahead and
353 # retry it.
354 # A 500 may indicate transient error in Amazon S3
355 # A 520 or 527 - may indicate transient error in CloudFlare
356 status_forcelist=[500, 503, 520, 527],
357
358 # Add a small amount of back off between failed requests in
359 # order to prevent hammering the service.
360 backoff_factor=0.25,
361 )
362
363 # We want to _only_ cache responses on securely fetched origins. We do
364 # this because we can't validate the response of an insecurely fetched
365 # origin, and we don't want someone to be able to poison the cache and
366 # require manual eviction from the cache to fix it.
367 if cache:
368 secure_adapter = CacheControlAdapter(
369 cache=SafeFileCache(cache, use_dir_lock=True),
370 max_retries=retries,
371 )
372 else:
373 secure_adapter = HTTPAdapter(max_retries=retries)
374
375 # Our Insecure HTTPAdapter disables HTTPS validation. It does not
376 # support caching (see above) so we'll use it for all http:// URLs as
377 # well as any https:// host that we've marked as ignoring TLS errors
378 # for.
379 insecure_adapter = InsecureHTTPAdapter(max_retries=retries)
380
381 self.mount("https://", secure_adapter)
382 self.mount("http://", insecure_adapter)
383
384 # Enable file:// urls
385 self.mount("file://", LocalFSAdapter())
386
387 # We want to use a non-validating adapter for any requests which are
388 # deemed insecure.
389 for host in insecure_hosts:
390 self.mount("https://{}/".format(host), insecure_adapter)
391
392 def request(self, method, url, *args, **kwargs):
393 # Allow setting a default timeout on a session
394 kwargs.setdefault("timeout", self.timeout)
395
396 # Dispatch the actual request
397 return super(PipSession, self).request(method, url, *args, **kwargs)
398
399
400def get_file_content(url, comes_from=None, session=None):
401 """Gets the content of a file; it may be a filename, file: URL, or
402 http: URL. Returns (location, content). Content is unicode.
403
404 :param url: File path or url.
405 :param comes_from: Origin description of requirements.
406 :param session: Instance of pip.download.PipSession.
407 """
408 if session is None:
409 raise TypeError(
410 "get_file_content() missing 1 required keyword argument: 'session'"
411 )
412
413 match = _scheme_re.search(url)
414 if match:
415 scheme = match.group(1).lower()
416 if (scheme == 'file' and comes_from and
417 comes_from.startswith('http')):
418 raise InstallationError(
419 'Requirements file %s references URL %s, which is local'
420 % (comes_from, url))
421 if scheme == 'file':
422 path = url.split(':', 1)[1]
423 path = path.replace('\\', '/')
424 match = _url_slash_drive_re.match(path)
425 if match:
426 path = match.group(1) + ':' + path.split('|', 1)[1]
427 path = urllib_parse.unquote(path)
428 if path.startswith('/'):
429 path = '/' + path.lstrip('/')
430 url = path
431 else:
432 # FIXME: catch some errors
433 resp = session.get(url)
434 resp.raise_for_status()
435 return resp.url, resp.text
436 try:
437 with open(url, 'rb') as f:
438 content = auto_decode(f.read())
439 except IOError as exc:
440 raise InstallationError(
441 'Could not open requirements file: %s' % str(exc)
442 )
443 return url, content
444
445
446_scheme_re = re.compile(r'^(http|https|file):', re.I)
447_url_slash_drive_re = re.compile(r'/*([a-z])\|', re.I)
448
449
450def is_url(name):
451 """Returns true if the name looks like a URL"""
452 if ':' not in name:
453 return False
454 scheme = name.split(':', 1)[0].lower()
455 return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes
456
457
458def url_to_path(url):
459 """
460 Convert a file: URL to a path.
461 """
462 assert url.startswith('file:'), (
463 "You can only turn file: urls into filenames (not %r)" % url)
464
465 _, netloc, path, _, _ = urllib_parse.urlsplit(url)
466
467 # if we have a UNC path, prepend UNC share notation
468 if netloc:
469 netloc = '\\\\' + netloc
470
471 path = urllib_request.url2pathname(netloc + path)
472 return path
473
474
475def path_to_url(path):
476 """
477 Convert a path to a file: URL. The path will be made absolute and have
478 quoted path parts.
479 """
480 path = os.path.normpath(os.path.abspath(path))
481 url = urllib_parse.urljoin('file:', urllib_request.pathname2url(path))
482 return url
483
484
485def is_archive_file(name):
486 """Return True if `name` is a considered as an archive file."""
487 ext = splitext(name)[1].lower()
488 if ext in ARCHIVE_EXTENSIONS:
489 return True
490 return False
491
492
493def unpack_vcs_link(link, location):
494 vcs_backend = _get_used_vcs_backend(link)
495 vcs_backend.unpack(location)
496
497
498def _get_used_vcs_backend(link):
499 for backend in vcs.backends:
500 if link.scheme in backend.schemes:
501 vcs_backend = backend(link.url)
502 return vcs_backend
503
504
505def is_vcs_url(link):
506 return bool(_get_used_vcs_backend(link))
507
508
509def is_file_url(link):
510 return link.url.lower().startswith('file:')
511
512
513def is_dir_url(link):
514 """Return whether a file:// Link points to a directory.
515
516 ``link`` must not have any other scheme but file://. Call is_file_url()
517 first.
518
519 """
520 link_path = url_to_path(link.url_without_fragment)
521 return os.path.isdir(link_path)
522
523
524def _progress_indicator(iterable, *args, **kwargs):
525 return iterable
526
527
528def _download_url(resp, link, content_file, hashes, progress_bar):
529 try:
530 total_length = int(resp.headers['content-length'])
531 except (ValueError, KeyError, TypeError):
532 total_length = 0
533
534 cached_resp = getattr(resp, "from_cache", False)
535 if logger.getEffectiveLevel() > logging.INFO:
536 show_progress = False
537 elif cached_resp:
538 show_progress = False
539 elif total_length > (40 * 1000):
540 show_progress = True
541 elif not total_length:
542 show_progress = True
543 else:
544 show_progress = False
545
546 show_url = link.show_url
547
548 def resp_read(chunk_size):
549 try:
550 # Special case for urllib3.
551 for chunk in resp.raw.stream(
552 chunk_size,
553 # We use decode_content=False here because we don't
554 # want urllib3 to mess with the raw bytes we get
555 # from the server. If we decompress inside of
556 # urllib3 then we cannot verify the checksum
557 # because the checksum will be of the compressed
558 # file. This breakage will only occur if the
559 # server adds a Content-Encoding header, which
560 # depends on how the server was configured:
561 # - Some servers will notice that the file isn't a
562 # compressible file and will leave the file alone
563 # and with an empty Content-Encoding
564 # - Some servers will notice that the file is
565 # already compressed and will leave the file
566 # alone and will add a Content-Encoding: gzip
567 # header
568 # - Some servers won't notice anything at all and
569 # will take a file that's already been compressed
570 # and compress it again and set the
571 # Content-Encoding: gzip header
572 #
573 # By setting this not to decode automatically we
574 # hope to eliminate problems with the second case.
575 decode_content=False):
576 yield chunk
577 except AttributeError:
578 # Standard file-like object.
579 while True:
580 chunk = resp.raw.read(chunk_size)
581 if not chunk:
582 break
583 yield chunk
584
585 def written_chunks(chunks):
586 for chunk in chunks:
587 content_file.write(chunk)
588 yield chunk
589
590 progress_indicator = _progress_indicator
591
592 if link.netloc == PyPI.netloc:
593 url = show_url
594 else:
595 url = link.url_without_fragment
596
597 if show_progress: # We don't show progress on cached responses
598 progress_indicator = DownloadProgressProvider(progress_bar,
599 max=total_length)
600 if total_length:
601 logger.info("Downloading %s (%s)", url, format_size(total_length))
602 else:
603 logger.info("Downloading %s", url)
604 elif cached_resp:
605 logger.info("Using cached %s", url)
606 else:
607 logger.info("Downloading %s", url)
608
609 logger.debug('Downloading from URL %s', link)
610
611 downloaded_chunks = written_chunks(
612 progress_indicator(
613 resp_read(CONTENT_CHUNK_SIZE),
614 CONTENT_CHUNK_SIZE
615 )
616 )
617 if hashes:
618 hashes.check_against_chunks(downloaded_chunks)
619 else:
620 consume(downloaded_chunks)
621
622
623def _copy_file(filename, location, link):
624 copy = True
625 download_location = os.path.join(location, link.filename)
626 if os.path.exists(download_location):
627 response = ask_path_exists(
628 'The file %s exists. (i)gnore, (w)ipe, (b)ackup, (a)abort' %
629 display_path(download_location), ('i', 'w', 'b', 'a'))
630 if response == 'i':
631 copy = False
632 elif response == 'w':
633 logger.warning('Deleting %s', display_path(download_location))
634 os.remove(download_location)
635 elif response == 'b':
636 dest_file = backup_dir(download_location)
637 logger.warning(
638 'Backing up %s to %s',
639 display_path(download_location),
640 display_path(dest_file),
641 )
642 shutil.move(download_location, dest_file)
643 elif response == 'a':
644 sys.exit(-1)
645 if copy:
646 shutil.copy(filename, download_location)
647 logger.info('Saved %s', display_path(download_location))
648
649
650def unpack_http_url(link, location, download_dir=None,
651 session=None, hashes=None, progress_bar="on"):
652 if session is None:
653 raise TypeError(
654 "unpack_http_url() missing 1 required keyword argument: 'session'"
655 )
656
657 with TempDirectory(kind="unpack") as temp_dir:
658 # If a download dir is specified, is the file already downloaded there?
659 already_downloaded_path = None
660 if download_dir:
661 already_downloaded_path = _check_download_dir(link,
662 download_dir,
663 hashes)
664
665 if already_downloaded_path:
666 from_path = already_downloaded_path
667 content_type = mimetypes.guess_type(from_path)[0]
668 else:
669 # let's download to a tmp dir
670 from_path, content_type = _download_http_url(link,
671 session,
672 temp_dir.path,
673 hashes,
674 progress_bar)
675
676 # unpack the archive to the build dir location. even when only
677 # downloading archives, they have to be unpacked to parse dependencies
678 unpack_file(from_path, location, content_type, link)
679
680 # a download dir is specified; let's copy the archive there
681 if download_dir and not already_downloaded_path:
682 _copy_file(from_path, download_dir, link)
683
684 if not already_downloaded_path:
685 os.unlink(from_path)
686
687
688def unpack_file_url(link, location, download_dir=None, hashes=None):
689 """Unpack link into location.
690
691 If download_dir is provided and link points to a file, make a copy
692 of the link file inside download_dir.
693 """
694 link_path = url_to_path(link.url_without_fragment)
695
696 # If it's a url to a local directory
697 if is_dir_url(link):
698 if os.path.isdir(location):
699 rmtree(location)
700 shutil.copytree(link_path, location, symlinks=True)
701 if download_dir:
702 logger.info('Link is a directory, ignoring download_dir')
703 return
704
705 # If --require-hashes is off, `hashes` is either empty, the
706 # link's embedded hash, or MissingHashes; it is required to
707 # match. If --require-hashes is on, we are satisfied by any
708 # hash in `hashes` matching: a URL-based or an option-based
709 # one; no internet-sourced hash will be in `hashes`.
710 if hashes:
711 hashes.check_against_path(link_path)
712
713 # If a download dir is specified, is the file already there and valid?
714 already_downloaded_path = None
715 if download_dir:
716 already_downloaded_path = _check_download_dir(link,
717 download_dir,
718 hashes)
719
720 if already_downloaded_path:
721 from_path = already_downloaded_path
722 else:
723 from_path = link_path
724
725 content_type = mimetypes.guess_type(from_path)[0]
726
727 # unpack the archive to the build dir location. even when only downloading
728 # archives, they have to be unpacked to parse dependencies
729 unpack_file(from_path, location, content_type, link)
730
731 # a download dir is specified and not already downloaded
732 if download_dir and not already_downloaded_path:
733 _copy_file(from_path, download_dir, link)
734
735
736def _copy_dist_from_dir(link_path, location):
737 """Copy distribution files in `link_path` to `location`.
738
739 Invoked when user requests to install a local directory. E.g.:
740
741 pip install .
742 pip install ~/dev/git-repos/python-prompt-toolkit
743
744 """
745
746 # Note: This is currently VERY SLOW if you have a lot of data in the
747 # directory, because it copies everything with `shutil.copytree`.
748 # What it should really do is build an sdist and install that.
749 # See https://github.com/pypa/pip/issues/2195
750
751 if os.path.isdir(location):
752 rmtree(location)
753
754 # build an sdist
755 setup_py = 'setup.py'
756 sdist_args = [sys.executable]
757 sdist_args.append('-c')
758 sdist_args.append(SETUPTOOLS_SHIM % setup_py)
759 sdist_args.append('sdist')
760 sdist_args += ['--dist-dir', location]
761 logger.info('Running setup.py sdist for %s', link_path)
762
763 with indent_log():
764 call_subprocess(sdist_args, cwd=link_path, show_stdout=False)
765
766 # unpack sdist into `location`
767 sdist = os.path.join(location, os.listdir(location)[0])
768 logger.info('Unpacking sdist %s into %s', sdist, location)
769 unpack_file(sdist, location, content_type=None, link=None)
770
771
772class PipXmlrpcTransport(xmlrpc_client.Transport):
773 """Provide a `xmlrpclib.Transport` implementation via a `PipSession`
774 object.
775 """
776
777 def __init__(self, index_url, session, use_datetime=False):
778 xmlrpc_client.Transport.__init__(self, use_datetime)
779 index_parts = urllib_parse.urlparse(index_url)
780 self._scheme = index_parts.scheme
781 self._session = session
782
783 def request(self, host, handler, request_body, verbose=False):
784 parts = (self._scheme, host, handler, None, None, None)
785 url = urllib_parse.urlunparse(parts)
786 try:
787 headers = {'Content-Type': 'text/xml'}
788 response = self._session.post(url, data=request_body,
789 headers=headers, stream=True)
790 response.raise_for_status()
791 self.verbose = verbose
792 return self.parse_response(response.raw)
793 except requests.HTTPError as exc:
794 logger.critical(
795 "HTTP error %s while getting %s",
796 exc.response.status_code, url,
797 )
798 raise
799
800
801def unpack_url(link, location, download_dir=None,
802 only_download=False, session=None, hashes=None,
803 progress_bar="on"):
804 """Unpack link.
805 If link is a VCS link:
806 if only_download, export into download_dir and ignore location
807 else unpack into location
808 for other types of link:
809 - unpack into location
810 - if download_dir, copy the file into download_dir
811 - if only_download, mark location for deletion
812
813 :param hashes: A Hashes object, one of whose embedded hashes must match,
814 or HashMismatch will be raised. If the Hashes is empty, no matches are
815 required, and unhashable types of requirements (like VCS ones, which
816 would ordinarily raise HashUnsupported) are allowed.
817 """
818 # non-editable vcs urls
819 if is_vcs_url(link):
820 unpack_vcs_link(link, location)
821
822 # file urls
823 elif is_file_url(link):
824 unpack_file_url(link, location, download_dir, hashes=hashes)
825
826 # http urls
827 else:
828 if session is None:
829 session = PipSession()
830
831 unpack_http_url(
832 link,
833 location,
834 download_dir,
835 session,
836 hashes=hashes,
837 progress_bar=progress_bar
838 )
839 if only_download:
840 write_delete_marker_file(location)
841
842
843def _download_http_url(link, session, temp_dir, hashes, progress_bar):
844 """Download link url into temp_dir using provided session"""
845 target_url = link.url.split('#', 1)[0]
846 try:
847 resp = session.get(
848 target_url,
849 # We use Accept-Encoding: identity here because requests
850 # defaults to accepting compressed responses. This breaks in
851 # a variety of ways depending on how the server is configured.
852 # - Some servers will notice that the file isn't a compressible
853 # file and will leave the file alone and with an empty
854 # Content-Encoding
855 # - Some servers will notice that the file is already
856 # compressed and will leave the file alone and will add a
857 # Content-Encoding: gzip header
858 # - Some servers won't notice anything at all and will take
859 # a file that's already been compressed and compress it again
860 # and set the Content-Encoding: gzip header
861 # By setting this to request only the identity encoding We're
862 # hoping to eliminate the third case. Hopefully there does not
863 # exist a server which when given a file will notice it is
864 # already compressed and that you're not asking for a
865 # compressed file and will then decompress it before sending
866 # because if that's the case I don't think it'll ever be
867 # possible to make this work.
868 headers={"Accept-Encoding": "identity"},
869 stream=True,
870 )
871 resp.raise_for_status()
872 except requests.HTTPError as exc:
873 logger.critical(
874 "HTTP error %s while getting %s", exc.response.status_code, link,
875 )
876 raise
877
878 content_type = resp.headers.get('content-type', '')
879 filename = link.filename # fallback
880 # Have a look at the Content-Disposition header for a better guess
881 content_disposition = resp.headers.get('content-disposition')
882 if content_disposition:
883 type, params = cgi.parse_header(content_disposition)
884 # We use ``or`` here because we don't want to use an "empty" value
885 # from the filename param.
886 filename = params.get('filename') or filename
887 ext = splitext(filename)[1]
888 if not ext:
889 ext = mimetypes.guess_extension(content_type)
890 if ext:
891 filename += ext
892 if not ext and link.url != resp.url:
893 ext = os.path.splitext(resp.url)[1]
894 if ext:
895 filename += ext
896 file_path = os.path.join(temp_dir, filename)
897 with open(file_path, 'wb') as content_file:
898 _download_url(resp, link, content_file, hashes, progress_bar)
899 return file_path, content_type
900
901
902def _check_download_dir(link, download_dir, hashes):
903 """ Check download_dir for previously downloaded file with correct hash
904 If a correct file is found return its path else None
905 """
906 download_path = os.path.join(download_dir, link.filename)
907 if os.path.exists(download_path):
908 # If already downloaded, does its hash match?
909 logger.info('File was already downloaded %s', download_path)
910 if hashes:
911 try:
912 hashes.check_against_path(download_path)
913 except HashMismatch:
914 logger.warning(
915 'Previously-downloaded file %s has bad hash. '
916 'Re-downloading.',
917 download_path
918 )
919 os.unlink(download_path)
920 return None
921 return download_path
922 return None