From 68df54d6629ec019142eb149dd037774f2d11e7c Mon Sep 17 00:00:00 2001 From: Shubham Saini Date: Tue, 11 Dec 2018 15:31:23 +0530 Subject: First commit --- .../pip/_internal/operations/__init__.py | 0 .../pip/_internal/operations/check.py | 106 ++++++ .../pip/_internal/operations/freeze.py | 252 ++++++++++++++ .../pip/_internal/operations/prepare.py | 380 +++++++++++++++++++++ 4 files changed, 738 insertions(+) create mode 100644 venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/operations/__init__.py create mode 100644 venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/operations/check.py create mode 100644 venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/operations/freeze.py create mode 100644 venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/operations/prepare.py (limited to 'venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/operations') diff --git a/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/operations/__init__.py b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/operations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/operations/check.py b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/operations/check.py new file mode 100644 index 0000000..bab6b9f --- /dev/null +++ b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/operations/check.py @@ -0,0 +1,106 @@ +"""Validation of dependencies of packages +""" + +from collections import namedtuple + +from pip._vendor.packaging.utils import canonicalize_name + +from pip._internal.operations.prepare import make_abstract_dist + +from pip._internal.utils.misc import get_installed_distributions +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from pip._internal.req.req_install import InstallRequirement + from typing import Any, Dict, Iterator, Set, Tuple, List + + # Shorthands + PackageSet = Dict[str, 'PackageDetails'] + Missing = Tuple[str, Any] + Conflicting = Tuple[str, str, Any] + + MissingDict = Dict[str, List[Missing]] + ConflictingDict = Dict[str, List[Conflicting]] + CheckResult = Tuple[MissingDict, ConflictingDict] + +PackageDetails = namedtuple('PackageDetails', ['version', 'requires']) + + +def create_package_set_from_installed(**kwargs): + # type: (**Any) -> PackageSet + """Converts a list of distributions into a PackageSet. + """ + # Default to using all packages installed on the system + if kwargs == {}: + kwargs = {"local_only": False, "skip": ()} + retval = {} + for dist in get_installed_distributions(**kwargs): + name = canonicalize_name(dist.project_name) + retval[name] = PackageDetails(dist.version, dist.requires()) + return retval + + +def check_package_set(package_set): + # type: (PackageSet) -> CheckResult + """Check if a package set is consistent + """ + missing = dict() + conflicting = dict() + + for package_name in package_set: + # Info about dependencies of package_name + missing_deps = set() # type: Set[Missing] + conflicting_deps = set() # type: Set[Conflicting] + + for req in package_set[package_name].requires: + name = canonicalize_name(req.project_name) # type: str + + # Check if it's missing + if name not in package_set: + missed = True + if req.marker is not None: + missed = req.marker.evaluate() + if missed: + missing_deps.add((name, req)) + continue + + # Check if there's a conflict + version = package_set[name].version # type: str + if not req.specifier.contains(version, prereleases=True): + conflicting_deps.add((name, version, req)) + + def str_key(x): + return str(x) + + if missing_deps: + missing[package_name] = sorted(missing_deps, key=str_key) + if conflicting_deps: + conflicting[package_name] = sorted(conflicting_deps, key=str_key) + + return missing, conflicting + + +def check_install_conflicts(to_install): + # type: (List[InstallRequirement]) -> Tuple[PackageSet, CheckResult] + """For checking if the dependency graph would be consistent after \ + installing given requirements + """ + # Start from the current state + state = create_package_set_from_installed() + _simulate_installation_of(to_install, state) + return state, check_package_set(state) + + +# NOTE from @pradyunsg +# This required a minor update in dependency link handling logic over at +# operations.prepare.IsSDist.dist() to get it working +def _simulate_installation_of(to_install, state): + # type: (List[InstallRequirement], PackageSet) -> None + """Computes the version of packages after installing to_install. + """ + + # Modify it as installing requirement_set would (assuming no errors) + for inst_req in to_install: + dist = make_abstract_dist(inst_req).dist(finder=None) + name = canonicalize_name(dist.key) + state[name] = PackageDetails(dist.version, dist.requires()) diff --git a/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/operations/freeze.py b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/operations/freeze.py new file mode 100644 index 0000000..000102d --- /dev/null +++ b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/operations/freeze.py @@ -0,0 +1,252 @@ +from __future__ import absolute_import + +import collections +import logging +import os +import re +import warnings + +from pip._vendor import pkg_resources, six +from pip._vendor.packaging.utils import canonicalize_name +from pip._vendor.pkg_resources import RequirementParseError + +from pip._internal.exceptions import InstallationError +from pip._internal.req import InstallRequirement +from pip._internal.req.req_file import COMMENT_RE +from pip._internal.utils.deprecation import RemovedInPip11Warning +from pip._internal.utils.misc import ( + dist_is_editable, get_installed_distributions, +) + +logger = logging.getLogger(__name__) + + +def freeze( + requirement=None, + find_links=None, local_only=None, user_only=None, skip_regex=None, + isolated=False, + wheel_cache=None, + exclude_editable=False, + skip=()): + find_links = find_links or [] + skip_match = None + + if skip_regex: + skip_match = re.compile(skip_regex).search + + dependency_links = [] + + for dist in pkg_resources.working_set: + if dist.has_metadata('dependency_links.txt'): + dependency_links.extend( + dist.get_metadata_lines('dependency_links.txt') + ) + for link in find_links: + if '#egg=' in link: + dependency_links.append(link) + for link in find_links: + yield '-f %s' % link + installations = {} + for dist in get_installed_distributions(local_only=local_only, + skip=(), + user_only=user_only): + try: + req = FrozenRequirement.from_dist( + dist, + dependency_links + ) + except RequirementParseError: + logger.warning( + "Could not parse requirement: %s", + dist.project_name + ) + continue + if exclude_editable and req.editable: + continue + installations[req.name] = req + + if requirement: + # the options that don't get turned into an InstallRequirement + # should only be emitted once, even if the same option is in multiple + # requirements files, so we need to keep track of what has been emitted + # so that we don't emit it again if it's seen again + emitted_options = set() + # keep track of which files a requirement is in so that we can + # give an accurate warning if a requirement appears multiple times. + req_files = collections.defaultdict(list) + for req_file_path in requirement: + with open(req_file_path) as req_file: + for line in req_file: + if (not line.strip() or + line.strip().startswith('#') or + (skip_match and skip_match(line)) or + line.startswith(( + '-r', '--requirement', + '-Z', '--always-unzip', + '-f', '--find-links', + '-i', '--index-url', + '--pre', + '--trusted-host', + '--process-dependency-links', + '--extra-index-url'))): + line = line.rstrip() + if line not in emitted_options: + emitted_options.add(line) + yield line + continue + + if line.startswith('-e') or line.startswith('--editable'): + if line.startswith('-e'): + line = line[2:].strip() + else: + line = line[len('--editable'):].strip().lstrip('=') + line_req = InstallRequirement.from_editable( + line, + isolated=isolated, + wheel_cache=wheel_cache, + ) + else: + line_req = InstallRequirement.from_line( + COMMENT_RE.sub('', line).strip(), + isolated=isolated, + wheel_cache=wheel_cache, + ) + + if not line_req.name: + logger.info( + "Skipping line in requirement file [%s] because " + "it's not clear what it would install: %s", + req_file_path, line.strip(), + ) + logger.info( + " (add #egg=PackageName to the URL to avoid" + " this warning)" + ) + elif line_req.name not in installations: + # either it's not installed, or it is installed + # but has been processed already + if not req_files[line_req.name]: + logger.warning( + "Requirement file [%s] contains %s, but that " + "package is not installed", + req_file_path, + COMMENT_RE.sub('', line).strip(), + ) + else: + req_files[line_req.name].append(req_file_path) + else: + yield str(installations[line_req.name]).rstrip() + del installations[line_req.name] + req_files[line_req.name].append(req_file_path) + + # Warn about requirements that were included multiple times (in a + # single requirements file or in different requirements files). + for name, files in six.iteritems(req_files): + if len(files) > 1: + logger.warning("Requirement %s included multiple times [%s]", + name, ', '.join(sorted(set(files)))) + + yield( + '## The following requirements were added by ' + 'pip freeze:' + ) + for installation in sorted( + installations.values(), key=lambda x: x.name.lower()): + if canonicalize_name(installation.name) not in skip: + yield str(installation).rstrip() + + +class FrozenRequirement(object): + def __init__(self, name, req, editable, comments=()): + self.name = name + self.req = req + self.editable = editable + self.comments = comments + + _rev_re = re.compile(r'-r(\d+)$') + _date_re = re.compile(r'-(20\d\d\d\d\d\d)$') + + @classmethod + def from_dist(cls, dist, dependency_links): + location = os.path.normcase(os.path.abspath(dist.location)) + comments = [] + from pip._internal.vcs import vcs, get_src_requirement + if dist_is_editable(dist) and vcs.get_backend_name(location): + editable = True + try: + req = get_src_requirement(dist, location) + except InstallationError as exc: + logger.warning( + "Error when trying to get requirement for VCS system %s, " + "falling back to uneditable format", exc + ) + req = None + if req is None: + logger.warning( + 'Could not determine repository location of %s', location + ) + comments.append( + '## !! Could not determine repository location' + ) + req = dist.as_requirement() + editable = False + else: + editable = False + req = dist.as_requirement() + specs = req.specs + assert len(specs) == 1 and specs[0][0] in ["==", "==="], \ + 'Expected 1 spec with == or ===; specs = %r; dist = %r' % \ + (specs, dist) + version = specs[0][1] + ver_match = cls._rev_re.search(version) + date_match = cls._date_re.search(version) + if ver_match or date_match: + svn_backend = vcs.get_backend('svn') + if svn_backend: + svn_location = svn_backend().get_location( + dist, + dependency_links, + ) + if not svn_location: + logger.warning( + 'Warning: cannot find svn location for %s', req, + ) + comments.append( + '## FIXME: could not find svn URL in dependency_links ' + 'for this package:' + ) + else: + warnings.warn( + "SVN editable detection based on dependency links " + "will be dropped in the future.", + RemovedInPip11Warning, + ) + comments.append( + '# Installing as editable to satisfy requirement %s:' % + req + ) + if ver_match: + rev = ver_match.group(1) + else: + rev = '{%s}' % date_match.group(1) + editable = True + req = '%s@%s#egg=%s' % ( + svn_location, + rev, + cls.egg_name(dist) + ) + return cls(dist.project_name, req, editable, comments) + + @staticmethod + def egg_name(dist): + name = dist.egg_name() + match = re.search(r'-py\d\.\d$', name) + if match: + name = name[:match.start()] + return name + + def __str__(self): + req = self.req + if self.editable: + req = '-e %s' % req + return '\n'.join(list(self.comments) + [str(req)]) + '\n' diff --git a/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/operations/prepare.py b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/operations/prepare.py new file mode 100644 index 0000000..c1e8158 --- /dev/null +++ b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/operations/prepare.py @@ -0,0 +1,380 @@ +"""Prepares a distribution for installation +""" + +import itertools +import logging +import os +import sys +from copy import copy + +from pip._vendor import pkg_resources, requests + +from pip._internal.build_env import NoOpBuildEnvironment +from pip._internal.compat import expanduser +from pip._internal.download import ( + is_dir_url, is_file_url, is_vcs_url, unpack_url, url_to_path, +) +from pip._internal.exceptions import ( + DirectoryUrlHashUnsupported, HashUnpinned, InstallationError, + PreviousBuildDirError, VcsHashUnsupported, +) +from pip._internal.index import FormatControl +from pip._internal.req.req_install import InstallRequirement +from pip._internal.utils.hashes import MissingHashes +from pip._internal.utils.logging import indent_log +from pip._internal.utils.misc import ( + call_subprocess, display_path, normalize_path, +) +from pip._internal.utils.ui import open_spinner +from pip._internal.vcs import vcs + +logger = logging.getLogger(__name__) + + +def make_abstract_dist(req): + """Factory to make an abstract dist object. + + Preconditions: Either an editable req with a source_dir, or satisfied_by or + a wheel link, or a non-editable req with a source_dir. + + :return: A concrete DistAbstraction. + """ + if req.editable: + return IsSDist(req) + elif req.link and req.link.is_wheel: + return IsWheel(req) + else: + return IsSDist(req) + + +def _install_build_reqs(finder, prefix, build_requirements): + # NOTE: What follows is not a very good thing. + # Eventually, this should move into the BuildEnvironment class and + # that should handle all the isolation and sub-process invocation. + finder = copy(finder) + finder.format_control = FormatControl(set(), set([":all:"])) + urls = [ + finder.find_requirement( + InstallRequirement.from_line(r), upgrade=False).url + for r in build_requirements + ] + args = [ + sys.executable, '-m', 'pip', 'install', '--ignore-installed', + '--no-user', '--prefix', prefix, + ] + list(urls) + + with open_spinner("Installing build dependencies") as spinner: + call_subprocess(args, show_stdout=False, spinner=spinner) + + +class DistAbstraction(object): + """Abstracts out the wheel vs non-wheel Resolver.resolve() logic. + + The requirements for anything installable are as follows: + - we must be able to determine the requirement name + (or we can't correctly handle the non-upgrade case). + - we must be able to generate a list of run-time dependencies + without installing any additional packages (or we would + have to either burn time by doing temporary isolated installs + or alternatively violate pips 'don't start installing unless + all requirements are available' rule - neither of which are + desirable). + - for packages with setup requirements, we must also be able + to determine their requirements without installing additional + packages (for the same reason as run-time dependencies) + - we must be able to create a Distribution object exposing the + above metadata. + """ + + def __init__(self, req): + self.req = req + + def dist(self, finder): + """Return a setuptools Dist object.""" + raise NotImplementedError(self.dist) + + def prep_for_dist(self, finder): + """Ensure that we can get a Dist for this requirement.""" + raise NotImplementedError(self.dist) + + +class IsWheel(DistAbstraction): + + def dist(self, finder): + return list(pkg_resources.find_distributions( + self.req.source_dir))[0] + + def prep_for_dist(self, finder, build_isolation): + # FIXME:https://github.com/pypa/pip/issues/1112 + pass + + +class IsSDist(DistAbstraction): + + def dist(self, finder): + dist = self.req.get_dist() + # FIXME: shouldn't be globally added. + if finder and dist.has_metadata('dependency_links.txt'): + finder.add_dependency_links( + dist.get_metadata_lines('dependency_links.txt') + ) + return dist + + def prep_for_dist(self, finder, build_isolation): + # Before calling "setup.py egg_info", we need to set-up the build + # environment. + build_requirements, isolate = self.req.get_pep_518_info() + should_isolate = build_isolation and isolate + + minimum_requirements = ('setuptools', 'wheel') + missing_requirements = set(minimum_requirements) - set( + pkg_resources.Requirement(r).key + for r in build_requirements + ) + if missing_requirements: + def format_reqs(rs): + return ' and '.join(map(repr, sorted(rs))) + logger.warning( + "Missing build time requirements in pyproject.toml for %s: " + "%s.", self.req, format_reqs(missing_requirements) + ) + logger.warning( + "This version of pip does not implement PEP 517 so it cannot " + "build a wheel without %s.", format_reqs(minimum_requirements) + ) + + if should_isolate: + with self.req.build_env: + pass + _install_build_reqs(finder, self.req.build_env.path, + build_requirements) + else: + self.req.build_env = NoOpBuildEnvironment(no_clean=False) + + self.req.run_egg_info() + self.req.assert_source_matches_version() + + +class Installed(DistAbstraction): + + def dist(self, finder): + return self.req.satisfied_by + + def prep_for_dist(self, finder): + pass + + +class RequirementPreparer(object): + """Prepares a Requirement + """ + + def __init__(self, build_dir, download_dir, src_dir, wheel_download_dir, + progress_bar, build_isolation): + super(RequirementPreparer, self).__init__() + + self.src_dir = src_dir + self.build_dir = build_dir + + # Where still packed archives should be written to. If None, they are + # not saved, and are deleted immediately after unpacking. + self.download_dir = download_dir + + # Where still-packed .whl files should be written to. If None, they are + # written to the download_dir parameter. Separate to download_dir to + # permit only keeping wheel archives for pip wheel. + if wheel_download_dir: + wheel_download_dir = normalize_path(wheel_download_dir) + self.wheel_download_dir = wheel_download_dir + + # NOTE + # download_dir and wheel_download_dir overlap semantically and may + # be combined if we're willing to have non-wheel archives present in + # the wheelhouse output by 'pip wheel'. + + self.progress_bar = progress_bar + + # Is build isolation allowed? + self.build_isolation = build_isolation + + @property + def _download_should_save(self): + # TODO: Modify to reduce indentation needed + if self.download_dir: + self.download_dir = expanduser(self.download_dir) + if os.path.exists(self.download_dir): + return True + else: + logger.critical('Could not find download directory') + raise InstallationError( + "Could not find or access download directory '%s'" + % display_path(self.download_dir)) + return False + + def prepare_linked_requirement(self, req, session, finder, + upgrade_allowed, require_hashes): + """Prepare a requirement that would be obtained from req.link + """ + # TODO: Breakup into smaller functions + if req.link and req.link.scheme == 'file': + path = url_to_path(req.link.url) + logger.info('Processing %s', display_path(path)) + else: + logger.info('Collecting %s', req) + + with indent_log(): + # @@ if filesystem packages are not marked + # editable in a req, a non deterministic error + # occurs when the script attempts to unpack the + # build directory + req.ensure_has_source_dir(self.build_dir) + # If a checkout exists, it's unwise to keep going. version + # inconsistencies are logged later, but do not fail the + # installation. + # FIXME: this won't upgrade when there's an existing + # package unpacked in `req.source_dir` + # package unpacked in `req.source_dir` + if os.path.exists(os.path.join(req.source_dir, 'setup.py')): + raise PreviousBuildDirError( + "pip can't proceed with requirements '%s' due to a" + " pre-existing build directory (%s). This is " + "likely due to a previous installation that failed" + ". pip is being responsible and not assuming it " + "can delete this. Please delete it and try again." + % (req, req.source_dir) + ) + req.populate_link(finder, upgrade_allowed, require_hashes) + + # We can't hit this spot and have populate_link return None. + # req.satisfied_by is None here (because we're + # guarded) and upgrade has no impact except when satisfied_by + # is not None. + # Then inside find_requirement existing_applicable -> False + # If no new versions are found, DistributionNotFound is raised, + # otherwise a result is guaranteed. + assert req.link + link = req.link + + # Now that we have the real link, we can tell what kind of + # requirements we have and raise some more informative errors + # than otherwise. (For example, we can raise VcsHashUnsupported + # for a VCS URL rather than HashMissing.) + if require_hashes: + # We could check these first 2 conditions inside + # unpack_url and save repetition of conditions, but then + # we would report less-useful error messages for + # unhashable requirements, complaining that there's no + # hash provided. + if is_vcs_url(link): + raise VcsHashUnsupported() + elif is_file_url(link) and is_dir_url(link): + raise DirectoryUrlHashUnsupported() + if not req.original_link and not req.is_pinned: + # Unpinned packages are asking for trouble when a new + # version is uploaded. This isn't a security check, but + # it saves users a surprising hash mismatch in the + # future. + # + # file:/// URLs aren't pinnable, so don't complain + # about them not being pinned. + raise HashUnpinned() + + hashes = req.hashes(trust_internet=not require_hashes) + if require_hashes and not hashes: + # Known-good hashes are missing for this requirement, so + # shim it with a facade object that will provoke hash + # computation and then raise a HashMissing exception + # showing the user what the hash should be. + hashes = MissingHashes() + + try: + download_dir = self.download_dir + # We always delete unpacked sdists after pip ran. + autodelete_unpacked = True + if req.link.is_wheel and self.wheel_download_dir: + # when doing 'pip wheel` we download wheels to a + # dedicated dir. + download_dir = self.wheel_download_dir + if req.link.is_wheel: + if download_dir: + # When downloading, we only unpack wheels to get + # metadata. + autodelete_unpacked = True + else: + # When installing a wheel, we use the unpacked + # wheel. + autodelete_unpacked = False + unpack_url( + req.link, req.source_dir, + download_dir, autodelete_unpacked, + session=session, hashes=hashes, + progress_bar=self.progress_bar + ) + except requests.HTTPError as exc: + logger.critical( + 'Could not install requirement %s because of error %s', + req, + exc, + ) + raise InstallationError( + 'Could not install requirement %s because of HTTP ' + 'error %s for URL %s' % + (req, exc, req.link) + ) + abstract_dist = make_abstract_dist(req) + abstract_dist.prep_for_dist(finder, self.build_isolation) + if self._download_should_save: + # Make a .zip of the source_dir we already created. + if req.link.scheme in vcs.all_schemes: + req.archive(self.download_dir) + return abstract_dist + + def prepare_editable_requirement(self, req, require_hashes, use_user_site, + finder): + """Prepare an editable requirement + """ + assert req.editable, "cannot prepare a non-editable req as editable" + + logger.info('Obtaining %s', req) + + with indent_log(): + if require_hashes: + raise InstallationError( + 'The editable requirement %s cannot be installed when ' + 'requiring hashes, because there is no single file to ' + 'hash.' % req + ) + req.ensure_has_source_dir(self.src_dir) + req.update_editable(not self._download_should_save) + + abstract_dist = make_abstract_dist(req) + abstract_dist.prep_for_dist(finder, self.build_isolation) + + if self._download_should_save: + req.archive(self.download_dir) + req.check_if_exists(use_user_site) + + return abstract_dist + + def prepare_installed_requirement(self, req, require_hashes, skip_reason): + """Prepare an already-installed requirement + """ + assert req.satisfied_by, "req should have been satisfied but isn't" + assert skip_reason is not None, ( + "did not get skip reason skipped but req.satisfied_by " + "is set to %r" % (req.satisfied_by,) + ) + logger.info( + 'Requirement %s: %s (%s)', + skip_reason, req, req.satisfied_by.version + ) + with indent_log(): + if require_hashes: + logger.debug( + 'Since it is already installed, we are trusting this ' + 'package without checking its hash. To ensure a ' + 'completely repeatable environment, install into an ' + 'empty virtualenv.' + ) + abstract_dist = Installed(req) + + return abstract_dist -- cgit v1.2.3