diff options
Diffstat (limited to 'venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/distlib/index.py')
-rw-r--r-- | venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/distlib/index.py | 516 |
1 files changed, 516 insertions, 0 deletions
diff --git a/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/distlib/index.py b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/distlib/index.py new file mode 100644 index 0000000..7197238 --- /dev/null +++ b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/distlib/index.py | |||
@@ -0,0 +1,516 @@ | |||
1 | # -*- coding: utf-8 -*- | ||
2 | # | ||
3 | # Copyright (C) 2013 Vinay Sajip. | ||
4 | # Licensed to the Python Software Foundation under a contributor agreement. | ||
5 | # See LICENSE.txt and CONTRIBUTORS.txt. | ||
6 | # | ||
7 | import hashlib | ||
8 | import logging | ||
9 | import os | ||
10 | import shutil | ||
11 | import subprocess | ||
12 | import tempfile | ||
13 | try: | ||
14 | from threading import Thread | ||
15 | except ImportError: | ||
16 | from dummy_threading import Thread | ||
17 | |||
18 | from . import DistlibException | ||
19 | from .compat import (HTTPBasicAuthHandler, Request, HTTPPasswordMgr, | ||
20 | urlparse, build_opener, string_types) | ||
21 | from .util import cached_property, zip_dir, ServerProxy | ||
22 | |||
23 | logger = logging.getLogger(__name__) | ||
24 | |||
25 | DEFAULT_INDEX = 'https://pypi.python.org/pypi' | ||
26 | DEFAULT_REALM = 'pypi' | ||
27 | |||
28 | class PackageIndex(object): | ||
29 | """ | ||
30 | This class represents a package index compatible with PyPI, the Python | ||
31 | Package Index. | ||
32 | """ | ||
33 | |||
34 | boundary = b'----------ThIs_Is_tHe_distlib_index_bouNdaRY_$' | ||
35 | |||
36 | def __init__(self, url=None): | ||
37 | """ | ||
38 | Initialise an instance. | ||
39 | |||
40 | :param url: The URL of the index. If not specified, the URL for PyPI is | ||
41 | used. | ||
42 | """ | ||
43 | self.url = url or DEFAULT_INDEX | ||
44 | self.read_configuration() | ||
45 | scheme, netloc, path, params, query, frag = urlparse(self.url) | ||
46 | if params or query or frag or scheme not in ('http', 'https'): | ||
47 | raise DistlibException('invalid repository: %s' % self.url) | ||
48 | self.password_handler = None | ||
49 | self.ssl_verifier = None | ||
50 | self.gpg = None | ||
51 | self.gpg_home = None | ||
52 | with open(os.devnull, 'w') as sink: | ||
53 | # Use gpg by default rather than gpg2, as gpg2 insists on | ||
54 | # prompting for passwords | ||
55 | for s in ('gpg', 'gpg2'): | ||
56 | try: | ||
57 | rc = subprocess.check_call([s, '--version'], stdout=sink, | ||
58 | stderr=sink) | ||
59 | if rc == 0: | ||
60 | self.gpg = s | ||
61 | break | ||
62 | except OSError: | ||
63 | pass | ||
64 | |||
65 | def _get_pypirc_command(self): | ||
66 | """ | ||
67 | Get the distutils command for interacting with PyPI configurations. | ||
68 | :return: the command. | ||
69 | """ | ||
70 | from distutils.core import Distribution | ||
71 | from distutils.config import PyPIRCCommand | ||
72 | d = Distribution() | ||
73 | return PyPIRCCommand(d) | ||
74 | |||
75 | def read_configuration(self): | ||
76 | """ | ||
77 | Read the PyPI access configuration as supported by distutils, getting | ||
78 | PyPI to do the actual work. This populates ``username``, ``password``, | ||
79 | ``realm`` and ``url`` attributes from the configuration. | ||
80 | """ | ||
81 | # get distutils to do the work | ||
82 | c = self._get_pypirc_command() | ||
83 | c.repository = self.url | ||
84 | cfg = c._read_pypirc() | ||
85 | self.username = cfg.get('username') | ||
86 | self.password = cfg.get('password') | ||
87 | self.realm = cfg.get('realm', 'pypi') | ||
88 | self.url = cfg.get('repository', self.url) | ||
89 | |||
90 | def save_configuration(self): | ||
91 | """ | ||
92 | Save the PyPI access configuration. You must have set ``username`` and | ||
93 | ``password`` attributes before calling this method. | ||
94 | |||
95 | Again, distutils is used to do the actual work. | ||
96 | """ | ||
97 | self.check_credentials() | ||
98 | # get distutils to do the work | ||
99 | c = self._get_pypirc_command() | ||
100 | c._store_pypirc(self.username, self.password) | ||
101 | |||
102 | def check_credentials(self): | ||
103 | """ | ||
104 | Check that ``username`` and ``password`` have been set, and raise an | ||
105 | exception if not. | ||
106 | """ | ||
107 | if self.username is None or self.password is None: | ||
108 | raise DistlibException('username and password must be set') | ||
109 | pm = HTTPPasswordMgr() | ||
110 | _, netloc, _, _, _, _ = urlparse(self.url) | ||
111 | pm.add_password(self.realm, netloc, self.username, self.password) | ||
112 | self.password_handler = HTTPBasicAuthHandler(pm) | ||
113 | |||
114 | def register(self, metadata): | ||
115 | """ | ||
116 | Register a distribution on PyPI, using the provided metadata. | ||
117 | |||
118 | :param metadata: A :class:`Metadata` instance defining at least a name | ||
119 | and version number for the distribution to be | ||
120 | registered. | ||
121 | :return: The HTTP response received from PyPI upon submission of the | ||
122 | request. | ||
123 | """ | ||
124 | self.check_credentials() | ||
125 | metadata.validate() | ||
126 | d = metadata.todict() | ||
127 | d[':action'] = 'verify' | ||
128 | request = self.encode_request(d.items(), []) | ||
129 | response = self.send_request(request) | ||
130 | d[':action'] = 'submit' | ||
131 | request = self.encode_request(d.items(), []) | ||
132 | return self.send_request(request) | ||
133 | |||
134 | def _reader(self, name, stream, outbuf): | ||
135 | """ | ||
136 | Thread runner for reading lines of from a subprocess into a buffer. | ||
137 | |||
138 | :param name: The logical name of the stream (used for logging only). | ||
139 | :param stream: The stream to read from. This will typically a pipe | ||
140 | connected to the output stream of a subprocess. | ||
141 | :param outbuf: The list to append the read lines to. | ||
142 | """ | ||
143 | while True: | ||
144 | s = stream.readline() | ||
145 | if not s: | ||
146 | break | ||
147 | s = s.decode('utf-8').rstrip() | ||
148 | outbuf.append(s) | ||
149 | logger.debug('%s: %s' % (name, s)) | ||
150 | stream.close() | ||
151 | |||
152 | def get_sign_command(self, filename, signer, sign_password, | ||
153 | keystore=None): | ||
154 | """ | ||
155 | Return a suitable command for signing a file. | ||
156 | |||
157 | :param filename: The pathname to the file to be signed. | ||
158 | :param signer: The identifier of the signer of the file. | ||
159 | :param sign_password: The passphrase for the signer's | ||
160 | private key used for signing. | ||
161 | :param keystore: The path to a directory which contains the keys | ||
162 | used in verification. If not specified, the | ||
163 | instance's ``gpg_home`` attribute is used instead. | ||
164 | :return: The signing command as a list suitable to be | ||
165 | passed to :class:`subprocess.Popen`. | ||
166 | """ | ||
167 | cmd = [self.gpg, '--status-fd', '2', '--no-tty'] | ||
168 | if keystore is None: | ||
169 | keystore = self.gpg_home | ||
170 | if keystore: | ||
171 | cmd.extend(['--homedir', keystore]) | ||
172 | if sign_password is not None: | ||
173 | cmd.extend(['--batch', '--passphrase-fd', '0']) | ||
174 | td = tempfile.mkdtemp() | ||
175 | sf = os.path.join(td, os.path.basename(filename) + '.asc') | ||
176 | cmd.extend(['--detach-sign', '--armor', '--local-user', | ||
177 | signer, '--output', sf, filename]) | ||
178 | logger.debug('invoking: %s', ' '.join(cmd)) | ||
179 | return cmd, sf | ||
180 | |||
181 | def run_command(self, cmd, input_data=None): | ||
182 | """ | ||
183 | Run a command in a child process , passing it any input data specified. | ||
184 | |||
185 | :param cmd: The command to run. | ||
186 | :param input_data: If specified, this must be a byte string containing | ||
187 | data to be sent to the child process. | ||
188 | :return: A tuple consisting of the subprocess' exit code, a list of | ||
189 | lines read from the subprocess' ``stdout``, and a list of | ||
190 | lines read from the subprocess' ``stderr``. | ||
191 | """ | ||
192 | kwargs = { | ||
193 | 'stdout': subprocess.PIPE, | ||
194 | 'stderr': subprocess.PIPE, | ||
195 | } | ||
196 | if input_data is not None: | ||
197 | kwargs['stdin'] = subprocess.PIPE | ||
198 | stdout = [] | ||
199 | stderr = [] | ||
200 | p = subprocess.Popen(cmd, **kwargs) | ||
201 | # We don't use communicate() here because we may need to | ||
202 | # get clever with interacting with the command | ||
203 | t1 = Thread(target=self._reader, args=('stdout', p.stdout, stdout)) | ||
204 | t1.start() | ||
205 | t2 = Thread(target=self._reader, args=('stderr', p.stderr, stderr)) | ||
206 | t2.start() | ||
207 | if input_data is not None: | ||
208 | p.stdin.write(input_data) | ||
209 | p.stdin.close() | ||
210 | |||
211 | p.wait() | ||
212 | t1.join() | ||
213 | t2.join() | ||
214 | return p.returncode, stdout, stderr | ||
215 | |||
216 | def sign_file(self, filename, signer, sign_password, keystore=None): | ||
217 | """ | ||
218 | Sign a file. | ||
219 | |||
220 | :param filename: The pathname to the file to be signed. | ||
221 | :param signer: The identifier of the signer of the file. | ||
222 | :param sign_password: The passphrase for the signer's | ||
223 | private key used for signing. | ||
224 | :param keystore: The path to a directory which contains the keys | ||
225 | used in signing. If not specified, the instance's | ||
226 | ``gpg_home`` attribute is used instead. | ||
227 | :return: The absolute pathname of the file where the signature is | ||
228 | stored. | ||
229 | """ | ||
230 | cmd, sig_file = self.get_sign_command(filename, signer, sign_password, | ||
231 | keystore) | ||
232 | rc, stdout, stderr = self.run_command(cmd, | ||
233 | sign_password.encode('utf-8')) | ||
234 | if rc != 0: | ||
235 | raise DistlibException('sign command failed with error ' | ||
236 | 'code %s' % rc) | ||
237 | return sig_file | ||
238 | |||
239 | def upload_file(self, metadata, filename, signer=None, sign_password=None, | ||
240 | filetype='sdist', pyversion='source', keystore=None): | ||
241 | """ | ||
242 | Upload a release file to the index. | ||
243 | |||
244 | :param metadata: A :class:`Metadata` instance defining at least a name | ||
245 | and version number for the file to be uploaded. | ||
246 | :param filename: The pathname of the file to be uploaded. | ||
247 | :param signer: The identifier of the signer of the file. | ||
248 | :param sign_password: The passphrase for the signer's | ||
249 | private key used for signing. | ||
250 | :param filetype: The type of the file being uploaded. This is the | ||
251 | distutils command which produced that file, e.g. | ||
252 | ``sdist`` or ``bdist_wheel``. | ||
253 | :param pyversion: The version of Python which the release relates | ||
254 | to. For code compatible with any Python, this would | ||
255 | be ``source``, otherwise it would be e.g. ``3.2``. | ||
256 | :param keystore: The path to a directory which contains the keys | ||
257 | used in signing. If not specified, the instance's | ||
258 | ``gpg_home`` attribute is used instead. | ||
259 | :return: The HTTP response received from PyPI upon submission of the | ||
260 | request. | ||
261 | """ | ||
262 | self.check_credentials() | ||
263 | if not os.path.exists(filename): | ||
264 | raise DistlibException('not found: %s' % filename) | ||
265 | metadata.validate() | ||
266 | d = metadata.todict() | ||
267 | sig_file = None | ||
268 | if signer: | ||
269 | if not self.gpg: | ||
270 | logger.warning('no signing program available - not signed') | ||
271 | else: | ||
272 | sig_file = self.sign_file(filename, signer, sign_password, | ||
273 | keystore) | ||
274 | with open(filename, 'rb') as f: | ||
275 | file_data = f.read() | ||
276 | md5_digest = hashlib.md5(file_data).hexdigest() | ||
277 | sha256_digest = hashlib.sha256(file_data).hexdigest() | ||
278 | d.update({ | ||
279 | ':action': 'file_upload', | ||
280 | 'protocol_version': '1', | ||
281 | 'filetype': filetype, | ||
282 | 'pyversion': pyversion, | ||
283 | 'md5_digest': md5_digest, | ||
284 | 'sha256_digest': sha256_digest, | ||
285 | }) | ||
286 | files = [('content', os.path.basename(filename), file_data)] | ||
287 | if sig_file: | ||
288 | with open(sig_file, 'rb') as f: | ||
289 | sig_data = f.read() | ||
290 | files.append(('gpg_signature', os.path.basename(sig_file), | ||
291 | sig_data)) | ||
292 | shutil.rmtree(os.path.dirname(sig_file)) | ||
293 | request = self.encode_request(d.items(), files) | ||
294 | return self.send_request(request) | ||
295 | |||
296 | def upload_documentation(self, metadata, doc_dir): | ||
297 | """ | ||
298 | Upload documentation to the index. | ||
299 | |||
300 | :param metadata: A :class:`Metadata` instance defining at least a name | ||
301 | and version number for the documentation to be | ||
302 | uploaded. | ||
303 | :param doc_dir: The pathname of the directory which contains the | ||
304 | documentation. This should be the directory that | ||
305 | contains the ``index.html`` for the documentation. | ||
306 | :return: The HTTP response received from PyPI upon submission of the | ||
307 | request. | ||
308 | """ | ||
309 | self.check_credentials() | ||
310 | if not os.path.isdir(doc_dir): | ||
311 | raise DistlibException('not a directory: %r' % doc_dir) | ||
312 | fn = os.path.join(doc_dir, 'index.html') | ||
313 | if not os.path.exists(fn): | ||
314 | raise DistlibException('not found: %r' % fn) | ||
315 | metadata.validate() | ||
316 | name, version = metadata.name, metadata.version | ||
317 | zip_data = zip_dir(doc_dir).getvalue() | ||
318 | fields = [(':action', 'doc_upload'), | ||
319 | ('name', name), ('version', version)] | ||
320 | files = [('content', name, zip_data)] | ||
321 | request = self.encode_request(fields, files) | ||
322 | return self.send_request(request) | ||
323 | |||
324 | def get_verify_command(self, signature_filename, data_filename, | ||
325 | keystore=None): | ||
326 | """ | ||
327 | Return a suitable command for verifying a file. | ||
328 | |||
329 | :param signature_filename: The pathname to the file containing the | ||
330 | signature. | ||
331 | :param data_filename: The pathname to the file containing the | ||
332 | signed data. | ||
333 | :param keystore: The path to a directory which contains the keys | ||
334 | used in verification. If not specified, the | ||
335 | instance's ``gpg_home`` attribute is used instead. | ||
336 | :return: The verifying command as a list suitable to be | ||
337 | passed to :class:`subprocess.Popen`. | ||
338 | """ | ||
339 | cmd = [self.gpg, '--status-fd', '2', '--no-tty'] | ||
340 | if keystore is None: | ||
341 | keystore = self.gpg_home | ||
342 | if keystore: | ||
343 | cmd.extend(['--homedir', keystore]) | ||
344 | cmd.extend(['--verify', signature_filename, data_filename]) | ||
345 | logger.debug('invoking: %s', ' '.join(cmd)) | ||
346 | return cmd | ||
347 | |||
348 | def verify_signature(self, signature_filename, data_filename, | ||
349 | keystore=None): | ||
350 | """ | ||
351 | Verify a signature for a file. | ||
352 | |||
353 | :param signature_filename: The pathname to the file containing the | ||
354 | signature. | ||
355 | :param data_filename: The pathname to the file containing the | ||
356 | signed data. | ||
357 | :param keystore: The path to a directory which contains the keys | ||
358 | used in verification. If not specified, the | ||
359 | instance's ``gpg_home`` attribute is used instead. | ||
360 | :return: True if the signature was verified, else False. | ||
361 | """ | ||
362 | if not self.gpg: | ||
363 | raise DistlibException('verification unavailable because gpg ' | ||
364 | 'unavailable') | ||
365 | cmd = self.get_verify_command(signature_filename, data_filename, | ||
366 | keystore) | ||
367 | rc, stdout, stderr = self.run_command(cmd) | ||
368 | if rc not in (0, 1): | ||
369 | raise DistlibException('verify command failed with error ' | ||
370 | 'code %s' % rc) | ||
371 | return rc == 0 | ||
372 | |||
373 | def download_file(self, url, destfile, digest=None, reporthook=None): | ||
374 | """ | ||
375 | This is a convenience method for downloading a file from an URL. | ||
376 | Normally, this will be a file from the index, though currently | ||
377 | no check is made for this (i.e. a file can be downloaded from | ||
378 | anywhere). | ||
379 | |||
380 | The method is just like the :func:`urlretrieve` function in the | ||
381 | standard library, except that it allows digest computation to be | ||
382 | done during download and checking that the downloaded data | ||
383 | matched any expected value. | ||
384 | |||
385 | :param url: The URL of the file to be downloaded (assumed to be | ||
386 | available via an HTTP GET request). | ||
387 | :param destfile: The pathname where the downloaded file is to be | ||
388 | saved. | ||
389 | :param digest: If specified, this must be a (hasher, value) | ||
390 | tuple, where hasher is the algorithm used (e.g. | ||
391 | ``'md5'``) and ``value`` is the expected value. | ||
392 | :param reporthook: The same as for :func:`urlretrieve` in the | ||
393 | standard library. | ||
394 | """ | ||
395 | if digest is None: | ||
396 | digester = None | ||
397 | logger.debug('No digest specified') | ||
398 | else: | ||
399 | if isinstance(digest, (list, tuple)): | ||
400 | hasher, digest = digest | ||
401 | else: | ||
402 | hasher = 'md5' | ||
403 | digester = getattr(hashlib, hasher)() | ||
404 | logger.debug('Digest specified: %s' % digest) | ||
405 | # The following code is equivalent to urlretrieve. | ||
406 | # We need to do it this way so that we can compute the | ||
407 | # digest of the file as we go. | ||
408 | with open(destfile, 'wb') as dfp: | ||
409 | # addinfourl is not a context manager on 2.x | ||
410 | # so we have to use try/finally | ||
411 | sfp = self.send_request(Request(url)) | ||
412 | try: | ||
413 | headers = sfp.info() | ||
414 | blocksize = 8192 | ||
415 | size = -1 | ||
416 | read = 0 | ||
417 | blocknum = 0 | ||
418 | if "content-length" in headers: | ||
419 | size = int(headers["Content-Length"]) | ||
420 | if reporthook: | ||
421 | reporthook(blocknum, blocksize, size) | ||
422 | while True: | ||
423 | block = sfp.read(blocksize) | ||
424 | if not block: | ||
425 | break | ||
426 | read += len(block) | ||
427 | dfp.write(block) | ||
428 | if digester: | ||
429 | digester.update(block) | ||
430 | blocknum += 1 | ||
431 | if reporthook: | ||
432 | reporthook(blocknum, blocksize, size) | ||
433 | finally: | ||
434 | sfp.close() | ||
435 | |||
436 | # check that we got the whole file, if we can | ||
437 | if size >= 0 and read < size: | ||
438 | raise DistlibException( | ||
439 | 'retrieval incomplete: got only %d out of %d bytes' | ||
440 | % (read, size)) | ||
441 | # if we have a digest, it must match. | ||
442 | if digester: | ||
443 | actual = digester.hexdigest() | ||
444 | if digest != actual: | ||
445 | raise DistlibException('%s digest mismatch for %s: expected ' | ||
446 | '%s, got %s' % (hasher, destfile, | ||
447 | digest, actual)) | ||
448 | logger.debug('Digest verified: %s', digest) | ||
449 | |||
450 | def send_request(self, req): | ||
451 | """ | ||
452 | Send a standard library :class:`Request` to PyPI and return its | ||
453 | response. | ||
454 | |||
455 | :param req: The request to send. | ||
456 | :return: The HTTP response from PyPI (a standard library HTTPResponse). | ||
457 | """ | ||
458 | handlers = [] | ||
459 | if self.password_handler: | ||
460 | handlers.append(self.password_handler) | ||
461 | if self.ssl_verifier: | ||
462 | handlers.append(self.ssl_verifier) | ||
463 | opener = build_opener(*handlers) | ||
464 | return opener.open(req) | ||
465 | |||
466 | def encode_request(self, fields, files): | ||
467 | """ | ||
468 | Encode fields and files for posting to an HTTP server. | ||
469 | |||
470 | :param fields: The fields to send as a list of (fieldname, value) | ||
471 | tuples. | ||
472 | :param files: The files to send as a list of (fieldname, filename, | ||
473 | file_bytes) tuple. | ||
474 | """ | ||
475 | # Adapted from packaging, which in turn was adapted from | ||
476 | # http://code.activestate.com/recipes/146306 | ||
477 | |||
478 | parts = [] | ||
479 | boundary = self.boundary | ||
480 | for k, values in fields: | ||
481 | if not isinstance(values, (list, tuple)): | ||
482 | values = [values] | ||
483 | |||
484 | for v in values: | ||
485 | parts.extend(( | ||
486 | b'--' + boundary, | ||
487 | ('Content-Disposition: form-data; name="%s"' % | ||
488 | k).encode('utf-8'), | ||
489 | b'', | ||
490 | v.encode('utf-8'))) | ||
491 | for key, filename, value in files: | ||
492 | parts.extend(( | ||
493 | b'--' + boundary, | ||
494 | ('Content-Disposition: form-data; name="%s"; filename="%s"' % | ||
495 | (key, filename)).encode('utf-8'), | ||
496 | b'', | ||
497 | value)) | ||
498 | |||
499 | parts.extend((b'--' + boundary + b'--', b'')) | ||
500 | |||
501 | body = b'\r\n'.join(parts) | ||
502 | ct = b'multipart/form-data; boundary=' + boundary | ||
503 | headers = { | ||
504 | 'Content-type': ct, | ||
505 | 'Content-length': str(len(body)) | ||
506 | } | ||
507 | return Request(self.url, body, headers) | ||
508 | |||
509 | def search(self, terms, operator=None): | ||
510 | if isinstance(terms, string_types): | ||
511 | terms = {'name': terms} | ||
512 | rpc_proxy = ServerProxy(self.url, timeout=3.0) | ||
513 | try: | ||
514 | return rpc_proxy.search(terms, operator or 'and') | ||
515 | finally: | ||
516 | rpc_proxy('close')() | ||