diff options
author | Shubham Saini <shubham6405@gmail.com> | 2018-12-11 10:01:23 +0000 |
---|---|---|
committer | Shubham Saini <shubham6405@gmail.com> | 2018-12-11 10:01:23 +0000 |
commit | 68df54d6629ec019142eb149dd037774f2d11e7c (patch) | |
tree | 345bc22d46b4e01a4ba8303b94278952a4ed2b9e /venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/distlib/_backport |
First commit
Diffstat (limited to 'venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/distlib/_backport')
6 files changed, 4287 insertions, 0 deletions
diff --git a/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/distlib/_backport/__init__.py b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/distlib/_backport/__init__.py new file mode 100644 index 0000000..e6143f1 --- /dev/null +++ b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/distlib/_backport/__init__.py | |||
@@ -0,0 +1,6 @@ | |||
1 | """Modules copied from Python 3 standard libraries, for internal use only. | ||
2 | |||
3 | Individual classes and functions are found in d2._backport.misc. Intended | ||
4 | usage is to always import things missing from 3.1 from that module: the | ||
5 | built-in/stdlib objects will be used if found. | ||
6 | """ | ||
diff --git a/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/distlib/_backport/misc.py b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/distlib/_backport/misc.py new file mode 100644 index 0000000..6eb7b86 --- /dev/null +++ b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/distlib/_backport/misc.py | |||
@@ -0,0 +1,41 @@ | |||
1 | # -*- coding: utf-8 -*- | ||
2 | # | ||
3 | # Copyright (C) 2012 The Python Software Foundation. | ||
4 | # See LICENSE.txt and CONTRIBUTORS.txt. | ||
5 | # | ||
6 | """Backports for individual classes and functions.""" | ||
7 | |||
8 | import os | ||
9 | import sys | ||
10 | |||
11 | __all__ = ['cache_from_source', 'callable', 'fsencode'] | ||
12 | |||
13 | |||
14 | try: | ||
15 | from imp import cache_from_source | ||
16 | except ImportError: | ||
17 | def cache_from_source(py_file, debug=__debug__): | ||
18 | ext = debug and 'c' or 'o' | ||
19 | return py_file + ext | ||
20 | |||
21 | |||
22 | try: | ||
23 | callable = callable | ||
24 | except NameError: | ||
25 | from collections import Callable | ||
26 | |||
27 | def callable(obj): | ||
28 | return isinstance(obj, Callable) | ||
29 | |||
30 | |||
31 | try: | ||
32 | fsencode = os.fsencode | ||
33 | except AttributeError: | ||
34 | def fsencode(filename): | ||
35 | if isinstance(filename, bytes): | ||
36 | return filename | ||
37 | elif isinstance(filename, str): | ||
38 | return filename.encode(sys.getfilesystemencoding()) | ||
39 | else: | ||
40 | raise TypeError("expect bytes or str, not %s" % | ||
41 | type(filename).__name__) | ||
diff --git a/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/distlib/_backport/shutil.py b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/distlib/_backport/shutil.py new file mode 100644 index 0000000..becbfd7 --- /dev/null +++ b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/distlib/_backport/shutil.py | |||
@@ -0,0 +1,761 @@ | |||
1 | # -*- coding: utf-8 -*- | ||
2 | # | ||
3 | # Copyright (C) 2012 The Python Software Foundation. | ||
4 | # See LICENSE.txt and CONTRIBUTORS.txt. | ||
5 | # | ||
6 | """Utility functions for copying and archiving files and directory trees. | ||
7 | |||
8 | XXX The functions here don't copy the resource fork or other metadata on Mac. | ||
9 | |||
10 | """ | ||
11 | |||
12 | import os | ||
13 | import sys | ||
14 | import stat | ||
15 | from os.path import abspath | ||
16 | import fnmatch | ||
17 | import collections | ||
18 | import errno | ||
19 | from . import tarfile | ||
20 | |||
21 | try: | ||
22 | import bz2 | ||
23 | _BZ2_SUPPORTED = True | ||
24 | except ImportError: | ||
25 | _BZ2_SUPPORTED = False | ||
26 | |||
27 | try: | ||
28 | from pwd import getpwnam | ||
29 | except ImportError: | ||
30 | getpwnam = None | ||
31 | |||
32 | try: | ||
33 | from grp import getgrnam | ||
34 | except ImportError: | ||
35 | getgrnam = None | ||
36 | |||
37 | __all__ = ["copyfileobj", "copyfile", "copymode", "copystat", "copy", "copy2", | ||
38 | "copytree", "move", "rmtree", "Error", "SpecialFileError", | ||
39 | "ExecError", "make_archive", "get_archive_formats", | ||
40 | "register_archive_format", "unregister_archive_format", | ||
41 | "get_unpack_formats", "register_unpack_format", | ||
42 | "unregister_unpack_format", "unpack_archive", "ignore_patterns"] | ||
43 | |||
44 | class Error(EnvironmentError): | ||
45 | pass | ||
46 | |||
47 | class SpecialFileError(EnvironmentError): | ||
48 | """Raised when trying to do a kind of operation (e.g. copying) which is | ||
49 | not supported on a special file (e.g. a named pipe)""" | ||
50 | |||
51 | class ExecError(EnvironmentError): | ||
52 | """Raised when a command could not be executed""" | ||
53 | |||
54 | class ReadError(EnvironmentError): | ||
55 | """Raised when an archive cannot be read""" | ||
56 | |||
57 | class RegistryError(Exception): | ||
58 | """Raised when a registry operation with the archiving | ||
59 | and unpacking registries fails""" | ||
60 | |||
61 | |||
62 | try: | ||
63 | WindowsError | ||
64 | except NameError: | ||
65 | WindowsError = None | ||
66 | |||
67 | def copyfileobj(fsrc, fdst, length=16*1024): | ||
68 | """copy data from file-like object fsrc to file-like object fdst""" | ||
69 | while 1: | ||
70 | buf = fsrc.read(length) | ||
71 | if not buf: | ||
72 | break | ||
73 | fdst.write(buf) | ||
74 | |||
75 | def _samefile(src, dst): | ||
76 | # Macintosh, Unix. | ||
77 | if hasattr(os.path, 'samefile'): | ||
78 | try: | ||
79 | return os.path.samefile(src, dst) | ||
80 | except OSError: | ||
81 | return False | ||
82 | |||
83 | # All other platforms: check for same pathname. | ||
84 | return (os.path.normcase(os.path.abspath(src)) == | ||
85 | os.path.normcase(os.path.abspath(dst))) | ||
86 | |||
87 | def copyfile(src, dst): | ||
88 | """Copy data from src to dst""" | ||
89 | if _samefile(src, dst): | ||
90 | raise Error("`%s` and `%s` are the same file" % (src, dst)) | ||
91 | |||
92 | for fn in [src, dst]: | ||
93 | try: | ||
94 | st = os.stat(fn) | ||
95 | except OSError: | ||
96 | # File most likely does not exist | ||
97 | pass | ||
98 | else: | ||
99 | # XXX What about other special files? (sockets, devices...) | ||
100 | if stat.S_ISFIFO(st.st_mode): | ||
101 | raise SpecialFileError("`%s` is a named pipe" % fn) | ||
102 | |||
103 | with open(src, 'rb') as fsrc: | ||
104 | with open(dst, 'wb') as fdst: | ||
105 | copyfileobj(fsrc, fdst) | ||
106 | |||
107 | def copymode(src, dst): | ||
108 | """Copy mode bits from src to dst""" | ||
109 | if hasattr(os, 'chmod'): | ||
110 | st = os.stat(src) | ||
111 | mode = stat.S_IMODE(st.st_mode) | ||
112 | os.chmod(dst, mode) | ||
113 | |||
114 | def copystat(src, dst): | ||
115 | """Copy all stat info (mode bits, atime, mtime, flags) from src to dst""" | ||
116 | st = os.stat(src) | ||
117 | mode = stat.S_IMODE(st.st_mode) | ||
118 | if hasattr(os, 'utime'): | ||
119 | os.utime(dst, (st.st_atime, st.st_mtime)) | ||
120 | if hasattr(os, 'chmod'): | ||
121 | os.chmod(dst, mode) | ||
122 | if hasattr(os, 'chflags') and hasattr(st, 'st_flags'): | ||
123 | try: | ||
124 | os.chflags(dst, st.st_flags) | ||
125 | except OSError as why: | ||
126 | if (not hasattr(errno, 'EOPNOTSUPP') or | ||
127 | why.errno != errno.EOPNOTSUPP): | ||
128 | raise | ||
129 | |||
130 | def copy(src, dst): | ||
131 | """Copy data and mode bits ("cp src dst"). | ||
132 | |||
133 | The destination may be a directory. | ||
134 | |||
135 | """ | ||
136 | if os.path.isdir(dst): | ||
137 | dst = os.path.join(dst, os.path.basename(src)) | ||
138 | copyfile(src, dst) | ||
139 | copymode(src, dst) | ||
140 | |||
141 | def copy2(src, dst): | ||
142 | """Copy data and all stat info ("cp -p src dst"). | ||
143 | |||
144 | The destination may be a directory. | ||
145 | |||
146 | """ | ||
147 | if os.path.isdir(dst): | ||
148 | dst = os.path.join(dst, os.path.basename(src)) | ||
149 | copyfile(src, dst) | ||
150 | copystat(src, dst) | ||
151 | |||
152 | def ignore_patterns(*patterns): | ||
153 | """Function that can be used as copytree() ignore parameter. | ||
154 | |||
155 | Patterns is a sequence of glob-style patterns | ||
156 | that are used to exclude files""" | ||
157 | def _ignore_patterns(path, names): | ||
158 | ignored_names = [] | ||
159 | for pattern in patterns: | ||
160 | ignored_names.extend(fnmatch.filter(names, pattern)) | ||
161 | return set(ignored_names) | ||
162 | return _ignore_patterns | ||
163 | |||
164 | def copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2, | ||
165 | ignore_dangling_symlinks=False): | ||
166 | """Recursively copy a directory tree. | ||
167 | |||
168 | The destination directory must not already exist. | ||
169 | If exception(s) occur, an Error is raised with a list of reasons. | ||
170 | |||
171 | If the optional symlinks flag is true, symbolic links in the | ||
172 | source tree result in symbolic links in the destination tree; if | ||
173 | it is false, the contents of the files pointed to by symbolic | ||
174 | links are copied. If the file pointed by the symlink doesn't | ||
175 | exist, an exception will be added in the list of errors raised in | ||
176 | an Error exception at the end of the copy process. | ||
177 | |||
178 | You can set the optional ignore_dangling_symlinks flag to true if you | ||
179 | want to silence this exception. Notice that this has no effect on | ||
180 | platforms that don't support os.symlink. | ||
181 | |||
182 | The optional ignore argument is a callable. If given, it | ||
183 | is called with the `src` parameter, which is the directory | ||
184 | being visited by copytree(), and `names` which is the list of | ||
185 | `src` contents, as returned by os.listdir(): | ||
186 | |||
187 | callable(src, names) -> ignored_names | ||
188 | |||
189 | Since copytree() is called recursively, the callable will be | ||
190 | called once for each directory that is copied. It returns a | ||
191 | list of names relative to the `src` directory that should | ||
192 | not be copied. | ||
193 | |||
194 | The optional copy_function argument is a callable that will be used | ||
195 | to copy each file. It will be called with the source path and the | ||
196 | destination path as arguments. By default, copy2() is used, but any | ||
197 | function that supports the same signature (like copy()) can be used. | ||
198 | |||
199 | """ | ||
200 | names = os.listdir(src) | ||
201 | if ignore is not None: | ||
202 | ignored_names = ignore(src, names) | ||
203 | else: | ||
204 | ignored_names = set() | ||
205 | |||
206 | os.makedirs(dst) | ||
207 | errors = [] | ||
208 | for name in names: | ||
209 | if name in ignored_names: | ||
210 | continue | ||
211 | srcname = os.path.join(src, name) | ||
212 | dstname = os.path.join(dst, name) | ||
213 | try: | ||
214 | if os.path.islink(srcname): | ||
215 | linkto = os.readlink(srcname) | ||
216 | if symlinks: | ||
217 | os.symlink(linkto, dstname) | ||
218 | else: | ||
219 | # ignore dangling symlink if the flag is on | ||
220 | if not os.path.exists(linkto) and ignore_dangling_symlinks: | ||
221 | continue | ||
222 | # otherwise let the copy occurs. copy2 will raise an error | ||
223 | copy_function(srcname, dstname) | ||
224 | elif os.path.isdir(srcname): | ||
225 | copytree(srcname, dstname, symlinks, ignore, copy_function) | ||
226 | else: | ||
227 | # Will raise a SpecialFileError for unsupported file types | ||
228 | copy_function(srcname, dstname) | ||
229 | # catch the Error from the recursive copytree so that we can | ||
230 | # continue with other files | ||
231 | except Error as err: | ||
232 | errors.extend(err.args[0]) | ||
233 | except EnvironmentError as why: | ||
234 | errors.append((srcname, dstname, str(why))) | ||
235 | try: | ||
236 | copystat(src, dst) | ||
237 | except OSError as why: | ||
238 | if WindowsError is not None and isinstance(why, WindowsError): | ||
239 | # Copying file access times may fail on Windows | ||
240 | pass | ||
241 | else: | ||
242 | errors.extend((src, dst, str(why))) | ||
243 | if errors: | ||
244 | raise Error(errors) | ||
245 | |||
246 | def rmtree(path, ignore_errors=False, onerror=None): | ||
247 | """Recursively delete a directory tree. | ||
248 | |||
249 | If ignore_errors is set, errors are ignored; otherwise, if onerror | ||
250 | is set, it is called to handle the error with arguments (func, | ||
251 | path, exc_info) where func is os.listdir, os.remove, or os.rmdir; | ||
252 | path is the argument to that function that caused it to fail; and | ||
253 | exc_info is a tuple returned by sys.exc_info(). If ignore_errors | ||
254 | is false and onerror is None, an exception is raised. | ||
255 | |||
256 | """ | ||
257 | if ignore_errors: | ||
258 | def onerror(*args): | ||
259 | pass | ||
260 | elif onerror is None: | ||
261 | def onerror(*args): | ||
262 | raise | ||
263 | try: | ||
264 | if os.path.islink(path): | ||
265 | # symlinks to directories are forbidden, see bug #1669 | ||
266 | raise OSError("Cannot call rmtree on a symbolic link") | ||
267 | except OSError: | ||
268 | onerror(os.path.islink, path, sys.exc_info()) | ||
269 | # can't continue even if onerror hook returns | ||
270 | return | ||
271 | names = [] | ||
272 | try: | ||
273 | names = os.listdir(path) | ||
274 | except os.error: | ||
275 | onerror(os.listdir, path, sys.exc_info()) | ||
276 | for name in names: | ||
277 | fullname = os.path.join(path, name) | ||
278 | try: | ||
279 | mode = os.lstat(fullname).st_mode | ||
280 | except os.error: | ||
281 | mode = 0 | ||
282 | if stat.S_ISDIR(mode): | ||
283 | rmtree(fullname, ignore_errors, onerror) | ||
284 | else: | ||
285 | try: | ||
286 | os.remove(fullname) | ||
287 | except os.error: | ||
288 | onerror(os.remove, fullname, sys.exc_info()) | ||
289 | try: | ||
290 | os.rmdir(path) | ||
291 | except os.error: | ||
292 | onerror(os.rmdir, path, sys.exc_info()) | ||
293 | |||
294 | |||
295 | def _basename(path): | ||
296 | # A basename() variant which first strips the trailing slash, if present. | ||
297 | # Thus we always get the last component of the path, even for directories. | ||
298 | return os.path.basename(path.rstrip(os.path.sep)) | ||
299 | |||
300 | def move(src, dst): | ||
301 | """Recursively move a file or directory to another location. This is | ||
302 | similar to the Unix "mv" command. | ||
303 | |||
304 | If the destination is a directory or a symlink to a directory, the source | ||
305 | is moved inside the directory. The destination path must not already | ||
306 | exist. | ||
307 | |||
308 | If the destination already exists but is not a directory, it may be | ||
309 | overwritten depending on os.rename() semantics. | ||
310 | |||
311 | If the destination is on our current filesystem, then rename() is used. | ||
312 | Otherwise, src is copied to the destination and then removed. | ||
313 | A lot more could be done here... A look at a mv.c shows a lot of | ||
314 | the issues this implementation glosses over. | ||
315 | |||
316 | """ | ||
317 | real_dst = dst | ||
318 | if os.path.isdir(dst): | ||
319 | if _samefile(src, dst): | ||
320 | # We might be on a case insensitive filesystem, | ||
321 | # perform the rename anyway. | ||
322 | os.rename(src, dst) | ||
323 | return | ||
324 | |||
325 | real_dst = os.path.join(dst, _basename(src)) | ||
326 | if os.path.exists(real_dst): | ||
327 | raise Error("Destination path '%s' already exists" % real_dst) | ||
328 | try: | ||
329 | os.rename(src, real_dst) | ||
330 | except OSError: | ||
331 | if os.path.isdir(src): | ||
332 | if _destinsrc(src, dst): | ||
333 | raise Error("Cannot move a directory '%s' into itself '%s'." % (src, dst)) | ||
334 | copytree(src, real_dst, symlinks=True) | ||
335 | rmtree(src) | ||
336 | else: | ||
337 | copy2(src, real_dst) | ||
338 | os.unlink(src) | ||
339 | |||
340 | def _destinsrc(src, dst): | ||
341 | src = abspath(src) | ||
342 | dst = abspath(dst) | ||
343 | if not src.endswith(os.path.sep): | ||
344 | src += os.path.sep | ||
345 | if not dst.endswith(os.path.sep): | ||
346 | dst += os.path.sep | ||
347 | return dst.startswith(src) | ||
348 | |||
349 | def _get_gid(name): | ||
350 | """Returns a gid, given a group name.""" | ||
351 | if getgrnam is None or name is None: | ||
352 | return None | ||
353 | try: | ||
354 | result = getgrnam(name) | ||
355 | except KeyError: | ||
356 | result = None | ||
357 | if result is not None: | ||
358 | return result[2] | ||
359 | return None | ||
360 | |||
361 | def _get_uid(name): | ||
362 | """Returns an uid, given a user name.""" | ||
363 | if getpwnam is None or name is None: | ||
364 | return None | ||
365 | try: | ||
366 | result = getpwnam(name) | ||
367 | except KeyError: | ||
368 | result = None | ||
369 | if result is not None: | ||
370 | return result[2] | ||
371 | return None | ||
372 | |||
373 | def _make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0, | ||
374 | owner=None, group=None, logger=None): | ||
375 | """Create a (possibly compressed) tar file from all the files under | ||
376 | 'base_dir'. | ||
377 | |||
378 | 'compress' must be "gzip" (the default), "bzip2", or None. | ||
379 | |||
380 | 'owner' and 'group' can be used to define an owner and a group for the | ||
381 | archive that is being built. If not provided, the current owner and group | ||
382 | will be used. | ||
383 | |||
384 | The output tar file will be named 'base_name' + ".tar", possibly plus | ||
385 | the appropriate compression extension (".gz", or ".bz2"). | ||
386 | |||
387 | Returns the output filename. | ||
388 | """ | ||
389 | tar_compression = {'gzip': 'gz', None: ''} | ||
390 | compress_ext = {'gzip': '.gz'} | ||
391 | |||
392 | if _BZ2_SUPPORTED: | ||
393 | tar_compression['bzip2'] = 'bz2' | ||
394 | compress_ext['bzip2'] = '.bz2' | ||
395 | |||
396 | # flags for compression program, each element of list will be an argument | ||
397 | if compress is not None and compress not in compress_ext: | ||
398 | raise ValueError("bad value for 'compress', or compression format not " | ||
399 | "supported : {0}".format(compress)) | ||
400 | |||
401 | archive_name = base_name + '.tar' + compress_ext.get(compress, '') | ||
402 | archive_dir = os.path.dirname(archive_name) | ||
403 | |||
404 | if not os.path.exists(archive_dir): | ||
405 | if logger is not None: | ||
406 | logger.info("creating %s", archive_dir) | ||
407 | if not dry_run: | ||
408 | os.makedirs(archive_dir) | ||
409 | |||
410 | # creating the tarball | ||
411 | if logger is not None: | ||
412 | logger.info('Creating tar archive') | ||
413 | |||
414 | uid = _get_uid(owner) | ||
415 | gid = _get_gid(group) | ||
416 | |||
417 | def _set_uid_gid(tarinfo): | ||
418 | if gid is not None: | ||
419 | tarinfo.gid = gid | ||
420 | tarinfo.gname = group | ||
421 | if uid is not None: | ||
422 | tarinfo.uid = uid | ||
423 | tarinfo.uname = owner | ||
424 | return tarinfo | ||
425 | |||
426 | if not dry_run: | ||
427 | tar = tarfile.open(archive_name, 'w|%s' % tar_compression[compress]) | ||
428 | try: | ||
429 | tar.add(base_dir, filter=_set_uid_gid) | ||
430 | finally: | ||
431 | tar.close() | ||
432 | |||
433 | return archive_name | ||
434 | |||
435 | def _call_external_zip(base_dir, zip_filename, verbose=False, dry_run=False): | ||
436 | # XXX see if we want to keep an external call here | ||
437 | if verbose: | ||
438 | zipoptions = "-r" | ||
439 | else: | ||
440 | zipoptions = "-rq" | ||
441 | from distutils.errors import DistutilsExecError | ||
442 | from distutils.spawn import spawn | ||
443 | try: | ||
444 | spawn(["zip", zipoptions, zip_filename, base_dir], dry_run=dry_run) | ||
445 | except DistutilsExecError: | ||
446 | # XXX really should distinguish between "couldn't find | ||
447 | # external 'zip' command" and "zip failed". | ||
448 | raise ExecError("unable to create zip file '%s': " | ||
449 | "could neither import the 'zipfile' module nor " | ||
450 | "find a standalone zip utility") % zip_filename | ||
451 | |||
452 | def _make_zipfile(base_name, base_dir, verbose=0, dry_run=0, logger=None): | ||
453 | """Create a zip file from all the files under 'base_dir'. | ||
454 | |||
455 | The output zip file will be named 'base_name' + ".zip". Uses either the | ||
456 | "zipfile" Python module (if available) or the InfoZIP "zip" utility | ||
457 | (if installed and found on the default search path). If neither tool is | ||
458 | available, raises ExecError. Returns the name of the output zip | ||
459 | file. | ||
460 | """ | ||
461 | zip_filename = base_name + ".zip" | ||
462 | archive_dir = os.path.dirname(base_name) | ||
463 | |||
464 | if not os.path.exists(archive_dir): | ||
465 | if logger is not None: | ||
466 | logger.info("creating %s", archive_dir) | ||
467 | if not dry_run: | ||
468 | os.makedirs(archive_dir) | ||
469 | |||
470 | # If zipfile module is not available, try spawning an external 'zip' | ||
471 | # command. | ||
472 | try: | ||
473 | import zipfile | ||
474 | except ImportError: | ||
475 | zipfile = None | ||
476 | |||
477 | if zipfile is None: | ||
478 | _call_external_zip(base_dir, zip_filename, verbose, dry_run) | ||
479 | else: | ||
480 | if logger is not None: | ||
481 | logger.info("creating '%s' and adding '%s' to it", | ||
482 | zip_filename, base_dir) | ||
483 | |||
484 | if not dry_run: | ||
485 | zip = zipfile.ZipFile(zip_filename, "w", | ||
486 | compression=zipfile.ZIP_DEFLATED) | ||
487 | |||
488 | for dirpath, dirnames, filenames in os.walk(base_dir): | ||
489 | for name in filenames: | ||
490 | path = os.path.normpath(os.path.join(dirpath, name)) | ||
491 | if os.path.isfile(path): | ||
492 | zip.write(path, path) | ||
493 | if logger is not None: | ||
494 | logger.info("adding '%s'", path) | ||
495 | zip.close() | ||
496 | |||
497 | return zip_filename | ||
498 | |||
499 | _ARCHIVE_FORMATS = { | ||
500 | 'gztar': (_make_tarball, [('compress', 'gzip')], "gzip'ed tar-file"), | ||
501 | 'bztar': (_make_tarball, [('compress', 'bzip2')], "bzip2'ed tar-file"), | ||
502 | 'tar': (_make_tarball, [('compress', None)], "uncompressed tar file"), | ||
503 | 'zip': (_make_zipfile, [], "ZIP file"), | ||
504 | } | ||
505 | |||
506 | if _BZ2_SUPPORTED: | ||
507 | _ARCHIVE_FORMATS['bztar'] = (_make_tarball, [('compress', 'bzip2')], | ||
508 | "bzip2'ed tar-file") | ||
509 | |||
510 | def get_archive_formats(): | ||
511 | """Returns a list of supported formats for archiving and unarchiving. | ||
512 | |||
513 | Each element of the returned sequence is a tuple (name, description) | ||
514 | """ | ||
515 | formats = [(name, registry[2]) for name, registry in | ||
516 | _ARCHIVE_FORMATS.items()] | ||
517 | formats.sort() | ||
518 | return formats | ||
519 | |||
520 | def register_archive_format(name, function, extra_args=None, description=''): | ||
521 | """Registers an archive format. | ||
522 | |||
523 | name is the name of the format. function is the callable that will be | ||
524 | used to create archives. If provided, extra_args is a sequence of | ||
525 | (name, value) tuples that will be passed as arguments to the callable. | ||
526 | description can be provided to describe the format, and will be returned | ||
527 | by the get_archive_formats() function. | ||
528 | """ | ||
529 | if extra_args is None: | ||
530 | extra_args = [] | ||
531 | if not isinstance(function, collections.Callable): | ||
532 | raise TypeError('The %s object is not callable' % function) | ||
533 | if not isinstance(extra_args, (tuple, list)): | ||
534 | raise TypeError('extra_args needs to be a sequence') | ||
535 | for element in extra_args: | ||
536 | if not isinstance(element, (tuple, list)) or len(element) !=2: | ||
537 | raise TypeError('extra_args elements are : (arg_name, value)') | ||
538 | |||
539 | _ARCHIVE_FORMATS[name] = (function, extra_args, description) | ||
540 | |||
541 | def unregister_archive_format(name): | ||
542 | del _ARCHIVE_FORMATS[name] | ||
543 | |||
544 | def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0, | ||
545 | dry_run=0, owner=None, group=None, logger=None): | ||
546 | """Create an archive file (eg. zip or tar). | ||
547 | |||
548 | 'base_name' is the name of the file to create, minus any format-specific | ||
549 | extension; 'format' is the archive format: one of "zip", "tar", "bztar" | ||
550 | or "gztar". | ||
551 | |||
552 | 'root_dir' is a directory that will be the root directory of the | ||
553 | archive; ie. we typically chdir into 'root_dir' before creating the | ||
554 | archive. 'base_dir' is the directory where we start archiving from; | ||
555 | ie. 'base_dir' will be the common prefix of all files and | ||
556 | directories in the archive. 'root_dir' and 'base_dir' both default | ||
557 | to the current directory. Returns the name of the archive file. | ||
558 | |||
559 | 'owner' and 'group' are used when creating a tar archive. By default, | ||
560 | uses the current owner and group. | ||
561 | """ | ||
562 | save_cwd = os.getcwd() | ||
563 | if root_dir is not None: | ||
564 | if logger is not None: | ||
565 | logger.debug("changing into '%s'", root_dir) | ||
566 | base_name = os.path.abspath(base_name) | ||
567 | if not dry_run: | ||
568 | os.chdir(root_dir) | ||
569 | |||
570 | if base_dir is None: | ||
571 | base_dir = os.curdir | ||
572 | |||
573 | kwargs = {'dry_run': dry_run, 'logger': logger} | ||
574 | |||
575 | try: | ||
576 | format_info = _ARCHIVE_FORMATS[format] | ||
577 | except KeyError: | ||
578 | raise ValueError("unknown archive format '%s'" % format) | ||
579 | |||
580 | func = format_info[0] | ||
581 | for arg, val in format_info[1]: | ||
582 | kwargs[arg] = val | ||
583 | |||
584 | if format != 'zip': | ||
585 | kwargs['owner'] = owner | ||
586 | kwargs['group'] = group | ||
587 | |||
588 | try: | ||
589 | filename = func(base_name, base_dir, **kwargs) | ||
590 | finally: | ||
591 | if root_dir is not None: | ||
592 | if logger is not None: | ||
593 | logger.debug("changing back to '%s'", save_cwd) | ||
594 | os.chdir(save_cwd) | ||
595 | |||
596 | return filename | ||
597 | |||
598 | |||
599 | def get_unpack_formats(): | ||
600 | """Returns a list of supported formats for unpacking. | ||
601 | |||
602 | Each element of the returned sequence is a tuple | ||
603 | (name, extensions, description) | ||
604 | """ | ||
605 | formats = [(name, info[0], info[3]) for name, info in | ||
606 | _UNPACK_FORMATS.items()] | ||
607 | formats.sort() | ||
608 | return formats | ||
609 | |||
610 | def _check_unpack_options(extensions, function, extra_args): | ||
611 | """Checks what gets registered as an unpacker.""" | ||
612 | # first make sure no other unpacker is registered for this extension | ||
613 | existing_extensions = {} | ||
614 | for name, info in _UNPACK_FORMATS.items(): | ||
615 | for ext in info[0]: | ||
616 | existing_extensions[ext] = name | ||
617 | |||
618 | for extension in extensions: | ||
619 | if extension in existing_extensions: | ||
620 | msg = '%s is already registered for "%s"' | ||
621 | raise RegistryError(msg % (extension, | ||
622 | existing_extensions[extension])) | ||
623 | |||
624 | if not isinstance(function, collections.Callable): | ||
625 | raise TypeError('The registered function must be a callable') | ||
626 | |||
627 | |||
628 | def register_unpack_format(name, extensions, function, extra_args=None, | ||
629 | description=''): | ||
630 | """Registers an unpack format. | ||
631 | |||
632 | `name` is the name of the format. `extensions` is a list of extensions | ||
633 | corresponding to the format. | ||
634 | |||
635 | `function` is the callable that will be | ||
636 | used to unpack archives. The callable will receive archives to unpack. | ||
637 | If it's unable to handle an archive, it needs to raise a ReadError | ||
638 | exception. | ||
639 | |||
640 | If provided, `extra_args` is a sequence of | ||
641 | (name, value) tuples that will be passed as arguments to the callable. | ||
642 | description can be provided to describe the format, and will be returned | ||
643 | by the get_unpack_formats() function. | ||
644 | """ | ||
645 | if extra_args is None: | ||
646 | extra_args = [] | ||
647 | _check_unpack_options(extensions, function, extra_args) | ||
648 | _UNPACK_FORMATS[name] = extensions, function, extra_args, description | ||
649 | |||
650 | def unregister_unpack_format(name): | ||
651 | """Removes the pack format from the registry.""" | ||
652 | del _UNPACK_FORMATS[name] | ||
653 | |||
654 | def _ensure_directory(path): | ||
655 | """Ensure that the parent directory of `path` exists""" | ||
656 | dirname = os.path.dirname(path) | ||
657 | if not os.path.isdir(dirname): | ||
658 | os.makedirs(dirname) | ||
659 | |||
660 | def _unpack_zipfile(filename, extract_dir): | ||
661 | """Unpack zip `filename` to `extract_dir` | ||
662 | """ | ||
663 | try: | ||
664 | import zipfile | ||
665 | except ImportError: | ||
666 | raise ReadError('zlib not supported, cannot unpack this archive.') | ||
667 | |||
668 | if not zipfile.is_zipfile(filename): | ||
669 | raise ReadError("%s is not a zip file" % filename) | ||
670 | |||
671 | zip = zipfile.ZipFile(filename) | ||
672 | try: | ||
673 | for info in zip.infolist(): | ||
674 | name = info.filename | ||
675 | |||
676 | # don't extract absolute paths or ones with .. in them | ||
677 | if name.startswith('/') or '..' in name: | ||
678 | continue | ||
679 | |||
680 | target = os.path.join(extract_dir, *name.split('/')) | ||
681 | if not target: | ||
682 | continue | ||
683 | |||
684 | _ensure_directory(target) | ||
685 | if not name.endswith('/'): | ||
686 | # file | ||
687 | data = zip.read(info.filename) | ||
688 | f = open(target, 'wb') | ||
689 | try: | ||
690 | f.write(data) | ||
691 | finally: | ||
692 | f.close() | ||
693 | del data | ||
694 | finally: | ||
695 | zip.close() | ||
696 | |||
697 | def _unpack_tarfile(filename, extract_dir): | ||
698 | """Unpack tar/tar.gz/tar.bz2 `filename` to `extract_dir` | ||
699 | """ | ||
700 | try: | ||
701 | tarobj = tarfile.open(filename) | ||
702 | except tarfile.TarError: | ||
703 | raise ReadError( | ||
704 | "%s is not a compressed or uncompressed tar file" % filename) | ||
705 | try: | ||
706 | tarobj.extractall(extract_dir) | ||
707 | finally: | ||
708 | tarobj.close() | ||
709 | |||
710 | _UNPACK_FORMATS = { | ||
711 | 'gztar': (['.tar.gz', '.tgz'], _unpack_tarfile, [], "gzip'ed tar-file"), | ||
712 | 'tar': (['.tar'], _unpack_tarfile, [], "uncompressed tar file"), | ||
713 | 'zip': (['.zip'], _unpack_zipfile, [], "ZIP file") | ||
714 | } | ||
715 | |||
716 | if _BZ2_SUPPORTED: | ||
717 | _UNPACK_FORMATS['bztar'] = (['.bz2'], _unpack_tarfile, [], | ||
718 | "bzip2'ed tar-file") | ||
719 | |||
720 | def _find_unpack_format(filename): | ||
721 | for name, info in _UNPACK_FORMATS.items(): | ||
722 | for extension in info[0]: | ||
723 | if filename.endswith(extension): | ||
724 | return name | ||
725 | return None | ||
726 | |||
727 | def unpack_archive(filename, extract_dir=None, format=None): | ||
728 | """Unpack an archive. | ||
729 | |||
730 | `filename` is the name of the archive. | ||
731 | |||
732 | `extract_dir` is the name of the target directory, where the archive | ||
733 | is unpacked. If not provided, the current working directory is used. | ||
734 | |||
735 | `format` is the archive format: one of "zip", "tar", or "gztar". Or any | ||
736 | other registered format. If not provided, unpack_archive will use the | ||
737 | filename extension and see if an unpacker was registered for that | ||
738 | extension. | ||
739 | |||
740 | In case none is found, a ValueError is raised. | ||
741 | """ | ||
742 | if extract_dir is None: | ||
743 | extract_dir = os.getcwd() | ||
744 | |||
745 | if format is not None: | ||
746 | try: | ||
747 | format_info = _UNPACK_FORMATS[format] | ||
748 | except KeyError: | ||
749 | raise ValueError("Unknown unpack format '{0}'".format(format)) | ||
750 | |||
751 | func = format_info[1] | ||
752 | func(filename, extract_dir, **dict(format_info[2])) | ||
753 | else: | ||
754 | # we need to look at the registered unpackers supported extensions | ||
755 | format = _find_unpack_format(filename) | ||
756 | if format is None: | ||
757 | raise ReadError("Unknown archive format '{0}'".format(filename)) | ||
758 | |||
759 | func = _UNPACK_FORMATS[format][1] | ||
760 | kwargs = dict(_UNPACK_FORMATS[format][2]) | ||
761 | func(filename, extract_dir, **kwargs) | ||
diff --git a/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/distlib/_backport/sysconfig.cfg b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/distlib/_backport/sysconfig.cfg new file mode 100644 index 0000000..c92cd48 --- /dev/null +++ b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/distlib/_backport/sysconfig.cfg | |||
@@ -0,0 +1,84 @@ | |||
1 | [posix_prefix] | ||
2 | # Configuration directories. Some of these come straight out of the | ||
3 | # configure script. They are for implementing the other variables, not to | ||
4 | # be used directly in [resource_locations]. | ||
5 | confdir = /etc | ||
6 | datadir = /usr/share | ||
7 | libdir = /usr/lib | ||
8 | statedir = /var | ||
9 | # User resource directory | ||
10 | local = ~/.local/{distribution.name} | ||
11 | |||
12 | stdlib = {base}/lib/python{py_version_short} | ||
13 | platstdlib = {platbase}/lib/python{py_version_short} | ||
14 | purelib = {base}/lib/python{py_version_short}/site-packages | ||
15 | platlib = {platbase}/lib/python{py_version_short}/site-packages | ||
16 | include = {base}/include/python{py_version_short}{abiflags} | ||
17 | platinclude = {platbase}/include/python{py_version_short}{abiflags} | ||
18 | data = {base} | ||
19 | |||
20 | [posix_home] | ||
21 | stdlib = {base}/lib/python | ||
22 | platstdlib = {base}/lib/python | ||
23 | purelib = {base}/lib/python | ||
24 | platlib = {base}/lib/python | ||
25 | include = {base}/include/python | ||
26 | platinclude = {base}/include/python | ||
27 | scripts = {base}/bin | ||
28 | data = {base} | ||
29 | |||
30 | [nt] | ||
31 | stdlib = {base}/Lib | ||
32 | platstdlib = {base}/Lib | ||
33 | purelib = {base}/Lib/site-packages | ||
34 | platlib = {base}/Lib/site-packages | ||
35 | include = {base}/Include | ||
36 | platinclude = {base}/Include | ||
37 | scripts = {base}/Scripts | ||
38 | data = {base} | ||
39 | |||
40 | [os2] | ||
41 | stdlib = {base}/Lib | ||
42 | platstdlib = {base}/Lib | ||
43 | purelib = {base}/Lib/site-packages | ||
44 | platlib = {base}/Lib/site-packages | ||
45 | include = {base}/Include | ||
46 | platinclude = {base}/Include | ||
47 | scripts = {base}/Scripts | ||
48 | data = {base} | ||
49 | |||
50 | [os2_home] | ||
51 | stdlib = {userbase}/lib/python{py_version_short} | ||
52 | platstdlib = {userbase}/lib/python{py_version_short} | ||
53 | purelib = {userbase}/lib/python{py_version_short}/site-packages | ||
54 | platlib = {userbase}/lib/python{py_version_short}/site-packages | ||
55 | include = {userbase}/include/python{py_version_short} | ||
56 | scripts = {userbase}/bin | ||
57 | data = {userbase} | ||
58 | |||
59 | [nt_user] | ||
60 | stdlib = {userbase}/Python{py_version_nodot} | ||
61 | platstdlib = {userbase}/Python{py_version_nodot} | ||
62 | purelib = {userbase}/Python{py_version_nodot}/site-packages | ||
63 | platlib = {userbase}/Python{py_version_nodot}/site-packages | ||
64 | include = {userbase}/Python{py_version_nodot}/Include | ||
65 | scripts = {userbase}/Scripts | ||
66 | data = {userbase} | ||
67 | |||
68 | [posix_user] | ||
69 | stdlib = {userbase}/lib/python{py_version_short} | ||
70 | platstdlib = {userbase}/lib/python{py_version_short} | ||
71 | purelib = {userbase}/lib/python{py_version_short}/site-packages | ||
72 | platlib = {userbase}/lib/python{py_version_short}/site-packages | ||
73 | include = {userbase}/include/python{py_version_short} | ||
74 | scripts = {userbase}/bin | ||
75 | data = {userbase} | ||
76 | |||
77 | [osx_framework_user] | ||
78 | stdlib = {userbase}/lib/python | ||
79 | platstdlib = {userbase}/lib/python | ||
80 | purelib = {userbase}/lib/python/site-packages | ||
81 | platlib = {userbase}/lib/python/site-packages | ||
82 | include = {userbase}/include | ||
83 | scripts = {userbase}/bin | ||
84 | data = {userbase} | ||
diff --git a/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/distlib/_backport/sysconfig.py b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/distlib/_backport/sysconfig.py new file mode 100644 index 0000000..b243da3 --- /dev/null +++ b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/distlib/_backport/sysconfig.py | |||
@@ -0,0 +1,788 @@ | |||
1 | # -*- coding: utf-8 -*- | ||
2 | # | ||
3 | # Copyright (C) 2012 The Python Software Foundation. | ||
4 | # See LICENSE.txt and CONTRIBUTORS.txt. | ||
5 | # | ||
6 | """Access to Python's configuration information.""" | ||
7 | |||
8 | import codecs | ||
9 | import os | ||
10 | import re | ||
11 | import sys | ||
12 | from os.path import pardir, realpath | ||
13 | try: | ||
14 | import configparser | ||
15 | except ImportError: | ||
16 | import ConfigParser as configparser | ||
17 | |||
18 | |||
19 | __all__ = [ | ||
20 | 'get_config_h_filename', | ||
21 | 'get_config_var', | ||
22 | 'get_config_vars', | ||
23 | 'get_makefile_filename', | ||
24 | 'get_path', | ||
25 | 'get_path_names', | ||
26 | 'get_paths', | ||
27 | 'get_platform', | ||
28 | 'get_python_version', | ||
29 | 'get_scheme_names', | ||
30 | 'parse_config_h', | ||
31 | ] | ||
32 | |||
33 | |||
34 | def _safe_realpath(path): | ||
35 | try: | ||
36 | return realpath(path) | ||
37 | except OSError: | ||
38 | return path | ||
39 | |||
40 | |||
41 | if sys.executable: | ||
42 | _PROJECT_BASE = os.path.dirname(_safe_realpath(sys.executable)) | ||
43 | else: | ||
44 | # sys.executable can be empty if argv[0] has been changed and Python is | ||
45 | # unable to retrieve the real program name | ||
46 | _PROJECT_BASE = _safe_realpath(os.getcwd()) | ||
47 | |||
48 | if os.name == "nt" and "pcbuild" in _PROJECT_BASE[-8:].lower(): | ||
49 | _PROJECT_BASE = _safe_realpath(os.path.join(_PROJECT_BASE, pardir)) | ||
50 | # PC/VS7.1 | ||
51 | if os.name == "nt" and "\\pc\\v" in _PROJECT_BASE[-10:].lower(): | ||
52 | _PROJECT_BASE = _safe_realpath(os.path.join(_PROJECT_BASE, pardir, pardir)) | ||
53 | # PC/AMD64 | ||
54 | if os.name == "nt" and "\\pcbuild\\amd64" in _PROJECT_BASE[-14:].lower(): | ||
55 | _PROJECT_BASE = _safe_realpath(os.path.join(_PROJECT_BASE, pardir, pardir)) | ||
56 | |||
57 | |||
58 | def is_python_build(): | ||
59 | for fn in ("Setup.dist", "Setup.local"): | ||
60 | if os.path.isfile(os.path.join(_PROJECT_BASE, "Modules", fn)): | ||
61 | return True | ||
62 | return False | ||
63 | |||
64 | _PYTHON_BUILD = is_python_build() | ||
65 | |||
66 | _cfg_read = False | ||
67 | |||
68 | def _ensure_cfg_read(): | ||
69 | global _cfg_read | ||
70 | if not _cfg_read: | ||
71 | from ..resources import finder | ||
72 | backport_package = __name__.rsplit('.', 1)[0] | ||
73 | _finder = finder(backport_package) | ||
74 | _cfgfile = _finder.find('sysconfig.cfg') | ||
75 | assert _cfgfile, 'sysconfig.cfg exists' | ||
76 | with _cfgfile.as_stream() as s: | ||
77 | _SCHEMES.readfp(s) | ||
78 | if _PYTHON_BUILD: | ||
79 | for scheme in ('posix_prefix', 'posix_home'): | ||
80 | _SCHEMES.set(scheme, 'include', '{srcdir}/Include') | ||
81 | _SCHEMES.set(scheme, 'platinclude', '{projectbase}/.') | ||
82 | |||
83 | _cfg_read = True | ||
84 | |||
85 | |||
86 | _SCHEMES = configparser.RawConfigParser() | ||
87 | _VAR_REPL = re.compile(r'\{([^{]*?)\}') | ||
88 | |||
89 | def _expand_globals(config): | ||
90 | _ensure_cfg_read() | ||
91 | if config.has_section('globals'): | ||
92 | globals = config.items('globals') | ||
93 | else: | ||
94 | globals = tuple() | ||
95 | |||
96 | sections = config.sections() | ||
97 | for section in sections: | ||
98 | if section == 'globals': | ||
99 | continue | ||
100 | for option, value in globals: | ||
101 | if config.has_option(section, option): | ||
102 | continue | ||
103 | config.set(section, option, value) | ||
104 | config.remove_section('globals') | ||
105 | |||
106 | # now expanding local variables defined in the cfg file | ||
107 | # | ||
108 | for section in config.sections(): | ||
109 | variables = dict(config.items(section)) | ||
110 | |||
111 | def _replacer(matchobj): | ||
112 | name = matchobj.group(1) | ||
113 | if name in variables: | ||
114 | return variables[name] | ||
115 | return matchobj.group(0) | ||
116 | |||
117 | for option, value in config.items(section): | ||
118 | config.set(section, option, _VAR_REPL.sub(_replacer, value)) | ||
119 | |||
120 | #_expand_globals(_SCHEMES) | ||
121 | |||
122 | # FIXME don't rely on sys.version here, its format is an implementation detail | ||
123 | # of CPython, use sys.version_info or sys.hexversion | ||
124 | _PY_VERSION = sys.version.split()[0] | ||
125 | _PY_VERSION_SHORT = sys.version[:3] | ||
126 | _PY_VERSION_SHORT_NO_DOT = _PY_VERSION[0] + _PY_VERSION[2] | ||
127 | _PREFIX = os.path.normpath(sys.prefix) | ||
128 | _EXEC_PREFIX = os.path.normpath(sys.exec_prefix) | ||
129 | _CONFIG_VARS = None | ||
130 | _USER_BASE = None | ||
131 | |||
132 | |||
133 | def _subst_vars(path, local_vars): | ||
134 | """In the string `path`, replace tokens like {some.thing} with the | ||
135 | corresponding value from the map `local_vars`. | ||
136 | |||
137 | If there is no corresponding value, leave the token unchanged. | ||
138 | """ | ||
139 | def _replacer(matchobj): | ||
140 | name = matchobj.group(1) | ||
141 | if name in local_vars: | ||
142 | return local_vars[name] | ||
143 | elif name in os.environ: | ||
144 | return os.environ[name] | ||
145 | return matchobj.group(0) | ||
146 | return _VAR_REPL.sub(_replacer, path) | ||
147 | |||
148 | |||
149 | def _extend_dict(target_dict, other_dict): | ||
150 | target_keys = target_dict.keys() | ||
151 | for key, value in other_dict.items(): | ||
152 | if key in target_keys: | ||
153 | continue | ||
154 | target_dict[key] = value | ||
155 | |||
156 | |||
157 | def _expand_vars(scheme, vars): | ||
158 | res = {} | ||
159 | if vars is None: | ||
160 | vars = {} | ||
161 | _extend_dict(vars, get_config_vars()) | ||
162 | |||
163 | for key, value in _SCHEMES.items(scheme): | ||
164 | if os.name in ('posix', 'nt'): | ||
165 | value = os.path.expanduser(value) | ||
166 | res[key] = os.path.normpath(_subst_vars(value, vars)) | ||
167 | return res | ||
168 | |||
169 | |||
170 | def format_value(value, vars): | ||
171 | def _replacer(matchobj): | ||
172 | name = matchobj.group(1) | ||
173 | if name in vars: | ||
174 | return vars[name] | ||
175 | return matchobj.group(0) | ||
176 | return _VAR_REPL.sub(_replacer, value) | ||
177 | |||
178 | |||
179 | def _get_default_scheme(): | ||
180 | if os.name == 'posix': | ||
181 | # the default scheme for posix is posix_prefix | ||
182 | return 'posix_prefix' | ||
183 | return os.name | ||
184 | |||
185 | |||
186 | def _getuserbase(): | ||
187 | env_base = os.environ.get("PYTHONUSERBASE", None) | ||
188 | |||
189 | def joinuser(*args): | ||
190 | return os.path.expanduser(os.path.join(*args)) | ||
191 | |||
192 | # what about 'os2emx', 'riscos' ? | ||
193 | if os.name == "nt": | ||
194 | base = os.environ.get("APPDATA") or "~" | ||
195 | if env_base: | ||
196 | return env_base | ||
197 | else: | ||
198 | return joinuser(base, "Python") | ||
199 | |||
200 | if sys.platform == "darwin": | ||
201 | framework = get_config_var("PYTHONFRAMEWORK") | ||
202 | if framework: | ||
203 | if env_base: | ||
204 | return env_base | ||
205 | else: | ||
206 | return joinuser("~", "Library", framework, "%d.%d" % | ||
207 | sys.version_info[:2]) | ||
208 | |||
209 | if env_base: | ||
210 | return env_base | ||
211 | else: | ||
212 | return joinuser("~", ".local") | ||
213 | |||
214 | |||
215 | def _parse_makefile(filename, vars=None): | ||
216 | """Parse a Makefile-style file. | ||
217 | |||
218 | A dictionary containing name/value pairs is returned. If an | ||
219 | optional dictionary is passed in as the second argument, it is | ||
220 | used instead of a new dictionary. | ||
221 | """ | ||
222 | # Regexes needed for parsing Makefile (and similar syntaxes, | ||
223 | # like old-style Setup files). | ||
224 | _variable_rx = re.compile(r"([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)") | ||
225 | _findvar1_rx = re.compile(r"\$\(([A-Za-z][A-Za-z0-9_]*)\)") | ||
226 | _findvar2_rx = re.compile(r"\${([A-Za-z][A-Za-z0-9_]*)}") | ||
227 | |||
228 | if vars is None: | ||
229 | vars = {} | ||
230 | done = {} | ||
231 | notdone = {} | ||
232 | |||
233 | with codecs.open(filename, encoding='utf-8', errors="surrogateescape") as f: | ||
234 | lines = f.readlines() | ||
235 | |||
236 | for line in lines: | ||
237 | if line.startswith('#') or line.strip() == '': | ||
238 | continue | ||
239 | m = _variable_rx.match(line) | ||
240 | if m: | ||
241 | n, v = m.group(1, 2) | ||
242 | v = v.strip() | ||
243 | # `$$' is a literal `$' in make | ||
244 | tmpv = v.replace('$$', '') | ||
245 | |||
246 | if "$" in tmpv: | ||
247 | notdone[n] = v | ||
248 | else: | ||
249 | try: | ||
250 | v = int(v) | ||
251 | except ValueError: | ||
252 | # insert literal `$' | ||
253 | done[n] = v.replace('$$', '$') | ||
254 | else: | ||
255 | done[n] = v | ||
256 | |||
257 | # do variable interpolation here | ||
258 | variables = list(notdone.keys()) | ||
259 | |||
260 | # Variables with a 'PY_' prefix in the makefile. These need to | ||
261 | # be made available without that prefix through sysconfig. | ||
262 | # Special care is needed to ensure that variable expansion works, even | ||
263 | # if the expansion uses the name without a prefix. | ||
264 | renamed_variables = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS') | ||
265 | |||
266 | while len(variables) > 0: | ||
267 | for name in tuple(variables): | ||
268 | value = notdone[name] | ||
269 | m = _findvar1_rx.search(value) or _findvar2_rx.search(value) | ||
270 | if m is not None: | ||
271 | n = m.group(1) | ||
272 | found = True | ||
273 | if n in done: | ||
274 | item = str(done[n]) | ||
275 | elif n in notdone: | ||
276 | # get it on a subsequent round | ||
277 | found = False | ||
278 | elif n in os.environ: | ||
279 | # do it like make: fall back to environment | ||
280 | item = os.environ[n] | ||
281 | |||
282 | elif n in renamed_variables: | ||
283 | if (name.startswith('PY_') and | ||
284 | name[3:] in renamed_variables): | ||
285 | item = "" | ||
286 | |||
287 | elif 'PY_' + n in notdone: | ||
288 | found = False | ||
289 | |||
290 | else: | ||
291 | item = str(done['PY_' + n]) | ||
292 | |||
293 | else: | ||
294 | done[n] = item = "" | ||
295 | |||
296 | if found: | ||
297 | after = value[m.end():] | ||
298 | value = value[:m.start()] + item + after | ||
299 | if "$" in after: | ||
300 | notdone[name] = value | ||
301 | else: | ||
302 | try: | ||
303 | value = int(value) | ||
304 | except ValueError: | ||
305 | done[name] = value.strip() | ||
306 | else: | ||
307 | done[name] = value | ||
308 | variables.remove(name) | ||
309 | |||
310 | if (name.startswith('PY_') and | ||
311 | name[3:] in renamed_variables): | ||
312 | |||
313 | name = name[3:] | ||
314 | if name not in done: | ||
315 | done[name] = value | ||
316 | |||
317 | else: | ||
318 | # bogus variable reference (e.g. "prefix=$/opt/python"); | ||
319 | # just drop it since we can't deal | ||
320 | done[name] = value | ||
321 | variables.remove(name) | ||
322 | |||
323 | # strip spurious spaces | ||
324 | for k, v in done.items(): | ||
325 | if isinstance(v, str): | ||
326 | done[k] = v.strip() | ||
327 | |||
328 | # save the results in the global dictionary | ||
329 | vars.update(done) | ||
330 | return vars | ||
331 | |||
332 | |||
333 | def get_makefile_filename(): | ||
334 | """Return the path of the Makefile.""" | ||
335 | if _PYTHON_BUILD: | ||
336 | return os.path.join(_PROJECT_BASE, "Makefile") | ||
337 | if hasattr(sys, 'abiflags'): | ||
338 | config_dir_name = 'config-%s%s' % (_PY_VERSION_SHORT, sys.abiflags) | ||
339 | else: | ||
340 | config_dir_name = 'config' | ||
341 | return os.path.join(get_path('stdlib'), config_dir_name, 'Makefile') | ||
342 | |||
343 | |||
344 | def _init_posix(vars): | ||
345 | """Initialize the module as appropriate for POSIX systems.""" | ||
346 | # load the installed Makefile: | ||
347 | makefile = get_makefile_filename() | ||
348 | try: | ||
349 | _parse_makefile(makefile, vars) | ||
350 | except IOError as e: | ||
351 | msg = "invalid Python installation: unable to open %s" % makefile | ||
352 | if hasattr(e, "strerror"): | ||
353 | msg = msg + " (%s)" % e.strerror | ||
354 | raise IOError(msg) | ||
355 | # load the installed pyconfig.h: | ||
356 | config_h = get_config_h_filename() | ||
357 | try: | ||
358 | with open(config_h) as f: | ||
359 | parse_config_h(f, vars) | ||
360 | except IOError as e: | ||
361 | msg = "invalid Python installation: unable to open %s" % config_h | ||
362 | if hasattr(e, "strerror"): | ||
363 | msg = msg + " (%s)" % e.strerror | ||
364 | raise IOError(msg) | ||
365 | # On AIX, there are wrong paths to the linker scripts in the Makefile | ||
366 | # -- these paths are relative to the Python source, but when installed | ||
367 | # the scripts are in another directory. | ||
368 | if _PYTHON_BUILD: | ||
369 | vars['LDSHARED'] = vars['BLDSHARED'] | ||
370 | |||
371 | |||
372 | def _init_non_posix(vars): | ||
373 | """Initialize the module as appropriate for NT""" | ||
374 | # set basic install directories | ||
375 | vars['LIBDEST'] = get_path('stdlib') | ||
376 | vars['BINLIBDEST'] = get_path('platstdlib') | ||
377 | vars['INCLUDEPY'] = get_path('include') | ||
378 | vars['SO'] = '.pyd' | ||
379 | vars['EXE'] = '.exe' | ||
380 | vars['VERSION'] = _PY_VERSION_SHORT_NO_DOT | ||
381 | vars['BINDIR'] = os.path.dirname(_safe_realpath(sys.executable)) | ||
382 | |||
383 | # | ||
384 | # public APIs | ||
385 | # | ||
386 | |||
387 | |||
388 | def parse_config_h(fp, vars=None): | ||
389 | """Parse a config.h-style file. | ||
390 | |||
391 | A dictionary containing name/value pairs is returned. If an | ||
392 | optional dictionary is passed in as the second argument, it is | ||
393 | used instead of a new dictionary. | ||
394 | """ | ||
395 | if vars is None: | ||
396 | vars = {} | ||
397 | define_rx = re.compile("#define ([A-Z][A-Za-z0-9_]+) (.*)\n") | ||
398 | undef_rx = re.compile("/[*] #undef ([A-Z][A-Za-z0-9_]+) [*]/\n") | ||
399 | |||
400 | while True: | ||
401 | line = fp.readline() | ||
402 | if not line: | ||
403 | break | ||
404 | m = define_rx.match(line) | ||
405 | if m: | ||
406 | n, v = m.group(1, 2) | ||
407 | try: | ||
408 | v = int(v) | ||
409 | except ValueError: | ||
410 | pass | ||
411 | vars[n] = v | ||
412 | else: | ||
413 | m = undef_rx.match(line) | ||
414 | if m: | ||
415 | vars[m.group(1)] = 0 | ||
416 | return vars | ||
417 | |||
418 | |||
419 | def get_config_h_filename(): | ||
420 | """Return the path of pyconfig.h.""" | ||
421 | if _PYTHON_BUILD: | ||
422 | if os.name == "nt": | ||
423 | inc_dir = os.path.join(_PROJECT_BASE, "PC") | ||
424 | else: | ||
425 | inc_dir = _PROJECT_BASE | ||
426 | else: | ||
427 | inc_dir = get_path('platinclude') | ||
428 | return os.path.join(inc_dir, 'pyconfig.h') | ||
429 | |||
430 | |||
431 | def get_scheme_names(): | ||
432 | """Return a tuple containing the schemes names.""" | ||
433 | return tuple(sorted(_SCHEMES.sections())) | ||
434 | |||
435 | |||
436 | def get_path_names(): | ||
437 | """Return a tuple containing the paths names.""" | ||
438 | # xxx see if we want a static list | ||
439 | return _SCHEMES.options('posix_prefix') | ||
440 | |||
441 | |||
442 | def get_paths(scheme=_get_default_scheme(), vars=None, expand=True): | ||
443 | """Return a mapping containing an install scheme. | ||
444 | |||
445 | ``scheme`` is the install scheme name. If not provided, it will | ||
446 | return the default scheme for the current platform. | ||
447 | """ | ||
448 | _ensure_cfg_read() | ||
449 | if expand: | ||
450 | return _expand_vars(scheme, vars) | ||
451 | else: | ||
452 | return dict(_SCHEMES.items(scheme)) | ||
453 | |||
454 | |||
455 | def get_path(name, scheme=_get_default_scheme(), vars=None, expand=True): | ||
456 | """Return a path corresponding to the scheme. | ||
457 | |||
458 | ``scheme`` is the install scheme name. | ||
459 | """ | ||
460 | return get_paths(scheme, vars, expand)[name] | ||
461 | |||
462 | |||
463 | def get_config_vars(*args): | ||
464 | """With no arguments, return a dictionary of all configuration | ||
465 | variables relevant for the current platform. | ||
466 | |||
467 | On Unix, this means every variable defined in Python's installed Makefile; | ||
468 | On Windows and Mac OS it's a much smaller set. | ||
469 | |||
470 | With arguments, return a list of values that result from looking up | ||
471 | each argument in the configuration variable dictionary. | ||
472 | """ | ||
473 | global _CONFIG_VARS | ||
474 | if _CONFIG_VARS is None: | ||
475 | _CONFIG_VARS = {} | ||
476 | # Normalized versions of prefix and exec_prefix are handy to have; | ||
477 | # in fact, these are the standard versions used most places in the | ||
478 | # distutils2 module. | ||
479 | _CONFIG_VARS['prefix'] = _PREFIX | ||
480 | _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX | ||
481 | _CONFIG_VARS['py_version'] = _PY_VERSION | ||
482 | _CONFIG_VARS['py_version_short'] = _PY_VERSION_SHORT | ||
483 | _CONFIG_VARS['py_version_nodot'] = _PY_VERSION[0] + _PY_VERSION[2] | ||
484 | _CONFIG_VARS['base'] = _PREFIX | ||
485 | _CONFIG_VARS['platbase'] = _EXEC_PREFIX | ||
486 | _CONFIG_VARS['projectbase'] = _PROJECT_BASE | ||
487 | try: | ||
488 | _CONFIG_VARS['abiflags'] = sys.abiflags | ||
489 | except AttributeError: | ||
490 | # sys.abiflags may not be defined on all platforms. | ||
491 | _CONFIG_VARS['abiflags'] = '' | ||
492 | |||
493 | if os.name in ('nt', 'os2'): | ||
494 | _init_non_posix(_CONFIG_VARS) | ||
495 | if os.name == 'posix': | ||
496 | _init_posix(_CONFIG_VARS) | ||
497 | # Setting 'userbase' is done below the call to the | ||
498 | # init function to enable using 'get_config_var' in | ||
499 | # the init-function. | ||
500 | if sys.version >= '2.6': | ||
501 | _CONFIG_VARS['userbase'] = _getuserbase() | ||
502 | |||
503 | if 'srcdir' not in _CONFIG_VARS: | ||
504 | _CONFIG_VARS['srcdir'] = _PROJECT_BASE | ||
505 | else: | ||
506 | _CONFIG_VARS['srcdir'] = _safe_realpath(_CONFIG_VARS['srcdir']) | ||
507 | |||
508 | # Convert srcdir into an absolute path if it appears necessary. | ||
509 | # Normally it is relative to the build directory. However, during | ||
510 | # testing, for example, we might be running a non-installed python | ||
511 | # from a different directory. | ||
512 | if _PYTHON_BUILD and os.name == "posix": | ||
513 | base = _PROJECT_BASE | ||
514 | try: | ||
515 | cwd = os.getcwd() | ||
516 | except OSError: | ||
517 | cwd = None | ||
518 | if (not os.path.isabs(_CONFIG_VARS['srcdir']) and | ||
519 | base != cwd): | ||
520 | # srcdir is relative and we are not in the same directory | ||
521 | # as the executable. Assume executable is in the build | ||
522 | # directory and make srcdir absolute. | ||
523 | srcdir = os.path.join(base, _CONFIG_VARS['srcdir']) | ||
524 | _CONFIG_VARS['srcdir'] = os.path.normpath(srcdir) | ||
525 | |||
526 | if sys.platform == 'darwin': | ||
527 | kernel_version = os.uname()[2] # Kernel version (8.4.3) | ||
528 | major_version = int(kernel_version.split('.')[0]) | ||
529 | |||
530 | if major_version < 8: | ||
531 | # On Mac OS X before 10.4, check if -arch and -isysroot | ||
532 | # are in CFLAGS or LDFLAGS and remove them if they are. | ||
533 | # This is needed when building extensions on a 10.3 system | ||
534 | # using a universal build of python. | ||
535 | for key in ('LDFLAGS', 'BASECFLAGS', | ||
536 | # a number of derived variables. These need to be | ||
537 | # patched up as well. | ||
538 | 'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'): | ||
539 | flags = _CONFIG_VARS[key] | ||
540 | flags = re.sub(r'-arch\s+\w+\s', ' ', flags) | ||
541 | flags = re.sub('-isysroot [^ \t]*', ' ', flags) | ||
542 | _CONFIG_VARS[key] = flags | ||
543 | else: | ||
544 | # Allow the user to override the architecture flags using | ||
545 | # an environment variable. | ||
546 | # NOTE: This name was introduced by Apple in OSX 10.5 and | ||
547 | # is used by several scripting languages distributed with | ||
548 | # that OS release. | ||
549 | if 'ARCHFLAGS' in os.environ: | ||
550 | arch = os.environ['ARCHFLAGS'] | ||
551 | for key in ('LDFLAGS', 'BASECFLAGS', | ||
552 | # a number of derived variables. These need to be | ||
553 | # patched up as well. | ||
554 | 'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'): | ||
555 | |||
556 | flags = _CONFIG_VARS[key] | ||
557 | flags = re.sub(r'-arch\s+\w+\s', ' ', flags) | ||
558 | flags = flags + ' ' + arch | ||
559 | _CONFIG_VARS[key] = flags | ||
560 | |||
561 | # If we're on OSX 10.5 or later and the user tries to | ||
562 | # compiles an extension using an SDK that is not present | ||
563 | # on the current machine it is better to not use an SDK | ||
564 | # than to fail. | ||
565 | # | ||
566 | # The major usecase for this is users using a Python.org | ||
567 | # binary installer on OSX 10.6: that installer uses | ||
568 | # the 10.4u SDK, but that SDK is not installed by default | ||
569 | # when you install Xcode. | ||
570 | # | ||
571 | CFLAGS = _CONFIG_VARS.get('CFLAGS', '') | ||
572 | m = re.search(r'-isysroot\s+(\S+)', CFLAGS) | ||
573 | if m is not None: | ||
574 | sdk = m.group(1) | ||
575 | if not os.path.exists(sdk): | ||
576 | for key in ('LDFLAGS', 'BASECFLAGS', | ||
577 | # a number of derived variables. These need to be | ||
578 | # patched up as well. | ||
579 | 'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'): | ||
580 | |||
581 | flags = _CONFIG_VARS[key] | ||
582 | flags = re.sub(r'-isysroot\s+\S+(\s|$)', ' ', flags) | ||
583 | _CONFIG_VARS[key] = flags | ||
584 | |||
585 | if args: | ||
586 | vals = [] | ||
587 | for name in args: | ||
588 | vals.append(_CONFIG_VARS.get(name)) | ||
589 | return vals | ||
590 | else: | ||
591 | return _CONFIG_VARS | ||
592 | |||
593 | |||
594 | def get_config_var(name): | ||
595 | """Return the value of a single variable using the dictionary returned by | ||
596 | 'get_config_vars()'. | ||
597 | |||
598 | Equivalent to get_config_vars().get(name) | ||
599 | """ | ||
600 | return get_config_vars().get(name) | ||
601 | |||
602 | |||
603 | def get_platform(): | ||
604 | """Return a string that identifies the current platform. | ||
605 | |||
606 | This is used mainly to distinguish platform-specific build directories and | ||
607 | platform-specific built distributions. Typically includes the OS name | ||
608 | and version and the architecture (as supplied by 'os.uname()'), | ||
609 | although the exact information included depends on the OS; eg. for IRIX | ||
610 | the architecture isn't particularly important (IRIX only runs on SGI | ||
611 | hardware), but for Linux the kernel version isn't particularly | ||
612 | important. | ||
613 | |||
614 | Examples of returned values: | ||
615 | linux-i586 | ||
616 | linux-alpha (?) | ||
617 | solaris-2.6-sun4u | ||
618 | irix-5.3 | ||
619 | irix64-6.2 | ||
620 | |||
621 | Windows will return one of: | ||
622 | win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc) | ||
623 | win-ia64 (64bit Windows on Itanium) | ||
624 | win32 (all others - specifically, sys.platform is returned) | ||
625 | |||
626 | For other non-POSIX platforms, currently just returns 'sys.platform'. | ||
627 | """ | ||
628 | if os.name == 'nt': | ||
629 | # sniff sys.version for architecture. | ||
630 | prefix = " bit (" | ||
631 | i = sys.version.find(prefix) | ||
632 | if i == -1: | ||
633 | return sys.platform | ||
634 | j = sys.version.find(")", i) | ||
635 | look = sys.version[i+len(prefix):j].lower() | ||
636 | if look == 'amd64': | ||
637 | return 'win-amd64' | ||
638 | if look == 'itanium': | ||
639 | return 'win-ia64' | ||
640 | return sys.platform | ||
641 | |||
642 | if os.name != "posix" or not hasattr(os, 'uname'): | ||
643 | # XXX what about the architecture? NT is Intel or Alpha, | ||
644 | # Mac OS is M68k or PPC, etc. | ||
645 | return sys.platform | ||
646 | |||
647 | # Try to distinguish various flavours of Unix | ||
648 | osname, host, release, version, machine = os.uname() | ||
649 | |||
650 | # Convert the OS name to lowercase, remove '/' characters | ||
651 | # (to accommodate BSD/OS), and translate spaces (for "Power Macintosh") | ||
652 | osname = osname.lower().replace('/', '') | ||
653 | machine = machine.replace(' ', '_') | ||
654 | machine = machine.replace('/', '-') | ||
655 | |||
656 | if osname[:5] == "linux": | ||
657 | # At least on Linux/Intel, 'machine' is the processor -- | ||
658 | # i386, etc. | ||
659 | # XXX what about Alpha, SPARC, etc? | ||
660 | return "%s-%s" % (osname, machine) | ||
661 | elif osname[:5] == "sunos": | ||
662 | if release[0] >= "5": # SunOS 5 == Solaris 2 | ||
663 | osname = "solaris" | ||
664 | release = "%d.%s" % (int(release[0]) - 3, release[2:]) | ||
665 | # fall through to standard osname-release-machine representation | ||
666 | elif osname[:4] == "irix": # could be "irix64"! | ||
667 | return "%s-%s" % (osname, release) | ||
668 | elif osname[:3] == "aix": | ||
669 | return "%s-%s.%s" % (osname, version, release) | ||
670 | elif osname[:6] == "cygwin": | ||
671 | osname = "cygwin" | ||
672 | rel_re = re.compile(r'[\d.]+') | ||
673 | m = rel_re.match(release) | ||
674 | if m: | ||
675 | release = m.group() | ||
676 | elif osname[:6] == "darwin": | ||
677 | # | ||
678 | # For our purposes, we'll assume that the system version from | ||
679 | # distutils' perspective is what MACOSX_DEPLOYMENT_TARGET is set | ||
680 | # to. This makes the compatibility story a bit more sane because the | ||
681 | # machine is going to compile and link as if it were | ||
682 | # MACOSX_DEPLOYMENT_TARGET. | ||
683 | cfgvars = get_config_vars() | ||
684 | macver = cfgvars.get('MACOSX_DEPLOYMENT_TARGET') | ||
685 | |||
686 | if True: | ||
687 | # Always calculate the release of the running machine, | ||
688 | # needed to determine if we can build fat binaries or not. | ||
689 | |||
690 | macrelease = macver | ||
691 | # Get the system version. Reading this plist is a documented | ||
692 | # way to get the system version (see the documentation for | ||
693 | # the Gestalt Manager) | ||
694 | try: | ||
695 | f = open('/System/Library/CoreServices/SystemVersion.plist') | ||
696 | except IOError: | ||
697 | # We're on a plain darwin box, fall back to the default | ||
698 | # behaviour. | ||
699 | pass | ||
700 | else: | ||
701 | try: | ||
702 | m = re.search(r'<key>ProductUserVisibleVersion</key>\s*' | ||
703 | r'<string>(.*?)</string>', f.read()) | ||
704 | finally: | ||
705 | f.close() | ||
706 | if m is not None: | ||
707 | macrelease = '.'.join(m.group(1).split('.')[:2]) | ||
708 | # else: fall back to the default behaviour | ||
709 | |||
710 | if not macver: | ||
711 | macver = macrelease | ||
712 | |||
713 | if macver: | ||
714 | release = macver | ||
715 | osname = "macosx" | ||
716 | |||
717 | if ((macrelease + '.') >= '10.4.' and | ||
718 | '-arch' in get_config_vars().get('CFLAGS', '').strip()): | ||
719 | # The universal build will build fat binaries, but not on | ||
720 | # systems before 10.4 | ||
721 | # | ||
722 | # Try to detect 4-way universal builds, those have machine-type | ||
723 | # 'universal' instead of 'fat'. | ||
724 | |||
725 | machine = 'fat' | ||
726 | cflags = get_config_vars().get('CFLAGS') | ||
727 | |||
728 | archs = re.findall(r'-arch\s+(\S+)', cflags) | ||
729 | archs = tuple(sorted(set(archs))) | ||
730 | |||
731 | if len(archs) == 1: | ||
732 | machine = archs[0] | ||
733 | elif archs == ('i386', 'ppc'): | ||
734 | machine = 'fat' | ||
735 | elif archs == ('i386', 'x86_64'): | ||
736 | machine = 'intel' | ||
737 | elif archs == ('i386', 'ppc', 'x86_64'): | ||
738 | machine = 'fat3' | ||
739 | elif archs == ('ppc64', 'x86_64'): | ||
740 | machine = 'fat64' | ||
741 | elif archs == ('i386', 'ppc', 'ppc64', 'x86_64'): | ||
742 | machine = 'universal' | ||
743 | else: | ||
744 | raise ValueError( | ||
745 | "Don't know machine value for archs=%r" % (archs,)) | ||
746 | |||
747 | elif machine == 'i386': | ||
748 | # On OSX the machine type returned by uname is always the | ||
749 | # 32-bit variant, even if the executable architecture is | ||
750 | # the 64-bit variant | ||
751 | if sys.maxsize >= 2**32: | ||
752 | machine = 'x86_64' | ||
753 | |||
754 | elif machine in ('PowerPC', 'Power_Macintosh'): | ||
755 | # Pick a sane name for the PPC architecture. | ||
756 | # See 'i386' case | ||
757 | if sys.maxsize >= 2**32: | ||
758 | machine = 'ppc64' | ||
759 | else: | ||
760 | machine = 'ppc' | ||
761 | |||
762 | return "%s-%s-%s" % (osname, release, machine) | ||
763 | |||
764 | |||
765 | def get_python_version(): | ||
766 | return _PY_VERSION_SHORT | ||
767 | |||
768 | |||
769 | def _print_dict(title, data): | ||
770 | for index, (key, value) in enumerate(sorted(data.items())): | ||
771 | if index == 0: | ||
772 | print('%s: ' % (title)) | ||
773 | print('\t%s = "%s"' % (key, value)) | ||
774 | |||
775 | |||
776 | def _main(): | ||
777 | """Display all information sysconfig detains.""" | ||
778 | print('Platform: "%s"' % get_platform()) | ||
779 | print('Python version: "%s"' % get_python_version()) | ||
780 | print('Current installation scheme: "%s"' % _get_default_scheme()) | ||
781 | print() | ||
782 | _print_dict('Paths', get_paths()) | ||
783 | print() | ||
784 | _print_dict('Variables', get_config_vars()) | ||
785 | |||
786 | |||
787 | if __name__ == '__main__': | ||
788 | _main() | ||
diff --git a/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/distlib/_backport/tarfile.py b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/distlib/_backport/tarfile.py new file mode 100644 index 0000000..b0599bc --- /dev/null +++ b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/distlib/_backport/tarfile.py | |||
@@ -0,0 +1,2607 @@ | |||
1 | #------------------------------------------------------------------- | ||
2 | # tarfile.py | ||
3 | #------------------------------------------------------------------- | ||
4 | # Copyright (C) 2002 Lars Gustaebel <lars@gustaebel.de> | ||
5 | # All rights reserved. | ||
6 | # | ||
7 | # Permission is hereby granted, free of charge, to any person | ||
8 | # obtaining a copy of this software and associated documentation | ||
9 | # files (the "Software"), to deal in the Software without | ||
10 | # restriction, including without limitation the rights to use, | ||
11 | # copy, modify, merge, publish, distribute, sublicense, and/or sell | ||
12 | # copies of the Software, and to permit persons to whom the | ||
13 | # Software is furnished to do so, subject to the following | ||
14 | # conditions: | ||
15 | # | ||
16 | # The above copyright notice and this permission notice shall be | ||
17 | # included in all copies or substantial portions of the Software. | ||
18 | # | ||
19 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | ||
20 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES | ||
21 | # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | ||
22 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT | ||
23 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, | ||
24 | # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
25 | # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
26 | # OTHER DEALINGS IN THE SOFTWARE. | ||
27 | # | ||
28 | from __future__ import print_function | ||
29 | |||
30 | """Read from and write to tar format archives. | ||
31 | """ | ||
32 | |||
33 | __version__ = "$Revision$" | ||
34 | |||
35 | version = "0.9.0" | ||
36 | __author__ = "Lars Gust\u00e4bel (lars@gustaebel.de)" | ||
37 | __date__ = "$Date: 2011-02-25 17:42:01 +0200 (Fri, 25 Feb 2011) $" | ||
38 | __cvsid__ = "$Id: tarfile.py 88586 2011-02-25 15:42:01Z marc-andre.lemburg $" | ||
39 | __credits__ = "Gustavo Niemeyer, Niels Gust\u00e4bel, Richard Townsend." | ||
40 | |||
41 | #--------- | ||
42 | # Imports | ||
43 | #--------- | ||
44 | import sys | ||
45 | import os | ||
46 | import stat | ||
47 | import errno | ||
48 | import time | ||
49 | import struct | ||
50 | import copy | ||
51 | import re | ||
52 | |||
53 | try: | ||
54 | import grp, pwd | ||
55 | except ImportError: | ||
56 | grp = pwd = None | ||
57 | |||
58 | # os.symlink on Windows prior to 6.0 raises NotImplementedError | ||
59 | symlink_exception = (AttributeError, NotImplementedError) | ||
60 | try: | ||
61 | # WindowsError (1314) will be raised if the caller does not hold the | ||
62 | # SeCreateSymbolicLinkPrivilege privilege | ||
63 | symlink_exception += (WindowsError,) | ||
64 | except NameError: | ||
65 | pass | ||
66 | |||
67 | # from tarfile import * | ||
68 | __all__ = ["TarFile", "TarInfo", "is_tarfile", "TarError"] | ||
69 | |||
70 | if sys.version_info[0] < 3: | ||
71 | import __builtin__ as builtins | ||
72 | else: | ||
73 | import builtins | ||
74 | |||
75 | _open = builtins.open # Since 'open' is TarFile.open | ||
76 | |||
77 | #--------------------------------------------------------- | ||
78 | # tar constants | ||
79 | #--------------------------------------------------------- | ||
80 | NUL = b"\0" # the null character | ||
81 | BLOCKSIZE = 512 # length of processing blocks | ||
82 | RECORDSIZE = BLOCKSIZE * 20 # length of records | ||
83 | GNU_MAGIC = b"ustar \0" # magic gnu tar string | ||
84 | POSIX_MAGIC = b"ustar\x0000" # magic posix tar string | ||
85 | |||
86 | LENGTH_NAME = 100 # maximum length of a filename | ||
87 | LENGTH_LINK = 100 # maximum length of a linkname | ||
88 | LENGTH_PREFIX = 155 # maximum length of the prefix field | ||
89 | |||
90 | REGTYPE = b"0" # regular file | ||
91 | AREGTYPE = b"\0" # regular file | ||
92 | LNKTYPE = b"1" # link (inside tarfile) | ||
93 | SYMTYPE = b"2" # symbolic link | ||
94 | CHRTYPE = b"3" # character special device | ||
95 | BLKTYPE = b"4" # block special device | ||
96 | DIRTYPE = b"5" # directory | ||
97 | FIFOTYPE = b"6" # fifo special device | ||
98 | CONTTYPE = b"7" # contiguous file | ||
99 | |||
100 | GNUTYPE_LONGNAME = b"L" # GNU tar longname | ||
101 | GNUTYPE_LONGLINK = b"K" # GNU tar longlink | ||
102 | GNUTYPE_SPARSE = b"S" # GNU tar sparse file | ||
103 | |||
104 | XHDTYPE = b"x" # POSIX.1-2001 extended header | ||
105 | XGLTYPE = b"g" # POSIX.1-2001 global header | ||
106 | SOLARIS_XHDTYPE = b"X" # Solaris extended header | ||
107 | |||
108 | USTAR_FORMAT = 0 # POSIX.1-1988 (ustar) format | ||
109 | GNU_FORMAT = 1 # GNU tar format | ||
110 | PAX_FORMAT = 2 # POSIX.1-2001 (pax) format | ||
111 | DEFAULT_FORMAT = GNU_FORMAT | ||
112 | |||
113 | #--------------------------------------------------------- | ||
114 | # tarfile constants | ||
115 | #--------------------------------------------------------- | ||
116 | # File types that tarfile supports: | ||
117 | SUPPORTED_TYPES = (REGTYPE, AREGTYPE, LNKTYPE, | ||
118 | SYMTYPE, DIRTYPE, FIFOTYPE, | ||
119 | CONTTYPE, CHRTYPE, BLKTYPE, | ||
120 | GNUTYPE_LONGNAME, GNUTYPE_LONGLINK, | ||
121 | GNUTYPE_SPARSE) | ||
122 | |||
123 | # File types that will be treated as a regular file. | ||
124 | REGULAR_TYPES = (REGTYPE, AREGTYPE, | ||
125 | CONTTYPE, GNUTYPE_SPARSE) | ||
126 | |||
127 | # File types that are part of the GNU tar format. | ||
128 | GNU_TYPES = (GNUTYPE_LONGNAME, GNUTYPE_LONGLINK, | ||
129 | GNUTYPE_SPARSE) | ||
130 | |||
131 | # Fields from a pax header that override a TarInfo attribute. | ||
132 | PAX_FIELDS = ("path", "linkpath", "size", "mtime", | ||
133 | "uid", "gid", "uname", "gname") | ||
134 | |||
135 | # Fields from a pax header that are affected by hdrcharset. | ||
136 | PAX_NAME_FIELDS = set(("path", "linkpath", "uname", "gname")) | ||
137 | |||
138 | # Fields in a pax header that are numbers, all other fields | ||
139 | # are treated as strings. | ||
140 | PAX_NUMBER_FIELDS = { | ||
141 | "atime": float, | ||
142 | "ctime": float, | ||
143 | "mtime": float, | ||
144 | "uid": int, | ||
145 | "gid": int, | ||
146 | "size": int | ||
147 | } | ||
148 | |||
149 | #--------------------------------------------------------- | ||
150 | # Bits used in the mode field, values in octal. | ||
151 | #--------------------------------------------------------- | ||
152 | S_IFLNK = 0o120000 # symbolic link | ||
153 | S_IFREG = 0o100000 # regular file | ||
154 | S_IFBLK = 0o060000 # block device | ||
155 | S_IFDIR = 0o040000 # directory | ||
156 | S_IFCHR = 0o020000 # character device | ||
157 | S_IFIFO = 0o010000 # fifo | ||
158 | |||
159 | TSUID = 0o4000 # set UID on execution | ||
160 | TSGID = 0o2000 # set GID on execution | ||
161 | TSVTX = 0o1000 # reserved | ||
162 | |||
163 | TUREAD = 0o400 # read by owner | ||
164 | TUWRITE = 0o200 # write by owner | ||
165 | TUEXEC = 0o100 # execute/search by owner | ||
166 | TGREAD = 0o040 # read by group | ||
167 | TGWRITE = 0o020 # write by group | ||
168 | TGEXEC = 0o010 # execute/search by group | ||
169 | TOREAD = 0o004 # read by other | ||
170 | TOWRITE = 0o002 # write by other | ||
171 | TOEXEC = 0o001 # execute/search by other | ||
172 | |||
173 | #--------------------------------------------------------- | ||
174 | # initialization | ||
175 | #--------------------------------------------------------- | ||
176 | if os.name in ("nt", "ce"): | ||
177 | ENCODING = "utf-8" | ||
178 | else: | ||
179 | ENCODING = sys.getfilesystemencoding() | ||
180 | |||
181 | #--------------------------------------------------------- | ||
182 | # Some useful functions | ||
183 | #--------------------------------------------------------- | ||
184 | |||
185 | def stn(s, length, encoding, errors): | ||
186 | """Convert a string to a null-terminated bytes object. | ||
187 | """ | ||
188 | s = s.encode(encoding, errors) | ||
189 | return s[:length] + (length - len(s)) * NUL | ||
190 | |||
191 | def nts(s, encoding, errors): | ||
192 | """Convert a null-terminated bytes object to a string. | ||
193 | """ | ||
194 | p = s.find(b"\0") | ||
195 | if p != -1: | ||
196 | s = s[:p] | ||
197 | return s.decode(encoding, errors) | ||
198 | |||
199 | def nti(s): | ||
200 | """Convert a number field to a python number. | ||
201 | """ | ||
202 | # There are two possible encodings for a number field, see | ||
203 | # itn() below. | ||
204 | if s[0] != chr(0o200): | ||
205 | try: | ||
206 | n = int(nts(s, "ascii", "strict") or "0", 8) | ||
207 | except ValueError: | ||
208 | raise InvalidHeaderError("invalid header") | ||
209 | else: | ||
210 | n = 0 | ||
211 | for i in range(len(s) - 1): | ||
212 | n <<= 8 | ||
213 | n += ord(s[i + 1]) | ||
214 | return n | ||
215 | |||
216 | def itn(n, digits=8, format=DEFAULT_FORMAT): | ||
217 | """Convert a python number to a number field. | ||
218 | """ | ||
219 | # POSIX 1003.1-1988 requires numbers to be encoded as a string of | ||
220 | # octal digits followed by a null-byte, this allows values up to | ||
221 | # (8**(digits-1))-1. GNU tar allows storing numbers greater than | ||
222 | # that if necessary. A leading 0o200 byte indicates this particular | ||
223 | # encoding, the following digits-1 bytes are a big-endian | ||
224 | # representation. This allows values up to (256**(digits-1))-1. | ||
225 | if 0 <= n < 8 ** (digits - 1): | ||
226 | s = ("%0*o" % (digits - 1, n)).encode("ascii") + NUL | ||
227 | else: | ||
228 | if format != GNU_FORMAT or n >= 256 ** (digits - 1): | ||
229 | raise ValueError("overflow in number field") | ||
230 | |||
231 | if n < 0: | ||
232 | # XXX We mimic GNU tar's behaviour with negative numbers, | ||
233 | # this could raise OverflowError. | ||
234 | n = struct.unpack("L", struct.pack("l", n))[0] | ||
235 | |||
236 | s = bytearray() | ||
237 | for i in range(digits - 1): | ||
238 | s.insert(0, n & 0o377) | ||
239 | n >>= 8 | ||
240 | s.insert(0, 0o200) | ||
241 | return s | ||
242 | |||
243 | def calc_chksums(buf): | ||
244 | """Calculate the checksum for a member's header by summing up all | ||
245 | characters except for the chksum field which is treated as if | ||
246 | it was filled with spaces. According to the GNU tar sources, | ||
247 | some tars (Sun and NeXT) calculate chksum with signed char, | ||
248 | which will be different if there are chars in the buffer with | ||
249 | the high bit set. So we calculate two checksums, unsigned and | ||
250 | signed. | ||
251 | """ | ||
252 | unsigned_chksum = 256 + sum(struct.unpack("148B", buf[:148]) + struct.unpack("356B", buf[156:512])) | ||
253 | signed_chksum = 256 + sum(struct.unpack("148b", buf[:148]) + struct.unpack("356b", buf[156:512])) | ||
254 | return unsigned_chksum, signed_chksum | ||
255 | |||
256 | def copyfileobj(src, dst, length=None): | ||
257 | """Copy length bytes from fileobj src to fileobj dst. | ||
258 | If length is None, copy the entire content. | ||
259 | """ | ||
260 | if length == 0: | ||
261 | return | ||
262 | if length is None: | ||
263 | while True: | ||
264 | buf = src.read(16*1024) | ||
265 | if not buf: | ||
266 | break | ||
267 | dst.write(buf) | ||
268 | return | ||
269 | |||
270 | BUFSIZE = 16 * 1024 | ||
271 | blocks, remainder = divmod(length, BUFSIZE) | ||
272 | for b in range(blocks): | ||
273 | buf = src.read(BUFSIZE) | ||
274 | if len(buf) < BUFSIZE: | ||
275 | raise IOError("end of file reached") | ||
276 | dst.write(buf) | ||
277 | |||
278 | if remainder != 0: | ||
279 | buf = src.read(remainder) | ||
280 | if len(buf) < remainder: | ||
281 | raise IOError("end of file reached") | ||
282 | dst.write(buf) | ||
283 | return | ||
284 | |||
285 | filemode_table = ( | ||
286 | ((S_IFLNK, "l"), | ||
287 | (S_IFREG, "-"), | ||
288 | (S_IFBLK, "b"), | ||
289 | (S_IFDIR, "d"), | ||
290 | (S_IFCHR, "c"), | ||
291 | (S_IFIFO, "p")), | ||
292 | |||
293 | ((TUREAD, "r"),), | ||
294 | ((TUWRITE, "w"),), | ||
295 | ((TUEXEC|TSUID, "s"), | ||
296 | (TSUID, "S"), | ||
297 | (TUEXEC, "x")), | ||
298 | |||
299 | ((TGREAD, "r"),), | ||
300 | ((TGWRITE, "w"),), | ||
301 | ((TGEXEC|TSGID, "s"), | ||
302 | (TSGID, "S"), | ||
303 | (TGEXEC, "x")), | ||
304 | |||
305 | ((TOREAD, "r"),), | ||
306 | ((TOWRITE, "w"),), | ||
307 | ((TOEXEC|TSVTX, "t"), | ||
308 | (TSVTX, "T"), | ||
309 | (TOEXEC, "x")) | ||
310 | ) | ||
311 | |||
312 | def filemode(mode): | ||
313 | """Convert a file's mode to a string of the form | ||
314 | -rwxrwxrwx. | ||
315 | Used by TarFile.list() | ||
316 | """ | ||
317 | perm = [] | ||
318 | for table in filemode_table: | ||
319 | for bit, char in table: | ||
320 | if mode & bit == bit: | ||
321 | perm.append(char) | ||
322 | break | ||
323 | else: | ||
324 | perm.append("-") | ||
325 | return "".join(perm) | ||
326 | |||
327 | class TarError(Exception): | ||
328 | """Base exception.""" | ||
329 | pass | ||
330 | class ExtractError(TarError): | ||
331 | """General exception for extract errors.""" | ||
332 | pass | ||
333 | class ReadError(TarError): | ||
334 | """Exception for unreadable tar archives.""" | ||
335 | pass | ||
336 | class CompressionError(TarError): | ||
337 | """Exception for unavailable compression methods.""" | ||
338 | pass | ||
339 | class StreamError(TarError): | ||
340 | """Exception for unsupported operations on stream-like TarFiles.""" | ||
341 | pass | ||
342 | class HeaderError(TarError): | ||
343 | """Base exception for header errors.""" | ||
344 | pass | ||
345 | class EmptyHeaderError(HeaderError): | ||
346 | """Exception for empty headers.""" | ||
347 | pass | ||
348 | class TruncatedHeaderError(HeaderError): | ||
349 | """Exception for truncated headers.""" | ||
350 | pass | ||
351 | class EOFHeaderError(HeaderError): | ||
352 | """Exception for end of file headers.""" | ||
353 | pass | ||
354 | class InvalidHeaderError(HeaderError): | ||
355 | """Exception for invalid headers.""" | ||
356 | pass | ||
357 | class SubsequentHeaderError(HeaderError): | ||
358 | """Exception for missing and invalid extended headers.""" | ||
359 | pass | ||
360 | |||
361 | #--------------------------- | ||
362 | # internal stream interface | ||
363 | #--------------------------- | ||
364 | class _LowLevelFile(object): | ||
365 | """Low-level file object. Supports reading and writing. | ||
366 | It is used instead of a regular file object for streaming | ||
367 | access. | ||
368 | """ | ||
369 | |||
370 | def __init__(self, name, mode): | ||
371 | mode = { | ||
372 | "r": os.O_RDONLY, | ||
373 | "w": os.O_WRONLY | os.O_CREAT | os.O_TRUNC, | ||
374 | }[mode] | ||
375 | if hasattr(os, "O_BINARY"): | ||
376 | mode |= os.O_BINARY | ||
377 | self.fd = os.open(name, mode, 0o666) | ||
378 | |||
379 | def close(self): | ||
380 | os.close(self.fd) | ||
381 | |||
382 | def read(self, size): | ||
383 | return os.read(self.fd, size) | ||
384 | |||
385 | def write(self, s): | ||
386 | os.write(self.fd, s) | ||
387 | |||
388 | class _Stream(object): | ||
389 | """Class that serves as an adapter between TarFile and | ||
390 | a stream-like object. The stream-like object only | ||
391 | needs to have a read() or write() method and is accessed | ||
392 | blockwise. Use of gzip or bzip2 compression is possible. | ||
393 | A stream-like object could be for example: sys.stdin, | ||
394 | sys.stdout, a socket, a tape device etc. | ||
395 | |||
396 | _Stream is intended to be used only internally. | ||
397 | """ | ||
398 | |||
399 | def __init__(self, name, mode, comptype, fileobj, bufsize): | ||
400 | """Construct a _Stream object. | ||
401 | """ | ||
402 | self._extfileobj = True | ||
403 | if fileobj is None: | ||
404 | fileobj = _LowLevelFile(name, mode) | ||
405 | self._extfileobj = False | ||
406 | |||
407 | if comptype == '*': | ||
408 | # Enable transparent compression detection for the | ||
409 | # stream interface | ||
410 | fileobj = _StreamProxy(fileobj) | ||
411 | comptype = fileobj.getcomptype() | ||
412 | |||
413 | self.name = name or "" | ||
414 | self.mode = mode | ||
415 | self.comptype = comptype | ||
416 | self.fileobj = fileobj | ||
417 | self.bufsize = bufsize | ||
418 | self.buf = b"" | ||
419 | self.pos = 0 | ||
420 | self.closed = False | ||
421 | |||
422 | try: | ||
423 | if comptype == "gz": | ||
424 | try: | ||
425 | import zlib | ||
426 | except ImportError: | ||
427 | raise CompressionError("zlib module is not available") | ||
428 | self.zlib = zlib | ||
429 | self.crc = zlib.crc32(b"") | ||
430 | if mode == "r": | ||
431 | self._init_read_gz() | ||
432 | else: | ||
433 | self._init_write_gz() | ||
434 | |||
435 | if comptype == "bz2": | ||
436 | try: | ||
437 | import bz2 | ||
438 | except ImportError: | ||
439 | raise CompressionError("bz2 module is not available") | ||
440 | if mode == "r": | ||
441 | self.dbuf = b"" | ||
442 | self.cmp = bz2.BZ2Decompressor() | ||
443 | else: | ||
444 | self.cmp = bz2.BZ2Compressor() | ||
445 | except: | ||
446 | if not self._extfileobj: | ||
447 | self.fileobj.close() | ||
448 | self.closed = True | ||
449 | raise | ||
450 | |||
451 | def __del__(self): | ||
452 | if hasattr(self, "closed") and not self.closed: | ||
453 | self.close() | ||
454 | |||
455 | def _init_write_gz(self): | ||
456 | """Initialize for writing with gzip compression. | ||
457 | """ | ||
458 | self.cmp = self.zlib.compressobj(9, self.zlib.DEFLATED, | ||
459 | -self.zlib.MAX_WBITS, | ||
460 | self.zlib.DEF_MEM_LEVEL, | ||
461 | 0) | ||
462 | timestamp = struct.pack("<L", int(time.time())) | ||
463 | self.__write(b"\037\213\010\010" + timestamp + b"\002\377") | ||
464 | if self.name.endswith(".gz"): | ||
465 | self.name = self.name[:-3] | ||
466 | # RFC1952 says we must use ISO-8859-1 for the FNAME field. | ||
467 | self.__write(self.name.encode("iso-8859-1", "replace") + NUL) | ||
468 | |||
469 | def write(self, s): | ||
470 | """Write string s to the stream. | ||
471 | """ | ||
472 | if self.comptype == "gz": | ||
473 | self.crc = self.zlib.crc32(s, self.crc) | ||
474 | self.pos += len(s) | ||
475 | if self.comptype != "tar": | ||
476 | s = self.cmp.compress(s) | ||
477 | self.__write(s) | ||
478 | |||
479 | def __write(self, s): | ||
480 | """Write string s to the stream if a whole new block | ||
481 | is ready to be written. | ||
482 | """ | ||
483 | self.buf += s | ||
484 | while len(self.buf) > self.bufsize: | ||
485 | self.fileobj.write(self.buf[:self.bufsize]) | ||
486 | self.buf = self.buf[self.bufsize:] | ||
487 | |||
488 | def close(self): | ||
489 | """Close the _Stream object. No operation should be | ||
490 | done on it afterwards. | ||
491 | """ | ||
492 | if self.closed: | ||
493 | return | ||
494 | |||
495 | if self.mode == "w" and self.comptype != "tar": | ||
496 | self.buf += self.cmp.flush() | ||
497 | |||
498 | if self.mode == "w" and self.buf: | ||
499 | self.fileobj.write(self.buf) | ||
500 | self.buf = b"" | ||
501 | if self.comptype == "gz": | ||
502 | # The native zlib crc is an unsigned 32-bit integer, but | ||
503 | # the Python wrapper implicitly casts that to a signed C | ||
504 | # long. So, on a 32-bit box self.crc may "look negative", | ||
505 | # while the same crc on a 64-bit box may "look positive". | ||
506 | # To avoid irksome warnings from the `struct` module, force | ||
507 | # it to look positive on all boxes. | ||
508 | self.fileobj.write(struct.pack("<L", self.crc & 0xffffffff)) | ||
509 | self.fileobj.write(struct.pack("<L", self.pos & 0xffffFFFF)) | ||
510 | |||
511 | if not self._extfileobj: | ||
512 | self.fileobj.close() | ||
513 | |||
514 | self.closed = True | ||
515 | |||
516 | def _init_read_gz(self): | ||
517 | """Initialize for reading a gzip compressed fileobj. | ||
518 | """ | ||
519 | self.cmp = self.zlib.decompressobj(-self.zlib.MAX_WBITS) | ||
520 | self.dbuf = b"" | ||
521 | |||
522 | # taken from gzip.GzipFile with some alterations | ||
523 | if self.__read(2) != b"\037\213": | ||
524 | raise ReadError("not a gzip file") | ||
525 | if self.__read(1) != b"\010": | ||
526 | raise CompressionError("unsupported compression method") | ||
527 | |||
528 | flag = ord(self.__read(1)) | ||
529 | self.__read(6) | ||
530 | |||
531 | if flag & 4: | ||
532 | xlen = ord(self.__read(1)) + 256 * ord(self.__read(1)) | ||
533 | self.read(xlen) | ||
534 | if flag & 8: | ||
535 | while True: | ||
536 | s = self.__read(1) | ||
537 | if not s or s == NUL: | ||
538 | break | ||
539 | if flag & 16: | ||
540 | while True: | ||
541 | s = self.__read(1) | ||
542 | if not s or s == NUL: | ||
543 | break | ||
544 | if flag & 2: | ||
545 | self.__read(2) | ||
546 | |||
547 | def tell(self): | ||
548 | """Return the stream's file pointer position. | ||
549 | """ | ||
550 | return self.pos | ||
551 | |||
552 | def seek(self, pos=0): | ||
553 | """Set the stream's file pointer to pos. Negative seeking | ||
554 | is forbidden. | ||
555 | """ | ||
556 | if pos - self.pos >= 0: | ||
557 | blocks, remainder = divmod(pos - self.pos, self.bufsize) | ||
558 | for i in range(blocks): | ||
559 | self.read(self.bufsize) | ||
560 | self.read(remainder) | ||
561 | else: | ||
562 | raise StreamError("seeking backwards is not allowed") | ||
563 | return self.pos | ||
564 | |||
565 | def read(self, size=None): | ||
566 | """Return the next size number of bytes from the stream. | ||
567 | If size is not defined, return all bytes of the stream | ||
568 | up to EOF. | ||
569 | """ | ||
570 | if size is None: | ||
571 | t = [] | ||
572 | while True: | ||
573 | buf = self._read(self.bufsize) | ||
574 | if not buf: | ||
575 | break | ||
576 | t.append(buf) | ||
577 | buf = "".join(t) | ||
578 | else: | ||
579 | buf = self._read(size) | ||
580 | self.pos += len(buf) | ||
581 | return buf | ||
582 | |||
583 | def _read(self, size): | ||
584 | """Return size bytes from the stream. | ||
585 | """ | ||
586 | if self.comptype == "tar": | ||
587 | return self.__read(size) | ||
588 | |||
589 | c = len(self.dbuf) | ||
590 | while c < size: | ||
591 | buf = self.__read(self.bufsize) | ||
592 | if not buf: | ||
593 | break | ||
594 | try: | ||
595 | buf = self.cmp.decompress(buf) | ||
596 | except IOError: | ||
597 | raise ReadError("invalid compressed data") | ||
598 | self.dbuf += buf | ||
599 | c += len(buf) | ||
600 | buf = self.dbuf[:size] | ||
601 | self.dbuf = self.dbuf[size:] | ||
602 | return buf | ||
603 | |||
604 | def __read(self, size): | ||
605 | """Return size bytes from stream. If internal buffer is empty, | ||
606 | read another block from the stream. | ||
607 | """ | ||
608 | c = len(self.buf) | ||
609 | while c < size: | ||
610 | buf = self.fileobj.read(self.bufsize) | ||
611 | if not buf: | ||
612 | break | ||
613 | self.buf += buf | ||
614 | c += len(buf) | ||
615 | buf = self.buf[:size] | ||
616 | self.buf = self.buf[size:] | ||
617 | return buf | ||
618 | # class _Stream | ||
619 | |||
620 | class _StreamProxy(object): | ||
621 | """Small proxy class that enables transparent compression | ||
622 | detection for the Stream interface (mode 'r|*'). | ||
623 | """ | ||
624 | |||
625 | def __init__(self, fileobj): | ||
626 | self.fileobj = fileobj | ||
627 | self.buf = self.fileobj.read(BLOCKSIZE) | ||
628 | |||
629 | def read(self, size): | ||
630 | self.read = self.fileobj.read | ||
631 | return self.buf | ||
632 | |||
633 | def getcomptype(self): | ||
634 | if self.buf.startswith(b"\037\213\010"): | ||
635 | return "gz" | ||
636 | if self.buf.startswith(b"BZh91"): | ||
637 | return "bz2" | ||
638 | return "tar" | ||
639 | |||
640 | def close(self): | ||
641 | self.fileobj.close() | ||
642 | # class StreamProxy | ||
643 | |||
644 | class _BZ2Proxy(object): | ||
645 | """Small proxy class that enables external file object | ||
646 | support for "r:bz2" and "w:bz2" modes. This is actually | ||
647 | a workaround for a limitation in bz2 module's BZ2File | ||
648 | class which (unlike gzip.GzipFile) has no support for | ||
649 | a file object argument. | ||
650 | """ | ||
651 | |||
652 | blocksize = 16 * 1024 | ||
653 | |||
654 | def __init__(self, fileobj, mode): | ||
655 | self.fileobj = fileobj | ||
656 | self.mode = mode | ||
657 | self.name = getattr(self.fileobj, "name", None) | ||
658 | self.init() | ||
659 | |||
660 | def init(self): | ||
661 | import bz2 | ||
662 | self.pos = 0 | ||
663 | if self.mode == "r": | ||
664 | self.bz2obj = bz2.BZ2Decompressor() | ||
665 | self.fileobj.seek(0) | ||
666 | self.buf = b"" | ||
667 | else: | ||
668 | self.bz2obj = bz2.BZ2Compressor() | ||
669 | |||
670 | def read(self, size): | ||
671 | x = len(self.buf) | ||
672 | while x < size: | ||
673 | raw = self.fileobj.read(self.blocksize) | ||
674 | if not raw: | ||
675 | break | ||
676 | data = self.bz2obj.decompress(raw) | ||
677 | self.buf += data | ||
678 | x += len(data) | ||
679 | |||
680 | buf = self.buf[:size] | ||
681 | self.buf = self.buf[size:] | ||
682 | self.pos += len(buf) | ||
683 | return buf | ||
684 | |||
685 | def seek(self, pos): | ||
686 | if pos < self.pos: | ||
687 | self.init() | ||
688 | self.read(pos - self.pos) | ||
689 | |||
690 | def tell(self): | ||
691 | return self.pos | ||
692 | |||
693 | def write(self, data): | ||
694 | self.pos += len(data) | ||
695 | raw = self.bz2obj.compress(data) | ||
696 | self.fileobj.write(raw) | ||
697 | |||
698 | def close(self): | ||
699 | if self.mode == "w": | ||
700 | raw = self.bz2obj.flush() | ||
701 | self.fileobj.write(raw) | ||
702 | # class _BZ2Proxy | ||
703 | |||
704 | #------------------------ | ||
705 | # Extraction file object | ||
706 | #------------------------ | ||
707 | class _FileInFile(object): | ||
708 | """A thin wrapper around an existing file object that | ||
709 | provides a part of its data as an individual file | ||
710 | object. | ||
711 | """ | ||
712 | |||
713 | def __init__(self, fileobj, offset, size, blockinfo=None): | ||
714 | self.fileobj = fileobj | ||
715 | self.offset = offset | ||
716 | self.size = size | ||
717 | self.position = 0 | ||
718 | |||
719 | if blockinfo is None: | ||
720 | blockinfo = [(0, size)] | ||
721 | |||
722 | # Construct a map with data and zero blocks. | ||
723 | self.map_index = 0 | ||
724 | self.map = [] | ||
725 | lastpos = 0 | ||
726 | realpos = self.offset | ||
727 | for offset, size in blockinfo: | ||
728 | if offset > lastpos: | ||
729 | self.map.append((False, lastpos, offset, None)) | ||
730 | self.map.append((True, offset, offset + size, realpos)) | ||
731 | realpos += size | ||
732 | lastpos = offset + size | ||
733 | if lastpos < self.size: | ||
734 | self.map.append((False, lastpos, self.size, None)) | ||
735 | |||
736 | def seekable(self): | ||
737 | if not hasattr(self.fileobj, "seekable"): | ||
738 | # XXX gzip.GzipFile and bz2.BZ2File | ||
739 | return True | ||
740 | return self.fileobj.seekable() | ||
741 | |||
742 | def tell(self): | ||
743 | """Return the current file position. | ||
744 | """ | ||
745 | return self.position | ||
746 | |||
747 | def seek(self, position): | ||
748 | """Seek to a position in the file. | ||
749 | """ | ||
750 | self.position = position | ||
751 | |||
752 | def read(self, size=None): | ||
753 | """Read data from the file. | ||
754 | """ | ||
755 | if size is None: | ||
756 | size = self.size - self.position | ||
757 | else: | ||
758 | size = min(size, self.size - self.position) | ||
759 | |||
760 | buf = b"" | ||
761 | while size > 0: | ||
762 | while True: | ||
763 | data, start, stop, offset = self.map[self.map_index] | ||
764 | if start <= self.position < stop: | ||
765 | break | ||
766 | else: | ||
767 | self.map_index += 1 | ||
768 | if self.map_index == len(self.map): | ||
769 | self.map_index = 0 | ||
770 | length = min(size, stop - self.position) | ||
771 | if data: | ||
772 | self.fileobj.seek(offset + (self.position - start)) | ||
773 | buf += self.fileobj.read(length) | ||
774 | else: | ||
775 | buf += NUL * length | ||
776 | size -= length | ||
777 | self.position += length | ||
778 | return buf | ||
779 | #class _FileInFile | ||
780 | |||
781 | |||
782 | class ExFileObject(object): | ||
783 | """File-like object for reading an archive member. | ||
784 | Is returned by TarFile.extractfile(). | ||
785 | """ | ||
786 | blocksize = 1024 | ||
787 | |||
788 | def __init__(self, tarfile, tarinfo): | ||
789 | self.fileobj = _FileInFile(tarfile.fileobj, | ||
790 | tarinfo.offset_data, | ||
791 | tarinfo.size, | ||
792 | tarinfo.sparse) | ||
793 | self.name = tarinfo.name | ||
794 | self.mode = "r" | ||
795 | self.closed = False | ||
796 | self.size = tarinfo.size | ||
797 | |||
798 | self.position = 0 | ||
799 | self.buffer = b"" | ||
800 | |||
801 | def readable(self): | ||
802 | return True | ||
803 | |||
804 | def writable(self): | ||
805 | return False | ||
806 | |||
807 | def seekable(self): | ||
808 | return self.fileobj.seekable() | ||
809 | |||
810 | def read(self, size=None): | ||
811 | """Read at most size bytes from the file. If size is not | ||
812 | present or None, read all data until EOF is reached. | ||
813 | """ | ||
814 | if self.closed: | ||
815 | raise ValueError("I/O operation on closed file") | ||
816 | |||
817 | buf = b"" | ||
818 | if self.buffer: | ||
819 | if size is None: | ||
820 | buf = self.buffer | ||
821 | self.buffer = b"" | ||
822 | else: | ||
823 | buf = self.buffer[:size] | ||
824 | self.buffer = self.buffer[size:] | ||
825 | |||
826 | if size is None: | ||
827 | buf += self.fileobj.read() | ||
828 | else: | ||
829 | buf += self.fileobj.read(size - len(buf)) | ||
830 | |||
831 | self.position += len(buf) | ||
832 | return buf | ||
833 | |||
834 | # XXX TextIOWrapper uses the read1() method. | ||
835 | read1 = read | ||
836 | |||
837 | def readline(self, size=-1): | ||
838 | """Read one entire line from the file. If size is present | ||
839 | and non-negative, return a string with at most that | ||
840 | size, which may be an incomplete line. | ||
841 | """ | ||
842 | if self.closed: | ||
843 | raise ValueError("I/O operation on closed file") | ||
844 | |||
845 | pos = self.buffer.find(b"\n") + 1 | ||
846 | if pos == 0: | ||
847 | # no newline found. | ||
848 | while True: | ||
849 | buf = self.fileobj.read(self.blocksize) | ||
850 | self.buffer += buf | ||
851 | if not buf or b"\n" in buf: | ||
852 | pos = self.buffer.find(b"\n") + 1 | ||
853 | if pos == 0: | ||
854 | # no newline found. | ||
855 | pos = len(self.buffer) | ||
856 | break | ||
857 | |||
858 | if size != -1: | ||
859 | pos = min(size, pos) | ||
860 | |||
861 | buf = self.buffer[:pos] | ||
862 | self.buffer = self.buffer[pos:] | ||
863 | self.position += len(buf) | ||
864 | return buf | ||
865 | |||
866 | def readlines(self): | ||
867 | """Return a list with all remaining lines. | ||
868 | """ | ||
869 | result = [] | ||
870 | while True: | ||
871 | line = self.readline() | ||
872 | if not line: break | ||
873 | result.append(line) | ||
874 | return result | ||
875 | |||
876 | def tell(self): | ||
877 | """Return the current file position. | ||
878 | """ | ||
879 | if self.closed: | ||
880 | raise ValueError("I/O operation on closed file") | ||
881 | |||
882 | return self.position | ||
883 | |||
884 | def seek(self, pos, whence=os.SEEK_SET): | ||
885 | """Seek to a position in the file. | ||
886 | """ | ||
887 | if self.closed: | ||
888 | raise ValueError("I/O operation on closed file") | ||
889 | |||
890 | if whence == os.SEEK_SET: | ||
891 | self.position = min(max(pos, 0), self.size) | ||
892 | elif whence == os.SEEK_CUR: | ||
893 | if pos < 0: | ||
894 | self.position = max(self.position + pos, 0) | ||
895 | else: | ||
896 | self.position = min(self.position + pos, self.size) | ||
897 | elif whence == os.SEEK_END: | ||
898 | self.position = max(min(self.size + pos, self.size), 0) | ||
899 | else: | ||
900 | raise ValueError("Invalid argument") | ||
901 | |||
902 | self.buffer = b"" | ||
903 | self.fileobj.seek(self.position) | ||
904 | |||
905 | def close(self): | ||
906 | """Close the file object. | ||
907 | """ | ||
908 | self.closed = True | ||
909 | |||
910 | def __iter__(self): | ||
911 | """Get an iterator over the file's lines. | ||
912 | """ | ||
913 | while True: | ||
914 | line = self.readline() | ||
915 | if not line: | ||
916 | break | ||
917 | yield line | ||
918 | #class ExFileObject | ||
919 | |||
920 | #------------------ | ||
921 | # Exported Classes | ||
922 | #------------------ | ||
923 | class TarInfo(object): | ||
924 | """Informational class which holds the details about an | ||
925 | archive member given by a tar header block. | ||
926 | TarInfo objects are returned by TarFile.getmember(), | ||
927 | TarFile.getmembers() and TarFile.gettarinfo() and are | ||
928 | usually created internally. | ||
929 | """ | ||
930 | |||
931 | __slots__ = ("name", "mode", "uid", "gid", "size", "mtime", | ||
932 | "chksum", "type", "linkname", "uname", "gname", | ||
933 | "devmajor", "devminor", | ||
934 | "offset", "offset_data", "pax_headers", "sparse", | ||
935 | "tarfile", "_sparse_structs", "_link_target") | ||
936 | |||
937 | def __init__(self, name=""): | ||
938 | """Construct a TarInfo object. name is the optional name | ||
939 | of the member. | ||
940 | """ | ||
941 | self.name = name # member name | ||
942 | self.mode = 0o644 # file permissions | ||
943 | self.uid = 0 # user id | ||
944 | self.gid = 0 # group id | ||
945 | self.size = 0 # file size | ||
946 | self.mtime = 0 # modification time | ||
947 | self.chksum = 0 # header checksum | ||
948 | self.type = REGTYPE # member type | ||
949 | self.linkname = "" # link name | ||
950 | self.uname = "" # user name | ||
951 | self.gname = "" # group name | ||
952 | self.devmajor = 0 # device major number | ||
953 | self.devminor = 0 # device minor number | ||
954 | |||
955 | self.offset = 0 # the tar header starts here | ||
956 | self.offset_data = 0 # the file's data starts here | ||
957 | |||
958 | self.sparse = None # sparse member information | ||
959 | self.pax_headers = {} # pax header information | ||
960 | |||
961 | # In pax headers the "name" and "linkname" field are called | ||
962 | # "path" and "linkpath". | ||
963 | def _getpath(self): | ||
964 | return self.name | ||
965 | def _setpath(self, name): | ||
966 | self.name = name | ||
967 | path = property(_getpath, _setpath) | ||
968 | |||
969 | def _getlinkpath(self): | ||
970 | return self.linkname | ||
971 | def _setlinkpath(self, linkname): | ||
972 | self.linkname = linkname | ||
973 | linkpath = property(_getlinkpath, _setlinkpath) | ||
974 | |||
975 | def __repr__(self): | ||
976 | return "<%s %r at %#x>" % (self.__class__.__name__,self.name,id(self)) | ||
977 | |||
978 | def get_info(self): | ||
979 | """Return the TarInfo's attributes as a dictionary. | ||
980 | """ | ||
981 | info = { | ||
982 | "name": self.name, | ||
983 | "mode": self.mode & 0o7777, | ||
984 | "uid": self.uid, | ||
985 | "gid": self.gid, | ||
986 | "size": self.size, | ||
987 | "mtime": self.mtime, | ||
988 | "chksum": self.chksum, | ||
989 | "type": self.type, | ||
990 | "linkname": self.linkname, | ||
991 | "uname": self.uname, | ||
992 | "gname": self.gname, | ||
993 | "devmajor": self.devmajor, | ||
994 | "devminor": self.devminor | ||
995 | } | ||
996 | |||
997 | if info["type"] == DIRTYPE and not info["name"].endswith("/"): | ||
998 | info["name"] += "/" | ||
999 | |||
1000 | return info | ||
1001 | |||
1002 | def tobuf(self, format=DEFAULT_FORMAT, encoding=ENCODING, errors="surrogateescape"): | ||
1003 | """Return a tar header as a string of 512 byte blocks. | ||
1004 | """ | ||
1005 | info = self.get_info() | ||
1006 | |||
1007 | if format == USTAR_FORMAT: | ||
1008 | return self.create_ustar_header(info, encoding, errors) | ||
1009 | elif format == GNU_FORMAT: | ||
1010 | return self.create_gnu_header(info, encoding, errors) | ||
1011 | elif format == PAX_FORMAT: | ||
1012 | return self.create_pax_header(info, encoding) | ||
1013 | else: | ||
1014 | raise ValueError("invalid format") | ||
1015 | |||
1016 | def create_ustar_header(self, info, encoding, errors): | ||
1017 | """Return the object as a ustar header block. | ||
1018 | """ | ||
1019 | info["magic"] = POSIX_MAGIC | ||
1020 | |||
1021 | if len(info["linkname"]) > LENGTH_LINK: | ||
1022 | raise ValueError("linkname is too long") | ||
1023 | |||
1024 | if len(info["name"]) > LENGTH_NAME: | ||
1025 | info["prefix"], info["name"] = self._posix_split_name(info["name"]) | ||
1026 | |||
1027 | return self._create_header(info, USTAR_FORMAT, encoding, errors) | ||
1028 | |||
1029 | def create_gnu_header(self, info, encoding, errors): | ||
1030 | """Return the object as a GNU header block sequence. | ||
1031 | """ | ||
1032 | info["magic"] = GNU_MAGIC | ||
1033 | |||
1034 | buf = b"" | ||
1035 | if len(info["linkname"]) > LENGTH_LINK: | ||
1036 | buf += self._create_gnu_long_header(info["linkname"], GNUTYPE_LONGLINK, encoding, errors) | ||
1037 | |||
1038 | if len(info["name"]) > LENGTH_NAME: | ||
1039 | buf += self._create_gnu_long_header(info["name"], GNUTYPE_LONGNAME, encoding, errors) | ||
1040 | |||
1041 | return buf + self._create_header(info, GNU_FORMAT, encoding, errors) | ||
1042 | |||
1043 | def create_pax_header(self, info, encoding): | ||
1044 | """Return the object as a ustar header block. If it cannot be | ||
1045 | represented this way, prepend a pax extended header sequence | ||
1046 | with supplement information. | ||
1047 | """ | ||
1048 | info["magic"] = POSIX_MAGIC | ||
1049 | pax_headers = self.pax_headers.copy() | ||
1050 | |||
1051 | # Test string fields for values that exceed the field length or cannot | ||
1052 | # be represented in ASCII encoding. | ||
1053 | for name, hname, length in ( | ||
1054 | ("name", "path", LENGTH_NAME), ("linkname", "linkpath", LENGTH_LINK), | ||
1055 | ("uname", "uname", 32), ("gname", "gname", 32)): | ||
1056 | |||
1057 | if hname in pax_headers: | ||
1058 | # The pax header has priority. | ||
1059 | continue | ||
1060 | |||
1061 | # Try to encode the string as ASCII. | ||
1062 | try: | ||
1063 | info[name].encode("ascii", "strict") | ||
1064 | except UnicodeEncodeError: | ||
1065 | pax_headers[hname] = info[name] | ||
1066 | continue | ||
1067 | |||
1068 | if len(info[name]) > length: | ||
1069 | pax_headers[hname] = info[name] | ||
1070 | |||
1071 | # Test number fields for values that exceed the field limit or values | ||
1072 | # that like to be stored as float. | ||
1073 | for name, digits in (("uid", 8), ("gid", 8), ("size", 12), ("mtime", 12)): | ||
1074 | if name in pax_headers: | ||
1075 | # The pax header has priority. Avoid overflow. | ||
1076 | info[name] = 0 | ||
1077 | continue | ||
1078 | |||
1079 | val = info[name] | ||
1080 | if not 0 <= val < 8 ** (digits - 1) or isinstance(val, float): | ||
1081 | pax_headers[name] = str(val) | ||
1082 | info[name] = 0 | ||
1083 | |||
1084 | # Create a pax extended header if necessary. | ||
1085 | if pax_headers: | ||
1086 | buf = self._create_pax_generic_header(pax_headers, XHDTYPE, encoding) | ||
1087 | else: | ||
1088 | buf = b"" | ||
1089 | |||
1090 | return buf + self._create_header(info, USTAR_FORMAT, "ascii", "replace") | ||
1091 | |||
1092 | @classmethod | ||
1093 | def create_pax_global_header(cls, pax_headers): | ||
1094 | """Return the object as a pax global header block sequence. | ||
1095 | """ | ||
1096 | return cls._create_pax_generic_header(pax_headers, XGLTYPE, "utf8") | ||
1097 | |||
1098 | def _posix_split_name(self, name): | ||
1099 | """Split a name longer than 100 chars into a prefix | ||
1100 | and a name part. | ||
1101 | """ | ||
1102 | prefix = name[:LENGTH_PREFIX + 1] | ||
1103 | while prefix and prefix[-1] != "/": | ||
1104 | prefix = prefix[:-1] | ||
1105 | |||
1106 | name = name[len(prefix):] | ||
1107 | prefix = prefix[:-1] | ||
1108 | |||
1109 | if not prefix or len(name) > LENGTH_NAME: | ||
1110 | raise ValueError("name is too long") | ||
1111 | return prefix, name | ||
1112 | |||
1113 | @staticmethod | ||
1114 | def _create_header(info, format, encoding, errors): | ||
1115 | """Return a header block. info is a dictionary with file | ||
1116 | information, format must be one of the *_FORMAT constants. | ||
1117 | """ | ||
1118 | parts = [ | ||
1119 | stn(info.get("name", ""), 100, encoding, errors), | ||
1120 | itn(info.get("mode", 0) & 0o7777, 8, format), | ||
1121 | itn(info.get("uid", 0), 8, format), | ||
1122 | itn(info.get("gid", 0), 8, format), | ||
1123 | itn(info.get("size", 0), 12, format), | ||
1124 | itn(info.get("mtime", 0), 12, format), | ||
1125 | b" ", # checksum field | ||
1126 | info.get("type", REGTYPE), | ||
1127 | stn(info.get("linkname", ""), 100, encoding, errors), | ||
1128 | info.get("magic", POSIX_MAGIC), | ||
1129 | stn(info.get("uname", ""), 32, encoding, errors), | ||
1130 | stn(info.get("gname", ""), 32, encoding, errors), | ||
1131 | itn(info.get("devmajor", 0), 8, format), | ||
1132 | itn(info.get("devminor", 0), 8, format), | ||
1133 | stn(info.get("prefix", ""), 155, encoding, errors) | ||
1134 | ] | ||
1135 | |||
1136 | buf = struct.pack("%ds" % BLOCKSIZE, b"".join(parts)) | ||
1137 | chksum = calc_chksums(buf[-BLOCKSIZE:])[0] | ||
1138 | buf = buf[:-364] + ("%06o\0" % chksum).encode("ascii") + buf[-357:] | ||
1139 | return buf | ||
1140 | |||
1141 | @staticmethod | ||
1142 | def _create_payload(payload): | ||
1143 | """Return the string payload filled with zero bytes | ||
1144 | up to the next 512 byte border. | ||
1145 | """ | ||
1146 | blocks, remainder = divmod(len(payload), BLOCKSIZE) | ||
1147 | if remainder > 0: | ||
1148 | payload += (BLOCKSIZE - remainder) * NUL | ||
1149 | return payload | ||
1150 | |||
1151 | @classmethod | ||
1152 | def _create_gnu_long_header(cls, name, type, encoding, errors): | ||
1153 | """Return a GNUTYPE_LONGNAME or GNUTYPE_LONGLINK sequence | ||
1154 | for name. | ||
1155 | """ | ||
1156 | name = name.encode(encoding, errors) + NUL | ||
1157 | |||
1158 | info = {} | ||
1159 | info["name"] = "././@LongLink" | ||
1160 | info["type"] = type | ||
1161 | info["size"] = len(name) | ||
1162 | info["magic"] = GNU_MAGIC | ||
1163 | |||
1164 | # create extended header + name blocks. | ||
1165 | return cls._create_header(info, USTAR_FORMAT, encoding, errors) + \ | ||
1166 | cls._create_payload(name) | ||
1167 | |||
1168 | @classmethod | ||
1169 | def _create_pax_generic_header(cls, pax_headers, type, encoding): | ||
1170 | """Return a POSIX.1-2008 extended or global header sequence | ||
1171 | that contains a list of keyword, value pairs. The values | ||
1172 | must be strings. | ||
1173 | """ | ||
1174 | # Check if one of the fields contains surrogate characters and thereby | ||
1175 | # forces hdrcharset=BINARY, see _proc_pax() for more information. | ||
1176 | binary = False | ||
1177 | for keyword, value in pax_headers.items(): | ||
1178 | try: | ||
1179 | value.encode("utf8", "strict") | ||
1180 | except UnicodeEncodeError: | ||
1181 | binary = True | ||
1182 | break | ||
1183 | |||
1184 | records = b"" | ||
1185 | if binary: | ||
1186 | # Put the hdrcharset field at the beginning of the header. | ||
1187 | records += b"21 hdrcharset=BINARY\n" | ||
1188 | |||
1189 | for keyword, value in pax_headers.items(): | ||
1190 | keyword = keyword.encode("utf8") | ||
1191 | if binary: | ||
1192 | # Try to restore the original byte representation of `value'. | ||
1193 | # Needless to say, that the encoding must match the string. | ||
1194 | value = value.encode(encoding, "surrogateescape") | ||
1195 | else: | ||
1196 | value = value.encode("utf8") | ||
1197 | |||
1198 | l = len(keyword) + len(value) + 3 # ' ' + '=' + '\n' | ||
1199 | n = p = 0 | ||
1200 | while True: | ||
1201 | n = l + len(str(p)) | ||
1202 | if n == p: | ||
1203 | break | ||
1204 | p = n | ||
1205 | records += bytes(str(p), "ascii") + b" " + keyword + b"=" + value + b"\n" | ||
1206 | |||
1207 | # We use a hardcoded "././@PaxHeader" name like star does | ||
1208 | # instead of the one that POSIX recommends. | ||
1209 | info = {} | ||
1210 | info["name"] = "././@PaxHeader" | ||
1211 | info["type"] = type | ||
1212 | info["size"] = len(records) | ||
1213 | info["magic"] = POSIX_MAGIC | ||
1214 | |||
1215 | # Create pax header + record blocks. | ||
1216 | return cls._create_header(info, USTAR_FORMAT, "ascii", "replace") + \ | ||
1217 | cls._create_payload(records) | ||
1218 | |||
1219 | @classmethod | ||
1220 | def frombuf(cls, buf, encoding, errors): | ||
1221 | """Construct a TarInfo object from a 512 byte bytes object. | ||
1222 | """ | ||
1223 | if len(buf) == 0: | ||
1224 | raise EmptyHeaderError("empty header") | ||
1225 | if len(buf) != BLOCKSIZE: | ||
1226 | raise TruncatedHeaderError("truncated header") | ||
1227 | if buf.count(NUL) == BLOCKSIZE: | ||
1228 | raise EOFHeaderError("end of file header") | ||
1229 | |||
1230 | chksum = nti(buf[148:156]) | ||
1231 | if chksum not in calc_chksums(buf): | ||
1232 | raise InvalidHeaderError("bad checksum") | ||
1233 | |||
1234 | obj = cls() | ||
1235 | obj.name = nts(buf[0:100], encoding, errors) | ||
1236 | obj.mode = nti(buf[100:108]) | ||
1237 | obj.uid = nti(buf[108:116]) | ||
1238 | obj.gid = nti(buf[116:124]) | ||
1239 | obj.size = nti(buf[124:136]) | ||
1240 | obj.mtime = nti(buf[136:148]) | ||
1241 | obj.chksum = chksum | ||
1242 | obj.type = buf[156:157] | ||
1243 | obj.linkname = nts(buf[157:257], encoding, errors) | ||
1244 | obj.uname = nts(buf[265:297], encoding, errors) | ||
1245 | obj.gname = nts(buf[297:329], encoding, errors) | ||
1246 | obj.devmajor = nti(buf[329:337]) | ||
1247 | obj.devminor = nti(buf[337:345]) | ||
1248 | prefix = nts(buf[345:500], encoding, errors) | ||
1249 | |||
1250 | # Old V7 tar format represents a directory as a regular | ||
1251 | # file with a trailing slash. | ||
1252 | if obj.type == AREGTYPE and obj.name.endswith("/"): | ||
1253 | obj.type = DIRTYPE | ||
1254 | |||
1255 | # The old GNU sparse format occupies some of the unused | ||
1256 | # space in the buffer for up to 4 sparse structures. | ||
1257 | # Save the them for later processing in _proc_sparse(). | ||
1258 | if obj.type == GNUTYPE_SPARSE: | ||
1259 | pos = 386 | ||
1260 | structs = [] | ||
1261 | for i in range(4): | ||
1262 | try: | ||
1263 | offset = nti(buf[pos:pos + 12]) | ||
1264 | numbytes = nti(buf[pos + 12:pos + 24]) | ||
1265 | except ValueError: | ||
1266 | break | ||
1267 | structs.append((offset, numbytes)) | ||
1268 | pos += 24 | ||
1269 | isextended = bool(buf[482]) | ||
1270 | origsize = nti(buf[483:495]) | ||
1271 | obj._sparse_structs = (structs, isextended, origsize) | ||
1272 | |||
1273 | # Remove redundant slashes from directories. | ||
1274 | if obj.isdir(): | ||
1275 | obj.name = obj.name.rstrip("/") | ||
1276 | |||
1277 | # Reconstruct a ustar longname. | ||
1278 | if prefix and obj.type not in GNU_TYPES: | ||
1279 | obj.name = prefix + "/" + obj.name | ||
1280 | return obj | ||
1281 | |||
1282 | @classmethod | ||
1283 | def fromtarfile(cls, tarfile): | ||
1284 | """Return the next TarInfo object from TarFile object | ||
1285 | tarfile. | ||
1286 | """ | ||
1287 | buf = tarfile.fileobj.read(BLOCKSIZE) | ||
1288 | obj = cls.frombuf(buf, tarfile.encoding, tarfile.errors) | ||
1289 | obj.offset = tarfile.fileobj.tell() - BLOCKSIZE | ||
1290 | return obj._proc_member(tarfile) | ||
1291 | |||
1292 | #-------------------------------------------------------------------------- | ||
1293 | # The following are methods that are called depending on the type of a | ||
1294 | # member. The entry point is _proc_member() which can be overridden in a | ||
1295 | # subclass to add custom _proc_*() methods. A _proc_*() method MUST | ||
1296 | # implement the following | ||
1297 | # operations: | ||
1298 | # 1. Set self.offset_data to the position where the data blocks begin, | ||
1299 | # if there is data that follows. | ||
1300 | # 2. Set tarfile.offset to the position where the next member's header will | ||
1301 | # begin. | ||
1302 | # 3. Return self or another valid TarInfo object. | ||
1303 | def _proc_member(self, tarfile): | ||
1304 | """Choose the right processing method depending on | ||
1305 | the type and call it. | ||
1306 | """ | ||
1307 | if self.type in (GNUTYPE_LONGNAME, GNUTYPE_LONGLINK): | ||
1308 | return self._proc_gnulong(tarfile) | ||
1309 | elif self.type == GNUTYPE_SPARSE: | ||
1310 | return self._proc_sparse(tarfile) | ||
1311 | elif self.type in (XHDTYPE, XGLTYPE, SOLARIS_XHDTYPE): | ||
1312 | return self._proc_pax(tarfile) | ||
1313 | else: | ||
1314 | return self._proc_builtin(tarfile) | ||
1315 | |||
1316 | def _proc_builtin(self, tarfile): | ||
1317 | """Process a builtin type or an unknown type which | ||
1318 | will be treated as a regular file. | ||
1319 | """ | ||
1320 | self.offset_data = tarfile.fileobj.tell() | ||
1321 | offset = self.offset_data | ||
1322 | if self.isreg() or self.type not in SUPPORTED_TYPES: | ||
1323 | # Skip the following data blocks. | ||
1324 | offset += self._block(self.size) | ||
1325 | tarfile.offset = offset | ||
1326 | |||
1327 | # Patch the TarInfo object with saved global | ||
1328 | # header information. | ||
1329 | self._apply_pax_info(tarfile.pax_headers, tarfile.encoding, tarfile.errors) | ||
1330 | |||
1331 | return self | ||
1332 | |||
1333 | def _proc_gnulong(self, tarfile): | ||
1334 | """Process the blocks that hold a GNU longname | ||
1335 | or longlink member. | ||
1336 | """ | ||
1337 | buf = tarfile.fileobj.read(self._block(self.size)) | ||
1338 | |||
1339 | # Fetch the next header and process it. | ||
1340 | try: | ||
1341 | next = self.fromtarfile(tarfile) | ||
1342 | except HeaderError: | ||
1343 | raise SubsequentHeaderError("missing or bad subsequent header") | ||
1344 | |||
1345 | # Patch the TarInfo object from the next header with | ||
1346 | # the longname information. | ||
1347 | next.offset = self.offset | ||
1348 | if self.type == GNUTYPE_LONGNAME: | ||
1349 | next.name = nts(buf, tarfile.encoding, tarfile.errors) | ||
1350 | elif self.type == GNUTYPE_LONGLINK: | ||
1351 | next.linkname = nts(buf, tarfile.encoding, tarfile.errors) | ||
1352 | |||
1353 | return next | ||
1354 | |||
1355 | def _proc_sparse(self, tarfile): | ||
1356 | """Process a GNU sparse header plus extra headers. | ||
1357 | """ | ||
1358 | # We already collected some sparse structures in frombuf(). | ||
1359 | structs, isextended, origsize = self._sparse_structs | ||
1360 | del self._sparse_structs | ||
1361 | |||
1362 | # Collect sparse structures from extended header blocks. | ||
1363 | while isextended: | ||
1364 | buf = tarfile.fileobj.read(BLOCKSIZE) | ||
1365 | pos = 0 | ||
1366 | for i in range(21): | ||
1367 | try: | ||
1368 | offset = nti(buf[pos:pos + 12]) | ||
1369 | numbytes = nti(buf[pos + 12:pos + 24]) | ||
1370 | except ValueError: | ||
1371 | break | ||
1372 | if offset and numbytes: | ||
1373 | structs.append((offset, numbytes)) | ||
1374 | pos += 24 | ||
1375 | isextended = bool(buf[504]) | ||
1376 | self.sparse = structs | ||
1377 | |||
1378 | self.offset_data = tarfile.fileobj.tell() | ||
1379 | tarfile.offset = self.offset_data + self._block(self.size) | ||
1380 | self.size = origsize | ||
1381 | return self | ||
1382 | |||
1383 | def _proc_pax(self, tarfile): | ||
1384 | """Process an extended or global header as described in | ||
1385 | POSIX.1-2008. | ||
1386 | """ | ||
1387 | # Read the header information. | ||
1388 | buf = tarfile.fileobj.read(self._block(self.size)) | ||
1389 | |||
1390 | # A pax header stores supplemental information for either | ||
1391 | # the following file (extended) or all following files | ||
1392 | # (global). | ||
1393 | if self.type == XGLTYPE: | ||
1394 | pax_headers = tarfile.pax_headers | ||
1395 | else: | ||
1396 | pax_headers = tarfile.pax_headers.copy() | ||
1397 | |||
1398 | # Check if the pax header contains a hdrcharset field. This tells us | ||
1399 | # the encoding of the path, linkpath, uname and gname fields. Normally, | ||
1400 | # these fields are UTF-8 encoded but since POSIX.1-2008 tar | ||
1401 | # implementations are allowed to store them as raw binary strings if | ||
1402 | # the translation to UTF-8 fails. | ||
1403 | match = re.search(br"\d+ hdrcharset=([^\n]+)\n", buf) | ||
1404 | if match is not None: | ||
1405 | pax_headers["hdrcharset"] = match.group(1).decode("utf8") | ||
1406 | |||
1407 | # For the time being, we don't care about anything other than "BINARY". | ||
1408 | # The only other value that is currently allowed by the standard is | ||
1409 | # "ISO-IR 10646 2000 UTF-8" in other words UTF-8. | ||
1410 | hdrcharset = pax_headers.get("hdrcharset") | ||
1411 | if hdrcharset == "BINARY": | ||
1412 | encoding = tarfile.encoding | ||
1413 | else: | ||
1414 | encoding = "utf8" | ||
1415 | |||
1416 | # Parse pax header information. A record looks like that: | ||
1417 | # "%d %s=%s\n" % (length, keyword, value). length is the size | ||
1418 | # of the complete record including the length field itself and | ||
1419 | # the newline. keyword and value are both UTF-8 encoded strings. | ||
1420 | regex = re.compile(br"(\d+) ([^=]+)=") | ||
1421 | pos = 0 | ||
1422 | while True: | ||
1423 | match = regex.match(buf, pos) | ||
1424 | if not match: | ||
1425 | break | ||
1426 | |||
1427 | length, keyword = match.groups() | ||
1428 | length = int(length) | ||
1429 | value = buf[match.end(2) + 1:match.start(1) + length - 1] | ||
1430 | |||
1431 | # Normally, we could just use "utf8" as the encoding and "strict" | ||
1432 | # as the error handler, but we better not take the risk. For | ||
1433 | # example, GNU tar <= 1.23 is known to store filenames it cannot | ||
1434 | # translate to UTF-8 as raw strings (unfortunately without a | ||
1435 | # hdrcharset=BINARY header). | ||
1436 | # We first try the strict standard encoding, and if that fails we | ||
1437 | # fall back on the user's encoding and error handler. | ||
1438 | keyword = self._decode_pax_field(keyword, "utf8", "utf8", | ||
1439 | tarfile.errors) | ||
1440 | if keyword in PAX_NAME_FIELDS: | ||
1441 | value = self._decode_pax_field(value, encoding, tarfile.encoding, | ||
1442 | tarfile.errors) | ||
1443 | else: | ||
1444 | value = self._decode_pax_field(value, "utf8", "utf8", | ||
1445 | tarfile.errors) | ||
1446 | |||
1447 | pax_headers[keyword] = value | ||
1448 | pos += length | ||
1449 | |||
1450 | # Fetch the next header. | ||
1451 | try: | ||
1452 | next = self.fromtarfile(tarfile) | ||
1453 | except HeaderError: | ||
1454 | raise SubsequentHeaderError("missing or bad subsequent header") | ||
1455 | |||
1456 | # Process GNU sparse information. | ||
1457 | if "GNU.sparse.map" in pax_headers: | ||
1458 | # GNU extended sparse format version 0.1. | ||
1459 | self._proc_gnusparse_01(next, pax_headers) | ||
1460 | |||
1461 | elif "GNU.sparse.size" in pax_headers: | ||
1462 | # GNU extended sparse format version 0.0. | ||
1463 | self._proc_gnusparse_00(next, pax_headers, buf) | ||
1464 | |||
1465 | elif pax_headers.get("GNU.sparse.major") == "1" and pax_headers.get("GNU.sparse.minor") == "0": | ||
1466 | # GNU extended sparse format version 1.0. | ||
1467 | self._proc_gnusparse_10(next, pax_headers, tarfile) | ||
1468 | |||
1469 | if self.type in (XHDTYPE, SOLARIS_XHDTYPE): | ||
1470 | # Patch the TarInfo object with the extended header info. | ||
1471 | next._apply_pax_info(pax_headers, tarfile.encoding, tarfile.errors) | ||
1472 | next.offset = self.offset | ||
1473 | |||
1474 | if "size" in pax_headers: | ||
1475 | # If the extended header replaces the size field, | ||
1476 | # we need to recalculate the offset where the next | ||
1477 | # header starts. | ||
1478 | offset = next.offset_data | ||
1479 | if next.isreg() or next.type not in SUPPORTED_TYPES: | ||
1480 | offset += next._block(next.size) | ||
1481 | tarfile.offset = offset | ||
1482 | |||
1483 | return next | ||
1484 | |||
1485 | def _proc_gnusparse_00(self, next, pax_headers, buf): | ||
1486 | """Process a GNU tar extended sparse header, version 0.0. | ||
1487 | """ | ||
1488 | offsets = [] | ||
1489 | for match in re.finditer(br"\d+ GNU.sparse.offset=(\d+)\n", buf): | ||
1490 | offsets.append(int(match.group(1))) | ||
1491 | numbytes = [] | ||
1492 | for match in re.finditer(br"\d+ GNU.sparse.numbytes=(\d+)\n", buf): | ||
1493 | numbytes.append(int(match.group(1))) | ||
1494 | next.sparse = list(zip(offsets, numbytes)) | ||
1495 | |||
1496 | def _proc_gnusparse_01(self, next, pax_headers): | ||
1497 | """Process a GNU tar extended sparse header, version 0.1. | ||
1498 | """ | ||
1499 | sparse = [int(x) for x in pax_headers["GNU.sparse.map"].split(",")] | ||
1500 | next.sparse = list(zip(sparse[::2], sparse[1::2])) | ||
1501 | |||
1502 | def _proc_gnusparse_10(self, next, pax_headers, tarfile): | ||
1503 | """Process a GNU tar extended sparse header, version 1.0. | ||
1504 | """ | ||
1505 | fields = None | ||
1506 | sparse = [] | ||
1507 | buf = tarfile.fileobj.read(BLOCKSIZE) | ||
1508 | fields, buf = buf.split(b"\n", 1) | ||
1509 | fields = int(fields) | ||
1510 | while len(sparse) < fields * 2: | ||
1511 | if b"\n" not in buf: | ||
1512 | buf += tarfile.fileobj.read(BLOCKSIZE) | ||
1513 | number, buf = buf.split(b"\n", 1) | ||
1514 | sparse.append(int(number)) | ||
1515 | next.offset_data = tarfile.fileobj.tell() | ||
1516 | next.sparse = list(zip(sparse[::2], sparse[1::2])) | ||
1517 | |||
1518 | def _apply_pax_info(self, pax_headers, encoding, errors): | ||
1519 | """Replace fields with supplemental information from a previous | ||
1520 | pax extended or global header. | ||
1521 | """ | ||
1522 | for keyword, value in pax_headers.items(): | ||
1523 | if keyword == "GNU.sparse.name": | ||
1524 | setattr(self, "path", value) | ||
1525 | elif keyword == "GNU.sparse.size": | ||
1526 | setattr(self, "size", int(value)) | ||
1527 | elif keyword == "GNU.sparse.realsize": | ||
1528 | setattr(self, "size", int(value)) | ||
1529 | elif keyword in PAX_FIELDS: | ||
1530 | if keyword in PAX_NUMBER_FIELDS: | ||
1531 | try: | ||
1532 | value = PAX_NUMBER_FIELDS[keyword](value) | ||
1533 | except ValueError: | ||
1534 | value = 0 | ||
1535 | if keyword == "path": | ||
1536 | value = value.rstrip("/") | ||
1537 | setattr(self, keyword, value) | ||
1538 | |||
1539 | self.pax_headers = pax_headers.copy() | ||
1540 | |||
1541 | def _decode_pax_field(self, value, encoding, fallback_encoding, fallback_errors): | ||
1542 | """Decode a single field from a pax record. | ||
1543 | """ | ||
1544 | try: | ||
1545 | return value.decode(encoding, "strict") | ||
1546 | except UnicodeDecodeError: | ||
1547 | return value.decode(fallback_encoding, fallback_errors) | ||
1548 | |||
1549 | def _block(self, count): | ||
1550 | """Round up a byte count by BLOCKSIZE and return it, | ||
1551 | e.g. _block(834) => 1024. | ||
1552 | """ | ||
1553 | blocks, remainder = divmod(count, BLOCKSIZE) | ||
1554 | if remainder: | ||
1555 | blocks += 1 | ||
1556 | return blocks * BLOCKSIZE | ||
1557 | |||
1558 | def isreg(self): | ||
1559 | return self.type in REGULAR_TYPES | ||
1560 | def isfile(self): | ||
1561 | return self.isreg() | ||
1562 | def isdir(self): | ||
1563 | return self.type == DIRTYPE | ||
1564 | def issym(self): | ||
1565 | return self.type == SYMTYPE | ||
1566 | def islnk(self): | ||
1567 | return self.type == LNKTYPE | ||
1568 | def ischr(self): | ||
1569 | return self.type == CHRTYPE | ||
1570 | def isblk(self): | ||
1571 | return self.type == BLKTYPE | ||
1572 | def isfifo(self): | ||
1573 | return self.type == FIFOTYPE | ||
1574 | def issparse(self): | ||
1575 | return self.sparse is not None | ||
1576 | def isdev(self): | ||
1577 | return self.type in (CHRTYPE, BLKTYPE, FIFOTYPE) | ||
1578 | # class TarInfo | ||
1579 | |||
1580 | class TarFile(object): | ||
1581 | """The TarFile Class provides an interface to tar archives. | ||
1582 | """ | ||
1583 | |||
1584 | debug = 0 # May be set from 0 (no msgs) to 3 (all msgs) | ||
1585 | |||
1586 | dereference = False # If true, add content of linked file to the | ||
1587 | # tar file, else the link. | ||
1588 | |||
1589 | ignore_zeros = False # If true, skips empty or invalid blocks and | ||
1590 | # continues processing. | ||
1591 | |||
1592 | errorlevel = 1 # If 0, fatal errors only appear in debug | ||
1593 | # messages (if debug >= 0). If > 0, errors | ||
1594 | # are passed to the caller as exceptions. | ||
1595 | |||
1596 | format = DEFAULT_FORMAT # The format to use when creating an archive. | ||
1597 | |||
1598 | encoding = ENCODING # Encoding for 8-bit character strings. | ||
1599 | |||
1600 | errors = None # Error handler for unicode conversion. | ||
1601 | |||
1602 | tarinfo = TarInfo # The default TarInfo class to use. | ||
1603 | |||
1604 | fileobject = ExFileObject # The default ExFileObject class to use. | ||
1605 | |||
1606 | def __init__(self, name=None, mode="r", fileobj=None, format=None, | ||
1607 | tarinfo=None, dereference=None, ignore_zeros=None, encoding=None, | ||
1608 | errors="surrogateescape", pax_headers=None, debug=None, errorlevel=None): | ||
1609 | """Open an (uncompressed) tar archive `name'. `mode' is either 'r' to | ||
1610 | read from an existing archive, 'a' to append data to an existing | ||
1611 | file or 'w' to create a new file overwriting an existing one. `mode' | ||
1612 | defaults to 'r'. | ||
1613 | If `fileobj' is given, it is used for reading or writing data. If it | ||
1614 | can be determined, `mode' is overridden by `fileobj's mode. | ||
1615 | `fileobj' is not closed, when TarFile is closed. | ||
1616 | """ | ||
1617 | if len(mode) > 1 or mode not in "raw": | ||
1618 | raise ValueError("mode must be 'r', 'a' or 'w'") | ||
1619 | self.mode = mode | ||
1620 | self._mode = {"r": "rb", "a": "r+b", "w": "wb"}[mode] | ||
1621 | |||
1622 | if not fileobj: | ||
1623 | if self.mode == "a" and not os.path.exists(name): | ||
1624 | # Create nonexistent files in append mode. | ||
1625 | self.mode = "w" | ||
1626 | self._mode = "wb" | ||
1627 | fileobj = bltn_open(name, self._mode) | ||
1628 | self._extfileobj = False | ||
1629 | else: | ||
1630 | if name is None and hasattr(fileobj, "name"): | ||
1631 | name = fileobj.name | ||
1632 | if hasattr(fileobj, "mode"): | ||
1633 | self._mode = fileobj.mode | ||
1634 | self._extfileobj = True | ||
1635 | self.name = os.path.abspath(name) if name else None | ||
1636 | self.fileobj = fileobj | ||
1637 | |||
1638 | # Init attributes. | ||
1639 | if format is not None: | ||
1640 | self.format = format | ||
1641 | if tarinfo is not None: | ||
1642 | self.tarinfo = tarinfo | ||
1643 | if dereference is not None: | ||
1644 | self.dereference = dereference | ||
1645 | if ignore_zeros is not None: | ||
1646 | self.ignore_zeros = ignore_zeros | ||
1647 | if encoding is not None: | ||
1648 | self.encoding = encoding | ||
1649 | self.errors = errors | ||
1650 | |||
1651 | if pax_headers is not None and self.format == PAX_FORMAT: | ||
1652 | self.pax_headers = pax_headers | ||
1653 | else: | ||
1654 | self.pax_headers = {} | ||
1655 | |||
1656 | if debug is not None: | ||
1657 | self.debug = debug | ||
1658 | if errorlevel is not None: | ||
1659 | self.errorlevel = errorlevel | ||
1660 | |||
1661 | # Init datastructures. | ||
1662 | self.closed = False | ||
1663 | self.members = [] # list of members as TarInfo objects | ||
1664 | self._loaded = False # flag if all members have been read | ||
1665 | self.offset = self.fileobj.tell() | ||
1666 | # current position in the archive file | ||
1667 | self.inodes = {} # dictionary caching the inodes of | ||
1668 | # archive members already added | ||
1669 | |||
1670 | try: | ||
1671 | if self.mode == "r": | ||
1672 | self.firstmember = None | ||
1673 | self.firstmember = self.next() | ||
1674 | |||
1675 | if self.mode == "a": | ||
1676 | # Move to the end of the archive, | ||
1677 | # before the first empty block. | ||
1678 | while True: | ||
1679 | self.fileobj.seek(self.offset) | ||
1680 | try: | ||
1681 | tarinfo = self.tarinfo.fromtarfile(self) | ||
1682 | self.members.append(tarinfo) | ||
1683 | except EOFHeaderError: | ||
1684 | self.fileobj.seek(self.offset) | ||
1685 | break | ||
1686 | except HeaderError as e: | ||
1687 | raise ReadError(str(e)) | ||
1688 | |||
1689 | if self.mode in "aw": | ||
1690 | self._loaded = True | ||
1691 | |||
1692 | if self.pax_headers: | ||
1693 | buf = self.tarinfo.create_pax_global_header(self.pax_headers.copy()) | ||
1694 | self.fileobj.write(buf) | ||
1695 | self.offset += len(buf) | ||
1696 | except: | ||
1697 | if not self._extfileobj: | ||
1698 | self.fileobj.close() | ||
1699 | self.closed = True | ||
1700 | raise | ||
1701 | |||
1702 | #-------------------------------------------------------------------------- | ||
1703 | # Below are the classmethods which act as alternate constructors to the | ||
1704 | # TarFile class. The open() method is the only one that is needed for | ||
1705 | # public use; it is the "super"-constructor and is able to select an | ||
1706 | # adequate "sub"-constructor for a particular compression using the mapping | ||
1707 | # from OPEN_METH. | ||
1708 | # | ||
1709 | # This concept allows one to subclass TarFile without losing the comfort of | ||
1710 | # the super-constructor. A sub-constructor is registered and made available | ||
1711 | # by adding it to the mapping in OPEN_METH. | ||
1712 | |||
1713 | @classmethod | ||
1714 | def open(cls, name=None, mode="r", fileobj=None, bufsize=RECORDSIZE, **kwargs): | ||
1715 | """Open a tar archive for reading, writing or appending. Return | ||
1716 | an appropriate TarFile class. | ||
1717 | |||
1718 | mode: | ||
1719 | 'r' or 'r:*' open for reading with transparent compression | ||
1720 | 'r:' open for reading exclusively uncompressed | ||
1721 | 'r:gz' open for reading with gzip compression | ||
1722 | 'r:bz2' open for reading with bzip2 compression | ||
1723 | 'a' or 'a:' open for appending, creating the file if necessary | ||
1724 | 'w' or 'w:' open for writing without compression | ||
1725 | 'w:gz' open for writing with gzip compression | ||
1726 | 'w:bz2' open for writing with bzip2 compression | ||
1727 | |||
1728 | 'r|*' open a stream of tar blocks with transparent compression | ||
1729 | 'r|' open an uncompressed stream of tar blocks for reading | ||
1730 | 'r|gz' open a gzip compressed stream of tar blocks | ||
1731 | 'r|bz2' open a bzip2 compressed stream of tar blocks | ||
1732 | 'w|' open an uncompressed stream for writing | ||
1733 | 'w|gz' open a gzip compressed stream for writing | ||
1734 | 'w|bz2' open a bzip2 compressed stream for writing | ||
1735 | """ | ||
1736 | |||
1737 | if not name and not fileobj: | ||
1738 | raise ValueError("nothing to open") | ||
1739 | |||
1740 | if mode in ("r", "r:*"): | ||
1741 | # Find out which *open() is appropriate for opening the file. | ||
1742 | for comptype in cls.OPEN_METH: | ||
1743 | func = getattr(cls, cls.OPEN_METH[comptype]) | ||
1744 | if fileobj is not None: | ||
1745 | saved_pos = fileobj.tell() | ||
1746 | try: | ||
1747 | return func(name, "r", fileobj, **kwargs) | ||
1748 | except (ReadError, CompressionError) as e: | ||
1749 | if fileobj is not None: | ||
1750 | fileobj.seek(saved_pos) | ||
1751 | continue | ||
1752 | raise ReadError("file could not be opened successfully") | ||
1753 | |||
1754 | elif ":" in mode: | ||
1755 | filemode, comptype = mode.split(":", 1) | ||
1756 | filemode = filemode or "r" | ||
1757 | comptype = comptype or "tar" | ||
1758 | |||
1759 | # Select the *open() function according to | ||
1760 | # given compression. | ||
1761 | if comptype in cls.OPEN_METH: | ||
1762 | func = getattr(cls, cls.OPEN_METH[comptype]) | ||
1763 | else: | ||
1764 | raise CompressionError("unknown compression type %r" % comptype) | ||
1765 | return func(name, filemode, fileobj, **kwargs) | ||
1766 | |||
1767 | elif "|" in mode: | ||
1768 | filemode, comptype = mode.split("|", 1) | ||
1769 | filemode = filemode or "r" | ||
1770 | comptype = comptype or "tar" | ||
1771 | |||
1772 | if filemode not in "rw": | ||
1773 | raise ValueError("mode must be 'r' or 'w'") | ||
1774 | |||
1775 | stream = _Stream(name, filemode, comptype, fileobj, bufsize) | ||
1776 | try: | ||
1777 | t = cls(name, filemode, stream, **kwargs) | ||
1778 | except: | ||
1779 | stream.close() | ||
1780 | raise | ||
1781 | t._extfileobj = False | ||
1782 | return t | ||
1783 | |||
1784 | elif mode in "aw": | ||
1785 | return cls.taropen(name, mode, fileobj, **kwargs) | ||
1786 | |||
1787 | raise ValueError("undiscernible mode") | ||
1788 | |||
1789 | @classmethod | ||
1790 | def taropen(cls, name, mode="r", fileobj=None, **kwargs): | ||
1791 | """Open uncompressed tar archive name for reading or writing. | ||
1792 | """ | ||
1793 | if len(mode) > 1 or mode not in "raw": | ||
1794 | raise ValueError("mode must be 'r', 'a' or 'w'") | ||
1795 | return cls(name, mode, fileobj, **kwargs) | ||
1796 | |||
1797 | @classmethod | ||
1798 | def gzopen(cls, name, mode="r", fileobj=None, compresslevel=9, **kwargs): | ||
1799 | """Open gzip compressed tar archive name for reading or writing. | ||
1800 | Appending is not allowed. | ||
1801 | """ | ||
1802 | if len(mode) > 1 or mode not in "rw": | ||
1803 | raise ValueError("mode must be 'r' or 'w'") | ||
1804 | |||
1805 | try: | ||
1806 | import gzip | ||
1807 | gzip.GzipFile | ||
1808 | except (ImportError, AttributeError): | ||
1809 | raise CompressionError("gzip module is not available") | ||
1810 | |||
1811 | extfileobj = fileobj is not None | ||
1812 | try: | ||
1813 | fileobj = gzip.GzipFile(name, mode + "b", compresslevel, fileobj) | ||
1814 | t = cls.taropen(name, mode, fileobj, **kwargs) | ||
1815 | except IOError: | ||
1816 | if not extfileobj and fileobj is not None: | ||
1817 | fileobj.close() | ||
1818 | if fileobj is None: | ||
1819 | raise | ||
1820 | raise ReadError("not a gzip file") | ||
1821 | except: | ||
1822 | if not extfileobj and fileobj is not None: | ||
1823 | fileobj.close() | ||
1824 | raise | ||
1825 | t._extfileobj = extfileobj | ||
1826 | return t | ||
1827 | |||
1828 | @classmethod | ||
1829 | def bz2open(cls, name, mode="r", fileobj=None, compresslevel=9, **kwargs): | ||
1830 | """Open bzip2 compressed tar archive name for reading or writing. | ||
1831 | Appending is not allowed. | ||
1832 | """ | ||
1833 | if len(mode) > 1 or mode not in "rw": | ||
1834 | raise ValueError("mode must be 'r' or 'w'.") | ||
1835 | |||
1836 | try: | ||
1837 | import bz2 | ||
1838 | except ImportError: | ||
1839 | raise CompressionError("bz2 module is not available") | ||
1840 | |||
1841 | if fileobj is not None: | ||
1842 | fileobj = _BZ2Proxy(fileobj, mode) | ||
1843 | else: | ||
1844 | fileobj = bz2.BZ2File(name, mode, compresslevel=compresslevel) | ||
1845 | |||
1846 | try: | ||
1847 | t = cls.taropen(name, mode, fileobj, **kwargs) | ||
1848 | except (IOError, EOFError): | ||
1849 | fileobj.close() | ||
1850 | raise ReadError("not a bzip2 file") | ||
1851 | t._extfileobj = False | ||
1852 | return t | ||
1853 | |||
1854 | # All *open() methods are registered here. | ||
1855 | OPEN_METH = { | ||
1856 | "tar": "taropen", # uncompressed tar | ||
1857 | "gz": "gzopen", # gzip compressed tar | ||
1858 | "bz2": "bz2open" # bzip2 compressed tar | ||
1859 | } | ||
1860 | |||
1861 | #-------------------------------------------------------------------------- | ||
1862 | # The public methods which TarFile provides: | ||
1863 | |||
1864 | def close(self): | ||
1865 | """Close the TarFile. In write-mode, two finishing zero blocks are | ||
1866 | appended to the archive. | ||
1867 | """ | ||
1868 | if self.closed: | ||
1869 | return | ||
1870 | |||
1871 | if self.mode in "aw": | ||
1872 | self.fileobj.write(NUL * (BLOCKSIZE * 2)) | ||
1873 | self.offset += (BLOCKSIZE * 2) | ||
1874 | # fill up the end with zero-blocks | ||
1875 | # (like option -b20 for tar does) | ||
1876 | blocks, remainder = divmod(self.offset, RECORDSIZE) | ||
1877 | if remainder > 0: | ||
1878 | self.fileobj.write(NUL * (RECORDSIZE - remainder)) | ||
1879 | |||
1880 | if not self._extfileobj: | ||
1881 | self.fileobj.close() | ||
1882 | self.closed = True | ||
1883 | |||
1884 | def getmember(self, name): | ||
1885 | """Return a TarInfo object for member `name'. If `name' can not be | ||
1886 | found in the archive, KeyError is raised. If a member occurs more | ||
1887 | than once in the archive, its last occurrence is assumed to be the | ||
1888 | most up-to-date version. | ||
1889 | """ | ||
1890 | tarinfo = self._getmember(name) | ||
1891 | if tarinfo is None: | ||
1892 | raise KeyError("filename %r not found" % name) | ||
1893 | return tarinfo | ||
1894 | |||
1895 | def getmembers(self): | ||
1896 | """Return the members of the archive as a list of TarInfo objects. The | ||
1897 | list has the same order as the members in the archive. | ||
1898 | """ | ||
1899 | self._check() | ||
1900 | if not self._loaded: # if we want to obtain a list of | ||
1901 | self._load() # all members, we first have to | ||
1902 | # scan the whole archive. | ||
1903 | return self.members | ||
1904 | |||
1905 | def getnames(self): | ||
1906 | """Return the members of the archive as a list of their names. It has | ||
1907 | the same order as the list returned by getmembers(). | ||
1908 | """ | ||
1909 | return [tarinfo.name for tarinfo in self.getmembers()] | ||
1910 | |||
1911 | def gettarinfo(self, name=None, arcname=None, fileobj=None): | ||
1912 | """Create a TarInfo object for either the file `name' or the file | ||
1913 | object `fileobj' (using os.fstat on its file descriptor). You can | ||
1914 | modify some of the TarInfo's attributes before you add it using | ||
1915 | addfile(). If given, `arcname' specifies an alternative name for the | ||
1916 | file in the archive. | ||
1917 | """ | ||
1918 | self._check("aw") | ||
1919 | |||
1920 | # When fileobj is given, replace name by | ||
1921 | # fileobj's real name. | ||
1922 | if fileobj is not None: | ||
1923 | name = fileobj.name | ||
1924 | |||
1925 | # Building the name of the member in the archive. | ||
1926 | # Backward slashes are converted to forward slashes, | ||
1927 | # Absolute paths are turned to relative paths. | ||
1928 | if arcname is None: | ||
1929 | arcname = name | ||
1930 | drv, arcname = os.path.splitdrive(arcname) | ||
1931 | arcname = arcname.replace(os.sep, "/") | ||
1932 | arcname = arcname.lstrip("/") | ||
1933 | |||
1934 | # Now, fill the TarInfo object with | ||
1935 | # information specific for the file. | ||
1936 | tarinfo = self.tarinfo() | ||
1937 | tarinfo.tarfile = self | ||
1938 | |||
1939 | # Use os.stat or os.lstat, depending on platform | ||
1940 | # and if symlinks shall be resolved. | ||
1941 | if fileobj is None: | ||
1942 | if hasattr(os, "lstat") and not self.dereference: | ||
1943 | statres = os.lstat(name) | ||
1944 | else: | ||
1945 | statres = os.stat(name) | ||
1946 | else: | ||
1947 | statres = os.fstat(fileobj.fileno()) | ||
1948 | linkname = "" | ||
1949 | |||
1950 | stmd = statres.st_mode | ||
1951 | if stat.S_ISREG(stmd): | ||
1952 | inode = (statres.st_ino, statres.st_dev) | ||
1953 | if not self.dereference and statres.st_nlink > 1 and \ | ||
1954 | inode in self.inodes and arcname != self.inodes[inode]: | ||
1955 | # Is it a hardlink to an already | ||
1956 | # archived file? | ||
1957 | type = LNKTYPE | ||
1958 | linkname = self.inodes[inode] | ||
1959 | else: | ||
1960 | # The inode is added only if its valid. | ||
1961 | # For win32 it is always 0. | ||
1962 | type = REGTYPE | ||
1963 | if inode[0]: | ||
1964 | self.inodes[inode] = arcname | ||
1965 | elif stat.S_ISDIR(stmd): | ||
1966 | type = DIRTYPE | ||
1967 | elif stat.S_ISFIFO(stmd): | ||
1968 | type = FIFOTYPE | ||
1969 | elif stat.S_ISLNK(stmd): | ||
1970 | type = SYMTYPE | ||
1971 | linkname = os.readlink(name) | ||
1972 | elif stat.S_ISCHR(stmd): | ||
1973 | type = CHRTYPE | ||
1974 | elif stat.S_ISBLK(stmd): | ||
1975 | type = BLKTYPE | ||
1976 | else: | ||
1977 | return None | ||
1978 | |||
1979 | # Fill the TarInfo object with all | ||
1980 | # information we can get. | ||
1981 | tarinfo.name = arcname | ||
1982 | tarinfo.mode = stmd | ||
1983 | tarinfo.uid = statres.st_uid | ||
1984 | tarinfo.gid = statres.st_gid | ||
1985 | if type == REGTYPE: | ||
1986 | tarinfo.size = statres.st_size | ||
1987 | else: | ||
1988 | tarinfo.size = 0 | ||
1989 | tarinfo.mtime = statres.st_mtime | ||
1990 | tarinfo.type = type | ||
1991 | tarinfo.linkname = linkname | ||
1992 | if pwd: | ||
1993 | try: | ||
1994 | tarinfo.uname = pwd.getpwuid(tarinfo.uid)[0] | ||
1995 | except KeyError: | ||
1996 | pass | ||
1997 | if grp: | ||
1998 | try: | ||
1999 | tarinfo.gname = grp.getgrgid(tarinfo.gid)[0] | ||
2000 | except KeyError: | ||
2001 | pass | ||
2002 | |||
2003 | if type in (CHRTYPE, BLKTYPE): | ||
2004 | if hasattr(os, "major") and hasattr(os, "minor"): | ||
2005 | tarinfo.devmajor = os.major(statres.st_rdev) | ||
2006 | tarinfo.devminor = os.minor(statres.st_rdev) | ||
2007 | return tarinfo | ||
2008 | |||
2009 | def list(self, verbose=True): | ||
2010 | """Print a table of contents to sys.stdout. If `verbose' is False, only | ||
2011 | the names of the members are printed. If it is True, an `ls -l'-like | ||
2012 | output is produced. | ||
2013 | """ | ||
2014 | self._check() | ||
2015 | |||
2016 | for tarinfo in self: | ||
2017 | if verbose: | ||
2018 | print(filemode(tarinfo.mode), end=' ') | ||
2019 | print("%s/%s" % (tarinfo.uname or tarinfo.uid, | ||
2020 | tarinfo.gname or tarinfo.gid), end=' ') | ||
2021 | if tarinfo.ischr() or tarinfo.isblk(): | ||
2022 | print("%10s" % ("%d,%d" \ | ||
2023 | % (tarinfo.devmajor, tarinfo.devminor)), end=' ') | ||
2024 | else: | ||
2025 | print("%10d" % tarinfo.size, end=' ') | ||
2026 | print("%d-%02d-%02d %02d:%02d:%02d" \ | ||
2027 | % time.localtime(tarinfo.mtime)[:6], end=' ') | ||
2028 | |||
2029 | print(tarinfo.name + ("/" if tarinfo.isdir() else ""), end=' ') | ||
2030 | |||
2031 | if verbose: | ||
2032 | if tarinfo.issym(): | ||
2033 | print("->", tarinfo.linkname, end=' ') | ||
2034 | if tarinfo.islnk(): | ||
2035 | print("link to", tarinfo.linkname, end=' ') | ||
2036 | print() | ||
2037 | |||
2038 | def add(self, name, arcname=None, recursive=True, exclude=None, filter=None): | ||
2039 | """Add the file `name' to the archive. `name' may be any type of file | ||
2040 | (directory, fifo, symbolic link, etc.). If given, `arcname' | ||
2041 | specifies an alternative name for the file in the archive. | ||
2042 | Directories are added recursively by default. This can be avoided by | ||
2043 | setting `recursive' to False. `exclude' is a function that should | ||
2044 | return True for each filename to be excluded. `filter' is a function | ||
2045 | that expects a TarInfo object argument and returns the changed | ||
2046 | TarInfo object, if it returns None the TarInfo object will be | ||
2047 | excluded from the archive. | ||
2048 | """ | ||
2049 | self._check("aw") | ||
2050 | |||
2051 | if arcname is None: | ||
2052 | arcname = name | ||
2053 | |||
2054 | # Exclude pathnames. | ||
2055 | if exclude is not None: | ||
2056 | import warnings | ||
2057 | warnings.warn("use the filter argument instead", | ||
2058 | DeprecationWarning, 2) | ||
2059 | if exclude(name): | ||
2060 | self._dbg(2, "tarfile: Excluded %r" % name) | ||
2061 | return | ||
2062 | |||
2063 | # Skip if somebody tries to archive the archive... | ||
2064 | if self.name is not None and os.path.abspath(name) == self.name: | ||
2065 | self._dbg(2, "tarfile: Skipped %r" % name) | ||
2066 | return | ||
2067 | |||
2068 | self._dbg(1, name) | ||
2069 | |||
2070 | # Create a TarInfo object from the file. | ||
2071 | tarinfo = self.gettarinfo(name, arcname) | ||
2072 | |||
2073 | if tarinfo is None: | ||
2074 | self._dbg(1, "tarfile: Unsupported type %r" % name) | ||
2075 | return | ||
2076 | |||
2077 | # Change or exclude the TarInfo object. | ||
2078 | if filter is not None: | ||
2079 | tarinfo = filter(tarinfo) | ||
2080 | if tarinfo is None: | ||
2081 | self._dbg(2, "tarfile: Excluded %r" % name) | ||
2082 | return | ||
2083 | |||
2084 | # Append the tar header and data to the archive. | ||
2085 | if tarinfo.isreg(): | ||
2086 | f = bltn_open(name, "rb") | ||
2087 | self.addfile(tarinfo, f) | ||
2088 | f.close() | ||
2089 | |||
2090 | elif tarinfo.isdir(): | ||
2091 | self.addfile(tarinfo) | ||
2092 | if recursive: | ||
2093 | for f in os.listdir(name): | ||
2094 | self.add(os.path.join(name, f), os.path.join(arcname, f), | ||
2095 | recursive, exclude, filter=filter) | ||
2096 | |||
2097 | else: | ||
2098 | self.addfile(tarinfo) | ||
2099 | |||
2100 | def addfile(self, tarinfo, fileobj=None): | ||
2101 | """Add the TarInfo object `tarinfo' to the archive. If `fileobj' is | ||
2102 | given, tarinfo.size bytes are read from it and added to the archive. | ||
2103 | You can create TarInfo objects using gettarinfo(). | ||
2104 | On Windows platforms, `fileobj' should always be opened with mode | ||
2105 | 'rb' to avoid irritation about the file size. | ||
2106 | """ | ||
2107 | self._check("aw") | ||
2108 | |||
2109 | tarinfo = copy.copy(tarinfo) | ||
2110 | |||
2111 | buf = tarinfo.tobuf(self.format, self.encoding, self.errors) | ||
2112 | self.fileobj.write(buf) | ||
2113 | self.offset += len(buf) | ||
2114 | |||
2115 | # If there's data to follow, append it. | ||
2116 | if fileobj is not None: | ||
2117 | copyfileobj(fileobj, self.fileobj, tarinfo.size) | ||
2118 | blocks, remainder = divmod(tarinfo.size, BLOCKSIZE) | ||
2119 | if remainder > 0: | ||
2120 | self.fileobj.write(NUL * (BLOCKSIZE - remainder)) | ||
2121 | blocks += 1 | ||
2122 | self.offset += blocks * BLOCKSIZE | ||
2123 | |||
2124 | self.members.append(tarinfo) | ||
2125 | |||
2126 | def extractall(self, path=".", members=None): | ||
2127 | """Extract all members from the archive to the current working | ||
2128 | directory and set owner, modification time and permissions on | ||
2129 | directories afterwards. `path' specifies a different directory | ||
2130 | to extract to. `members' is optional and must be a subset of the | ||
2131 | list returned by getmembers(). | ||
2132 | """ | ||
2133 | directories = [] | ||
2134 | |||
2135 | if members is None: | ||
2136 | members = self | ||
2137 | |||
2138 | for tarinfo in members: | ||
2139 | if tarinfo.isdir(): | ||
2140 | # Extract directories with a safe mode. | ||
2141 | directories.append(tarinfo) | ||
2142 | tarinfo = copy.copy(tarinfo) | ||
2143 | tarinfo.mode = 0o700 | ||
2144 | # Do not set_attrs directories, as we will do that further down | ||
2145 | self.extract(tarinfo, path, set_attrs=not tarinfo.isdir()) | ||
2146 | |||
2147 | # Reverse sort directories. | ||
2148 | directories.sort(key=lambda a: a.name) | ||
2149 | directories.reverse() | ||
2150 | |||
2151 | # Set correct owner, mtime and filemode on directories. | ||
2152 | for tarinfo in directories: | ||
2153 | dirpath = os.path.join(path, tarinfo.name) | ||
2154 | try: | ||
2155 | self.chown(tarinfo, dirpath) | ||
2156 | self.utime(tarinfo, dirpath) | ||
2157 | self.chmod(tarinfo, dirpath) | ||
2158 | except ExtractError as e: | ||
2159 | if self.errorlevel > 1: | ||
2160 | raise | ||
2161 | else: | ||
2162 | self._dbg(1, "tarfile: %s" % e) | ||
2163 | |||
2164 | def extract(self, member, path="", set_attrs=True): | ||
2165 | """Extract a member from the archive to the current working directory, | ||
2166 | using its full name. Its file information is extracted as accurately | ||
2167 | as possible. `member' may be a filename or a TarInfo object. You can | ||
2168 | specify a different directory using `path'. File attributes (owner, | ||
2169 | mtime, mode) are set unless `set_attrs' is False. | ||
2170 | """ | ||
2171 | self._check("r") | ||
2172 | |||
2173 | if isinstance(member, str): | ||
2174 | tarinfo = self.getmember(member) | ||
2175 | else: | ||
2176 | tarinfo = member | ||
2177 | |||
2178 | # Prepare the link target for makelink(). | ||
2179 | if tarinfo.islnk(): | ||
2180 | tarinfo._link_target = os.path.join(path, tarinfo.linkname) | ||
2181 | |||
2182 | try: | ||
2183 | self._extract_member(tarinfo, os.path.join(path, tarinfo.name), | ||
2184 | set_attrs=set_attrs) | ||
2185 | except EnvironmentError as e: | ||
2186 | if self.errorlevel > 0: | ||
2187 | raise | ||
2188 | else: | ||
2189 | if e.filename is None: | ||
2190 | self._dbg(1, "tarfile: %s" % e.strerror) | ||
2191 | else: | ||
2192 | self._dbg(1, "tarfile: %s %r" % (e.strerror, e.filename)) | ||
2193 | except ExtractError as e: | ||
2194 | if self.errorlevel > 1: | ||
2195 | raise | ||
2196 | else: | ||
2197 | self._dbg(1, "tarfile: %s" % e) | ||
2198 | |||
2199 | def extractfile(self, member): | ||
2200 | """Extract a member from the archive as a file object. `member' may be | ||
2201 | a filename or a TarInfo object. If `member' is a regular file, a | ||
2202 | file-like object is returned. If `member' is a link, a file-like | ||
2203 | object is constructed from the link's target. If `member' is none of | ||
2204 | the above, None is returned. | ||
2205 | The file-like object is read-only and provides the following | ||
2206 | methods: read(), readline(), readlines(), seek() and tell() | ||
2207 | """ | ||
2208 | self._check("r") | ||
2209 | |||
2210 | if isinstance(member, str): | ||
2211 | tarinfo = self.getmember(member) | ||
2212 | else: | ||
2213 | tarinfo = member | ||
2214 | |||
2215 | if tarinfo.isreg(): | ||
2216 | return self.fileobject(self, tarinfo) | ||
2217 | |||
2218 | elif tarinfo.type not in SUPPORTED_TYPES: | ||
2219 | # If a member's type is unknown, it is treated as a | ||
2220 | # regular file. | ||
2221 | return self.fileobject(self, tarinfo) | ||
2222 | |||
2223 | elif tarinfo.islnk() or tarinfo.issym(): | ||
2224 | if isinstance(self.fileobj, _Stream): | ||
2225 | # A small but ugly workaround for the case that someone tries | ||
2226 | # to extract a (sym)link as a file-object from a non-seekable | ||
2227 | # stream of tar blocks. | ||
2228 | raise StreamError("cannot extract (sym)link as file object") | ||
2229 | else: | ||
2230 | # A (sym)link's file object is its target's file object. | ||
2231 | return self.extractfile(self._find_link_target(tarinfo)) | ||
2232 | else: | ||
2233 | # If there's no data associated with the member (directory, chrdev, | ||
2234 | # blkdev, etc.), return None instead of a file object. | ||
2235 | return None | ||
2236 | |||
2237 | def _extract_member(self, tarinfo, targetpath, set_attrs=True): | ||
2238 | """Extract the TarInfo object tarinfo to a physical | ||
2239 | file called targetpath. | ||
2240 | """ | ||
2241 | # Fetch the TarInfo object for the given name | ||
2242 | # and build the destination pathname, replacing | ||
2243 | # forward slashes to platform specific separators. | ||
2244 | targetpath = targetpath.rstrip("/") | ||
2245 | targetpath = targetpath.replace("/", os.sep) | ||
2246 | |||
2247 | # Create all upper directories. | ||
2248 | upperdirs = os.path.dirname(targetpath) | ||
2249 | if upperdirs and not os.path.exists(upperdirs): | ||
2250 | # Create directories that are not part of the archive with | ||
2251 | # default permissions. | ||
2252 | os.makedirs(upperdirs) | ||
2253 | |||
2254 | if tarinfo.islnk() or tarinfo.issym(): | ||
2255 | self._dbg(1, "%s -> %s" % (tarinfo.name, tarinfo.linkname)) | ||
2256 | else: | ||
2257 | self._dbg(1, tarinfo.name) | ||
2258 | |||
2259 | if tarinfo.isreg(): | ||
2260 | self.makefile(tarinfo, targetpath) | ||
2261 | elif tarinfo.isdir(): | ||
2262 | self.makedir(tarinfo, targetpath) | ||
2263 | elif tarinfo.isfifo(): | ||
2264 | self.makefifo(tarinfo, targetpath) | ||
2265 | elif tarinfo.ischr() or tarinfo.isblk(): | ||
2266 | self.makedev(tarinfo, targetpath) | ||
2267 | elif tarinfo.islnk() or tarinfo.issym(): | ||
2268 | self.makelink(tarinfo, targetpath) | ||
2269 | elif tarinfo.type not in SUPPORTED_TYPES: | ||
2270 | self.makeunknown(tarinfo, targetpath) | ||
2271 | else: | ||
2272 | self.makefile(tarinfo, targetpath) | ||
2273 | |||
2274 | if set_attrs: | ||
2275 | self.chown(tarinfo, targetpath) | ||
2276 | if not tarinfo.issym(): | ||
2277 | self.chmod(tarinfo, targetpath) | ||
2278 | self.utime(tarinfo, targetpath) | ||
2279 | |||
2280 | #-------------------------------------------------------------------------- | ||
2281 | # Below are the different file methods. They are called via | ||
2282 | # _extract_member() when extract() is called. They can be replaced in a | ||
2283 | # subclass to implement other functionality. | ||
2284 | |||
2285 | def makedir(self, tarinfo, targetpath): | ||
2286 | """Make a directory called targetpath. | ||
2287 | """ | ||
2288 | try: | ||
2289 | # Use a safe mode for the directory, the real mode is set | ||
2290 | # later in _extract_member(). | ||
2291 | os.mkdir(targetpath, 0o700) | ||
2292 | except EnvironmentError as e: | ||
2293 | if e.errno != errno.EEXIST: | ||
2294 | raise | ||
2295 | |||
2296 | def makefile(self, tarinfo, targetpath): | ||
2297 | """Make a file called targetpath. | ||
2298 | """ | ||
2299 | source = self.fileobj | ||
2300 | source.seek(tarinfo.offset_data) | ||
2301 | target = bltn_open(targetpath, "wb") | ||
2302 | if tarinfo.sparse is not None: | ||
2303 | for offset, size in tarinfo.sparse: | ||
2304 | target.seek(offset) | ||
2305 | copyfileobj(source, target, size) | ||
2306 | else: | ||
2307 | copyfileobj(source, target, tarinfo.size) | ||
2308 | target.seek(tarinfo.size) | ||
2309 | target.truncate() | ||
2310 | target.close() | ||
2311 | |||
2312 | def makeunknown(self, tarinfo, targetpath): | ||
2313 | """Make a file from a TarInfo object with an unknown type | ||
2314 | at targetpath. | ||
2315 | """ | ||
2316 | self.makefile(tarinfo, targetpath) | ||
2317 | self._dbg(1, "tarfile: Unknown file type %r, " \ | ||
2318 | "extracted as regular file." % tarinfo.type) | ||
2319 | |||
2320 | def makefifo(self, tarinfo, targetpath): | ||
2321 | """Make a fifo called targetpath. | ||
2322 | """ | ||
2323 | if hasattr(os, "mkfifo"): | ||
2324 | os.mkfifo(targetpath) | ||
2325 | else: | ||
2326 | raise ExtractError("fifo not supported by system") | ||
2327 | |||
2328 | def makedev(self, tarinfo, targetpath): | ||
2329 | """Make a character or block device called targetpath. | ||
2330 | """ | ||
2331 | if not hasattr(os, "mknod") or not hasattr(os, "makedev"): | ||
2332 | raise ExtractError("special devices not supported by system") | ||
2333 | |||
2334 | mode = tarinfo.mode | ||
2335 | if tarinfo.isblk(): | ||
2336 | mode |= stat.S_IFBLK | ||
2337 | else: | ||
2338 | mode |= stat.S_IFCHR | ||
2339 | |||
2340 | os.mknod(targetpath, mode, | ||
2341 | os.makedev(tarinfo.devmajor, tarinfo.devminor)) | ||
2342 | |||
2343 | def makelink(self, tarinfo, targetpath): | ||
2344 | """Make a (symbolic) link called targetpath. If it cannot be created | ||
2345 | (platform limitation), we try to make a copy of the referenced file | ||
2346 | instead of a link. | ||
2347 | """ | ||
2348 | try: | ||
2349 | # For systems that support symbolic and hard links. | ||
2350 | if tarinfo.issym(): | ||
2351 | os.symlink(tarinfo.linkname, targetpath) | ||
2352 | else: | ||
2353 | # See extract(). | ||
2354 | if os.path.exists(tarinfo._link_target): | ||
2355 | os.link(tarinfo._link_target, targetpath) | ||
2356 | else: | ||
2357 | self._extract_member(self._find_link_target(tarinfo), | ||
2358 | targetpath) | ||
2359 | except symlink_exception: | ||
2360 | if tarinfo.issym(): | ||
2361 | linkpath = os.path.join(os.path.dirname(tarinfo.name), | ||
2362 | tarinfo.linkname) | ||
2363 | else: | ||
2364 | linkpath = tarinfo.linkname | ||
2365 | else: | ||
2366 | try: | ||
2367 | self._extract_member(self._find_link_target(tarinfo), | ||
2368 | targetpath) | ||
2369 | except KeyError: | ||
2370 | raise ExtractError("unable to resolve link inside archive") | ||
2371 | |||
2372 | def chown(self, tarinfo, targetpath): | ||
2373 | """Set owner of targetpath according to tarinfo. | ||
2374 | """ | ||
2375 | if pwd and hasattr(os, "geteuid") and os.geteuid() == 0: | ||
2376 | # We have to be root to do so. | ||
2377 | try: | ||
2378 | g = grp.getgrnam(tarinfo.gname)[2] | ||
2379 | except KeyError: | ||
2380 | g = tarinfo.gid | ||
2381 | try: | ||
2382 | u = pwd.getpwnam(tarinfo.uname)[2] | ||
2383 | except KeyError: | ||
2384 | u = tarinfo.uid | ||
2385 | try: | ||
2386 | if tarinfo.issym() and hasattr(os, "lchown"): | ||
2387 | os.lchown(targetpath, u, g) | ||
2388 | else: | ||
2389 | if sys.platform != "os2emx": | ||
2390 | os.chown(targetpath, u, g) | ||
2391 | except EnvironmentError as e: | ||
2392 | raise ExtractError("could not change owner") | ||
2393 | |||
2394 | def chmod(self, tarinfo, targetpath): | ||
2395 | """Set file permissions of targetpath according to tarinfo. | ||
2396 | """ | ||
2397 | if hasattr(os, 'chmod'): | ||
2398 | try: | ||
2399 | os.chmod(targetpath, tarinfo.mode) | ||
2400 | except EnvironmentError as e: | ||
2401 | raise ExtractError("could not change mode") | ||
2402 | |||
2403 | def utime(self, tarinfo, targetpath): | ||
2404 | """Set modification time of targetpath according to tarinfo. | ||
2405 | """ | ||
2406 | if not hasattr(os, 'utime'): | ||
2407 | return | ||
2408 | try: | ||
2409 | os.utime(targetpath, (tarinfo.mtime, tarinfo.mtime)) | ||
2410 | except EnvironmentError as e: | ||
2411 | raise ExtractError("could not change modification time") | ||
2412 | |||
2413 | #-------------------------------------------------------------------------- | ||
2414 | def next(self): | ||
2415 | """Return the next member of the archive as a TarInfo object, when | ||
2416 | TarFile is opened for reading. Return None if there is no more | ||
2417 | available. | ||
2418 | """ | ||
2419 | self._check("ra") | ||
2420 | if self.firstmember is not None: | ||
2421 | m = self.firstmember | ||
2422 | self.firstmember = None | ||
2423 | return m | ||
2424 | |||
2425 | # Read the next block. | ||
2426 | self.fileobj.seek(self.offset) | ||
2427 | tarinfo = None | ||
2428 | while True: | ||
2429 | try: | ||
2430 | tarinfo = self.tarinfo.fromtarfile(self) | ||
2431 | except EOFHeaderError as e: | ||
2432 | if self.ignore_zeros: | ||
2433 | self._dbg(2, "0x%X: %s" % (self.offset, e)) | ||
2434 | self.offset += BLOCKSIZE | ||
2435 | continue | ||
2436 | except InvalidHeaderError as e: | ||
2437 | if self.ignore_zeros: | ||
2438 | self._dbg(2, "0x%X: %s" % (self.offset, e)) | ||
2439 | self.offset += BLOCKSIZE | ||
2440 | continue | ||
2441 | elif self.offset == 0: | ||
2442 | raise ReadError(str(e)) | ||
2443 | except EmptyHeaderError: | ||
2444 | if self.offset == 0: | ||
2445 | raise ReadError("empty file") | ||
2446 | except TruncatedHeaderError as e: | ||
2447 | if self.offset == 0: | ||
2448 | raise ReadError(str(e)) | ||
2449 | except SubsequentHeaderError as e: | ||
2450 | raise ReadError(str(e)) | ||
2451 | break | ||
2452 | |||
2453 | if tarinfo is not None: | ||
2454 | self.members.append(tarinfo) | ||
2455 | else: | ||
2456 | self._loaded = True | ||
2457 | |||
2458 | return tarinfo | ||
2459 | |||
2460 | #-------------------------------------------------------------------------- | ||
2461 | # Little helper methods: | ||
2462 | |||
2463 | def _getmember(self, name, tarinfo=None, normalize=False): | ||
2464 | """Find an archive member by name from bottom to top. | ||
2465 | If tarinfo is given, it is used as the starting point. | ||
2466 | """ | ||
2467 | # Ensure that all members have been loaded. | ||
2468 | members = self.getmembers() | ||
2469 | |||
2470 | # Limit the member search list up to tarinfo. | ||
2471 | if tarinfo is not None: | ||
2472 | members = members[:members.index(tarinfo)] | ||
2473 | |||
2474 | if normalize: | ||
2475 | name = os.path.normpath(name) | ||
2476 | |||
2477 | for member in reversed(members): | ||
2478 | if normalize: | ||
2479 | member_name = os.path.normpath(member.name) | ||
2480 | else: | ||
2481 | member_name = member.name | ||
2482 | |||
2483 | if name == member_name: | ||
2484 | return member | ||
2485 | |||
2486 | def _load(self): | ||
2487 | """Read through the entire archive file and look for readable | ||
2488 | members. | ||
2489 | """ | ||
2490 | while True: | ||
2491 | tarinfo = self.next() | ||
2492 | if tarinfo is None: | ||
2493 | break | ||
2494 | self._loaded = True | ||
2495 | |||
2496 | def _check(self, mode=None): | ||
2497 | """Check if TarFile is still open, and if the operation's mode | ||
2498 | corresponds to TarFile's mode. | ||
2499 | """ | ||
2500 | if self.closed: | ||
2501 | raise IOError("%s is closed" % self.__class__.__name__) | ||
2502 | if mode is not None and self.mode not in mode: | ||
2503 | raise IOError("bad operation for mode %r" % self.mode) | ||
2504 | |||
2505 | def _find_link_target(self, tarinfo): | ||
2506 | """Find the target member of a symlink or hardlink member in the | ||
2507 | archive. | ||
2508 | """ | ||
2509 | if tarinfo.issym(): | ||
2510 | # Always search the entire archive. | ||
2511 | linkname = os.path.dirname(tarinfo.name) + "/" + tarinfo.linkname | ||
2512 | limit = None | ||
2513 | else: | ||
2514 | # Search the archive before the link, because a hard link is | ||
2515 | # just a reference to an already archived file. | ||
2516 | linkname = tarinfo.linkname | ||
2517 | limit = tarinfo | ||
2518 | |||
2519 | member = self._getmember(linkname, tarinfo=limit, normalize=True) | ||
2520 | if member is None: | ||
2521 | raise KeyError("linkname %r not found" % linkname) | ||
2522 | return member | ||
2523 | |||
2524 | def __iter__(self): | ||
2525 | """Provide an iterator object. | ||
2526 | """ | ||
2527 | if self._loaded: | ||
2528 | return iter(self.members) | ||
2529 | else: | ||
2530 | return TarIter(self) | ||
2531 | |||
2532 | def _dbg(self, level, msg): | ||
2533 | """Write debugging output to sys.stderr. | ||
2534 | """ | ||
2535 | if level <= self.debug: | ||
2536 | print(msg, file=sys.stderr) | ||
2537 | |||
2538 | def __enter__(self): | ||
2539 | self._check() | ||
2540 | return self | ||
2541 | |||
2542 | def __exit__(self, type, value, traceback): | ||
2543 | if type is None: | ||
2544 | self.close() | ||
2545 | else: | ||
2546 | # An exception occurred. We must not call close() because | ||
2547 | # it would try to write end-of-archive blocks and padding. | ||
2548 | if not self._extfileobj: | ||
2549 | self.fileobj.close() | ||
2550 | self.closed = True | ||
2551 | # class TarFile | ||
2552 | |||
2553 | class TarIter(object): | ||
2554 | """Iterator Class. | ||
2555 | |||
2556 | for tarinfo in TarFile(...): | ||
2557 | suite... | ||
2558 | """ | ||
2559 | |||
2560 | def __init__(self, tarfile): | ||
2561 | """Construct a TarIter object. | ||
2562 | """ | ||
2563 | self.tarfile = tarfile | ||
2564 | self.index = 0 | ||
2565 | def __iter__(self): | ||
2566 | """Return iterator object. | ||
2567 | """ | ||
2568 | return self | ||
2569 | |||
2570 | def __next__(self): | ||
2571 | """Return the next item using TarFile's next() method. | ||
2572 | When all members have been read, set TarFile as _loaded. | ||
2573 | """ | ||
2574 | # Fix for SF #1100429: Under rare circumstances it can | ||
2575 | # happen that getmembers() is called during iteration, | ||
2576 | # which will cause TarIter to stop prematurely. | ||
2577 | if not self.tarfile._loaded: | ||
2578 | tarinfo = self.tarfile.next() | ||
2579 | if not tarinfo: | ||
2580 | self.tarfile._loaded = True | ||
2581 | raise StopIteration | ||
2582 | else: | ||
2583 | try: | ||
2584 | tarinfo = self.tarfile.members[self.index] | ||
2585 | except IndexError: | ||
2586 | raise StopIteration | ||
2587 | self.index += 1 | ||
2588 | return tarinfo | ||
2589 | |||
2590 | next = __next__ # for Python 2.x | ||
2591 | |||
2592 | #-------------------- | ||
2593 | # exported functions | ||
2594 | #-------------------- | ||
2595 | def is_tarfile(name): | ||
2596 | """Return True if name points to a tar archive that we | ||
2597 | are able to handle, else return False. | ||
2598 | """ | ||
2599 | try: | ||
2600 | t = open(name) | ||
2601 | t.close() | ||
2602 | return True | ||
2603 | except TarError: | ||
2604 | return False | ||
2605 | |||
2606 | bltn_open = open | ||
2607 | open = TarFile.open | ||