First commit

author: Shubham Saini <shubham6405@gmail.com> 2018-12-11 10:01:23 +0000
committer: Shubham Saini <shubham6405@gmail.com> 2018-12-11 10:01:23 +0000
commit: 68df54d6629ec019142eb149dd037774f2d11e7c (patch)
tree: 345bc22d46b4e01a4ba8303b94278952a4ed2b9e /venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/urllib3/response.py
1 files changed, 626 insertions, 0 deletions
diff --git a/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/urllib3/response.py b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/urllib3/response.py
new file mode 100644
index 0000000..54799ba
--- /dev/null
+++ b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/urllib3/response.py
@@ -0,0 +1,626 @@
+from __future__ import absolute_import
+from contextlib import contextmanager
+import zlib
+import io
+import logging
+from socket import timeout as SocketTimeout
+from socket import error as SocketError
+from ._collections import HTTPHeaderDict
+from .exceptions import (
+    BodyNotHttplibCompatible, ProtocolError, DecodeError, ReadTimeoutError,
+    ResponseNotChunked, IncompleteRead, InvalidHeader
+)
+from .packages.six import string_types as basestring, binary_type, PY3
+from .packages.six.moves import http_client as httplib
+from .connection import HTTPException, BaseSSLError
+from .util.response import is_fp_closed, is_response_to_head
+log = logging.getLogger(__name__)
+class DeflateDecoder(object):
+    def __init__(self):
+        self._first_try = True
+        self._data = binary_type()
+        self._obj = zlib.decompressobj()
+    def __getattr__(self, name):
+        return getattr(self._obj, name)
+    def decompress(self, data):
+        if not data:
+            return data
+        if not self._first_try:
+            return self._obj.decompress(data)
+        self._data += data
+        try:
+            decompressed = self._obj.decompress(data)
+            if decompressed:
+                self._first_try = False
+                self._data = None
+            return decompressed
+        except zlib.error:
+            self._first_try = False
+            self._obj = zlib.decompressobj(-zlib.MAX_WBITS)
+            try:
+                return self.decompress(self._data)
+            finally:
+                self._data = None
+class GzipDecoder(object):
+    def __init__(self):
+        self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS)
+    def __getattr__(self, name):
+        return getattr(self._obj, name)
+    def decompress(self, data):
+        if not data:
+            return data
+        return self._obj.decompress(data)
+def _get_decoder(mode):
+    if mode == 'gzip':
+        return GzipDecoder()
+    return DeflateDecoder()
+class HTTPResponse(io.IOBase):
+    """
+    HTTP Response container.
+    Backwards-compatible to httplib's HTTPResponse but the response ``body`` is
+    loaded and decoded on-demand when the ``data`` property is accessed.  This
+    class is also compatible with the Python standard library's :mod:`io`
+    module, and can hence be treated as a readable object in the context of that
+    framework.
+    Extra parameters for behaviour not present in httplib.HTTPResponse:
+    :param preload_content:
+        If True, the response's body will be preloaded during construction.
+    :param decode_content:
+        If True, attempts to decode specific content-encoding's based on headers
+        (like 'gzip' and 'deflate') will be skipped and raw data will be used
+        instead.
+    :param original_response:
+        When this HTTPResponse wrapper is generated from an httplib.HTTPResponse
+        object, it's convenient to include the original for debug purposes. It's
+        otherwise unused.
+    :param retries:
+        The retries contains the last :class:`~urllib3.util.retry.Retry` that
+        was used during the request.
+    :param enforce_content_length:
+        Enforce content length checking. Body returned by server must match
+        value of Content-Length header, if present. Otherwise, raise error.
+    """
+    CONTENT_DECODERS = ['gzip', 'deflate']
+    REDIRECT_STATUSES = [301, 302, 303, 307, 308]
+    def __init__(self, body='', headers=None, status=0, version=0, reason=None,
+                 strict=0, preload_content=True, decode_content=True,
+                 original_response=None, pool=None, connection=None,
+                 retries=None, enforce_content_length=False, request_method=None):
+        if isinstance(headers, HTTPHeaderDict):
+            self.headers = headers
+        else:
+            self.headers = HTTPHeaderDict(headers)
+        self.status = status
+        self.version = version
+        self.reason = reason
+        self.strict = strict
+        self.decode_content = decode_content
+        self.retries = retries
+        self.enforce_content_length = enforce_content_length
+        self._decoder = None
+        self._body = None
+        self._fp = None
+        self._original_response = original_response
+        self._fp_bytes_read = 0
+        if body and isinstance(body, (basestring, binary_type)):
+            self._body = body
+        self._pool = pool
+        self._connection = connection
+        if hasattr(body, 'read'):
+            self._fp = body
+        # Are we using the chunked-style of transfer encoding?
+        self.chunked = False
+        self.chunk_left = None
+        tr_enc = self.headers.get('transfer-encoding', '').lower()
+        # Don't incur the penalty of creating a list and then discarding it
+        encodings = (enc.strip() for enc in tr_enc.split(","))
+        if "chunked" in encodings:
+            self.chunked = True
+        # Determine length of response
+        self.length_remaining = self._init_length(request_method)
+        # If requested, preload the body.
+        if preload_content and not self._body:
+            self._body = self.read(decode_content=decode_content)
+    def get_redirect_location(self):
+        """
+        Should we redirect and where to?
+        :returns: Truthy redirect location string if we got a redirect status
+            code and valid location. ``None`` if redirect status and no
+            location. ``False`` if not a redirect status code.
+        """
+        if self.status in self.REDIRECT_STATUSES:
+            return self.headers.get('location')
+        return False
+    def release_conn(self):
+        if not self._pool or not self._connection:
+            return
+        self._pool._put_conn(self._connection)
+        self._connection = None
+    @property
+    def data(self):
+        # For backwords-compat with earlier urllib3 0.4 and earlier.
+        if self._body:
+            return self._body
+        if self._fp:
+            return self.read(cache_content=True)
+    @property
+    def connection(self):
+        return self._connection
+    def tell(self):
+        """
+        Obtain the number of bytes pulled over the wire so far. May differ from
+        the amount of content returned by :meth:``HTTPResponse.read`` if bytes
+        are encoded on the wire (e.g, compressed).
+        """
+        return self._fp_bytes_read
+    def _init_length(self, request_method):
+        """
+        Set initial length value for Response content if available.
+        """
+        length = self.headers.get('content-length')
+        if length is not None and self.chunked:
+            # This Response will fail with an IncompleteRead if it can't be
+            # received as chunked. This method falls back to attempt reading
+            # the response before raising an exception.
+            log.warning("Received response with both Content-Length and "
+                        "Transfer-Encoding set. This is expressly forbidden "
+                        "by RFC 7230 sec 3.3.2. Ignoring Content-Length and "
+                        "attempting to process response as Transfer-Encoding: "
+                        "chunked.")
+            return None
+        elif length is not None:
+            try:
+                # RFC 7230 section 3.3.2 specifies multiple content lengths can
+                # be sent in a single Content-Length header
+                # (e.g. Content-Length: 42, 42). This line ensures the values
+                # are all valid ints and that as long as the `set` length is 1,
+                # all values are the same. Otherwise, the header is invalid.
+                lengths = set([int(val) for val in length.split(',')])
+                if len(lengths) > 1:
+                    raise InvalidHeader("Content-Length contained multiple "
+                                        "unmatching values (%s)" % length)
+                length = lengths.pop()
+            except ValueError:
+                length = None
+            else:
+                if length < 0:
+                    length = None
+        # Convert status to int for comparison
+        # In some cases, httplib returns a status of "_UNKNOWN"
+        try:
+            status = int(self.status)
+        except ValueError:
+            status = 0
+        # Check for responses that shouldn't include a body
+        if status in (204, 304) or 100 <= status < 200 or request_method == 'HEAD':
+            length = 0
+        return length
+    def _init_decoder(self):
+        """
+        Set-up the _decoder attribute if necessary.
+        """
+        # Note: content-encoding value should be case-insensitive, per RFC 7230
+        # Section 3.2
+        content_encoding = self.headers.get('content-encoding', '').lower()
+        if self._decoder is None and content_encoding in self.CONTENT_DECODERS:
+            self._decoder = _get_decoder(content_encoding)
+    def _decode(self, data, decode_content, flush_decoder):
+        """
+        Decode the data passed in and potentially flush the decoder.
+        """
+        try:
+            if decode_content and self._decoder:
+                data = self._decoder.decompress(data)
+        except (IOError, zlib.error) as e:
+            content_encoding = self.headers.get('content-encoding', '').lower()
+            raise DecodeError(
+                "Received response with content-encoding: %s, but "
+                "failed to decode it." % content_encoding, e)
+        if flush_decoder and decode_content:
+            data += self._flush_decoder()
+        return data
+    def _flush_decoder(self):
+        """
+        Flushes the decoder. Should only be called if the decoder is actually
+        being used.
+        """
+        if self._decoder:
+            buf = self._decoder.decompress(b'')
+            return buf + self._decoder.flush()
+        return b''
+    @contextmanager
+    def _error_catcher(self):
+        """
+        Catch low-level python exceptions, instead re-raising urllib3
+        variants, so that low-level exceptions are not leaked in the
+        high-level api.
+        On exit, release the connection back to the pool.
+        """
+        clean_exit = False
+        try:
+            try:
+                yield
+            except SocketTimeout:
+                # FIXME: Ideally we'd like to include the url in the ReadTimeoutError but
+                # there is yet no clean way to get at it from this context.
+                raise ReadTimeoutError(self._pool, None, 'Read timed out.')
+            except BaseSSLError as e:
+                # FIXME: Is there a better way to differentiate between SSLErrors?
+                if 'read operation timed out' not in str(e):  # Defensive:
+                    # This shouldn't happen but just in case we're missing an edge
+                    # case, let's avoid swallowing SSL errors.
+                    raise
+                raise ReadTimeoutError(self._pool, None, 'Read timed out.')
+            except (HTTPException, SocketError) as e:
+                # This includes IncompleteRead.
+                raise ProtocolError('Connection broken: %r' % e, e)
+            # If no exception is thrown, we should avoid cleaning up
+            # unnecessarily.
+            clean_exit = True
+        finally:
+            # If we didn't terminate cleanly, we need to throw away our
+            # connection.
+            if not clean_exit:
+                # The response may not be closed but we're not going to use it
+                # anymore so close it now to ensure that the connection is
+                # released back to the pool.
+                if self._original_response:
+                    self._original_response.close()
+                # Closing the response may not actually be sufficient to close
+                # everything, so if we have a hold of the connection close that
+                # too.
+                if self._connection:
+                    self._connection.close()
+            # If we hold the original response but it's closed now, we should
+            # return the connection back to the pool.
+            if self._original_response and self._original_response.isclosed():
+                self.release_conn()
+    def read(self, amt=None, decode_content=None, cache_content=False):
+        """
+        Similar to :meth:`httplib.HTTPResponse.read`, but with two additional
+        parameters: ``decode_content`` and ``cache_content``.
+        :param amt:
+            How much of the content to read. If specified, caching is skipped
+            because it doesn't make sense to cache partial content as the full
+            response.
+        :param decode_content:
+            If True, will attempt to decode the body based on the
+            'content-encoding' header.
+        :param cache_content:
+            If True, will save the returned data such that the same result is
+            returned despite of the state of the underlying file object. This
+            is useful if you want the ``.data`` property to continue working
+            after having ``.read()`` the file object. (Overridden if ``amt`` is
+            set.)
+        """
+        self._init_decoder()
+        if decode_content is None:
+            decode_content = self.decode_content
+        if self._fp is None:
+            return
+        flush_decoder = False
+        data = None
+        with self._error_catcher():
+            if amt is None:
+                # cStringIO doesn't like amt=None
+                data = self._fp.read()
+                flush_decoder = True
+            else:
+                cache_content = False
+                data = self._fp.read(amt)
+                if amt != 0 and not data:  # Platform-specific: Buggy versions of Python.
+                    # Close the connection when no data is returned
+                    #
+                    # This is redundant to what httplib/http.client _should_
+                    # already do.  However, versions of python released before
+                    # December 15, 2012 (http://bugs.python.org/issue16298) do
+                    # not properly close the connection in all cases. There is
+                    # no harm in redundantly calling close.
+                    self._fp.close()
+                    flush_decoder = True
+                    if self.enforce_content_length and self.length_remaining not in (0, None):
+                        # This is an edge case that httplib failed to cover due
+                        # to concerns of backward compatibility. We're
+                        # addressing it here to make sure IncompleteRead is
+                        # raised during streaming, so all calls with incorrect
+                        # Content-Length are caught.
+                        raise IncompleteRead(self._fp_bytes_read, self.length_remaining)
+        if data:
+            self._fp_bytes_read += len(data)
+            if self.length_remaining is not None:
+                self.length_remaining -= len(data)
+            data = self._decode(data, decode_content, flush_decoder)
+            if cache_content:
+                self._body = data
+        return data
+    def stream(self, amt=2**16, decode_content=None):
+        """
+        A generator wrapper for the read() method. A call will block until
+        ``amt`` bytes have been read from the connection or until the
+        connection is closed.
+        :param amt:
+            How much of the content to read. The generator will return up to
+            much data per iteration, but may return less. This is particularly
+            likely when using compressed data. However, the empty string will
+            never be returned.
+        :param decode_content:
+            If True, will attempt to decode the body based on the
+            'content-encoding' header.
+        """
+        if self.chunked and self.supports_chunked_reads():
+            for line in self.read_chunked(amt, decode_content=decode_content):
+                yield line
+        else:
+            while not is_fp_closed(self._fp):
+                data = self.read(amt=amt, decode_content=decode_content)
+                if data:
+                    yield data
+    @classmethod
+    def from_httplib(ResponseCls, r, **response_kw):
+        """
+        Given an :class:`httplib.HTTPResponse` instance ``r``, return a
+        corresponding :class:`urllib3.response.HTTPResponse` object.
+        Remaining parameters are passed to the HTTPResponse constructor, along
+        with ``original_response=r``.
+        """
+        headers = r.msg
+        if not isinstance(headers, HTTPHeaderDict):
+            if PY3:  # Python 3
+                headers = HTTPHeaderDict(headers.items())
+            else:  # Python 2
+                headers = HTTPHeaderDict.from_httplib(headers)
+        # HTTPResponse objects in Python 3 don't have a .strict attribute
+        strict = getattr(r, 'strict', 0)
+        resp = ResponseCls(body=r,
+                           headers=headers,
+                           status=r.status,
+                           version=r.version,
+                           reason=r.reason,
+                           strict=strict,
+                           original_response=r,
+                           **response_kw)
+        return resp
+    # Backwards-compatibility methods for httplib.HTTPResponse
+    def getheaders(self):
+        return self.headers
+    def getheader(self, name, default=None):
+        return self.headers.get(name, default)
+    # Backwards compatibility for http.cookiejar
+    def info(self):
+        return self.headers
+    # Overrides from io.IOBase
+    def close(self):
+        if not self.closed:
+            self._fp.close()
+        if self._connection:
+            self._connection.close()
+    @property
+    def closed(self):
+        if self._fp is None:
+            return True
+        elif hasattr(self._fp, 'isclosed'):
+            return self._fp.isclosed()
+        elif hasattr(self._fp, 'closed'):
+            return self._fp.closed
+        else:
+            return True
+    def fileno(self):
+        if self._fp is None:
+            raise IOError("HTTPResponse has no file to get a fileno from")
+        elif hasattr(self._fp, "fileno"):
+            return self._fp.fileno()
+        else:
+            raise IOError("The file-like object this HTTPResponse is wrapped "
+                          "around has no file descriptor")
+    def flush(self):
+        if self._fp is not None and hasattr(self._fp, 'flush'):
+            return self._fp.flush()
+    def readable(self):
+        # This method is required for `io` module compatibility.
+        return True
+    def readinto(self, b):
+        # This method is required for `io` module compatibility.
+        temp = self.read(len(b))
+        if len(temp) == 0:
+            return 0
+        else:
+            b[:len(temp)] = temp
+            return len(temp)
+    def supports_chunked_reads(self):
+        """
+        Checks if the underlying file-like object looks like a
+        httplib.HTTPResponse object. We do this by testing for the fp
+        attribute. If it is present we assume it returns raw chunks as
+        processed by read_chunked().
+        """
+        return hasattr(self._fp, 'fp')
+    def _update_chunk_length(self):
+        # First, we'll figure out length of a chunk and then
+        # we'll try to read it from socket.
+        if self.chunk_left is not None:
+            return
+        line = self._fp.fp.readline()
+        line = line.split(b';', 1)[0]
+        try:
+            self.chunk_left = int(line, 16)
+        except ValueError:
+            # Invalid chunked protocol response, abort.
+            self.close()
+            raise httplib.IncompleteRead(line)
+    def _handle_chunk(self, amt):
+        returned_chunk = None
+        if amt is None:
+            chunk = self._fp._safe_read(self.chunk_left)
+            returned_chunk = chunk
+            self._fp._safe_read(2)  # Toss the CRLF at the end of the chunk.
+            self.chunk_left = None
+        elif amt < self.chunk_left:
+            value = self._fp._safe_read(amt)
+            self.chunk_left = self.chunk_left - amt
+            returned_chunk = value
+        elif amt == self.chunk_left:
+            value = self._fp._safe_read(amt)
+            self._fp._safe_read(2)  # Toss the CRLF at the end of the chunk.
+            self.chunk_left = None
+            returned_chunk = value
+        else:  # amt > self.chunk_left
+            returned_chunk = self._fp._safe_read(self.chunk_left)
+            self._fp._safe_read(2)  # Toss the CRLF at the end of the chunk.
+            self.chunk_left = None
+        return returned_chunk
+    def read_chunked(self, amt=None, decode_content=None):
+        """
+        Similar to :meth:`HTTPResponse.read`, but with an additional
+        parameter: ``decode_content``.
+        :param decode_content:
+            If True, will attempt to decode the body based on the
+            'content-encoding' header.
+        """
+        self._init_decoder()
+        # FIXME: Rewrite this method and make it a class with a better structured logic.
+        if not self.chunked:
+            raise ResponseNotChunked(
+                "Response is not chunked. "
+                "Header 'transfer-encoding: chunked' is missing.")
+        if not self.supports_chunked_reads():
+            raise BodyNotHttplibCompatible(
+                "Body should be httplib.HTTPResponse like. "
+                "It should have have an fp attribute which returns raw chunks.")
+        # Don't bother reading the body of a HEAD request.
+        if self._original_response and is_response_to_head(self._original_response):
+            self._original_response.close()
+            return
+        with self._error_catcher():
+            while True:
+                self._update_chunk_length()
+                if self.chunk_left == 0:
+                    break
+                chunk = self._handle_chunk(amt)
+                decoded = self._decode(chunk, decode_content=decode_content,
+                                       flush_decoder=False)
+                if decoded:
+                    yield decoded
+            if decode_content:
+                # On CPython and PyPy, we should never need to flush the
+                # decoder. However, on Jython we *might* need to, so
+                # lets defensively do it anyway.
+                decoded = self._flush_decoder()
+                if decoded:  # Platform-specific: Jython.
+                    yield decoded
+            # Chunk content ends with \r\n: discard it.
+            while True:
+                line = self._fp.fp.readline()
+                if not line:
+                    # Some sites may not end with '\r\n'.
+                    break
+                if line == b'\r\n':
+                    break
+            # We read everything; close the "file".
+            if self._original_response:
+                self._original_response.close()
author	Shubham Saini <shubham6405@gmail.com>	2018-12-11 10:01:23 +0000
committer	Shubham Saini <shubham6405@gmail.com>	2018-12-11 10:01:23 +0000
commit	68df54d6629ec019142eb149dd037774f2d11e7c (patch)
tree	345bc22d46b4e01a4ba8303b94278952a4ed2b9e /venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/urllib3/response.py

diff --git a/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/urllib3/response.py b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/urllib3/response.py new file mode 100644 index 0000000..54799ba --- /dev/null +++ b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/urllib3/response.py
@@ -0,0 +1,626 @@
	1	from __future__ import absolute_import
	2	from contextlib import contextmanager
	3	import zlib
	4	import io
	5	import logging
	6	from socket import timeout as SocketTimeout
	7	from socket import error as SocketError
	8
	9	from ._collections import HTTPHeaderDict
	10	from .exceptions import (
	11	BodyNotHttplibCompatible, ProtocolError, DecodeError, ReadTimeoutError,
	12	ResponseNotChunked, IncompleteRead, InvalidHeader
	13	)
	14	from .packages.six import string_types as basestring, binary_type, PY3
	15	from .packages.six.moves import http_client as httplib
	16	from .connection import HTTPException, BaseSSLError
	17	from .util.response import is_fp_closed, is_response_to_head
	18
	19	log = logging.getLogger(__name__)
	20
	21
	22	class DeflateDecoder(object):
	23
	24	def __init__(self):
	25	self._first_try = True
	26	self._data = binary_type()
	27	self._obj = zlib.decompressobj()
	28
	29	def __getattr__(self, name):
	30	return getattr(self._obj, name)
	31
	32	def decompress(self, data):
	33	if not data:
	34	return data
	35
	36	if not self._first_try:
	37	return self._obj.decompress(data)
	38
	39	self._data += data
	40	try:
	41	decompressed = self._obj.decompress(data)
	42	if decompressed:
	43	self._first_try = False
	44	self._data = None
	45	return decompressed
	46	except zlib.error:
	47	self._first_try = False
	48	self._obj = zlib.decompressobj(-zlib.MAX_WBITS)
	49	try:
	50	return self.decompress(self._data)
	51	finally:
	52	self._data = None
	53
	54
	55	class GzipDecoder(object):
	56
	57	def __init__(self):
	58	self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS)
	59
	60	def __getattr__(self, name):
	61	return getattr(self._obj, name)
	62
	63	def decompress(self, data):
	64	if not data:
	65	return data
	66	return self._obj.decompress(data)
	67
	68
	69	def _get_decoder(mode):
	70	if mode == 'gzip':
	71	return GzipDecoder()
	72
	73	return DeflateDecoder()
	74
	75
	76	class HTTPResponse(io.IOBase):
	77	"""
	78	HTTP Response container.
	79
	80	Backwards-compatible to httplib's HTTPResponse but the response ``body`` is
	81	loaded and decoded on-demand when the ``data`` property is accessed. This
	82	class is also compatible with the Python standard library's :mod:`io`
	83	module, and can hence be treated as a readable object in the context of that
	84	framework.
	85
	86	Extra parameters for behaviour not present in httplib.HTTPResponse:
	87
	88	:param preload_content:
	89	If True, the response's body will be preloaded during construction.
	90
	91	:param decode_content:
	92	If True, attempts to decode specific content-encoding's based on headers
	93	(like 'gzip' and 'deflate') will be skipped and raw data will be used
	94	instead.
	95
	96	:param original_response:
	97	When this HTTPResponse wrapper is generated from an httplib.HTTPResponse
	98	object, it's convenient to include the original for debug purposes. It's
	99	otherwise unused.
	100
	101	:param retries:
	102	The retries contains the last :class:`~urllib3.util.retry.Retry` that
	103	was used during the request.
	104
	105	:param enforce_content_length:
	106	Enforce content length checking. Body returned by server must match
	107	value of Content-Length header, if present. Otherwise, raise error.
	108	"""
	109
	110	CONTENT_DECODERS = ['gzip', 'deflate']
	111	REDIRECT_STATUSES = [301, 302, 303, 307, 308]
	112
	113	def __init__(self, body='', headers=None, status=0, version=0, reason=None,
	114	strict=0, preload_content=True, decode_content=True,
	115	original_response=None, pool=None, connection=None,
	116	retries=None, enforce_content_length=False, request_method=None):
	117
	118	if isinstance(headers, HTTPHeaderDict):
	119	self.headers = headers
	120	else:
	121	self.headers = HTTPHeaderDict(headers)
	122	self.status = status
	123	self.version = version
	124	self.reason = reason
	125	self.strict = strict
	126	self.decode_content = decode_content
	127	self.retries = retries
	128	self.enforce_content_length = enforce_content_length
	129
	130	self._decoder = None
	131	self._body = None
	132	self._fp = None
	133	self._original_response = original_response
	134	self._fp_bytes_read = 0
	135
	136	if body and isinstance(body, (basestring, binary_type)):
	137	self._body = body
	138
	139	self._pool = pool
	140	self._connection = connection
	141
	142	if hasattr(body, 'read'):
	143	self._fp = body
	144
	145	# Are we using the chunked-style of transfer encoding?
	146	self.chunked = False
	147	self.chunk_left = None
	148	tr_enc = self.headers.get('transfer-encoding', '').lower()
	149	# Don't incur the penalty of creating a list and then discarding it
	150	encodings = (enc.strip() for enc in tr_enc.split(","))
	151	if "chunked" in encodings:
	152	self.chunked = True
	153
	154	# Determine length of response
	155	self.length_remaining = self._init_length(request_method)
	156
	157	# If requested, preload the body.
	158	if preload_content and not self._body:
	159	self._body = self.read(decode_content=decode_content)
	160
	161	def get_redirect_location(self):
	162	"""
	163	Should we redirect and where to?
	164
	165	:returns: Truthy redirect location string if we got a redirect status
	166	code and valid location. ``None`` if redirect status and no
	167	location. ``False`` if not a redirect status code.
	168	"""
	169	if self.status in self.REDIRECT_STATUSES:
	170	return self.headers.get('location')
	171
	172	return False
	173
	174	def release_conn(self):
	175	if not self._pool or not self._connection:
	176	return
	177
	178	self._pool._put_conn(self._connection)
	179	self._connection = None
	180
	181	@property
	182	def data(self):
	183	# For backwords-compat with earlier urllib3 0.4 and earlier.
	184	if self._body:
	185	return self._body
	186
	187	if self._fp:
	188	return self.read(cache_content=True)
	189
	190	@property
	191	def connection(self):
	192	return self._connection
	193
	194	def tell(self):
	195	"""
	196	Obtain the number of bytes pulled over the wire so far. May differ from
	197	the amount of content returned by :meth:``HTTPResponse.read`` if bytes
	198	are encoded on the wire (e.g, compressed).
	199	"""
	200	return self._fp_bytes_read
	201
	202	def _init_length(self, request_method):
	203	"""
	204	Set initial length value for Response content if available.
	205	"""
	206	length = self.headers.get('content-length')
	207
	208	if length is not None and self.chunked:
	209	# This Response will fail with an IncompleteRead if it can't be
	210	# received as chunked. This method falls back to attempt reading
	211	# the response before raising an exception.
	212	log.warning("Received response with both Content-Length and "
	213	"Transfer-Encoding set. This is expressly forbidden "
	214	"by RFC 7230 sec 3.3.2. Ignoring Content-Length and "
	215	"attempting to process response as Transfer-Encoding: "
	216	"chunked.")
	217	return None
	218
	219	elif length is not None:
	220	try:
	221	# RFC 7230 section 3.3.2 specifies multiple content lengths can
	222	# be sent in a single Content-Length header
	223	# (e.g. Content-Length: 42, 42). This line ensures the values
	224	# are all valid ints and that as long as the `set` length is 1,
	225	# all values are the same. Otherwise, the header is invalid.
	226	lengths = set([int(val) for val in length.split(',')])
	227	if len(lengths) > 1:
	228	raise InvalidHeader("Content-Length contained multiple "
	229	"unmatching values (%s)" % length)
	230	length = lengths.pop()
	231	except ValueError:
	232	length = None
	233	else:
	234	if length < 0:
	235	length = None
	236
	237	# Convert status to int for comparison
	238	# In some cases, httplib returns a status of "_UNKNOWN"
	239	try:
	240	status = int(self.status)
	241	except ValueError:
	242	status = 0
	243
	244	# Check for responses that shouldn't include a body
	245	if status in (204, 304) or 100 <= status < 200 or request_method == 'HEAD':
	246	length = 0
	247
	248	return length
	249
	250	def _init_decoder(self):
	251	"""
	252	Set-up the _decoder attribute if necessary.
	253	"""
	254	# Note: content-encoding value should be case-insensitive, per RFC 7230
	255	# Section 3.2
	256	content_encoding = self.headers.get('content-encoding', '').lower()
	257	if self._decoder is None and content_encoding in self.CONTENT_DECODERS:
	258	self._decoder = _get_decoder(content_encoding)
	259
	260	def _decode(self, data, decode_content, flush_decoder):
	261	"""
	262	Decode the data passed in and potentially flush the decoder.
	263	"""
	264	try:
	265	if decode_content and self._decoder:
	266	data = self._decoder.decompress(data)
	267	except (IOError, zlib.error) as e:
	268	content_encoding = self.headers.get('content-encoding', '').lower()
	269	raise DecodeError(
	270	"Received response with content-encoding: %s, but "
	271	"failed to decode it." % content_encoding, e)
	272
	273	if flush_decoder and decode_content:
	274	data += self._flush_decoder()
	275
	276	return data
	277
	278	def _flush_decoder(self):
	279	"""
	280	Flushes the decoder. Should only be called if the decoder is actually
	281	being used.
	282	"""
	283	if self._decoder:
	284	buf = self._decoder.decompress(b'')
	285	return buf + self._decoder.flush()
	286
	287	return b''
	288
	289	@contextmanager
	290	def _error_catcher(self):
	291	"""
	292	Catch low-level python exceptions, instead re-raising urllib3
	293	variants, so that low-level exceptions are not leaked in the
	294	high-level api.
	295
	296	On exit, release the connection back to the pool.
	297	"""
	298	clean_exit = False
	299
	300	try:
	301	try:
	302	yield
	303
	304	except SocketTimeout:
	305	# FIXME: Ideally we'd like to include the url in the ReadTimeoutError but
	306	# there is yet no clean way to get at it from this context.
	307	raise ReadTimeoutError(self._pool, None, 'Read timed out.')
	308
	309	except BaseSSLError as e:
	310	# FIXME: Is there a better way to differentiate between SSLErrors?
	311	if 'read operation timed out' not in str(e): # Defensive:
	312	# This shouldn't happen but just in case we're missing an edge
	313	# case, let's avoid swallowing SSL errors.
	314	raise
	315
	316	raise ReadTimeoutError(self._pool, None, 'Read timed out.')
	317
	318	except (HTTPException, SocketError) as e:
	319	# This includes IncompleteRead.
	320	raise ProtocolError('Connection broken: %r' % e, e)
	321
	322	# If no exception is thrown, we should avoid cleaning up
	323	# unnecessarily.
	324	clean_exit = True
	325	finally:
	326	# If we didn't terminate cleanly, we need to throw away our
	327	# connection.
	328	if not clean_exit:
	329	# The response may not be closed but we're not going to use it
	330	# anymore so close it now to ensure that the connection is
	331	# released back to the pool.
	332	if self._original_response:
	333	self._original_response.close()
	334
	335	# Closing the response may not actually be sufficient to close
	336	# everything, so if we have a hold of the connection close that
	337	# too.
	338	if self._connection:
	339	self._connection.close()
	340
	341	# If we hold the original response but it's closed now, we should
	342	# return the connection back to the pool.
	343	if self._original_response and self._original_response.isclosed():
	344	self.release_conn()
	345
	346	def read(self, amt=None, decode_content=None, cache_content=False):
	347	"""
	348	Similar to :meth:`httplib.HTTPResponse.read`, but with two additional
	349	parameters: ``decode_content`` and ``cache_content``.
	350
	351	:param amt:
	352	How much of the content to read. If specified, caching is skipped
	353	because it doesn't make sense to cache partial content as the full
	354	response.
	355
	356	:param decode_content:
	357	If True, will attempt to decode the body based on the
	358	'content-encoding' header.
	359
	360	:param cache_content:
	361	If True, will save the returned data such that the same result is
	362	returned despite of the state of the underlying file object. This
	363	is useful if you want the ``.data`` property to continue working
	364	after having ``.read()`` the file object. (Overridden if ``amt`` is
	365	set.)
	366	"""
	367	self._init_decoder()
	368	if decode_content is None:
	369	decode_content = self.decode_content
	370
	371	if self._fp is None:
	372	return
	373
	374	flush_decoder = False
	375	data = None
	376
	377	with self._error_catcher():
	378	if amt is None:
	379	# cStringIO doesn't like amt=None
	380	data = self._fp.read()
	381	flush_decoder = True
	382	else:
	383	cache_content = False
	384	data = self._fp.read(amt)
	385	if amt != 0 and not data: # Platform-specific: Buggy versions of Python.
	386	# Close the connection when no data is returned
	387	#
	388	# This is redundant to what httplib/http.client _should_
	389	# already do. However, versions of python released before
	390	# December 15, 2012 (http://bugs.python.org/issue16298) do
	391	# not properly close the connection in all cases. There is
	392	# no harm in redundantly calling close.
	393	self._fp.close()
	394	flush_decoder = True
	395	if self.enforce_content_length and self.length_remaining not in (0, None):
	396	# This is an edge case that httplib failed to cover due
	397	# to concerns of backward compatibility. We're
	398	# addressing it here to make sure IncompleteRead is
	399	# raised during streaming, so all calls with incorrect
	400	# Content-Length are caught.
	401	raise IncompleteRead(self._fp_bytes_read, self.length_remaining)
	402
	403	if data:
	404	self._fp_bytes_read += len(data)
	405	if self.length_remaining is not None:
	406	self.length_remaining -= len(data)
	407
	408	data = self._decode(data, decode_content, flush_decoder)
	409
	410	if cache_content:
	411	self._body = data
	412
	413	return data
	414
	415	def stream(self, amt=2**16, decode_content=None):
	416	"""
	417	A generator wrapper for the read() method. A call will block until
	418	``amt`` bytes have been read from the connection or until the
	419	connection is closed.
	420
	421	:param amt:
	422	How much of the content to read. The generator will return up to
	423	much data per iteration, but may return less. This is particularly
	424	likely when using compressed data. However, the empty string will
	425	never be returned.
	426
	427	:param decode_content:
	428	If True, will attempt to decode the body based on the
	429	'content-encoding' header.
	430	"""
	431	if self.chunked and self.supports_chunked_reads():
	432	for line in self.read_chunked(amt, decode_content=decode_content):
	433	yield line
	434	else:
	435	while not is_fp_closed(self._fp):
	436	data = self.read(amt=amt, decode_content=decode_content)
	437
	438	if data:
	439	yield data
	440
	441	@classmethod
	442	def from_httplib(ResponseCls, r, **response_kw):
	443	"""
	444	Given an :class:`httplib.HTTPResponse` instance ``r``, return a
	445	corresponding :class:`urllib3.response.HTTPResponse` object.
	446
	447	Remaining parameters are passed to the HTTPResponse constructor, along
	448	with ``original_response=r``.
	449	"""
	450	headers = r.msg
	451
	452	if not isinstance(headers, HTTPHeaderDict):
	453	if PY3: # Python 3
	454	headers = HTTPHeaderDict(headers.items())
	455	else: # Python 2
	456	headers = HTTPHeaderDict.from_httplib(headers)
	457
	458	# HTTPResponse objects in Python 3 don't have a .strict attribute
	459	strict = getattr(r, 'strict', 0)
	460	resp = ResponseCls(body=r,
	461	headers=headers,
	462	status=r.status,
	463	version=r.version,
	464	reason=r.reason,
	465	strict=strict,
	466	original_response=r,
	467	**response_kw)
	468	return resp
	469
	470	# Backwards-compatibility methods for httplib.HTTPResponse
	471	def getheaders(self):
	472	return self.headers
	473
	474	def getheader(self, name, default=None):
	475	return self.headers.get(name, default)
	476
	477	# Backwards compatibility for http.cookiejar
	478	def info(self):
	479	return self.headers
	480
	481	# Overrides from io.IOBase
	482	def close(self):
	483	if not self.closed:
	484	self._fp.close()
	485
	486	if self._connection:
	487	self._connection.close()
	488
	489	@property
	490	def closed(self):
	491	if self._fp is None:
	492	return True
	493	elif hasattr(self._fp, 'isclosed'):
	494	return self._fp.isclosed()
	495	elif hasattr(self._fp, 'closed'):
	496	return self._fp.closed
	497	else:
	498	return True
	499
	500	def fileno(self):
	501	if self._fp is None:
	502	raise IOError("HTTPResponse has no file to get a fileno from")
	503	elif hasattr(self._fp, "fileno"):
	504	return self._fp.fileno()
	505	else:
	506	raise IOError("The file-like object this HTTPResponse is wrapped "
	507	"around has no file descriptor")
	508
	509	def flush(self):
	510	if self._fp is not None and hasattr(self._fp, 'flush'):
	511	return self._fp.flush()
	512
	513	def readable(self):
	514	# This method is required for `io` module compatibility.
	515	return True
	516
	517	def readinto(self, b):
	518	# This method is required for `io` module compatibility.
	519	temp = self.read(len(b))
	520	if len(temp) == 0:
	521	return 0
	522	else:
	523	b[:len(temp)] = temp
	524	return len(temp)
	525
	526	def supports_chunked_reads(self):
	527	"""
	528	Checks if the underlying file-like object looks like a
	529	httplib.HTTPResponse object. We do this by testing for the fp
	530	attribute. If it is present we assume it returns raw chunks as
	531	processed by read_chunked().
	532	"""
	533	return hasattr(self._fp, 'fp')
	534
	535	def _update_chunk_length(self):
	536	# First, we'll figure out length of a chunk and then
	537	# we'll try to read it from socket.
	538	if self.chunk_left is not None:
	539	return
	540	line = self._fp.fp.readline()
	541	line = line.split(b';', 1)[0]
	542	try:
	543	self.chunk_left = int(line, 16)
	544	except ValueError:
	545	# Invalid chunked protocol response, abort.
	546	self.close()
	547	raise httplib.IncompleteRead(line)
	548
	549	def _handle_chunk(self, amt):
	550	returned_chunk = None
	551	if amt is None:
	552	chunk = self._fp._safe_read(self.chunk_left)
	553	returned_chunk = chunk
	554	self._fp._safe_read(2) # Toss the CRLF at the end of the chunk.
	555	self.chunk_left = None
	556	elif amt < self.chunk_left:
	557	value = self._fp._safe_read(amt)
	558	self.chunk_left = self.chunk_left - amt
	559	returned_chunk = value
	560	elif amt == self.chunk_left:
	561	value = self._fp._safe_read(amt)
	562	self._fp._safe_read(2) # Toss the CRLF at the end of the chunk.
	563	self.chunk_left = None
	564	returned_chunk = value
	565	else: # amt > self.chunk_left
	566	returned_chunk = self._fp._safe_read(self.chunk_left)
	567	self._fp._safe_read(2) # Toss the CRLF at the end of the chunk.
	568	self.chunk_left = None
	569	return returned_chunk
	570
	571	def read_chunked(self, amt=None, decode_content=None):
	572	"""
	573	Similar to :meth:`HTTPResponse.read`, but with an additional
	574	parameter: ``decode_content``.
	575
	576	:param decode_content:
	577	If True, will attempt to decode the body based on the
	578	'content-encoding' header.
	579	"""
	580	self._init_decoder()
	581	# FIXME: Rewrite this method and make it a class with a better structured logic.
	582	if not self.chunked:
	583	raise ResponseNotChunked(
	584	"Response is not chunked. "
	585	"Header 'transfer-encoding: chunked' is missing.")
	586	if not self.supports_chunked_reads():
	587	raise BodyNotHttplibCompatible(
	588	"Body should be httplib.HTTPResponse like. "
	589	"It should have have an fp attribute which returns raw chunks.")
	590
	591	# Don't bother reading the body of a HEAD request.
	592	if self._original_response and is_response_to_head(self._original_response):
	593	self._original_response.close()
	594	return
	595
	596	with self._error_catcher():
	597	while True:
	598	self._update_chunk_length()
	599	if self.chunk_left == 0:
	600	break
	601	chunk = self._handle_chunk(amt)
	602	decoded = self._decode(chunk, decode_content=decode_content,
	603	flush_decoder=False)
	604	if decoded:
	605	yield decoded
	606
	607	if decode_content:
	608	# On CPython and PyPy, we should never need to flush the
	609	# decoder. However, on Jython we might need to, so
	610	# lets defensively do it anyway.
	611	decoded = self._flush_decoder()
	612	if decoded: # Platform-specific: Jython.
	613	yield decoded
	614
	615	# Chunk content ends with \r\n: discard it.
	616	while True:
	617	line = self._fp.fp.readline()
	618	if not line:
	619	# Some sites may not end with '\r\n'.
	620	break
	621	if line == b'\r\n':
	622	break
	623
	624	# We read everything; close the "file".
	625	if self._original_response:
	626	self._original_response.close()