From 68df54d6629ec019142eb149dd037774f2d11e7c Mon Sep 17 00:00:00 2001 From: Shubham Saini Date: Tue, 11 Dec 2018 15:31:23 +0530 Subject: First commit --- .../pip/_internal/utils/encoding.py | 33 ++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/utils/encoding.py (limited to 'venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/utils/encoding.py') diff --git a/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/utils/encoding.py b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/utils/encoding.py new file mode 100644 index 0000000..831f3f6 --- /dev/null +++ b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_internal/utils/encoding.py @@ -0,0 +1,33 @@ +import codecs +import locale +import re +import sys + +BOMS = [ + (codecs.BOM_UTF8, 'utf8'), + (codecs.BOM_UTF16, 'utf16'), + (codecs.BOM_UTF16_BE, 'utf16-be'), + (codecs.BOM_UTF16_LE, 'utf16-le'), + (codecs.BOM_UTF32, 'utf32'), + (codecs.BOM_UTF32_BE, 'utf32-be'), + (codecs.BOM_UTF32_LE, 'utf32-le'), +] + +ENCODING_RE = re.compile(br'coding[:=]\s*([-\w.]+)') + + +def auto_decode(data): + """Check a bytes string for a BOM to correctly detect the encoding + + Fallback to locale.getpreferredencoding(False) like open() on Python3""" + for bom, encoding in BOMS: + if data.startswith(bom): + return data[len(bom):].decode(encoding) + # Lets check the first two lines as in PEP263 + for line in data.split(b'\n')[:2]: + if line[0:1] == b'#' and ENCODING_RE.search(line): + encoding = ENCODING_RE.search(line).groups()[0].decode('ascii') + return data.decode(encoding) + return data.decode( + locale.getpreferredencoding(False) or sys.getdefaultencoding(), + ) -- cgit v1.2.3