1 files changed, 132 insertions, 0 deletions
diff --git a/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/chardet/sbcharsetprober.py b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/chardet/sbcharsetprober.py
new file mode 100644
index 0000000..66e0dfc
--- /dev/null
+++ b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/chardet/sbcharsetprober.py
@@ -0,0 +1,132 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Universal charset detector code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2001
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#   Shy Shalom - original C code
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+from .charsetprober import CharSetProber
+from .enums import CharacterCategory, ProbingState, SequenceLikelihood
+class SingleByteCharSetProber(CharSetProber):
+    SAMPLE_SIZE = 64
+    SB_ENOUGH_REL_THRESHOLD = 1024  #  0.25 * SAMPLE_SIZE^2
+    POSITIVE_SHORTCUT_THRESHOLD = 0.95
+    NEGATIVE_SHORTCUT_THRESHOLD = 0.05
+    def __init__(self, model, reversed=False, name_prober=None):
+        super(SingleByteCharSetProber, self).__init__()
+        self._model = model
+        # TRUE if we need to reverse every pair in the model lookup
+        self._reversed = reversed
+        # Optional auxiliary prober for name decision
+        self._name_prober = name_prober
+        self._last_order = None
+        self._seq_counters = None
+        self._total_seqs = None
+        self._total_char = None
+        self._freq_char = None
+        self.reset()
+    def reset(self):
+        super(SingleByteCharSetProber, self).reset()
+        # char order of last character
+        self._last_order = 255
+        self._seq_counters = [0] * SequenceLikelihood.get_num_categories()
+        self._total_seqs = 0
+        self._total_char = 0
+        # characters that fall in our sampling range
+        self._freq_char = 0
+    @property
+    def charset_name(self):
+        if self._name_prober:
+            return self._name_prober.charset_name
+        else:
+            return self._model['charset_name']
+    @property
+    def language(self):
+        if self._name_prober:
+            return self._name_prober.language
+        else:
+            return self._model.get('language')
+    def feed(self, byte_str):
+        if not self._model['keep_english_letter']:
+            byte_str = self.filter_international_words(byte_str)
+        if not byte_str:
+            return self.state
+        char_to_order_map = self._model['char_to_order_map']
+        for i, c in enumerate(byte_str):
+            # XXX: Order is in range 1-64, so one would think we want 0-63 here,
+            #      but that leads to 27 more test failures than before.
+            order = char_to_order_map[c]
+            # XXX: This was SYMBOL_CAT_ORDER before, with a value of 250, but
+            #      CharacterCategory.SYMBOL is actually 253, so we use CONTROL
+            #      to make it closer to the original intent. The only difference
+            #      is whether or not we count digits and control characters for
+            #      _total_char purposes.
+            if order < CharacterCategory.CONTROL:
+                self._total_char += 1
+            if order < self.SAMPLE_SIZE:
+                self._freq_char += 1
+                if self._last_order < self.SAMPLE_SIZE:
+                    self._total_seqs += 1
+                    if not self._reversed:
+                        i = (self._last_order * self.SAMPLE_SIZE) + order
+                        model = self._model['precedence_matrix'][i]
+                    else:  # reverse the order of the letters in the lookup
+                        i = (order * self.SAMPLE_SIZE) + self._last_order
+                        model = self._model['precedence_matrix'][i]
+                    self._seq_counters[model] += 1
+            self._last_order = order
+        charset_name = self._model['charset_name']
+        if self.state == ProbingState.DETECTING:
+            if self._total_seqs > self.SB_ENOUGH_REL_THRESHOLD:
+                confidence = self.get_confidence()
+                if confidence > self.POSITIVE_SHORTCUT_THRESHOLD:
+                    self.logger.debug('%s confidence = %s, we have a winner',
+                                      charset_name, confidence)
+                    self._state = ProbingState.FOUND_IT
+                elif confidence < self.NEGATIVE_SHORTCUT_THRESHOLD:
+                    self.logger.debug('%s confidence = %s, below negative '
+                                      'shortcut threshhold %s', charset_name,
+                                      confidence,
+                                      self.NEGATIVE_SHORTCUT_THRESHOLD)
+                    self._state = ProbingState.NOT_ME
+        return self.state
+    def get_confidence(self):
+        r = 0.01
+        if self._total_seqs > 0:
+            r = ((1.0 * self._seq_counters[SequenceLikelihood.POSITIVE]) /
+                 self._total_seqs / self._model['typical_positive_ratio'])
+            r = r * self._freq_char / self._total_char
+            if r >= 1.0:
+                r = 0.99
+        return r

diff --git a/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/chardet/sbcharsetprober.py b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/chardet/sbcharsetprober.py new file mode 100644 index 0000000..66e0dfc --- /dev/null +++ b/venv/lib/python3.7/site-packages/pip-10.0.1-py3.7.egg/pip/_vendor/chardet/sbcharsetprober.py
@@ -0,0 +1,132 @@
	1	######################## BEGIN LICENSE BLOCK ########################
	2	# The Original Code is Mozilla Universal charset detector code.
	3	#
	4	# The Initial Developer of the Original Code is
	5	# Netscape Communications Corporation.
	6	# Portions created by the Initial Developer are Copyright (C) 2001
	7	# the Initial Developer. All Rights Reserved.
	8	#
	9	# Contributor(s):
	10	# Mark Pilgrim - port to Python
	11	# Shy Shalom - original C code
	12	#
	13	# This library is free software; you can redistribute it and/or
	14	# modify it under the terms of the GNU Lesser General Public
	15	# License as published by the Free Software Foundation; either
	16	# version 2.1 of the License, or (at your option) any later version.
	17	#
	18	# This library is distributed in the hope that it will be useful,
	19	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	20	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	21	# Lesser General Public License for more details.
	22	#
	23	# You should have received a copy of the GNU Lesser General Public
	24	# License along with this library; if not, write to the Free Software
	25	# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
	26	# 02110-1301 USA
	27	######################### END LICENSE BLOCK #########################
	28
	29	from .charsetprober import CharSetProber
	30	from .enums import CharacterCategory, ProbingState, SequenceLikelihood
	31
	32
	33	class SingleByteCharSetProber(CharSetProber):
	34	SAMPLE_SIZE = 64
	35	SB_ENOUGH_REL_THRESHOLD = 1024 # 0.25 * SAMPLE_SIZE^2
	36	POSITIVE_SHORTCUT_THRESHOLD = 0.95
	37	NEGATIVE_SHORTCUT_THRESHOLD = 0.05
	38
	39	def __init__(self, model, reversed=False, name_prober=None):
	40	super(SingleByteCharSetProber, self).__init__()
	41	self._model = model
	42	# TRUE if we need to reverse every pair in the model lookup
	43	self._reversed = reversed
	44	# Optional auxiliary prober for name decision
	45	self._name_prober = name_prober
	46	self._last_order = None
	47	self._seq_counters = None
	48	self._total_seqs = None
	49	self._total_char = None
	50	self._freq_char = None
	51	self.reset()
	52
	53	def reset(self):
	54	super(SingleByteCharSetProber, self).reset()
	55	# char order of last character
	56	self._last_order = 255
	57	self._seq_counters = [0] * SequenceLikelihood.get_num_categories()
	58	self._total_seqs = 0
	59	self._total_char = 0
	60	# characters that fall in our sampling range
	61	self._freq_char = 0
	62
	63	@property
	64	def charset_name(self):
	65	if self._name_prober:
	66	return self._name_prober.charset_name
	67	else:
	68	return self._model['charset_name']
	69
	70	@property
	71	def language(self):
	72	if self._name_prober:
	73	return self._name_prober.language
	74	else:
	75	return self._model.get('language')
	76
	77	def feed(self, byte_str):
	78	if not self._model['keep_english_letter']:
	79	byte_str = self.filter_international_words(byte_str)
	80	if not byte_str:
	81	return self.state
	82	char_to_order_map = self._model['char_to_order_map']
	83	for i, c in enumerate(byte_str):
	84	# XXX: Order is in range 1-64, so one would think we want 0-63 here,
	85	# but that leads to 27 more test failures than before.
	86	order = char_to_order_map[c]
	87	# XXX: This was SYMBOL_CAT_ORDER before, with a value of 250, but
	88	# CharacterCategory.SYMBOL is actually 253, so we use CONTROL
	89	# to make it closer to the original intent. The only difference
	90	# is whether or not we count digits and control characters for
	91	# _total_char purposes.
	92	if order < CharacterCategory.CONTROL:
	93	self._total_char += 1
	94	if order < self.SAMPLE_SIZE:
	95	self._freq_char += 1
	96	if self._last_order < self.SAMPLE_SIZE:
	97	self._total_seqs += 1
	98	if not self._reversed:
	99	i = (self._last_order * self.SAMPLE_SIZE) + order
	100	model = self._model['precedence_matrix'][i]
	101	else: # reverse the order of the letters in the lookup
	102	i = (order * self.SAMPLE_SIZE) + self._last_order
	103	model = self._model['precedence_matrix'][i]
	104	self._seq_counters[model] += 1
	105	self._last_order = order
	106
	107	charset_name = self._model['charset_name']
	108	if self.state == ProbingState.DETECTING:
	109	if self._total_seqs > self.SB_ENOUGH_REL_THRESHOLD:
	110	confidence = self.get_confidence()
	111	if confidence > self.POSITIVE_SHORTCUT_THRESHOLD:
	112	self.logger.debug('%s confidence = %s, we have a winner',
	113	charset_name, confidence)
	114	self._state = ProbingState.FOUND_IT
	115	elif confidence < self.NEGATIVE_SHORTCUT_THRESHOLD:
	116	self.logger.debug('%s confidence = %s, below negative '
	117	'shortcut threshhold %s', charset_name,
	118	confidence,
	119	self.NEGATIVE_SHORTCUT_THRESHOLD)
	120	self._state = ProbingState.NOT_ME
	121
	122	return self.state
	123
	124	def get_confidence(self):
	125	r = 0.01
	126	if self._total_seqs > 0:
	127	r = ((1.0 * self._seq_counters[SequenceLikelihood.POSITIVE]) /
	128	self._total_seqs / self._model['typical_positive_ratio'])
	129	r = r * self._freq_char / self._total_char
	130	if r >= 1.0:
	131	r = 0.99
	132	return r