



import numpy as np

from transformers import BatchFeature
from transformers.testing_utils import require_tf, require_torch

from .test_feature_extraction_common import FeatureExtractionSavingTestMixin



class     lowerCAmelCase_		(  a__							):
       # to overwrite at feature extractactor specific tests
       UpperCAmelCase__    : Tuple       					=  None
       UpperCAmelCase__    : List[Any]       					=  None
       @property
       def  snake_case_	(      self	)					->		Optional[Any]:
          return self.feat_extract_tester.prepare_feat_extract_dict()
       def  snake_case_	(      self	)					->		int:
          UpperCamelCase   :  int						     =     self.feature_extraction_class(**self.feat_extract_dict	)
          self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_,      'feature_size'	)	)
          self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_,      'sampling_rate'	)	)
          self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_,      'padding_value'	)	)
       def  snake_case_	(      self	)					->		Union[str, Any]:
          UpperCamelCase   :  Union[str, Any]						     =     self.feat_extract_tester.prepare_inputs_for_common()
          UpperCamelCase   :  str						     =     self.feature_extraction_class(**self.feat_extract_dict	)
          UpperCamelCase   :  Optional[int]						     =     feat_extract.model_input_names[0]

          UpperCamelCase   :  List[Any]						     =     BatchFeature({input_name: speech_inputs}	)

          self.assertTrue(all(len(SCREAMING_SNAKE_CASE_	) == len(SCREAMING_SNAKE_CASE_	) for x, y in zip(SCREAMING_SNAKE_CASE_,      processed_features[input_name]	)	)	)

          UpperCamelCase   :  List[str]						     =     self.feat_extract_tester.prepare_inputs_for_common(equal_length=SCREAMING_SNAKE_CASE_	)
          UpperCamelCase   :  int						     =     BatchFeature({input_name: speech_inputs},      tensor_type='np'	)

          UpperCamelCase   :  Optional[int]						     =     processed_features[input_name]

          if len(batch_features_input.shape	) < 3:
             UpperCamelCase   :  List[Any]						     =     batch_features_input[:, :, None]

          self.assertTrue(
              batch_features_input.shape
              == (self.feat_extract_tester.batch_size, len(speech_inputs[0]	), self.feat_extract_tester.feature_size)	)
       @require_torch
       def  snake_case_	(      self	)					->		Tuple:
          UpperCamelCase   :  str						     =     self.feat_extract_tester.prepare_inputs_for_common(equal_length=SCREAMING_SNAKE_CASE_	)
          UpperCamelCase   :  int						     =     self.feature_extraction_class(**self.feat_extract_dict	)
          UpperCamelCase   :  Dict						     =     feat_extract.model_input_names[0]

          UpperCamelCase   :  Union[str, Any]						     =     BatchFeature({input_name: speech_inputs},      tensor_type='pt'	)

          UpperCamelCase   :  Dict						     =     processed_features[input_name]

          if len(batch_features_input.shape	) < 3:
             UpperCamelCase   :  Union[str, Any]						     =     batch_features_input[:, :, None]

          self.assertTrue(
              batch_features_input.shape
              == (self.feat_extract_tester.batch_size, len(speech_inputs[0]	), self.feat_extract_tester.feature_size)	)
       @require_tf
       def  snake_case_	(      self	)					->		Any:
          UpperCamelCase   :  Optional[Any]						     =     self.feat_extract_tester.prepare_inputs_for_common(equal_length=SCREAMING_SNAKE_CASE_	)
          UpperCamelCase   :  Union[str, Any]						     =     self.feature_extraction_class(**self.feat_extract_dict	)
          UpperCamelCase   :  int						     =     feat_extract.model_input_names[0]

          UpperCamelCase   :  str						     =     BatchFeature({input_name: speech_inputs},      tensor_type='tf'	)

          UpperCamelCase   :  Dict						     =     processed_features[input_name]

          if len(batch_features_input.shape	) < 3:
             UpperCamelCase   :  Union[str, Any]						     =     batch_features_input[:, :, None]

          self.assertTrue(
              batch_features_input.shape
              == (self.feat_extract_tester.batch_size, len(speech_inputs[0]	), self.feat_extract_tester.feature_size)	)
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_=False	)					->		Any:
          def _inputs_have_equal_length(SCREAMING_SNAKE_CASE_	):
             UpperCamelCase   :  Tuple						     =     len(input[0]	)
             for input_slice in input[1:]:
                if len(SCREAMING_SNAKE_CASE_	) != length:
                   return False
             return True

          def _inputs_are_equal(SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_	):
             if len(SCREAMING_SNAKE_CASE_	) != len(SCREAMING_SNAKE_CASE_	):
                return False

             for input_slice_a, input_slice_a in zip(SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_	):
                if not np.allclose(np.asarray(SCREAMING_SNAKE_CASE_	),      np.asarray(SCREAMING_SNAKE_CASE_	),      atol=1e-3	):
                   return False
             return True

          UpperCamelCase   :  Optional[int]						     =     self.feature_extraction_class(**self.feat_extract_dict	)
          UpperCamelCase   :  int						     =     self.feat_extract_tester.prepare_inputs_for_common(numpify=SCREAMING_SNAKE_CASE_	)
          UpperCamelCase   :  str						     =     feat_extract.model_input_names[0]

          UpperCamelCase   :  Tuple						     =     BatchFeature({input_name: speech_inputs}	)

          UpperCamelCase   :  List[Any]						     =     self.feat_extract_tester.seq_length_diff
          UpperCamelCase   :  Any						     =     self.feat_extract_tester.max_seq_length + pad_diff
          UpperCamelCase   :  Tuple						     =     self.feat_extract_tester.min_seq_length
          UpperCamelCase   :  Dict						     =     self.feat_extract_tester.batch_size
          UpperCamelCase   :  str						     =     self.feat_extract_tester.feature_size

          # test padding for List[int] + numpy
          UpperCamelCase   :  Dict						     =     feat_extract.pad(SCREAMING_SNAKE_CASE_,      padding=SCREAMING_SNAKE_CASE_	)
          UpperCamelCase   :  str						     =     input_a[input_name]

          UpperCamelCase   :  str						     =     feat_extract.pad(SCREAMING_SNAKE_CASE_,      padding='longest'	)
          UpperCamelCase   :  Union[str, Any]						     =     input_a[input_name]

          UpperCamelCase   :  str						     =     feat_extract.pad(SCREAMING_SNAKE_CASE_,      padding='max_length',      max_length=len(speech_inputs[-1]	)	)
          UpperCamelCase   :  Any						     =     input_a[input_name]

          UpperCamelCase   :  Tuple						     =     feat_extract.pad(SCREAMING_SNAKE_CASE_,      padding='longest',      return_tensors='np'	)
          UpperCamelCase   :  Any						     =     input_a[input_name]

          # max_length parameter has to be provided when setting `padding="max_length"`
          with self.assertRaises(SCREAMING_SNAKE_CASE_	):
             feat_extract.pad(SCREAMING_SNAKE_CASE_,      padding='max_length'	)[input_name]

          UpperCamelCase   :  int						     =     feat_extract.pad(
              SCREAMING_SNAKE_CASE_,      padding='max_length',      max_length=SCREAMING_SNAKE_CASE_,      return_tensors='np'	)
          UpperCamelCase   :  Optional[int]						     =     input_a[input_name]

          self.assertFalse(_inputs_have_equal_length(SCREAMING_SNAKE_CASE_	)	)
          self.assertTrue(_inputs_have_equal_length(SCREAMING_SNAKE_CASE_	)	)
          self.assertTrue(_inputs_have_equal_length(SCREAMING_SNAKE_CASE_	)	)
          self.assertTrue(_inputs_are_equal(SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_	)	)
          self.assertTrue(len(input_a[0]	) == pad_min_length	)
          self.assertTrue(len(input_a[1]	) == pad_min_length + pad_diff	)
          self.assertTrue(input_a.shape[:2] == (batch_size, len(input_a[0]	))	)
          self.assertTrue(input_a.shape[:2] == (batch_size, pad_max_length)	)

          if feature_size > 1:
             self.assertTrue(input_a.shape[2] == input_a.shape[2] == feature_size	)

          # test padding for `pad_to_multiple_of` for List[int] + numpy
          UpperCamelCase   :  List[Any]						     =     feat_extract.pad(SCREAMING_SNAKE_CASE_,      pad_to_multiple_of=10	)
          UpperCamelCase   :  List[str]						     =     input_a[input_name]

          UpperCamelCase   :  Dict						     =     feat_extract.pad(SCREAMING_SNAKE_CASE_,      padding='longest',      pad_to_multiple_of=10	)
          UpperCamelCase   :  Dict						     =     input_a[input_name]

          UpperCamelCase   :  List[Any]						     =     feat_extract.pad(
              SCREAMING_SNAKE_CASE_,      padding='max_length',      pad_to_multiple_of=10,      max_length=SCREAMING_SNAKE_CASE_	)
          UpperCamelCase   :  List[str]						     =     input_a[input_name]

          UpperCamelCase   :  Optional[int]						     =     feat_extract.pad(
              SCREAMING_SNAKE_CASE_,      padding='max_length',      pad_to_multiple_of=10,      max_length=SCREAMING_SNAKE_CASE_,      return_tensors='np',      )
          UpperCamelCase   :  Dict						     =     input_a[input_name]

          self.assertTrue(all(len(SCREAMING_SNAKE_CASE_	) % 10 == 0 for x in input_a	)	)
          self.assertTrue(_inputs_are_equal(SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_	)	)

          UpperCamelCase   :  int						     =     pad_max_length if pad_max_length % 10 == 0 else (pad_max_length // 10 + 1) * 10
          self.assertTrue(all(len(SCREAMING_SNAKE_CASE_	) == expected_mult_pad_length for x in input_a	)	)
          self.assertEqual(input_a.shape[:2],      (batch_size, expected_mult_pad_length)	)

          if feature_size > 1:
             self.assertTrue(input_a.shape[2] == feature_size	)

          # Check padding value is correct
          UpperCamelCase   :  List[Any]						     =     (np.ones(self.feat_extract_tester.feature_size	) * feat_extract.padding_value).sum()
          self.assertTrue(
              abs(np.asarray(input_a[0]	)[pad_min_length:].sum() - padding_vector_sum * (pad_max_length - pad_min_length)	)
              < 1e-3	)
          self.assertTrue(
              abs(
                  np.asarray(input_a[1]	)[pad_min_length + pad_diff :].sum()
                  - padding_vector_sum * (pad_max_length - pad_min_length - pad_diff)	)
              < 1e-3	)
          self.assertTrue(
              abs(
                  np.asarray(input_a[2]	)[pad_min_length + 2 * pad_diff :].sum()
                  - padding_vector_sum * (pad_max_length - pad_min_length - 2 * pad_diff)	)
              < 1e-3	)
          self.assertTrue(
              abs(input_a[0, pad_min_length:].sum() - padding_vector_sum * (pad_max_length - pad_min_length)	) < 1e-3	)
          self.assertTrue(
              abs(input_a[0, pad_min_length:].sum() - padding_vector_sum * (expected_mult_pad_length - pad_min_length)	)
              < 1e-3	)
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_=False	)					->		int:
          def _inputs_have_equal_length(SCREAMING_SNAKE_CASE_	):
             UpperCamelCase   :  Optional[int]						     =     len(input[0]	)
             for input_slice in input[1:]:
                if len(SCREAMING_SNAKE_CASE_	) != length:
                   return False
             return True

          def _inputs_are_equal(SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_	):
             if len(SCREAMING_SNAKE_CASE_	) != len(SCREAMING_SNAKE_CASE_	):
                return False

             for input_slice_a, input_slice_a in zip(SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_	):
                if not np.allclose(np.asarray(SCREAMING_SNAKE_CASE_	),      np.asarray(SCREAMING_SNAKE_CASE_	),      atol=1e-3	):
                   return False
             return True

          UpperCamelCase   :  int						     =     self.feature_extraction_class(**self.feat_extract_dict	)
          UpperCamelCase   :  List[str]						     =     self.feat_extract_tester.prepare_inputs_for_common(numpify=SCREAMING_SNAKE_CASE_	)
          UpperCamelCase   :  List[Any]						     =     feat_extract.model_input_names[0]

          UpperCamelCase   :  Any						     =     BatchFeature({input_name: speech_inputs}	)

          # truncate to smallest
          UpperCamelCase   :  Union[str, Any]						     =     feat_extract.pad(
              SCREAMING_SNAKE_CASE_,      padding='max_length',      max_length=len(speech_inputs[0]	),      truncation=SCREAMING_SNAKE_CASE_	)
          UpperCamelCase   :  List[Any]						     =     input_a[input_name]

          UpperCamelCase   :  Optional[int]						     =     feat_extract.pad(SCREAMING_SNAKE_CASE_,      padding='max_length',      max_length=len(speech_inputs[0]	)	)
          UpperCamelCase   :  Any						     =     input_a[input_name]

          self.assertTrue(_inputs_have_equal_length(SCREAMING_SNAKE_CASE_	)	)
          self.assertFalse(_inputs_have_equal_length(SCREAMING_SNAKE_CASE_	)	)

          # truncate to smallest with np
          UpperCamelCase   :  Optional[int]						     =     feat_extract.pad(
              SCREAMING_SNAKE_CASE_,      padding='max_length',      max_length=len(speech_inputs[0]	),      return_tensors='np',      truncation=SCREAMING_SNAKE_CASE_,      )
          UpperCamelCase   :  Tuple						     =     input_a[input_name]

          UpperCamelCase   :  List[str]						     =     feat_extract.pad(
              SCREAMING_SNAKE_CASE_,      padding='max_length',      max_length=len(speech_inputs[0]	),      return_tensors='np'	)
          UpperCamelCase   :  Optional[Any]						     =     input_a[input_name]

          self.assertTrue(_inputs_have_equal_length(SCREAMING_SNAKE_CASE_	)	)
          self.assertTrue(input_a.shape[1] == len(speech_inputs[0]	)	)

          # since truncation forces padding to be smaller than longest input
          # function can't return `np.ndarray`, but has to return list
          self.assertFalse(_inputs_have_equal_length(SCREAMING_SNAKE_CASE_	)	)

          # truncate to middle
          UpperCamelCase   :  Optional[int]						     =     feat_extract.pad(
              SCREAMING_SNAKE_CASE_,      padding='max_length',      max_length=len(speech_inputs[1]	),      truncation=SCREAMING_SNAKE_CASE_,      return_tensors='np',      )
          UpperCamelCase   :  Any						     =     input_a[input_name]

          UpperCamelCase   :  Union[str, Any]						     =     feat_extract.pad(
              SCREAMING_SNAKE_CASE_,      padding='max_length',      max_length=len(speech_inputs[1]	),      truncation=SCREAMING_SNAKE_CASE_	)
          UpperCamelCase   :  Any						     =     input_a[input_name]

          UpperCamelCase   :  str						     =     feat_extract.pad(
              SCREAMING_SNAKE_CASE_,      padding='max_length',      max_length=len(speech_inputs[1]	),      return_tensors='np'	)
          UpperCamelCase   :  List[str]						     =     input_a[input_name]

          self.assertTrue(input_a.shape[1] == len(speech_inputs[1]	)	)
          self.assertTrue(_inputs_have_equal_length(SCREAMING_SNAKE_CASE_	)	)
          self.assertTrue(_inputs_have_equal_length(SCREAMING_SNAKE_CASE_	)	)
          self.assertTrue(_inputs_are_equal(SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_	)	)

          # since truncation forces padding to be smaller than longest input
          # function can't return `np.ndarray`, but has to return list
          self.assertFalse(_inputs_have_equal_length(SCREAMING_SNAKE_CASE_	)	)
          self.assertTrue(len(input_a[-1]	) == len(speech_inputs[-1]	)	)

          # padding has to be max_length when setting `truncation=True`
          with self.assertRaises(SCREAMING_SNAKE_CASE_	):
             feat_extract.pad(SCREAMING_SNAKE_CASE_,      truncation=SCREAMING_SNAKE_CASE_	)[input_name]

          # padding has to be max_length when setting `truncation=True`
          with self.assertRaises(SCREAMING_SNAKE_CASE_	):
             feat_extract.pad(SCREAMING_SNAKE_CASE_,      padding='longest',      truncation=SCREAMING_SNAKE_CASE_	)[input_name]

          # padding has to be max_length when setting `truncation=True`
          with self.assertRaises(SCREAMING_SNAKE_CASE_	):
             feat_extract.pad(SCREAMING_SNAKE_CASE_,      padding='longest',      truncation=SCREAMING_SNAKE_CASE_	)[input_name]

          # max_length parameter has to be provided when setting `truncation=True` and padding="max_length"
          with self.assertRaises(SCREAMING_SNAKE_CASE_	):
             feat_extract.pad(SCREAMING_SNAKE_CASE_,      padding='max_length',      truncation=SCREAMING_SNAKE_CASE_	)[input_name]

          # test truncation for `pad_to_multiple_of` for List[int] + numpy
          UpperCamelCase   :  Optional[Any]						     =     12
          UpperCamelCase   :  Optional[Any]						     =     feat_extract.pad(
              SCREAMING_SNAKE_CASE_,      padding='max_length',      max_length=len(speech_inputs[0]	),      pad_to_multiple_of=SCREAMING_SNAKE_CASE_,      truncation=SCREAMING_SNAKE_CASE_,      )
          UpperCamelCase   :  Tuple						     =     input_a[input_name]

          UpperCamelCase   :  List[Any]						     =     feat_extract.pad(
              SCREAMING_SNAKE_CASE_,      padding='max_length',      max_length=len(speech_inputs[0]	),      pad_to_multiple_of=SCREAMING_SNAKE_CASE_,      )
          UpperCamelCase   :  Union[str, Any]						     =     input_a[input_name]

          # retrieve expected_length as multiple of pad_to_multiple_of
          UpperCamelCase   :  Optional[int]						     =     len(speech_inputs[0]	)
          if expected_length % pad_to_multiple_of != 0:
             UpperCamelCase   :  Tuple						     =     ((len(speech_inputs[0]	) // pad_to_multiple_of) + 1) * pad_to_multiple_of

          self.assertTrue(len(input_a[0]	) == expected_length	)
          self.assertTrue(_inputs_have_equal_length(SCREAMING_SNAKE_CASE_	)	)
          self.assertFalse(_inputs_have_equal_length(SCREAMING_SNAKE_CASE_	)	)
       def  snake_case_	(      self	)					->		Optional[Any]:
          self._check_padding(numpify=SCREAMING_SNAKE_CASE_	)
       def  snake_case_	(      self	)					->		List[Any]:
          self._check_padding(numpify=SCREAMING_SNAKE_CASE_	)
       def  snake_case_	(      self	)					->		Any:
          self._check_truncation(numpify=SCREAMING_SNAKE_CASE_	)
       def  snake_case_	(      self	)					->		Any:
          self._check_truncation(numpify=SCREAMING_SNAKE_CASE_	)
       @require_torch
       def  snake_case_	(      self	)					->		Union[str, Any]:
          UpperCamelCase   :  Optional[Any]						     =     self.feature_extraction_class(**self.feat_extract_dict	)
          UpperCamelCase   :  Optional[int]						     =     self.feat_extract_tester.prepare_inputs_for_common()
          UpperCamelCase   :  Tuple						     =     feat_extract.model_input_names[0]

          UpperCamelCase   :  str						     =     BatchFeature({input_name: speech_inputs}	)

          UpperCamelCase   :  Optional[int]						     =     feat_extract.pad(SCREAMING_SNAKE_CASE_,      padding='longest',      return_tensors='np'	)[input_name]
          UpperCamelCase   :  Union[str, Any]						     =     feat_extract.pad(SCREAMING_SNAKE_CASE_,      padding='longest',      return_tensors='pt'	)[input_name]

          self.assertTrue(abs(input_np.astype(np.floataa	).sum() - input_pt.numpy().astype(np.floataa	).sum()	) < 1e-2	)
       @require_tf
       def  snake_case_	(      self	)					->		Tuple:
          UpperCamelCase   :  Optional[Any]						     =     self.feature_extraction_class(**self.feat_extract_dict	)
          UpperCamelCase   :  Dict						     =     self.feat_extract_tester.prepare_inputs_for_common()
          UpperCamelCase   :  str						     =     feat_extract.model_input_names[0]

          UpperCamelCase   :  Optional[Any]						     =     BatchFeature({input_name: speech_inputs}	)

          UpperCamelCase   :  int						     =     feat_extract.pad(SCREAMING_SNAKE_CASE_,      padding='longest',      return_tensors='np'	)[input_name]
          UpperCamelCase   :  Union[str, Any]						     =     feat_extract.pad(SCREAMING_SNAKE_CASE_,      padding='longest',      return_tensors='tf'	)[input_name]

          self.assertTrue(abs(input_np.astype(np.floataa	).sum() - input_tf.numpy().astype(np.floataa	).sum()	) < 1e-2	)
       def  snake_case_	(      self	)					->		List[Any]:
          UpperCamelCase   :  List[Any]						     =     self.feat_extract_dict
          UpperCamelCase   :  Optional[Any]						     =     True
          UpperCamelCase   :  List[str]						     =     self.feature_extraction_class(**SCREAMING_SNAKE_CASE_	)
          UpperCamelCase   :  Optional[int]						     =     self.feat_extract_tester.prepare_inputs_for_common()
          UpperCamelCase   :  Optional[int]						     =     [len(SCREAMING_SNAKE_CASE_	) for x in speech_inputs]
          UpperCamelCase   :  Union[str, Any]						     =     feat_extract.model_input_names[0]

          UpperCamelCase   :  Optional[Any]						     =     BatchFeature({input_name: speech_inputs}	)

          UpperCamelCase   :  List[Any]						     =     feat_extract.pad(SCREAMING_SNAKE_CASE_,      padding='longest',      return_tensors='np'	)
          self.assertIn('attention_mask',      SCREAMING_SNAKE_CASE_	)
          self.assertListEqual(list(processed.attention_mask.shape	),      list(processed[input_name].shape[:2]	)	)
          self.assertListEqual(processed.attention_mask.sum(-1	).tolist(),      SCREAMING_SNAKE_CASE_	)






       def  snake_case_	(      self	)					->		Dict:
          UpperCamelCase   :  List[str]						     =     self.feat_extract_dict
          UpperCamelCase   :  Optional[int]						     =     True
          UpperCamelCase   :  int						     =     self.feature_extraction_class(**SCREAMING_SNAKE_CASE_	)
          UpperCamelCase   :  Optional[Any]						     =     self.feat_extract_tester.prepare_inputs_for_common()
          UpperCamelCase   :  Union[str, Any]						     =     [len(SCREAMING_SNAKE_CASE_	) for x in speech_inputs]
          UpperCamelCase   :  int						     =     feat_extract.model_input_names[0]

          UpperCamelCase   :  str						     =     BatchFeature({input_name: speech_inputs}	)
          UpperCamelCase   :  Dict						     =     min(SCREAMING_SNAKE_CASE_	)

          UpperCamelCase   :  Dict						     =     feat_extract.pad(
              SCREAMING_SNAKE_CASE_,      padding='max_length',      max_length=SCREAMING_SNAKE_CASE_,      truncation=SCREAMING_SNAKE_CASE_,      return_tensors='np'	)
          self.assertIn('attention_mask',      SCREAMING_SNAKE_CASE_	)
          self.assertListEqual(
              list(processed_pad.attention_mask.shape	),      [processed_pad[input_name].shape[0], max_length]	)
          self.assertListEqual(
              processed_pad.attention_mask[:, :max_length].sum(-1	).tolist(),      [max_length for x in speech_inputs]	)








import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
    import torch


if TYPE_CHECKING:
    from transformers.pipelines.conversational import Conversation


__UpperCAmelCase	       =	logging.get_logger(__name__)
__UpperCAmelCase	       =	{'''vocab_file''': '''spiece.model'''}

__UpperCAmelCase	       =	{
    '''vocab_file''': {
        '''AI-Sweden/gpt-sw3-126m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-350m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-1.6b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-6.7b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-20b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model''',
    }
}

__UpperCAmelCase	       =	{
    '''AI-Sweden/gpt-sw3-126m''': 2_048,
    '''AI-Sweden/gpt-sw3-350m''': 2_048,
    '''AI-Sweden/gpt-sw3-1.6b''': 2_048,
    '''AI-Sweden/gpt-sw3-6.7b''': 2_048,
    '''AI-Sweden/gpt-sw3-20b''': 2_048,
}



class     lowerCAmelCase_		(  a__							):
       UpperCAmelCase__    : Union[str, Any]       					=  VOCAB_FILES_NAMES
       UpperCAmelCase__    : Any       					=  PRETRAINED_VOCAB_FILES_MAP
       UpperCAmelCase__    : int       					=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       UpperCAmelCase__    : Optional[int]       					=  ["input_ids", "attention_mask"]
       def __init__(      self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_=False,      SCREAMING_SNAKE_CASE_=False,      SCREAMING_SNAKE_CASE_=False,      SCREAMING_SNAKE_CASE_=None,      SCREAMING_SNAKE_CASE_=None,      SCREAMING_SNAKE_CASE_=None,      SCREAMING_SNAKE_CASE_=None,      SCREAMING_SNAKE_CASE_ = None,      **SCREAMING_SNAKE_CASE_,      )					->		None:
          UpperCamelCase   :  Optional[Any]						     =     {} if sp_model_kwargs is None else sp_model_kwargs

          UpperCamelCase   :  Dict						     =     kwargs.get('name_or_path'	)
          if name_or_path is None:
             logger.warning(
                 'name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'
                 ' you are testing the model, this can safely be ignored'	)
             UpperCamelCase   :  Tuple						     =     'None'

          # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
          UpperCamelCase   :  str						     =     '<|endoftext|>' if eos_token is None else eos_token
          UpperCamelCase   :  Tuple						     =     '<unk>' if unk_token is None else unk_token
          if "gpt-sw3-7b" in name_or_path:
             UpperCamelCase   :  str						     =     unk_token if pad_token is None else pad_token
             UpperCamelCase   :  List[str]						     =     eos_token if bos_token is None else bos_token
          else:
             UpperCamelCase   :  List[Any]						     =     '<pad>' if pad_token is None else pad_token
             UpperCamelCase   :  Dict						     =     '<s>' if bos_token is None else bos_token

          super().__init__(
              do_lower_case=SCREAMING_SNAKE_CASE_,      remove_space=SCREAMING_SNAKE_CASE_,      keep_accents=SCREAMING_SNAKE_CASE_,      bos_token=SCREAMING_SNAKE_CASE_,      eos_token=SCREAMING_SNAKE_CASE_,      unk_token=SCREAMING_SNAKE_CASE_,      pad_token=SCREAMING_SNAKE_CASE_,      sp_model_kwargs=self.sp_model_kwargs,      **SCREAMING_SNAKE_CASE_,      )

          UpperCamelCase   :  List[str]						     =     do_lower_case
          UpperCamelCase   :  List[str]						     =     remove_space
          UpperCamelCase   :  List[Any]						     =     keep_accents
          UpperCamelCase   :  List[str]						     =     vocab_file

          UpperCamelCase   :  Union[str, Any]						     =     spm.SentencePieceProcessor(**self.sp_model_kwargs	)
          self.sp_model.Load(SCREAMING_SNAKE_CASE_	)

          # Used for whitespace normalization in input texts
          # fmt : off
          UpperCamelCase   :  Dict						     =     {' ', ' ', ' ', ' ', ' ', '　', ' ', ' ', ' ', ' ', '￼', ''}
          # fmt : on

          # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
          UpperCamelCase   :  List[Any]						     =     re.compile(
              F"""[{"".join(map(SCREAMING_SNAKE_CASE_,      list(range(0,      9	)	) + list(range(11,      32	)	) + list(range(127,      160	)	) + [160, 173, 8203]	)	)}]"""	)
       def __getstate__(      self	)					->		Tuple:
          UpperCamelCase   :  List[Any]						     =     self.__dict__.copy()
          UpperCamelCase   :  Optional[int]						     =     None
          return state
       def __setstate__(      self,      SCREAMING_SNAKE_CASE_	)					->		Any:
          UpperCamelCase   :  Any						     =     d

          # for backward compatibility
          if not hasattr(self,      'sp_model_kwargs'	):
             UpperCamelCase   :  Optional[int]						     =     {}

          UpperCamelCase   :  int						     =     spm.SentencePieceProcessor(**self.sp_model_kwargs	)
          self.sp_model.Load(self.vocab_file	)
       @property
       # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
       def  snake_case_	(      self	)					->		int:
          return len(self.sp_model	)
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_	)					->		str:
          UpperCamelCase   :  Dict						     =     self.non_printing_characters_re.sub('',      SCREAMING_SNAKE_CASE_	)

          # Normalize whitespaces
          UpperCamelCase   :  Any						     =     ''.join([char if char not in self.whitespaces else ' ' for char in text]	)

          # NFC Unicode normalization
          UpperCamelCase   :  Dict						     =     unicodedata.normalize('NFC',      SCREAMING_SNAKE_CASE_	)
          return text
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_,      **SCREAMING_SNAKE_CASE_	)					->		List[str]:
          UpperCamelCase   :  Any						     =     self.preprocess_text(SCREAMING_SNAKE_CASE_	)
          return self.sp_model.encode(SCREAMING_SNAKE_CASE_,      out_type=SCREAMING_SNAKE_CASE_	)
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_	)					->		int:
          return self.sp_model.PieceToId(SCREAMING_SNAKE_CASE_	)
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_	)					->		str:
          return self.sp_model.IdToPiece(SCREAMING_SNAKE_CASE_	)
       @staticmethod
       def  snake_case_	(      SCREAMING_SNAKE_CASE_	)					->		str:
          return out_string
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_	)					->		str:
          UpperCamelCase   :  Optional[Any]						     =     []
          UpperCamelCase   :  List[Any]						     =     ''
          UpperCamelCase   :  str						     =     False
          for token in tokens:
             # make sure that special tokens are not decoded using sentencepiece model
             if token in self.all_special_tokens:
                # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
                if not prev_is_special:
                   out_string += " "

                out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE_	) + token
                UpperCamelCase   :  Dict						     =     True
                UpperCamelCase   :  Optional[Any]						     =     []
             else:
                current_sub_tokens.append(SCREAMING_SNAKE_CASE_	)
                UpperCamelCase   :  str						     =     False
          out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE_	)

          return out_string
       def  snake_case_	(      self	)					->		Dict[str, int]:
          UpperCamelCase   :  Tuple						     =     {self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_	): i for i in range(self.vocab_size	)}
          vocab.update(self.added_tokens_encoder	)
          return vocab
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_ = None	)					->		Tuple[str]:
          if not os.path.isdir(SCREAMING_SNAKE_CASE_	):
             logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""	)
             return
          UpperCamelCase   :  List[str]						     =     os.path.join(
              SCREAMING_SNAKE_CASE_,      (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']	)

          if os.path.abspath(self.vocab_file	) != os.path.abspath(SCREAMING_SNAKE_CASE_	) and os.path.isfile(self.vocab_file	):
             copyfile(self.vocab_file,      SCREAMING_SNAKE_CASE_	)
          elif not os.path.isfile(self.vocab_file	):
             with open(SCREAMING_SNAKE_CASE_,      'wb'	) as fi:
                UpperCamelCase   :  Any						     =     self.sp_model.serialized_model_proto()
                fi.write(SCREAMING_SNAKE_CASE_	)

          return (out_vocab_file,)
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_ = False	)					->		Union[List[int], List[List[int]], "torch.Tensor"]:

          if isinstance(SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_	):
             UpperCamelCase   :  List[str]						     =     self.preprocess_text(SCREAMING_SNAKE_CASE_	)
             UpperCamelCase   :  Optional[Any]						     =     self.sp_model.encode(SCREAMING_SNAKE_CASE_	)
          else:
             UpperCamelCase   :  Union[str, Any]						     =     [self.preprocess_text(SCREAMING_SNAKE_CASE_	) for t in text]
             UpperCamelCase   :  Any						     =     self.sp_model.encode(SCREAMING_SNAKE_CASE_	)

          if return_tensors is True or return_tensors == "pt":
             UpperCamelCase   :  List[Any]						     =     torch.tensor(SCREAMING_SNAKE_CASE_	)

          return token_ids
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_	)					->		str:
          return self.sp_model.decode(SCREAMING_SNAKE_CASE_	)






       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_	)					->		List[int]:
          UpperCamelCase   :  List[Any]						     =     [F"""User: {text}""" if is_user else F"""Bot: {text}""" for is_user, text in conversation.iter_texts()]
          UpperCamelCase   :  Optional[Any]						     =     (
              F"""{self.eos_token}{self.bos_token}""" + F"""{self.bos_token}""".join(SCREAMING_SNAKE_CASE_	) + F"""{self.bos_token}Bot:"""
          )
          return self.encode(text=SCREAMING_SNAKE_CASE_	)



