



import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


__UpperCAmelCase	       =	logging.get_logger(__name__)

__UpperCAmelCase	       =	{'''vocab_file''': '''spiece.model'''}

__UpperCAmelCase	       =	{
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    }
}

__UpperCAmelCase	       =	{
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

# Segments (not really needed)
__UpperCAmelCase	       =	0
__UpperCAmelCase	       =	1
__UpperCAmelCase	       =	2
__UpperCAmelCase	       =	3
__UpperCAmelCase	       =	4



class     lowerCAmelCase_		(  __lowercase							):
       UpperCAmelCase__    : List[str]       					=  VOCAB_FILES_NAMES
       UpperCAmelCase__    : int       					=  PRETRAINED_VOCAB_FILES_MAP
       UpperCAmelCase__    : Optional[int]       					=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       UpperCAmelCase__    : int       					=  '''left'''
       def __init__(      self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_=False,      SCREAMING_SNAKE_CASE_=True,      SCREAMING_SNAKE_CASE_=False,      SCREAMING_SNAKE_CASE_="<s>",      SCREAMING_SNAKE_CASE_="</s>",      SCREAMING_SNAKE_CASE_="<unk>",      SCREAMING_SNAKE_CASE_="<sep>",      SCREAMING_SNAKE_CASE_="<pad>",      SCREAMING_SNAKE_CASE_="<cls>",      SCREAMING_SNAKE_CASE_="<mask>",      SCREAMING_SNAKE_CASE_=["<eop>", "<eod>"],      SCREAMING_SNAKE_CASE_ = None,      **SCREAMING_SNAKE_CASE_,      )					->		None:
          UpperCamelCase   :  Tuple						     =     AddedToken(SCREAMING_SNAKE_CASE_,      lstrip=SCREAMING_SNAKE_CASE_,      rstrip=SCREAMING_SNAKE_CASE_	) if isinstance(SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_	) else mask_token

          UpperCamelCase   :  Union[str, Any]						     =     {} if sp_model_kwargs is None else sp_model_kwargs

          super().__init__(
              do_lower_case=SCREAMING_SNAKE_CASE_,      remove_space=SCREAMING_SNAKE_CASE_,      keep_accents=SCREAMING_SNAKE_CASE_,      bos_token=SCREAMING_SNAKE_CASE_,      eos_token=SCREAMING_SNAKE_CASE_,      unk_token=SCREAMING_SNAKE_CASE_,      sep_token=SCREAMING_SNAKE_CASE_,      pad_token=SCREAMING_SNAKE_CASE_,      cls_token=SCREAMING_SNAKE_CASE_,      mask_token=SCREAMING_SNAKE_CASE_,      additional_special_tokens=SCREAMING_SNAKE_CASE_,      sp_model_kwargs=self.sp_model_kwargs,      **SCREAMING_SNAKE_CASE_,      )

          UpperCamelCase   :  Tuple						     =     3

          UpperCamelCase   :  int						     =     do_lower_case
          UpperCamelCase   :  Optional[int]						     =     remove_space
          UpperCamelCase   :  Optional[int]						     =     keep_accents
          UpperCamelCase   :  List[Any]						     =     vocab_file

          UpperCamelCase   :  Optional[Any]						     =     spm.SentencePieceProcessor(**self.sp_model_kwargs	)
          self.sp_model.Load(SCREAMING_SNAKE_CASE_	)
       @property
       def  snake_case_	(      self	)					->		str:
          return len(self.sp_model	)
       def  snake_case_	(      self	)					->		Any:
          UpperCamelCase   :  Any						     =     {self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_	): i for i in range(self.vocab_size	)}
          vocab.update(self.added_tokens_encoder	)
          return vocab
       def __getstate__(      self	)					->		List[Any]:
          UpperCamelCase   :  Optional[Any]						     =     self.__dict__.copy()
          UpperCamelCase   :  List[Any]						     =     None
          return state
       def __setstate__(      self,      SCREAMING_SNAKE_CASE_	)					->		Union[str, Any]:
          UpperCamelCase   :  Union[str, Any]						     =     d

          # for backward compatibility
          if not hasattr(self,      'sp_model_kwargs'	):
             UpperCamelCase   :  int						     =     {}

          UpperCamelCase   :  List[str]						     =     spm.SentencePieceProcessor(**self.sp_model_kwargs	)
          self.sp_model.Load(self.vocab_file	)
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_	)					->		List[str]:

          if self.remove_space:
             UpperCamelCase   :  Optional[int]						     =     """ """.join(inputs.strip().split()	)
          else:
             UpperCamelCase   :  List[str]						     =     inputs
          UpperCamelCase   :  Optional[Any]						     =     outputs.replace('``',      '\"'	).replace('\'\'',      '\"'	)

          if not self.keep_accents:
             UpperCamelCase   :  int						     =     unicodedata.normalize('NFKD',      SCREAMING_SNAKE_CASE_	)
             UpperCamelCase   :  int						     =     """""".join([c for c in outputs if not unicodedata.combining(SCREAMING_SNAKE_CASE_	)]	)
          if self.do_lower_case:
             UpperCamelCase   :  Optional[Any]						     =     outputs.lower()

          return outputs
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_	)					->		List[str]:
          UpperCamelCase   :  List[Any]						     =     self.preprocess_text(SCREAMING_SNAKE_CASE_	)
          UpperCamelCase   :  Dict						     =     self.sp_model.encode(SCREAMING_SNAKE_CASE_,      out_type=SCREAMING_SNAKE_CASE_	)
          UpperCamelCase   :  Optional[Any]						     =     []
          for piece in pieces:
             if len(SCREAMING_SNAKE_CASE_	) > 1 and piece[-1] == str(','	) and piece[-2].isdigit():
                UpperCamelCase   :  Any						     =     self.sp_model.EncodeAsPieces(piece[:-1].replace(SCREAMING_SNAKE_CASE_,      ''	)	)
                if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                   if len(cur_pieces[0]	) == 1:
                      UpperCamelCase   :  List[str]						     =     cur_pieces[1:]
                   else:
                      UpperCamelCase   :  Any						     =     cur_pieces[0][1:]
                cur_pieces.append(piece[-1]	)
                new_pieces.extend(SCREAMING_SNAKE_CASE_	)
             else:
                new_pieces.append(SCREAMING_SNAKE_CASE_	)

          return new_pieces
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_	)					->		Any:
          return self.sp_model.PieceToId(SCREAMING_SNAKE_CASE_	)
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_	)					->		int:
          return self.sp_model.IdToPiece(SCREAMING_SNAKE_CASE_	)
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_	)					->		Dict:
          UpperCamelCase   :  Optional[int]						     =     """""".join(SCREAMING_SNAKE_CASE_	).replace(SCREAMING_SNAKE_CASE_,      ' '	).strip()
          return out_string
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_ = False,      SCREAMING_SNAKE_CASE_ = None,      SCREAMING_SNAKE_CASE_ = True,      **SCREAMING_SNAKE_CASE_,      )					->		str:
          UpperCamelCase   :  Union[str, Any]						     =     kwargs.pop('use_source_tokenizer',      SCREAMING_SNAKE_CASE_	)

          UpperCamelCase   :  Any						     =     self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_,      skip_special_tokens=SCREAMING_SNAKE_CASE_	)

          # To avoid mixing byte-level and unicode for byte-level BPT
          # we need to build string separately for added tokens and byte-level tokens
          # cf. https://github.com/huggingface/transformers/issues/1133
          UpperCamelCase   :  Dict						     =     []
          UpperCamelCase   :  Any						     =     []
          for token in filtered_tokens:
             if skip_special_tokens and token in self.all_special_ids:
                continue
             if token in self.added_tokens_encoder:
                if current_sub_text:
                   sub_texts.append(self.convert_tokens_to_string(SCREAMING_SNAKE_CASE_	)	)
                   UpperCamelCase   :  Optional[int]						     =     []
                sub_texts.append(SCREAMING_SNAKE_CASE_	)
             else:
                current_sub_text.append(SCREAMING_SNAKE_CASE_	)
          if current_sub_text:
             sub_texts.append(self.convert_tokens_to_string(SCREAMING_SNAKE_CASE_	)	)

          # Mimic the behavior of the Rust tokenizer:
          # By default, there are no spaces between special tokens
          UpperCamelCase   :  Tuple						     =     """""".join(SCREAMING_SNAKE_CASE_	)

          UpperCamelCase   :  Optional[Any]						     =     (
              clean_up_tokenization_spaces
              if clean_up_tokenization_spaces is not None
              else self.clean_up_tokenization_spaces
          )
          if clean_up_tokenization_spaces:
             UpperCamelCase   :  Dict						     =     self.clean_up_tokenization(SCREAMING_SNAKE_CASE_	)
             return clean_text
          else:
             return text
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_ = None	)					->		List[int]:
          UpperCamelCase   :  Union[str, Any]						     =     [self.sep_token_id]
          UpperCamelCase   :  Union[str, Any]						     =     [self.cls_token_id]
          if token_ids_a is None:
             return token_ids_a + sep + cls
          return token_ids_a + sep + token_ids_a + sep + cls
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_ = None,      SCREAMING_SNAKE_CASE_ = False	)					->		List[int]:

          if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=SCREAMING_SNAKE_CASE_,      token_ids_a=SCREAMING_SNAKE_CASE_,      already_has_special_tokens=SCREAMING_SNAKE_CASE_	)

          if token_ids_a is not None:
             return ([0] * len(SCREAMING_SNAKE_CASE_	)) + [1] + ([0] * len(SCREAMING_SNAKE_CASE_	)) + [1, 1]
          return ([0] * len(SCREAMING_SNAKE_CASE_	)) + [1, 1]
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_ = None	)					->		List[int]:
          UpperCamelCase   :  Any						     =     [self.sep_token_id]
          UpperCamelCase   :  Optional[int]						     =     [2]

          if token_ids_a is None:
             return len(token_ids_a + sep	) * [0] + cls_segment_id
          return len(token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1] + cls_segment_id






       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_ = None	)					->		Tuple[str]:

          if not os.path.isdir(SCREAMING_SNAKE_CASE_	):
             logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""	)
             return
          UpperCamelCase   :  int						     =     os.path.join(
              SCREAMING_SNAKE_CASE_,      (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']	)

          if os.path.abspath(self.vocab_file	) != os.path.abspath(SCREAMING_SNAKE_CASE_	) and os.path.isfile(self.vocab_file	):
             copyfile(self.vocab_file,      SCREAMING_SNAKE_CASE_	)
          elif not os.path.isfile(self.vocab_file	):
             with open(SCREAMING_SNAKE_CASE_,      'wb'	) as fi:
                UpperCamelCase   :  Dict						     =     self.sp_model.serialized_model_proto()
                fi.write(SCREAMING_SNAKE_CASE_	)

          return (out_vocab_file,)








import unittest

from transformers import (
    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TextClassificationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow

from .test_pipelines_common import ANY


# These 2 model types require different inputs than those of the usual text models.
__UpperCAmelCase	       =	{'''LayoutLMv2Config''', '''LayoutLMv3Config'''}



@is_pipeline_test
class     lowerCAmelCase_		(  unittest.TestCase							):
       UpperCAmelCase__    : Dict       					=  MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
       UpperCAmelCase__    : List[str]       					=  TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING

       if model_mapping is not None:
              UpperCAmelCase__    : Optional[Any]       					=  {config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
       if tf_model_mapping is not None:
              UpperCAmelCase__    : Dict       					=  {
                  config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
              }
       @require_torch
       def  snake_case_	(      self	)					->		int:
          UpperCamelCase   :  List[str]						     =     pipeline(
              task='text-classification',      model='hf-internal-testing/tiny-random-distilbert',      framework='pt'	)

          UpperCamelCase   :  str						     =     text_classifier('This is great !'	)
          self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE_	),      [{'label': 'LABEL_0', 'score': 0.5_04}]	)

          UpperCamelCase   :  Dict						     =     text_classifier('This is great !',      top_k=2	)
          self.assertEqual(
              nested_simplify(SCREAMING_SNAKE_CASE_	),      [{'label': 'LABEL_0', 'score': 0.5_04}, {'label': 'LABEL_1', 'score': 0.4_96}]	)

          UpperCamelCase   :  List[Any]						     =     text_classifier(['This is great !', 'This is bad'],      top_k=2	)
          self.assertEqual(
              nested_simplify(SCREAMING_SNAKE_CASE_	),      [
                  [{'label': 'LABEL_0', 'score': 0.5_04}, {'label': 'LABEL_1', 'score': 0.4_96}],
                  [{'label': 'LABEL_0', 'score': 0.5_04}, {'label': 'LABEL_1', 'score': 0.4_96}],
              ],      )

          UpperCamelCase   :  List[Any]						     =     text_classifier('This is great !',      top_k=1	)
          self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE_	),      [{'label': 'LABEL_0', 'score': 0.5_04}]	)

          # Legacy behavior
          UpperCamelCase   :  str						     =     text_classifier('This is great !',      return_all_scores=SCREAMING_SNAKE_CASE_	)
          self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE_	),      [{'label': 'LABEL_0', 'score': 0.5_04}]	)

          UpperCamelCase   :  Tuple						     =     text_classifier('This is great !',      return_all_scores=SCREAMING_SNAKE_CASE_	)
          self.assertEqual(
              nested_simplify(SCREAMING_SNAKE_CASE_	),      [[{'label': 'LABEL_0', 'score': 0.5_04}, {'label': 'LABEL_1', 'score': 0.4_96}]]	)

          UpperCamelCase   :  List[Any]						     =     text_classifier(['This is great !', 'Something else'],      return_all_scores=SCREAMING_SNAKE_CASE_	)
          self.assertEqual(
              nested_simplify(SCREAMING_SNAKE_CASE_	),      [
                  [{'label': 'LABEL_0', 'score': 0.5_04}, {'label': 'LABEL_1', 'score': 0.4_96}],
                  [{'label': 'LABEL_0', 'score': 0.5_04}, {'label': 'LABEL_1', 'score': 0.4_96}],
              ],      )

          UpperCamelCase   :  Optional[int]						     =     text_classifier(['This is great !', 'Something else'],      return_all_scores=SCREAMING_SNAKE_CASE_	)
          self.assertEqual(
              nested_simplify(SCREAMING_SNAKE_CASE_	),      [
                  {'label': 'LABEL_0', 'score': 0.5_04},
                  {'label': 'LABEL_0', 'score': 0.5_04},
              ],      )
       @require_torch
       def  snake_case_	(      self	)					->		Optional[Any]:
          import torch

          UpperCamelCase   :  str						     =     pipeline(
              task='text-classification',      model='hf-internal-testing/tiny-random-distilbert',      framework='pt',      device=torch.device('cpu'	),      )

          UpperCamelCase   :  List[Any]						     =     text_classifier('This is great !'	)
          self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE_	),      [{'label': 'LABEL_0', 'score': 0.5_04}]	)
       @require_tf
       def  snake_case_	(      self	)					->		Dict:
          UpperCamelCase   :  List[Any]						     =     pipeline(
              task='text-classification',      model='hf-internal-testing/tiny-random-distilbert',      framework='tf'	)

          UpperCamelCase   :  str						     =     text_classifier('This is great !'	)
          self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE_	),      [{'label': 'LABEL_0', 'score': 0.5_04}]	)
       @slow
       @require_torch
       def  snake_case_	(      self	)					->		int:
          UpperCamelCase   :  str						     =     pipeline('text-classification'	)

          UpperCamelCase   :  str						     =     text_classifier('This is great !'	)
          self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE_	),      [{'label': 'POSITIVE', 'score': 1.0}]	)
          UpperCamelCase   :  List[str]						     =     text_classifier('This is bad !'	)
          self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE_	),      [{'label': 'NEGATIVE', 'score': 1.0}]	)
          UpperCamelCase   :  int						     =     text_classifier('Birds are a type of animal'	)
          self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE_	),      [{'label': 'POSITIVE', 'score': 0.9_88}]	)
       @slow
       @require_tf
       def  snake_case_	(      self	)					->		Tuple:
          UpperCamelCase   :  int						     =     pipeline('text-classification',      framework='tf'	)

          UpperCamelCase   :  Tuple						     =     text_classifier('This is great !'	)
          self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE_	),      [{'label': 'POSITIVE', 'score': 1.0}]	)
          UpperCamelCase   :  Union[str, Any]						     =     text_classifier('This is bad !'	)
          self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE_	),      [{'label': 'NEGATIVE', 'score': 1.0}]	)
          UpperCamelCase   :  int						     =     text_classifier('Birds are a type of animal'	)
          self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE_	),      [{'label': 'POSITIVE', 'score': 0.9_88}]	)
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_	)					->		Union[str, Any]:
          UpperCamelCase   :  str						     =     TextClassificationPipeline(model=SCREAMING_SNAKE_CASE_,      tokenizer=SCREAMING_SNAKE_CASE_	)
          return text_classifier, ["HuggingFace is in", "This is another test"]






       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_	)					->		Any:
          UpperCamelCase   :  List[str]						     =     text_classifier.model
          # Small inputs because BartTokenizer tiny has maximum position embeddings = 22
          UpperCamelCase   :  List[str]						     =     'HuggingFace is in'
          UpperCamelCase   :  Tuple						     =     text_classifier(SCREAMING_SNAKE_CASE_	)

          self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE_	),      [{'label': ANY(SCREAMING_SNAKE_CASE_	), 'score': ANY(SCREAMING_SNAKE_CASE_	)}]	)
          self.assertTrue(outputs[0]['label'] in model.config.idalabel.values()	)

          UpperCamelCase   :  List[Any]						     =     ['HuggingFace is in ', 'Paris is in France']
          UpperCamelCase   :  List[str]						     =     text_classifier(SCREAMING_SNAKE_CASE_	)
          self.assertEqual(
              nested_simplify(SCREAMING_SNAKE_CASE_	),      [{'label': ANY(SCREAMING_SNAKE_CASE_	), 'score': ANY(SCREAMING_SNAKE_CASE_	)}, {'label': ANY(SCREAMING_SNAKE_CASE_	), 'score': ANY(SCREAMING_SNAKE_CASE_	)}],      )
          self.assertTrue(outputs[0]['label'] in model.config.idalabel.values()	)
          self.assertTrue(outputs[1]['label'] in model.config.idalabel.values()	)

          # Forcing to get all results with `top_k=None`
          # This is NOT the legacy format
          UpperCamelCase   :  Optional[int]						     =     text_classifier(SCREAMING_SNAKE_CASE_,      top_k=SCREAMING_SNAKE_CASE_	)
          UpperCamelCase   :  str						     =     len(model.config.idalabel.values()	)
          self.assertEqual(
              nested_simplify(SCREAMING_SNAKE_CASE_	),      [[{'label': ANY(SCREAMING_SNAKE_CASE_	), 'score': ANY(SCREAMING_SNAKE_CASE_	)}] * N, [{'label': ANY(SCREAMING_SNAKE_CASE_	), 'score': ANY(SCREAMING_SNAKE_CASE_	)}] * N],      )

          UpperCamelCase   :  str						     =     {'text': 'HuggingFace is in ', 'text_pair': 'Paris is in France'}
          UpperCamelCase   :  Union[str, Any]						     =     text_classifier(SCREAMING_SNAKE_CASE_	)
          self.assertEqual(
              nested_simplify(SCREAMING_SNAKE_CASE_	),      {'label': ANY(SCREAMING_SNAKE_CASE_	), 'score': ANY(SCREAMING_SNAKE_CASE_	)},      )
          self.assertTrue(outputs['label'] in model.config.idalabel.values()	)

          # This might be used a text pair, but tokenizer + pipe interaction
          # makes it hard to understand that it's not using the pair properly
          # https://github.com/huggingface/transformers/issues/17305
          # We disabled this usage instead as it was outputting wrong outputs.
          UpperCamelCase   :  List[str]						     =     [['HuggingFace is in ', 'Paris is in France']]
          with self.assertRaises(SCREAMING_SNAKE_CASE_	):
             text_classifier(SCREAMING_SNAKE_CASE_	)

          # This used to be valid for doing text pairs
          # We're keeping it working because of backward compatibility
          UpperCamelCase   :  List[str]						     =     text_classifier([[['HuggingFace is in ', 'Paris is in France']]]	)
          self.assertEqual(
              nested_simplify(SCREAMING_SNAKE_CASE_	),      [{'label': ANY(SCREAMING_SNAKE_CASE_	), 'score': ANY(SCREAMING_SNAKE_CASE_	)}],      )
          self.assertTrue(outputs[0]['label'] in model.config.idalabel.values()	)



