




import unittest

from transformers import BigBirdTokenizer, BigBirdTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase_						:      Union[str, Any]       =						"▁"

UpperCamelCase_						:      Tuple       =						get_tests_dir('''fixtures/test_sentencepiece.model''')

@require_sentencepiece
@require_tokenizers
class        _a    (						UpperCAmelCase__ , unittest.TestCase							):
     SCREAMING_SNAKE_CASE_							:   Optional[int]       =						BigBirdTokenizer
     SCREAMING_SNAKE_CASE_							:   Optional[int]       =						BigBirdTokenizerFast
     SCREAMING_SNAKE_CASE_							:   int       =						True
     SCREAMING_SNAKE_CASE_							:   Dict       =						True






     def     _lowercase     (			self   )							->      Dict:
          super().setUp()

          _snake_case							=				self.tokenizer_class(_SCREAMING_SNAKE_CASE ,keep_accents=_SCREAMING_SNAKE_CASE   )
          tokenizer.save_pretrained(self.tmpdirname   )






     def     _lowercase     (			self   )							->      Dict:
          _snake_case							=				"""<s>"""
          _snake_case							=				1

          self.assertEqual(self.get_tokenizer()._convert_token_to_id(_SCREAMING_SNAKE_CASE   ) ,_SCREAMING_SNAKE_CASE   )
          self.assertEqual(self.get_tokenizer()._convert_id_to_token(_SCREAMING_SNAKE_CASE   ) ,_SCREAMING_SNAKE_CASE   )






     def     _lowercase     (			self   )							->      Union[str, Any]:
          _snake_case							=				list(self.get_tokenizer().get_vocab().keys()   )

          self.assertEqual(vocab_keys[0] ,"<unk>"   )
          self.assertEqual(vocab_keys[1] ,"<s>"   )
          self.assertEqual(vocab_keys[-1] ,"[MASK]"   )
          self.assertEqual(len(_SCREAMING_SNAKE_CASE   ) ,1_004   )






     def     _lowercase     (			self   )							->      Union[str, Any]:
          self.assertEqual(self.get_tokenizer().vocab_size ,1_000   )






     def     _lowercase     (			self   )							->      Optional[Any]:




          if not self.test_rust_tokenizer:
               return

          _snake_case							=				self.get_tokenizer()
          _snake_case							=				self.get_rust_tokenizer()

          _snake_case							=				"""I was born in 92000, and this is falsé."""

          _snake_case							=				tokenizer.tokenize(_SCREAMING_SNAKE_CASE   )
          _snake_case							=				rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE   )
          self.assertListEqual(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE   )

          _snake_case							=				tokenizer.encode(_SCREAMING_SNAKE_CASE ,add_special_tokens=_SCREAMING_SNAKE_CASE   )
          _snake_case							=				rust_tokenizer.encode(_SCREAMING_SNAKE_CASE ,add_special_tokens=_SCREAMING_SNAKE_CASE   )
          self.assertListEqual(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE   )

          _snake_case							=				self.get_rust_tokenizer()
          _snake_case							=				tokenizer.encode(_SCREAMING_SNAKE_CASE   )
          _snake_case							=				rust_tokenizer.encode(_SCREAMING_SNAKE_CASE   )
          self.assertListEqual(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE   )






     def     _lowercase     (			self   )							->      Optional[Any]:
          _snake_case							=				BigBirdTokenizer(_SCREAMING_SNAKE_CASE ,keep_accents=_SCREAMING_SNAKE_CASE   )

          _snake_case							=				tokenizer.tokenize("This is a test"   )
          self.assertListEqual(_SCREAMING_SNAKE_CASE ,["▁This", "▁is", "▁a", "▁t", "est"]   )

          self.assertListEqual(
              tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE   ) ,[285, 46, 10, 170, 382] ,)

          _snake_case							=				tokenizer.tokenize("I was born in 92000, and this is falsé."   )
          self.assertListEqual(
              _SCREAMING_SNAKE_CASE ,[
                  SPIECE_UNDERLINE + "I",
                  SPIECE_UNDERLINE + "was",
                  SPIECE_UNDERLINE + "b",
                  "or",
                  "n",
                  SPIECE_UNDERLINE + "in",
                  SPIECE_UNDERLINE + "",
                  "9",
                  "2",
                  "0",
                  "0",
                  "0",
                  ",",
                  SPIECE_UNDERLINE + "and",
                  SPIECE_UNDERLINE + "this",
                  SPIECE_UNDERLINE + "is",
                  SPIECE_UNDERLINE + "f",
                  "al",
                  "s",
                  "é",
                  ".",
              ] ,)
          _snake_case							=				tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE   )
          self.assertListEqual(
              _SCREAMING_SNAKE_CASE ,[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4] ,)

          _snake_case							=				tokenizer.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE   )
          self.assertListEqual(
              _SCREAMING_SNAKE_CASE ,[
                  SPIECE_UNDERLINE + "I",
                  SPIECE_UNDERLINE + "was",
                  SPIECE_UNDERLINE + "b",
                  "or",
                  "n",
                  SPIECE_UNDERLINE + "in",
                  SPIECE_UNDERLINE + "",
                  "<unk>",
                  "2",
                  "0",
                  "0",
                  "0",
                  ",",
                  SPIECE_UNDERLINE + "and",
                  SPIECE_UNDERLINE + "this",
                  SPIECE_UNDERLINE + "is",
                  SPIECE_UNDERLINE + "f",
                  "al",
                  "s",
                  "<unk>",
                  ".",
              ] ,)






     @cached_property
     def     _lowercase     (			self   )							->      str:
          return BigBirdTokenizer.from_pretrained("google/bigbird-roberta-base"   )






     @slow
     def     _lowercase     (			self   )							->      Any:
          _snake_case							=				"""Hello World!"""
          _snake_case							=				[65, 18_536, 2_260, 101, 66]

          self.assertListEqual(_SCREAMING_SNAKE_CASE ,self.big_tokenizer.encode(_SCREAMING_SNAKE_CASE   )   )






     @slow
     def     _lowercase     (			self   )							->      List[str]:
          _snake_case							=				(
              """This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"""
              """ add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"""
          )
          # fmt: off
          _snake_case							=				[65, 871, 419, 358, 946, 991, 2_521, 452, 358, 1_357, 387, 7_751, 3_536, 112, 985, 456, 126, 865, 938, 5_400, 5_734, 458, 1_368, 467, 786, 2_462, 5_246, 1_159, 633, 865, 4_519, 457, 582, 852, 2_557, 427, 916, 508, 405, 34_324, 497, 391, 408, 11_342, 1_244, 385, 100, 938, 985, 456, 574, 362, 12_597, 3_200, 3_129, 1_172, 66]  # noqa: E231
          # fmt: on
          self.assertListEqual(_SCREAMING_SNAKE_CASE ,self.big_tokenizer.encode(_SCREAMING_SNAKE_CASE   )   )






     @require_torch
     @slow
     def     _lowercase     (			self   )							->      Optional[int]:
          import torch

          from transformers import BigBirdConfig, BigBirdModel

          # Build sequence
          _snake_case							=				list(self.big_tokenizer.get_vocab().keys()   )[:10]
          _snake_case							=				""" """.join(_SCREAMING_SNAKE_CASE   )
          _snake_case							=				self.big_tokenizer.encode_plus(_SCREAMING_SNAKE_CASE ,return_tensors="pt" ,return_token_type_ids=_SCREAMING_SNAKE_CASE   )
          _snake_case							=				self.big_tokenizer.batch_encode_plus(
              [sequence + " " + sequence] ,return_tensors="pt" ,return_token_type_ids=_SCREAMING_SNAKE_CASE   )

          _snake_case							=				BigBirdConfig(attention_type="original_full"   )
          _snake_case							=				BigBirdModel(_SCREAMING_SNAKE_CASE   )

          assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

          with torch.no_grad():
               model(**_SCREAMING_SNAKE_CASE   )
               model(**_SCREAMING_SNAKE_CASE   )






     @slow
     def     _lowercase     (			self   )							->      int:
          _snake_case							=				BigBirdTokenizer.from_pretrained("google/bigbird-roberta-base"   )
          _snake_case							=				tokenizer.decode(tokenizer("Paris is the [MASK]."   ).input_ids   )

          self.assertTrue(decoded_text == "[CLS] Paris is the[MASK].[SEP]"   )






     @slow
     def     _lowercase     (			self   )							->      Union[str, Any]:
          _snake_case							=				{"""input_ids""": [[65, 39_286, 458, 36_335, 2_001, 456, 13_073, 13_266, 455, 113, 7_746, 1_741, 11_157, 391, 13_073, 13_266, 455, 113, 3_967, 35_412, 113, 4_936, 109, 3_870, 2_377, 113, 30_084, 45_720, 458, 134, 17_496, 112, 503, 11_672, 113, 118, 112, 5_665, 13_347, 38_687, 112, 1_496, 31_389, 112, 3_268, 47_264, 134, 962, 112, 16_377, 8_035, 23_130, 430, 12_169, 15_518, 28_592, 458, 146, 41_697, 109, 391, 12_169, 15_518, 16_689, 458, 146, 41_358, 109, 452, 726, 4_034, 111, 763, 35_412, 5_082, 388, 1_903, 111, 9_051, 391, 2_870, 48_918, 1_900, 1_123, 550, 998, 112, 9_586, 15_985, 455, 391, 410, 22_955, 37_636, 114, 66], [65, 448, 17_496, 419, 3_663, 385, 763, 113, 27_533, 2_870, 3_283, 13_043, 1_639, 24_713, 523, 656, 24_013, 18_550, 2_521, 517, 27_014, 21_244, 420, 1_212, 1_465, 391, 927, 4_833, 388, 578, 11_786, 114, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [65, 484, 2_169, 7_687, 21_932, 18_146, 726, 363, 17_032, 3_391, 114, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
          # fmt: on

          self.tokenizer_integration_test_util(
              expected_encoding=_SCREAMING_SNAKE_CASE ,model_name="google/bigbird-roberta-base" ,revision="215c99f1600e06f83acce68422f2035b2b5c3510" ,)








'''simple docstring'''





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCamelCase_						:      List[Any]       =						{
    '''configuration_convbert''': ['''CONVBERT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''ConvBertConfig''', '''ConvBertOnnxConfig'''],
    '''tokenization_convbert''': ['''ConvBertTokenizer'''],
}

try:
   if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCamelCase_						:      Optional[Any]       =						['''ConvBertTokenizerFast''']

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCamelCase_						:      Optional[int]       =						[
       '''CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
       '''ConvBertForMaskedLM''',
       '''ConvBertForMultipleChoice''',
       '''ConvBertForQuestionAnswering''',
       '''ConvBertForSequenceClassification''',
       '''ConvBertForTokenClassification''',
       '''ConvBertLayer''',
       '''ConvBertModel''',
       '''ConvBertPreTrainedModel''',
       '''load_tf_weights_in_convbert''',
   ]


try:
   if not is_tf_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCamelCase_						:      Tuple       =						[
       '''TF_CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
       '''TFConvBertForMaskedLM''',
       '''TFConvBertForMultipleChoice''',
       '''TFConvBertForQuestionAnswering''',
       '''TFConvBertForSequenceClassification''',
       '''TFConvBertForTokenClassification''',
       '''TFConvBertLayer''',
       '''TFConvBertModel''',
       '''TFConvBertPreTrainedModel''',
   ]


if TYPE_CHECKING:
   from .configuration_convbert import CONVBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, ConvBertConfig, ConvBertOnnxConfig
   from .tokenization_convbert import ConvBertTokenizer

   try:
      if not is_tokenizers_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_convbert_fast import ConvBertTokenizerFast

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_convbert import (
          CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
          ConvBertForMaskedLM,
          ConvBertForMultipleChoice,
          ConvBertForQuestionAnswering,
          ConvBertForSequenceClassification,
          ConvBertForTokenClassification,
          ConvBertLayer,
          ConvBertModel,
          ConvBertPreTrainedModel,
          load_tf_weights_in_convbert,
      )

   try:
      if not is_tf_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_tf_convbert import (
          TF_CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
          TFConvBertForMaskedLM,
          TFConvBertForMultipleChoice,
          TFConvBertForQuestionAnswering,
          TFConvBertForSequenceClassification,
          TFConvBertForTokenClassification,
          TFConvBertLayer,
          TFConvBertModel,
          TFConvBertPreTrainedModel,
      )


else:
   import sys

   UpperCamelCase_						:      Tuple       =						_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)


