




'''simple docstring'''





import json
import os
import unittest

from transformers.models.xlm.tokenization_xlm import VOCAB_FILES_NAMES, XLMTokenizer
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin

class        _a    (						__lowerCAmelCase , unittest.TestCase							):
     SCREAMING_SNAKE_CASE_							:   Union[str, Any]       =						XLMTokenizer
     SCREAMING_SNAKE_CASE_							:   str       =						False






     def     _lowercase     (			self   )							->      Union[str, Any]:
          super().setUp()

          # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
          _snake_case							=				[
              "l",
              "o",
              "w",
              "e",
              "r",
              "s",
              "t",
              "i",
              "d",
              "n",
              "w</w>",
              "r</w>",
              "t</w>",
              "lo",
              "low",
              "er</w>",
              "low</w>",
              "lowest</w>",
              "newer</w>",
              "wider</w>",
              "<unk>",
          ]
          _snake_case							=				dict(zip(_SCREAMING_SNAKE_CASE ,range(len(_SCREAMING_SNAKE_CASE   )   )   )   )
          _snake_case							=				["l o 123", "lo w 1456", "e r</w> 1789", ""]

          _snake_case							=				os.path.join(self.tmpdirname ,VOCAB_FILES_NAMES["vocab_file"]   )
          _snake_case							=				os.path.join(self.tmpdirname ,VOCAB_FILES_NAMES["merges_file"]   )
          with open(self.vocab_file ,"w"   ) as fp:
               fp.write(json.dumps(_SCREAMING_SNAKE_CASE   )   )
          with open(self.merges_file ,"w"   ) as fp:
               fp.write("\n".join(_SCREAMING_SNAKE_CASE   )   )






     def     _lowercase     (			self ,_SCREAMING_SNAKE_CASE   )							->      Tuple:
          _snake_case							=				"lower newer"
          _snake_case							=				"lower newer"
          return input_text, output_text






     def     _lowercase     (			self   )							->      Any:
          _snake_case							=				XLMTokenizer(self.vocab_file ,self.merges_file   )

          _snake_case							=				"lower"
          _snake_case							=				["low", "er</w>"]
          _snake_case							=				tokenizer.tokenize(_SCREAMING_SNAKE_CASE   )
          self.assertListEqual(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE   )

          _snake_case							=				tokens + ["<unk>"]
          _snake_case							=				[14, 15, 20]
          self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE   ) ,_SCREAMING_SNAKE_CASE   )






     @slow
     def     _lowercase     (			self   )							->      List[Any]:
          _snake_case							=				XLMTokenizer.from_pretrained("xlm-mlm-en-2048"   )

          _snake_case							=				tokenizer.encode("sequence builders" ,add_special_tokens=_SCREAMING_SNAKE_CASE   )
          _snake_case							=				tokenizer.encode("multi-sequence build" ,add_special_tokens=_SCREAMING_SNAKE_CASE   )

          _snake_case							=				tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE   )
          _snake_case							=				tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE   )

          assert encoded_sentence == [0] + text + [1]
          assert encoded_pair == [0] + text + [1] + text_a + [1]








'''simple docstring'''





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCamelCase_						:      List[Any]       =						{
    '''configuration_convbert''': ['''CONVBERT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''ConvBertConfig''', '''ConvBertOnnxConfig'''],
    '''tokenization_convbert''': ['''ConvBertTokenizer'''],
}

try:
   if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCamelCase_						:      Optional[Any]       =						['''ConvBertTokenizerFast''']

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCamelCase_						:      Optional[int]       =						[
       '''CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
       '''ConvBertForMaskedLM''',
       '''ConvBertForMultipleChoice''',
       '''ConvBertForQuestionAnswering''',
       '''ConvBertForSequenceClassification''',
       '''ConvBertForTokenClassification''',
       '''ConvBertLayer''',
       '''ConvBertModel''',
       '''ConvBertPreTrainedModel''',
       '''load_tf_weights_in_convbert''',
   ]


try:
   if not is_tf_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCamelCase_						:      Tuple       =						[
       '''TF_CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
       '''TFConvBertForMaskedLM''',
       '''TFConvBertForMultipleChoice''',
       '''TFConvBertForQuestionAnswering''',
       '''TFConvBertForSequenceClassification''',
       '''TFConvBertForTokenClassification''',
       '''TFConvBertLayer''',
       '''TFConvBertModel''',
       '''TFConvBertPreTrainedModel''',
   ]


if TYPE_CHECKING:
   from .configuration_convbert import CONVBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, ConvBertConfig, ConvBertOnnxConfig
   from .tokenization_convbert import ConvBertTokenizer

   try:
      if not is_tokenizers_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_convbert_fast import ConvBertTokenizerFast

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_convbert import (
          CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
          ConvBertForMaskedLM,
          ConvBertForMultipleChoice,
          ConvBertForQuestionAnswering,
          ConvBertForSequenceClassification,
          ConvBertForTokenClassification,
          ConvBertLayer,
          ConvBertModel,
          ConvBertPreTrainedModel,
          load_tf_weights_in_convbert,
      )

   try:
      if not is_tf_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_tf_convbert import (
          TF_CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
          TFConvBertForMaskedLM,
          TFConvBertForMultipleChoice,
          TFConvBertForQuestionAnswering,
          TFConvBertForSequenceClassification,
          TFConvBertForTokenClassification,
          TFConvBertLayer,
          TFConvBertModel,
          TFConvBertPreTrainedModel,
      )


else:
   import sys

   UpperCamelCase_						:      Tuple       =						_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)


