'''simple docstring'''





from __future__ import annotations

import unittest

from transformers import BlenderbotSmallConfig, BlenderbotSmallTokenizer, is_tf_available
from transformers.testing_utils import require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
				import tensorflow as tf

				from transformers import TFAutoModelForSeqaSeqLM, TFBlenderbotSmallForConditionalGeneration, TFBlenderbotSmallModel




@require_tf
class 	__A				:
				a__	:						str	       =  BlenderbotSmallConfig
				a__	:						Optional[Any]	       =  {}
				a__	:						Optional[int]	       =  """gelu"""



				def __init__(self   :   List[str]       ,      __a   :   Optional[Any]       ,      __a   :   Union[str, Any]=13       ,      __a   :   Dict=7       ,      __a   :   Tuple=True       ,      __a   :   Union[str, Any]=False       ,      __a   :   Optional[Any]=99       ,      __a   :   List[str]=32       ,      __a   :   Tuple=2       ,      __a   :   Dict=4       ,      __a   :   Any=37       ,      __a   :   Any=0.1       ,      __a   :   Tuple=0.1       ,      __a   :   Any=20       ,      __a   :   Tuple=2       ,      __a   :   Any=1       ,      __a   :   Optional[Any]=0       ,      ):
									UpperCAmelCase_	   =       parent
									UpperCAmelCase_	   =       batch_size
									UpperCAmelCase_	   =       seq_length
									UpperCAmelCase_	   =       is_training
									UpperCAmelCase_	   =       use_labels
									UpperCAmelCase_	   =       vocab_size
									UpperCAmelCase_	   =       hidden_size
									UpperCAmelCase_	   =       num_hidden_layers
									UpperCAmelCase_	   =       num_attention_heads
									UpperCAmelCase_	   =       intermediate_size

									UpperCAmelCase_	   =       hidden_dropout_prob
									UpperCAmelCase_	   =       attention_probs_dropout_prob
									UpperCAmelCase_	   =       max_position_embeddings
									UpperCAmelCase_	   =       eos_token_id
									UpperCAmelCase_	   =       pad_token_id
									UpperCAmelCase_	   =       bos_token_id



				def  _lowercase						(self   :   Optional[Any]			):
									UpperCAmelCase_	   =       ids_tensor([self.batch_size, self.seq_length - 1]       ,      self.vocab_size			)
									UpperCAmelCase_	   =       tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size			)       ,      1			)
									UpperCAmelCase_	   =       tf.concat([input_ids, eos_tensor]       ,      axis=1			)

									UpperCAmelCase_	   =       ids_tensor([self.batch_size, self.seq_length]       ,      self.vocab_size			)

									UpperCAmelCase_	   =       self.config_cls(
									    vocab_size=self.vocab_size       ,      d_model=self.hidden_size       ,      encoder_layers=self.num_hidden_layers       ,      decoder_layers=self.num_hidden_layers       ,      encoder_attention_heads=self.num_attention_heads       ,      decoder_attention_heads=self.num_attention_heads       ,      encoder_ffn_dim=self.intermediate_size       ,      decoder_ffn_dim=self.intermediate_size       ,      dropout=self.hidden_dropout_prob       ,      attention_dropout=self.attention_probs_dropout_prob       ,      max_position_embeddings=self.max_position_embeddings       ,      eos_token_ids=[2]       ,      bos_token_id=self.bos_token_id       ,      pad_token_id=self.pad_token_id       ,      decoder_start_token_id=self.pad_token_id       ,      **self.config_updates       ,      )
									UpperCAmelCase_	   =       prepare_blenderbot_small_inputs_dict(__a       ,      __a       ,      __a			)
									return config, inputs_dict






				def  _lowercase						(self   :   Any       ,      __a   :   int       ,      __a   :   List[str]			):
									UpperCAmelCase_	   =       TFBlenderbotSmallModel(config=__a			).get_decoder()
									UpperCAmelCase_	   =       inputs_dict['input_ids']

									UpperCAmelCase_	   =       input_ids[:1, :]
									UpperCAmelCase_	   =       inputs_dict['attention_mask'][:1, :]
									UpperCAmelCase_	   =       inputs_dict['head_mask']
									UpperCAmelCase_	   =       1

									# first forward pass
									UpperCAmelCase_	   =       model(__a       ,      attention_mask=__a       ,      head_mask=__a       ,      use_cache=__a			)

									UpperCAmelCase_	   =       outputs.to_tuple()

									# create hypothetical next token and extent to next_input_ids
									UpperCAmelCase_	   =       ids_tensor((self.batch_size, 3)       ,      config.vocab_size			)
									UpperCAmelCase_	   =       tf.cast(ids_tensor((self.batch_size, 3)       ,      2			)       ,      tf.inta			)

									# append to next input_ids and
									UpperCAmelCase_	   =       tf.concat([input_ids, next_tokens]       ,      axis=-1			)
									UpperCAmelCase_	   =       tf.concat([attention_mask, next_attn_mask]       ,      axis=-1			)

									UpperCAmelCase_	   =       model(__a       ,      attention_mask=__a			)[0]
									UpperCAmelCase_	   =       model(__a       ,      attention_mask=__a       ,      past_key_values=__a			)[0]

									self.parent.assertEqual(next_tokens.shape[1]       ,      output_from_past.shape[1]			)

									# select random slice
									UpperCAmelCase_	   =       int(ids_tensor((1,)       ,      output_from_past.shape[-1]			)			)
									UpperCAmelCase_	   =       output_from_no_past[:, -3:, random_slice_idx]
									UpperCAmelCase_	   =       output_from_past[:, :, random_slice_idx]

									# test that outputs are equal for slice
									tf.debugging.assert_near(__a       ,      __a       ,      rtol=1E-3			)





def      lowerCAmelCase_    ( snake_case_	:			int ,   snake_case_	:			Union[str, Any] ,   snake_case_	:			Any ,   snake_case_	:			Union[str, Any]=None ,   snake_case_	:			Any=None ,   snake_case_	:			Tuple=None ,   snake_case_	:			Optional[int]=None ,   snake_case_	:			Tuple=None ,   )   ->				str:


					'''simple docstring'''




					if attention_mask is None:
										UpperCAmelCase_	   =       tf.cast(tf.math.not_equal(_UpperCAmelCase ,   config.pad_token_id      ) ,   tf.inta      )
					if decoder_attention_mask is None:
										UpperCAmelCase_	   =       tf.concat(
										    [
										        tf.ones(decoder_input_ids[:, :1].shape ,   dtype=tf.inta      ),
										        tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:] ,   config.pad_token_id      ) ,   tf.inta      ),
										    ] ,   axis=-1 ,   )
					if head_mask is None:
										UpperCAmelCase_	   =       tf.ones((config.encoder_layers, config.encoder_attention_heads)      )
					if decoder_head_mask is None:
										UpperCAmelCase_	   =       tf.ones((config.decoder_layers, config.decoder_attention_heads)      )
					if cross_attn_head_mask is None:
										UpperCAmelCase_	   =       tf.ones((config.decoder_layers, config.decoder_attention_heads)      )
					return {
					    "input_ids": input_ids,
					    "decoder_input_ids": decoder_input_ids,
					    "attention_mask": attention_mask,
					    "decoder_attention_mask": decoder_attention_mask,
					    "head_mask": head_mask,
					    "decoder_head_mask": decoder_head_mask,
					    "cross_attn_head_mask": cross_attn_head_mask,
					}




@require_tf
class 	__A				(			SCREAMING_SNAKE_CASE_		,     SCREAMING_SNAKE_CASE_		,     unittest.TestCase     ):
				a__	:						str	       =  (
				    (TFBlenderbotSmallForConditionalGeneration, TFBlenderbotSmallModel) if is_tf_available() else ()
				)
				a__	:						Tuple	       =  (TFBlenderbotSmallForConditionalGeneration,) if is_tf_available() else ()
				a__	:						Dict	       =  (
				    {
				        """conversational""": TFBlenderbotSmallForConditionalGeneration,
				        """feature-extraction""": TFBlenderbotSmallModel,
				        """summarization""": TFBlenderbotSmallForConditionalGeneration,
				        """text2text-generation""": TFBlenderbotSmallForConditionalGeneration,
				        """translation""": TFBlenderbotSmallForConditionalGeneration,
				    }
				    if is_tf_available()
				    else {}
				)
				a__	:						Tuple	       =  True
				a__	:						Union[str, Any]	       =  False
				a__	:						Dict	       =  False



				def  _lowercase						(self   :   Any			):
									UpperCAmelCase_	   =       TFBlenderbotSmallModelTester(self			)
									UpperCAmelCase_	   =       ConfigTester(self       ,      config_class=__a			)



				def  _lowercase						(self   :   Tuple			):
									self.config_tester.run_common_tests()






				def  _lowercase						(self   :   Union[str, Any]			):
									UpperCAmelCase_	   =       self.model_tester.prepare_config_and_inputs_for_common()
									self.model_tester.check_decoder_model_past_large_inputs(*__a			)




@require_tokenizers
@require_tf
class 	__A				(			unittest.TestCase     ):
				a__	:						Tuple	       =  [
				    """Social anxiety\nWow, I am never shy. Do you have anxiety?\nYes. I end up sweating and blushing and feel like """
				    """  i\'m going to throw up.\nand why is that?"""
				]
				a__	:						Any	       =  """facebook/blenderbot_small-90M"""



				@cached_property
				def  _lowercase						(self   :   int			):
									return BlenderbotSmallTokenizer.from_pretrained("facebook/blenderbot-90M"			)



				@cached_property
				def  _lowercase						(self   :   List[Any]			):
									UpperCAmelCase_	   =       TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name			)
									return model






				@slow
				def  _lowercase						(self   :   Union[str, Any]			):
									UpperCAmelCase_	   =       self.tokenizer(self.src_text       ,      return_tensors="tf"			)
									UpperCAmelCase_	   =       self.model.generate(
									    model_inputs.input_ids       ,      attention_mask=model_inputs.attention_mask       ,      num_beams=2       ,      use_cache=__a       ,      )
									UpperCAmelCase_	   =       self.tokenizer.batch_decode(generated_ids.numpy()       ,      skip_special_tokens=__a			)[0]
									assert generated_words in (
									    "i don't know. i just feel like i'm going to throw up. it's not fun.",
									    "i'm not sure. i just feel like i've been feeling like i have to be in a certain place",
									    "i'm not sure. i just feel like i've been in a bad situation.",
									)





'''simple docstring'''





import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class 	__A				(			UpperCamelCase__		,     unittest.TestCase     ):
    a__	:						List[Any]	       =  MobileBertTokenizer
    a__	:						str	       =  MobileBertTokenizerFast
    a__	:						List[str]	       =  True
    a__	:						Dict	       =  True
    a__	:						Optional[int]	       =  filter_non_english
    a__	:						int	       =  """google/mobilebert-uncased"""



    def  _lowercase						(self   :   List[str]			):
         super().setUp()

         UpperCAmelCase_	   =       [
             "[UNK]",
             "[CLS]",
             "[SEP]",
             "[PAD]",
             "[MASK]",
             "want",
             "##want",
             "##ed",
             "wa",
             "un",
             "runn",
             "##ing",
             ",",
             "low",
             "lowest",
         ]
         UpperCAmelCase_	   =       os.path.join(self.tmpdirname       ,      VOCAB_FILES_NAMES["vocab_file"]			)
         with open(self.vocab_file       ,      "w"       ,      encoding="utf-8"			) as vocab_writer:
              vocab_writer.write("".join([x + "\n" for x in vocab_tokens]			)			)

         UpperCAmelCase_	   =       [
             (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
             for tokenizer_def in self.tokenizers_list
         ]



    def  _lowercase						(self   :   Tuple       ,      __a   :   str			):
         UpperCAmelCase_	   =       "UNwant\u00E9d,running"
         UpperCAmelCase_	   =       "unwanted, running"
         return input_text, output_text



    def  _lowercase						(self   :   List[Any]			):
         UpperCAmelCase_	   =       self.tokenizer_class(self.vocab_file			)

         UpperCAmelCase_	   =       tokenizer.tokenize("UNwant\u00E9d,running"			)
         self.assertListEqual(__a       ,      ["un", "##want", "##ed", ",", "runn", "##ing"]			)
         self.assertListEqual(tokenizer.convert_tokens_to_ids(__a			)       ,      [9, 6, 7, 12, 10, 11]			)



    def  _lowercase						(self   :   Dict			):
         if not self.test_rust_tokenizer:
              return

         UpperCAmelCase_	   =       self.get_tokenizer()
         UpperCAmelCase_	   =       self.get_rust_tokenizer()

         UpperCAmelCase_	   =       "UNwant\u00E9d,running"

         UpperCAmelCase_	   =       tokenizer.tokenize(__a			)
         UpperCAmelCase_	   =       rust_tokenizer.tokenize(__a			)
         self.assertListEqual(__a       ,      __a			)

         UpperCAmelCase_	   =       tokenizer.encode(__a       ,      add_special_tokens=__a			)
         UpperCAmelCase_	   =       rust_tokenizer.encode(__a       ,      add_special_tokens=__a			)
         self.assertListEqual(__a       ,      __a			)

         UpperCAmelCase_	   =       self.get_rust_tokenizer()
         UpperCAmelCase_	   =       tokenizer.encode(__a			)
         UpperCAmelCase_	   =       rust_tokenizer.encode(__a			)
         self.assertListEqual(__a       ,      __a			)

         # With lower casing
         UpperCAmelCase_	   =       self.get_tokenizer(do_lower_case=__a			)
         UpperCAmelCase_	   =       self.get_rust_tokenizer(do_lower_case=__a			)

         UpperCAmelCase_	   =       "UNwant\u00E9d,running"

         UpperCAmelCase_	   =       tokenizer.tokenize(__a			)
         UpperCAmelCase_	   =       rust_tokenizer.tokenize(__a			)
         self.assertListEqual(__a       ,      __a			)

         UpperCAmelCase_	   =       tokenizer.encode(__a       ,      add_special_tokens=__a			)
         UpperCAmelCase_	   =       rust_tokenizer.encode(__a       ,      add_special_tokens=__a			)
         self.assertListEqual(__a       ,      __a			)

         UpperCAmelCase_	   =       self.get_rust_tokenizer()
         UpperCAmelCase_	   =       tokenizer.encode(__a			)
         UpperCAmelCase_	   =       rust_tokenizer.encode(__a			)
         self.assertListEqual(__a       ,      __a			)



    def  _lowercase						(self   :   Dict			):
         UpperCAmelCase_	   =       BasicTokenizer()

         self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"			)       ,      ["ah", "\u535A", "\u63A8", "zz"]			)



    def  _lowercase						(self   :   Optional[int]			):
         UpperCAmelCase_	   =       BasicTokenizer(do_lower_case=__a			)

         self.assertListEqual(
             tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "			)       ,      ["hello", "!", "how", "are", "you", "?"]			)
         self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)       ,      ["hello"]			)



    def  _lowercase						(self   :   Any			):
         UpperCAmelCase_	   =       BasicTokenizer(do_lower_case=__a       ,      strip_accents=__a			)

         self.assertListEqual(
             tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)       ,      ["hällo", "!", "how", "are", "you", "?"]			)
         self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)       ,      ["h\u00E9llo"]			)



    def  _lowercase						(self   :   Optional[int]			):
         UpperCAmelCase_	   =       BasicTokenizer(do_lower_case=__a       ,      strip_accents=__a			)

         self.assertListEqual(
             tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)       ,      ["hallo", "!", "how", "are", "you", "?"]			)
         self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)       ,      ["hello"]			)



    def  _lowercase						(self   :   Optional[int]			):
         UpperCAmelCase_	   =       BasicTokenizer(do_lower_case=__a			)

         self.assertListEqual(
             tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)       ,      ["hallo", "!", "how", "are", "you", "?"]			)
         self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)       ,      ["hello"]			)



    def  _lowercase						(self   :   Optional[Any]			):
         UpperCAmelCase_	   =       BasicTokenizer(do_lower_case=__a			)

         self.assertListEqual(
             tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "			)       ,      ["HeLLo", "!", "how", "Are", "yoU", "?"]			)



    def  _lowercase						(self   :   Optional[int]			):
         UpperCAmelCase_	   =       BasicTokenizer(do_lower_case=__a       ,      strip_accents=__a			)

         self.assertListEqual(
             tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)       ,      ["HäLLo", "!", "how", "Are", "yoU", "?"]			)



    def  _lowercase						(self   :   Tuple			):
         UpperCAmelCase_	   =       BasicTokenizer(do_lower_case=__a       ,      strip_accents=__a			)

         self.assertListEqual(
             tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)       ,      ["HaLLo", "!", "how", "Are", "yoU", "?"]			)



    def  _lowercase						(self   :   Tuple			):
         UpperCAmelCase_	   =       BasicTokenizer(do_lower_case=__a       ,      never_split=["[UNK]"]			)

         self.assertListEqual(
             tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"			)       ,      ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]			)



    def  _lowercase						(self   :   Any			):
         UpperCAmelCase_	   =       ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

         UpperCAmelCase_	   =       {}
         for i, token in enumerate(__a			):
              UpperCAmelCase_	   =       i
         UpperCAmelCase_	   =       WordpieceTokenizer(vocab=__a       ,      unk_token="[UNK]"			)

         self.assertListEqual(tokenizer.tokenize(""			)       ,      []			)

         self.assertListEqual(tokenizer.tokenize("unwanted running"			)       ,      ["un", "##want", "##ed", "runn", "##ing"]			)

         self.assertListEqual(tokenizer.tokenize("unwantedX running"			)       ,      ["[UNK]", "runn", "##ing"]			)



    def  _lowercase						(self   :   Optional[int]			):
         self.assertTrue(_is_whitespace(" "			)			)
         self.assertTrue(_is_whitespace("\t"			)			)
         self.assertTrue(_is_whitespace("\r"			)			)
         self.assertTrue(_is_whitespace("\n"			)			)
         self.assertTrue(_is_whitespace("\u00A0"			)			)

         self.assertFalse(_is_whitespace("A"			)			)
         self.assertFalse(_is_whitespace("-"			)			)



    def  _lowercase						(self   :   str			):
         self.assertTrue(_is_control("\u0005"			)			)

         self.assertFalse(_is_control("A"			)			)
         self.assertFalse(_is_control(" "			)			)
         self.assertFalse(_is_control("\t"			)			)
         self.assertFalse(_is_control("\r"			)			)



    def  _lowercase						(self   :   Any			):
         self.assertTrue(_is_punctuation("-"			)			)
         self.assertTrue(_is_punctuation("$"			)			)
         self.assertTrue(_is_punctuation("`"			)			)
         self.assertTrue(_is_punctuation("."			)			)

         self.assertFalse(_is_punctuation("A"			)			)
         self.assertFalse(_is_punctuation(" "			)			)



    def  _lowercase						(self   :   Any			):
         UpperCAmelCase_	   =       self.get_tokenizer()
         UpperCAmelCase_	   =       self.get_rust_tokenizer()

         # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
         self.assertListEqual([tokenizer.tokenize(__a			) for t in ["Test", "\xad", "test"]]       ,      [["[UNK]"], [], ["[UNK]"]]			)

         self.assertListEqual(
             [rust_tokenizer.tokenize(__a			) for t in ["Test", "\xad", "test"]]       ,      [["[UNK]"], [], ["[UNK]"]]			)



    @slow
    def  _lowercase						(self   :   Dict			):
         UpperCAmelCase_	   =       self.tokenizer_class.from_pretrained("google/mobilebert-uncased"			)

         UpperCAmelCase_	   =       tokenizer.encode("sequence builders"       ,      add_special_tokens=__a			)
         UpperCAmelCase_	   =       tokenizer.encode("multi-sequence build"       ,      add_special_tokens=__a			)

         UpperCAmelCase_	   =       tokenizer.build_inputs_with_special_tokens(__a			)
         UpperCAmelCase_	   =       tokenizer.build_inputs_with_special_tokens(__a       ,      __a			)

         assert encoded_sentence == [101] + text + [102]
         assert encoded_pair == [101] + text + [102] + text_a + [102]



    def  _lowercase						(self   :   Dict			):
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
                   UpperCAmelCase_	   =       self.rust_tokenizer_class.from_pretrained(__a       ,      **__a			)

                   UpperCAmelCase_	   =       f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
                   UpperCAmelCase_	   =       tokenizer_r.encode_plus(
                       __a       ,      return_attention_mask=__a       ,      return_token_type_ids=__a       ,      return_offsets_mapping=__a       ,      add_special_tokens=__a       ,      )

                   UpperCAmelCase_	   =       tokenizer_r.do_lower_case if hasattr(__a       ,      "do_lower_case"			) else False
                   UpperCAmelCase_	   =       (
                       [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), "A"),
                           ((1, 2), ","),
                           ((3, 5), "na"),
                           ((5, 6), "##ï"),
                           ((6, 8), "##ve"),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), "Allen"),
                           ((21, 23), "##NL"),
                           ((23, 24), "##P"),
                           ((25, 33), "sentence"),
                           ((33, 34), "."),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                       if not do_lower_case
                       else [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), "a"),
                           ((1, 2), ","),
                           ((3, 8), "naive"),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), "allen"),
                           ((21, 23), "##nl"),
                           ((23, 24), "##p"),
                           ((25, 33), "sentence"),
                           ((33, 34), "."),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                   )

                   self.assertEqual(
                       [e[1] for e in expected_results]       ,      tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]			)			)
                   self.assertEqual([e[0] for e in expected_results]       ,      tokens["offset_mapping"]			)






    def  _lowercase						(self   :   Optional[int]			):
         UpperCAmelCase_	   =       ["的", "人", "有"]
         UpperCAmelCase_	   =       "".join(__a			)
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
                   UpperCAmelCase_	   =       True
                   UpperCAmelCase_	   =       self.tokenizer_class.from_pretrained(__a       ,      **__a			)
                   UpperCAmelCase_	   =       self.rust_tokenizer_class.from_pretrained(__a       ,      **__a			)

                   UpperCAmelCase_	   =       tokenizer_p.encode(__a       ,      add_special_tokens=__a			)
                   UpperCAmelCase_	   =       tokenizer_r.encode(__a       ,      add_special_tokens=__a			)

                   UpperCAmelCase_	   =       tokenizer_r.convert_ids_to_tokens(__a			)
                   UpperCAmelCase_	   =       tokenizer_p.convert_ids_to_tokens(__a			)

                   # it is expected that each Chinese character is not preceded by "##"
                   self.assertListEqual(__a       ,      __a			)
                   self.assertListEqual(__a       ,      __a			)

                   UpperCAmelCase_	   =       False
                   UpperCAmelCase_	   =       self.rust_tokenizer_class.from_pretrained(__a       ,      **__a			)
                   UpperCAmelCase_	   =       self.tokenizer_class.from_pretrained(__a       ,      **__a			)

                   UpperCAmelCase_	   =       tokenizer_r.encode(__a       ,      add_special_tokens=__a			)
                   UpperCAmelCase_	   =       tokenizer_p.encode(__a       ,      add_special_tokens=__a			)

                   UpperCAmelCase_	   =       tokenizer_r.convert_ids_to_tokens(__a			)
                   UpperCAmelCase_	   =       tokenizer_p.convert_ids_to_tokens(__a			)

                   # it is expected that only the first Chinese character is not preceded by "##".
                   UpperCAmelCase_	   =       [
                       f"""##{token}""" if idx != 0 else token for idx, token in enumerate(__a			)
                   ]
                   self.assertListEqual(__a       ,      __a			)
                   self.assertListEqual(__a       ,      __a			)




