




'''simple docstring'''





import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_SCREAMING_SNAKE_CASE							:       str						=			get_tests_dir("fixtures/spiece.model")




@require_sentencepiece
@require_tokenizers
class 			_snake_case		(     lowercase_						,						unittest.TestCase							):
   lowerCAmelCase_					:				int							=   DebertaVaTokenizer
   lowerCAmelCase_					:				Optional[int]							=   DebertaVaTokenizerFast
   lowerCAmelCase_					:				Any							=   True
   lowerCAmelCase_					:				Optional[int]							=   True




   def        lowerCAmelCase__    ( self							)    ->    int:

          '''simple docstring'''




          super().setUp()

          # We have a SentencePiece fixture for testing
          snake_case_      =				DebertaVaTokenizer(a__    ,							unk_token="<unk>"							)
          tokenizer.save_pretrained(self.tmpdirname							)




   def        lowerCAmelCase__    ( self    ,							a__							)    ->    Any:

          '''simple docstring'''




          snake_case_      =				"this is a test"
          snake_case_      =				"this is a test"
          return input_text, output_text




   def        lowerCAmelCase__    ( self							)    ->    int:

          '''simple docstring'''




          snake_case_      =				"<pad>"
          snake_case_      =				0

          self.assertEqual(self.get_tokenizer()._convert_token_to_id(a__							)    ,							a__							)
          self.assertEqual(self.get_tokenizer()._convert_id_to_token(a__							)    ,							a__							)




   def        lowerCAmelCase__    ( self							)    ->    List[str]:

          '''simple docstring'''




          snake_case_      =				list(self.get_tokenizer().get_vocab().keys()							)
          self.assertEqual(vocab_keys[0]    ,							"<pad>"							)
          self.assertEqual(vocab_keys[1]    ,							"<unk>"							)
          self.assertEqual(vocab_keys[-1]    ,							"[PAD]"							)
          self.assertEqual(len(a__							)    ,							30_001							)




   def        lowerCAmelCase__    ( self							)    ->    str:

          '''simple docstring'''




          self.assertEqual(self.get_tokenizer().vocab_size    ,							30_000							)




   def        lowerCAmelCase__    ( self							)    ->    str:

          '''simple docstring'''




          snake_case_      =				" \tHeLLo!how  \n Are yoU?  "
          snake_case_      =				["▁hello", "!", "how", "▁are", "▁you", "?"]
          # fmt: on

          snake_case_      =				DebertaVaTokenizer(a__    ,							do_lower_case=a__							)
          snake_case_      =				tokenizer.convert_ids_to_tokens(tokenizer.encode(a__    ,							add_special_tokens=a__							)							)

          self.assertListEqual(a__    ,							a__							)

          snake_case_      =				DebertaVaTokenizerFast(a__    ,							do_lower_case=a__							)
          snake_case_      =				rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(a__    ,							add_special_tokens=a__							)							)

          self.assertListEqual(a__    ,							a__							)




   @unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."							)
   def        lowerCAmelCase__    ( self							)    ->    int:

          '''simple docstring'''




          pass




   @unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."							)
   def        lowerCAmelCase__    ( self							)    ->    Optional[Any]:

          '''simple docstring'''




          pass




   def        lowerCAmelCase__    ( self							)    ->    str:

          '''simple docstring'''




          snake_case_      =				"I was born in 92000, and this is falsé."
          snake_case_      =				["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
          # fmt: on

          snake_case_      =				DebertaVaTokenizer(a__    ,							split_by_punct=a__							)
          snake_case_      =				tokenizer.convert_ids_to_tokens(tokenizer.encode(a__    ,							add_special_tokens=a__							)							)

          self.assertListEqual(a__    ,							a__							)

          snake_case_      =				DebertaVaTokenizerFast(a__    ,							split_by_punct=a__							)
          snake_case_      =				rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(a__    ,							add_special_tokens=a__							)							)

          self.assertListEqual(a__    ,							a__							)




   def        lowerCAmelCase__    ( self							)    ->    Optional[Any]:

          '''simple docstring'''




          snake_case_      =				"I was born in 92000, and this is falsé."
          snake_case_      =				["▁i", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
          # fmt: on

          snake_case_      =				DebertaVaTokenizer(a__    ,							do_lower_case=a__    ,							split_by_punct=a__							)
          snake_case_      =				tokenizer.convert_ids_to_tokens(tokenizer.encode(a__    ,							add_special_tokens=a__							)							)
          self.assertListEqual(a__    ,							a__							)

          snake_case_      =				DebertaVaTokenizerFast(a__    ,							do_lower_case=a__    ,							split_by_punct=a__							)
          snake_case_      =				rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(a__    ,							add_special_tokens=a__							)							)
          self.assertListEqual(a__    ,							a__							)




   def        lowerCAmelCase__    ( self							)    ->    int:

          '''simple docstring'''




          snake_case_      =				"I was born in 92000, and this is falsé."
          snake_case_      =				["▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
          # fmt: on

          snake_case_      =				DebertaVaTokenizer(a__    ,							do_lower_case=a__    ,							split_by_punct=a__							)
          snake_case_      =				tokenizer.convert_ids_to_tokens(tokenizer.encode(a__    ,							add_special_tokens=a__							)							)

          self.assertListEqual(a__    ,							a__							)

          snake_case_      =				DebertaVaTokenizerFast(a__    ,							do_lower_case=a__    ,							split_by_punct=a__							)
          snake_case_      =				rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(a__    ,							add_special_tokens=a__							)							)

          self.assertListEqual(a__    ,							a__							)




   def        lowerCAmelCase__    ( self							)    ->    str:

          '''simple docstring'''




          snake_case_      =				"I was born in 92000, and this is falsé."
          snake_case_      =				["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
          # fmt: on

          snake_case_      =				DebertaVaTokenizer(a__    ,							do_lower_case=a__    ,							split_by_punct=a__							)
          snake_case_      =				tokenizer.convert_ids_to_tokens(tokenizer.encode(a__    ,							add_special_tokens=a__							)							)

          self.assertListEqual(a__    ,							a__							)

          snake_case_      =				DebertaVaTokenizerFast(a__    ,							do_lower_case=a__    ,							split_by_punct=a__							)
          snake_case_      =				rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(a__    ,							add_special_tokens=a__							)							)

          self.assertListEqual(a__    ,							a__							)




   def        lowerCAmelCase__    ( self							)    ->    Optional[int]:

          '''simple docstring'''




          snake_case_      =				" \tHeLLo!how  \n Are yoU?  "
          snake_case_      =				["▁", "<unk>", "e", "<unk>", "o", "!", "how", "▁", "<unk>", "re", "▁yo", "<unk>", "?"]
          # fmt: on

          snake_case_      =				DebertaVaTokenizer(a__    ,							do_lower_case=a__    ,							split_by_punct=a__							)
          snake_case_      =				tokenizer.convert_ids_to_tokens(tokenizer.encode(a__    ,							add_special_tokens=a__							)							)

          self.assertListEqual(a__    ,							a__							)

          snake_case_      =				DebertaVaTokenizerFast(a__    ,							do_lower_case=a__    ,							split_by_punct=a__							)
          snake_case_      =				rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(a__    ,							add_special_tokens=a__							)							)

          self.assertListEqual(a__    ,							a__							)




   def        lowerCAmelCase__    ( self							)    ->    str:

          '''simple docstring'''




          snake_case_      =				self.get_tokenizer()
          snake_case_      =				self.get_rust_tokenizer()

          snake_case_      =				"I was born in 92000, and this is falsé."

          snake_case_      =				tokenizer.convert_ids_to_tokens(tokenizer.encode(a__    ,							add_special_tokens=a__							)							)
          snake_case_      =				rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(a__    ,							add_special_tokens=a__							)							)
          self.assertListEqual(a__    ,							a__							)

          snake_case_      =				tokenizer.encode(a__    ,							add_special_tokens=a__							)
          snake_case_      =				rust_tokenizer.encode(a__    ,							add_special_tokens=a__							)
          self.assertListEqual(a__    ,							a__							)

          snake_case_      =				self.get_rust_tokenizer()
          snake_case_      =				tokenizer.encode(a__							)
          snake_case_      =				rust_tokenizer.encode(a__							)
          self.assertListEqual(a__    ,							a__							)




   def        lowerCAmelCase__    ( self							)    ->    Optional[int]:

          '''simple docstring'''




          snake_case_      =				"This is a test"
          snake_case_      =				[13, 1, 4_398, 25, 21, 1_289]
          snake_case_      =				["▁", "T", "his", "▁is", "▁a", "▁test"]
          snake_case_      =				["▁", "<unk>", "his", "▁is", "▁a", "▁test"]

          snake_case_      =				DebertaVaTokenizer(a__    ,							keep_accents=a__							)
          snake_case_      =				DebertaVaTokenizerFast(a__    ,							keep_accents=a__							)

          snake_case_      =				tokenizer.encode(a__    ,							add_special_tokens=a__							)
          self.assertListEqual(a__    ,							a__							)
          snake_case_      =				tokenizer.tokenize(a__							)
          self.assertListEqual(a__    ,							a__							)
          snake_case_      =				tokenizer.convert_ids_to_tokens(a__							)
          self.assertListEqual(a__    ,							a__							)

          snake_case_      =				rust_tokenizer.encode(a__    ,							add_special_tokens=a__							)
          self.assertListEqual(a__    ,							a__							)
          snake_case_      =				rust_tokenizer.tokenize(a__							)
          self.assertListEqual(a__    ,							a__							)
          snake_case_      =				rust_tokenizer.convert_ids_to_tokens(a__							)
          self.assertListEqual(a__    ,							a__							)

          # fmt: off
          snake_case_      =				"I was born in 92000, and this is falsé."
          snake_case_      =				[13, 1, 23, 386, 19, 561, 3_050, 15, 17, 48, 25, 8_256, 18, 1, 9]
          snake_case_      =				["▁", "I", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "é", ".", ]
          snake_case_      =				["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
          # fmt: on

          snake_case_      =				tokenizer.encode(a__    ,							add_special_tokens=a__							)
          self.assertListEqual(a__    ,							a__							)
          snake_case_      =				tokenizer.tokenize(a__							)
          self.assertListEqual(a__    ,							a__							)
          snake_case_      =				tokenizer.convert_ids_to_tokens(a__							)
          self.assertListEqual(a__    ,							a__							)

          snake_case_      =				rust_tokenizer.encode(a__    ,							add_special_tokens=a__							)
          self.assertListEqual(a__    ,							a__							)
          snake_case_      =				rust_tokenizer.tokenize(a__							)
          self.assertListEqual(a__    ,							a__							)
          snake_case_      =				rust_tokenizer.convert_ids_to_tokens(a__							)
          self.assertListEqual(a__    ,							a__							)




   def        lowerCAmelCase__    ( self							)    ->    List[str]:

          '''simple docstring'''




          snake_case_      =				DebertaVaTokenizer(a__							)

          snake_case_      =				tokenizer.encode("sequence builders"							)
          snake_case_      =				tokenizer.encode("multi-sequence build"							)

          snake_case_      =				tokenizer.build_inputs_with_special_tokens(a__							)
          snake_case_      =				tokenizer.build_inputs_with_special_tokens(a__    ,							a__							)

          self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]    ,							a__							)
          self.assertEqual(
              [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]    ,							a__    ,							)




   @slow
   def        lowerCAmelCase__    ( self							)    ->    Optional[int]:

          '''simple docstring'''




          snake_case_      =				{"input_ids": [[1, 39_867, 36, 19_390, 486, 27, 35_052, 81_436, 18, 60_685, 1_225, 7, 35_052, 81_436, 18, 9_367, 16_899, 18, 15_937, 53, 594, 773, 18, 16_287, 30_465, 36, 15_937, 6, 41_139, 38, 36_979, 60_763, 191, 6, 34_132, 99, 6, 50_538, 390, 43_230, 6, 34_132, 2_779, 20_850, 14, 699, 1_072, 1_194, 36, 382, 10_901, 53, 7, 699, 1_072, 2_084, 36, 20_422, 630, 53, 19, 105, 3_049, 1_896, 1_053, 16_899, 1_506, 11, 37_978, 4_243, 7, 1_237, 31_869, 200, 16_566, 654, 6, 35_052, 81_436, 7, 55_630, 13_593, 4, 2], [1, 26, 15_011, 13, 667, 8, 1_053, 18, 23_611, 1_237, 72_356, 12_820, 34, 104_134, 1_209, 35, 13_313, 6_627, 21, 202, 347, 7, 164, 2_399, 11, 46, 4_485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1_232, 2_864, 15_785, 14_951, 105, 5, 8_581, 1_250, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
          # fmt: on

          self.tokenizer_integration_test_util(
              expected_encoding=a__    ,							model_name="microsoft/deberta-v2-xlarge"    ,							revision="ad6e42c1532ddf3a15c39246b63f5559d558b670"    ,							)










'''simple docstring'''





from ...configuration_utils import PretrainedConfig
from ...utils import logging


_SCREAMING_SNAKE_CASE							:       Any						=			logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE							:       int						=			{
    "alibaba-damo/mgp-str-base": "https://huggingface.co/alibaba-damo/mgp-str-base/resolve/main/config.json",
}




class 			_snake_case		(     lowercase_							):
			lowerCAmelCase_					:				Union[str, Any]							=   "mgp-str"




			def __init__( self    ,							a__=[32, 128]    ,							a__=4    ,							a__=3    ,							a__=27    ,							a__=38    ,							a__=50_257    ,							a__=30_522    ,							a__=768    ,							a__=12    ,							a__=12    ,							a__=4.0    ,							a__=True    ,							a__=False    ,							a__=1e-5    ,							a__=0.0    ,							a__=0.0    ,							a__=0.0    ,							a__=False    ,							a__=0.0_2    ,							**a__    ,							)    ->    Union[str, Any]:

										'''simple docstring'''




										super().__init__(**a__							)

										snake_case_      =				image_size
										snake_case_      =				patch_size
										snake_case_      =				num_channels
										snake_case_      =				max_token_length
										snake_case_      =				num_character_labels
										snake_case_      =				num_bpe_labels
										snake_case_      =				num_wordpiece_labels
										snake_case_      =				hidden_size
										snake_case_      =				num_hidden_layers
										snake_case_      =				num_attention_heads
										snake_case_      =				mlp_ratio
										snake_case_      =				distilled
										snake_case_      =				layer_norm_eps
										snake_case_      =				drop_rate
										snake_case_      =				qkv_bias
										snake_case_      =				attn_drop_rate
										snake_case_      =				drop_path_rate
										snake_case_      =				output_aa_attentions
										snake_case_      =				initializer_range




