
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartaaTokenizer, MBartaaTokenizerFast, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)

from ...test_tokenization_common import TokenizerTesterMixin


_A   =			get_tests_dir("fixtures/test_sentencepiece.model")

if is_torch_available():
			from transformers.models.mbart.modeling_mbart import shift_tokens_right

_A   =			25_00_04
_A   =			25_00_20







@require_sentencepiece
@require_tokenizers
class 					lowerCamelCase		(  lowerCamelCase__							,			unittest.TestCase  ):
					UpperCAmelCase__    :							Union[str, Any]									=		MBartaaTokenizer
					UpperCAmelCase__    :							List[Any]									=		MBartaaTokenizerFast
					UpperCAmelCase__    :							Union[str, Any]									=		True
					UpperCAmelCase__    :							Optional[Any]									=		True





					def 		UpperCAmelCase(self				:   Any       )						->						List[str]:
									super().setUp()

									# We have a SentencePiece fixture for testing
									snake_case								= MBartaaTokenizer(snake_case__    ,   src_lang="en_XX"    ,   tgt_lang="ro_RO"    ,   keep_accents=snake_case__       )
									tokenizer.save_pretrained(self.tmpdirname       )





					def 		UpperCAmelCase(self				:   Dict       )						->						Optional[Any]:
									snake_case								= '''<s>'''
									snake_case								= 0

									self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case__       )    ,   snake_case__       )
									self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case__       )    ,   snake_case__       )





					def 		UpperCAmelCase(self				:   Union[str, Any]       )						->						List[str]:
									snake_case								= list(self.get_tokenizer().get_vocab().keys()       )

									self.assertEqual(vocab_keys[0]    ,   "<s>"       )
									self.assertEqual(vocab_keys[1]    ,   "<pad>"       )
									self.assertEqual(vocab_keys[-1]    ,   "<mask>"       )
									self.assertEqual(len(snake_case__       )    ,   1_0_5_4       )





					def 		UpperCAmelCase(self				:   str       )						->						int:
									self.assertEqual(self.get_tokenizer().vocab_size    ,   1_0_5_4       )





					def 		UpperCAmelCase(self				:   Union[str, Any]       )						->						Optional[int]:
									snake_case								= MBartaaTokenizer(snake_case__    ,   src_lang="en_XX"    ,   tgt_lang="ro_RO"    ,   keep_accents=snake_case__       )

									snake_case								= tokenizer.tokenize("This is a test"       )
									self.assertListEqual(snake_case__    ,   ["▁This", "▁is", "▁a", "▁t", "est"]       )

									self.assertListEqual(
									    tokenizer.convert_tokens_to_ids(snake_case__       )    ,   [value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]    ,   )

									snake_case								= tokenizer.tokenize("I was born in 92000, and this is falsé."       )
									self.assertListEqual(
									    snake_case__    ,   [SPIECE_UNDERLINE + "I", SPIECE_UNDERLINE + "was", SPIECE_UNDERLINE + "b", "or", "n", SPIECE_UNDERLINE + "in", SPIECE_UNDERLINE + "", "9", "2", "0", "0", "0", ",", SPIECE_UNDERLINE + "and", SPIECE_UNDERLINE + "this", SPIECE_UNDERLINE + "is", SPIECE_UNDERLINE + "f", "al", "s", "é", "."]    ,   )
									snake_case								= tokenizer.convert_tokens_to_ids(snake_case__       )
									self.assertListEqual(
									    snake_case__    ,   [
									        value + tokenizer.fairseq_offset
									        for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
									    ]    ,   )

									snake_case								= tokenizer.convert_ids_to_tokens(snake_case__       )
									self.assertListEqual(
									    snake_case__    ,   [SPIECE_UNDERLINE + "I", SPIECE_UNDERLINE + "was", SPIECE_UNDERLINE + "b", "or", "n", SPIECE_UNDERLINE + "in", SPIECE_UNDERLINE + "", "<unk>", "2", "0", "0", "0", ",", SPIECE_UNDERLINE + "and", SPIECE_UNDERLINE + "this", SPIECE_UNDERLINE + "is", SPIECE_UNDERLINE + "f", "al", "s", "<unk>", "."]    ,   )





					@slow
					def 		UpperCAmelCase(self				:   Tuple       )						->						Tuple:
									snake_case								= {'''input_ids''': [[2_5_0_0_0_4, 1_1_0_6_2, 8_2_7_7_2, 7, 1_5, 8_2_7_7_2, 5_3_8, 5_1_5_2_9, 2_3_7, 1_7_1_9_8, 1_2_9_0, 2_0_6, 9, 2_1_5_1_7_5, 1_3_1_4, 1_3_6, 1_7_1_9_8, 1_2_9_0, 2_0_6, 9, 5_6_3_5_9, 4_2, 1_2_2_0_0_9, 9, 1_6_4_6_6, 1_6, 8_7_3_4_4, 4_5_3_7, 9, 4_7_1_7, 7_8_3_8_1, 6, 1_5_9_9_5_8, 7, 1_5, 2_4_4_8_0, 6_1_8, 4, 5_2_7, 2_2_6_9_3, 5_4_2_8, 4, 2_7_7_7, 2_4_4_8_0, 9_8_7_4, 4, 4_3_5_2_3, 5_9_4, 4, 8_0_3, 1_8_3_9_2, 3_3_1_8_9, 1_8, 4, 4_3_5_2_3, 2_4_4_4_7, 1_2_3_9_9, 1_0_0, 2_4_9_5_5, 8_3_6_5_8, 9_6_2_6, 1_4_4_0_5_7, 1_5, 8_3_9, 2_2_3_3_5, 1_6, 1_3_6, 2_4_9_5_5, 8_3_6_5_8, 8_3_4_7_9, 1_5, 3_9_1_0_2, 7_2_4, 1_6, 6_7_8, 6_4_5, 2_7_8_9, 1_3_2_8, 4_5_8_9, 4_2, 1_2_2_0_0_9, 1_1_5_7_7_4, 2_3, 8_0_5, 1_3_2_8, 4_6_8_7_6, 7, 1_3_6, 5_3_8_9_4, 1_9_4_0, 4_2_2_2_7, 4_1_1_5_9, 1_7_7_2_1, 8_2_3, 4_2_5, 4, 2_7_5_1_2, 9_8_7_2_2, 2_0_6, 1_3_6, 5_5_3_1, 4_9_7_0, 9_1_9, 1_7_3_3_6, 5, 2], [2_5_0_0_0_4, 2_0_0_8_0, 6_1_8, 8_3, 8_2_7_7_5, 4_7, 4_7_9, 9, 1_5_1_7, 7_3, 5_3_8_9_4, 3_3_3, 8_0_5_8_1, 1_1_0_1_1_7, 1_8_8_1_1, 5_2_5_6, 1_2_9_5, 5_1, 1_5_2_5_2_6, 2_9_7, 7_9_8_6, 3_9_0, 1_2_4_4_1_6, 5_3_8, 3_5_4_3_1, 2_1_4, 9_8, 1_5_0_4_4, 2_5_7_3_7, 1_3_6, 7_1_0_8, 4_3_7_0_1, 2_3, 7_5_6, 1_3_5_3_5_5, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2_5_0_0_0_4, 5_8_1, 6_3_7_7_3, 1_1_9_4_5_5, 6, 1_4_7_7_9_7, 8_8_2_0_3, 7, 6_4_5, 7_0, 2_1, 3_2_8_5, 1_0_2_6_9, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
									# fmt: on

									self.tokenizer_integration_test_util(
									    expected_encoding=snake_case__    ,   model_name="facebook/mbart-large-50"    ,   revision="d3913889c59cd5c9e456b269c376325eabad57e2"    ,   )





					def 		UpperCAmelCase(self				:   Optional[int]       )						->						Any:





									if not self.test_slow_tokenizer:
													# as we don't have a slow version, we can't compare the outputs between slow and fast versions
													return

									snake_case								= (self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-mbart50''', {})
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'       ):
																	snake_case								= self.rust_tokenizer_class.from_pretrained(snake_case__    ,   **snake_case__       )
																	snake_case								= self.tokenizer_class.from_pretrained(snake_case__    ,   **snake_case__       )

																	snake_case								= tempfile.mkdtemp()

																	snake_case								= tokenizer_r.save_pretrained(snake_case__       )
																	snake_case								= tokenizer_p.save_pretrained(snake_case__       )

																	# Checks it save with the same files + the tokenizer.json file for the fast one
																	self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files       )       )
																	snake_case								= tuple(f for f in tokenizer_r_files if "tokenizer.json" not in f       )
																	self.assertSequenceEqual(snake_case__    ,   snake_case__       )

																	# Checks everything loads correctly in the same way
																	snake_case								= tokenizer_r.from_pretrained(snake_case__       )
																	snake_case								= tokenizer_p.from_pretrained(snake_case__       )

																	# Check special tokens are set accordingly on Rust and Python
																	for key in tokenizer_pp.special_tokens_map:
																					self.assertTrue(hasattr(snake_case__    ,   snake_case__       )       )
																					# self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
																					# self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

																	shutil.rmtree(snake_case__       )

																	# Save tokenizer rust, legacy_format=True
																	snake_case								= tempfile.mkdtemp()

																	snake_case								= tokenizer_r.save_pretrained(snake_case__    ,   legacy_format=snake_case__       )
																	snake_case								= tokenizer_p.save_pretrained(snake_case__       )

																	# Checks it save with the same files
																	self.assertSequenceEqual(snake_case__    ,   snake_case__       )

																	# Checks everything loads correctly in the same way
																	snake_case								= tokenizer_r.from_pretrained(snake_case__       )
																	snake_case								= tokenizer_p.from_pretrained(snake_case__       )

																	# Check special tokens are set accordingly on Rust and Python
																	for key in tokenizer_pp.special_tokens_map:
																					self.assertTrue(hasattr(snake_case__    ,   snake_case__       )       )

																	shutil.rmtree(snake_case__       )

																	# Save tokenizer rust, legacy_format=False
																	snake_case								= tempfile.mkdtemp()

																	snake_case								= tokenizer_r.save_pretrained(snake_case__    ,   legacy_format=snake_case__       )
																	snake_case								= tokenizer_p.save_pretrained(snake_case__       )

																	# Checks it saved the tokenizer.json file
																	self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files       )       )

																	# Checks everything loads correctly in the same way
																	snake_case								= tokenizer_r.from_pretrained(snake_case__       )
																	snake_case								= tokenizer_p.from_pretrained(snake_case__       )

																	# Check special tokens are set accordingly on Rust and Python
																	for key in tokenizer_pp.special_tokens_map:
																					self.assertTrue(hasattr(snake_case__    ,   snake_case__       )       )

																	shutil.rmtree(snake_case__       )







@require_torch
@require_sentencepiece
@require_tokenizers
class 					lowerCamelCase		(  unittest.TestCase  ):
					UpperCAmelCase__    :							List[Any]									=		"""facebook/mbart-large-50-one-to-many-mmt"""
					UpperCAmelCase__    :							str									=		[
					    """ UN Chief Says There Is No Military Solution in Syria""",
					    """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.""",
					]
					UpperCAmelCase__    :							int									=		[
					    """Şeful ONU declară că nu există o soluţie militară în Siria""",
					    """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"""
					    """ pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"""
					    """ face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
					]
					UpperCAmelCase__    :							str									=		[EN_CODE, 82_74, 12_78_73, 2_59_16, 7, 86_22, 20_71, 4_38, 6_74_85, 53, 18_78_95, 23, 5_17_12, 2]





					@classmethod
					def 		UpperCAmelCase(cls				:   int       )						->						Optional[int]:
									snake_case								= MBartaaTokenizer.from_pretrained(
									    cls.checkpoint_name    ,   src_lang="en_XX"    ,   tgt_lang="ro_RO"       )
									snake_case								= 1
									return cls





					def 		UpperCAmelCase(self				:   Dict       )						->						List[Any]:
									self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["ar_AR"]    ,   2_5_0_0_0_1       )
									self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["en_EN"]    ,   2_5_0_0_0_4       )
									self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["ro_RO"]    ,   2_5_0_0_2_0       )
									self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["mr_IN"]    ,   2_5_0_0_3_8       )





					def 		UpperCAmelCase(self				:   Tuple       )						->						Optional[int]:
									snake_case								= self.tokenizer.batch_encode_plus(self.src_text       ).input_ids[0]
									self.assertListEqual(self.expected_src_tokens    ,   snake_case__       )





					def 		UpperCAmelCase(self				:   Any       )						->						Dict:
									self.assertIn(snake_case__    ,   self.tokenizer.all_special_ids       )
									snake_case								= [RO_CODE, 8_8_4, 9_0_1_9, 9_6, 9, 9_1_6, 8_6_7_9_2, 3_6, 1_8_7_4_3, 1_5_5_9_6, 5, 2]
									snake_case								= self.tokenizer.decode(snake_case__    ,   skip_special_tokens=snake_case__       )
									snake_case								= self.tokenizer.decode(generated_ids[1:]    ,   skip_special_tokens=snake_case__       )
									self.assertEqual(snake_case__    ,   snake_case__       )
									self.assertNotIn(self.tokenizer.eos_token    ,   snake_case__       )





					def 		UpperCAmelCase(self				:   Optional[Any]       )						->						int:
									snake_case								= ['''this is gunna be a long sentence ''' * 2_0]
									assert isinstance(src_text[0]    ,   snake_case__       )
									snake_case								= 1_0
									snake_case								= self.tokenizer(snake_case__    ,   max_length=snake_case__    ,   truncation=snake_case__       ).input_ids[0]
									self.assertEqual(ids[0]    ,   snake_case__       )
									self.assertEqual(ids[-1]    ,   2       )
									self.assertEqual(len(snake_case__       )    ,   snake_case__       )





					def 		UpperCAmelCase(self				:   Tuple       )						->						Any:
									self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["<mask>", "ar_AR"]       )    ,   [2_5_0_0_5_3, 2_5_0_0_0_1]       )





					def 		UpperCAmelCase(self				:   Optional[int]       )						->						List[Any]:
									snake_case								= tempfile.mkdtemp()
									snake_case								= self.tokenizer.fairseq_tokens_to_ids
									self.tokenizer.save_pretrained(snake_case__       )
									snake_case								= MBartaaTokenizer.from_pretrained(snake_case__       )
									self.assertDictEqual(new_tok.fairseq_tokens_to_ids    ,   snake_case__       )





					@require_torch
					def 		UpperCAmelCase(self				:   Optional[Any]       )						->						str:
									snake_case								= self.tokenizer(self.src_text    ,   text_target=self.tgt_text    ,   padding=snake_case__    ,   return_tensors="pt"       )
									snake_case								= shift_tokens_right(batch["labels"]    ,   self.tokenizer.pad_token_id       )

									# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
									assert batch.input_ids[1][0] == EN_CODE
									assert batch.input_ids[1][-1] == 2
									assert batch.labels[1][0] == RO_CODE
									assert batch.labels[1][-1] == 2
									assert batch.decoder_input_ids[1][:2].tolist() == [2, RO_CODE]





					@require_torch
					def 		UpperCAmelCase(self				:   Optional[int]       )						->						Tuple:
									snake_case								= self.tokenizer(
									    self.src_text    ,   text_target=self.tgt_text    ,   padding=snake_case__    ,   truncation=snake_case__    ,   max_length=len(self.expected_src_tokens       )    ,   return_tensors="pt"    ,   )
									snake_case								= shift_tokens_right(batch["labels"]    ,   self.tokenizer.pad_token_id       )

									self.assertIsInstance(snake_case__    ,   snake_case__       )

									self.assertEqual((2, 1_4)    ,   batch.input_ids.shape       )
									self.assertEqual((2, 1_4)    ,   batch.attention_mask.shape       )
									snake_case								= batch.input_ids.tolist()[0]
									self.assertListEqual(self.expected_src_tokens    ,   snake_case__       )
									self.assertEqual(2    ,   batch.decoder_input_ids[0, 0]       )  # decoder_start_token_id
									# Test that special tokens are reset
									self.assertEqual(self.tokenizer.prefix_tokens    ,   [EN_CODE]       )
									self.assertEqual(self.tokenizer.suffix_tokens    ,   [self.tokenizer.eos_token_id]       )





					def 		UpperCAmelCase(self				:   Any       )						->						List[Any]:
									snake_case								= self.tokenizer(self.src_text    ,   padding=snake_case__    ,   truncation=snake_case__    ,   max_length=3    ,   return_tensors="pt"       )
									snake_case								= self.tokenizer(
									    text_target=self.tgt_text    ,   padding=snake_case__    ,   truncation=snake_case__    ,   max_length=1_0    ,   return_tensors="pt"       )
									snake_case								= targets['''input_ids''']
									snake_case								= shift_tokens_right(snake_case__    ,   self.tokenizer.pad_token_id       )

									self.assertEqual(batch.input_ids.shape[1]    ,   3       )
									self.assertEqual(batch.decoder_input_ids.shape[1]    ,   1_0       )





					@require_torch
					def 		UpperCAmelCase(self				:   Tuple       )						->						Optional[Any]:
									snake_case								= self.tokenizer._build_translation_inputs(
									    "A test"    ,   return_tensors="pt"    ,   src_lang="en_XX"    ,   tgt_lang="ar_AR"       )

									self.assertEqual(
									    nested_simplify(snake_case__       )    ,   {
									        # en_XX, A, test, EOS
									        "input_ids": [[2_5_0_0_0_4, 6_2, 3_0_3_4, 2]],
									        "attention_mask": [[1, 1, 1, 1]],
									        # ar_AR
									        "forced_bos_token_id": 2_5_0_0_0_1,
									    }    ,   )



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


_A   =			logging.get_logger(__name__)

_A   =			{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_A   =			{
    "vocab_file": {
        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt",
        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/vocab.txt",
        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/vocab.txt",
        "bert-base-multilingual-uncased": (
            "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt"
        ),
        "bert-base-multilingual-cased": "https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt",
        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt",
        "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt",
        "bert-large-uncased-whole-word-masking": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt"
        ),
        "bert-large-cased-whole-word-masking": (
            "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt"
        ),
        "bert-large-uncased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"
        ),
        "bert-large-cased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"
        ),
        "bert-base-cased-finetuned-mrpc": (
            "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt"
        ),
        "bert-base-german-dbmdz-cased": "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt",
        "bert-base-german-dbmdz-uncased": (
            "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt"
        ),
        "TurkuNLP/bert-base-finnish-cased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt"
        ),
        "TurkuNLP/bert-base-finnish-uncased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt"
        ),
        "wietsedv/bert-base-dutch-cased": (
            "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json",
        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json",
        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json",
        "bert-base-multilingual-uncased": (
            "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json"
        ),
        "bert-base-multilingual-cased": (
            "https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json",
        "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json",
        "bert-large-uncased-whole-word-masking": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json"
        ),
        "bert-large-cased-whole-word-masking": (
            "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json"
        ),
        "bert-large-uncased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"
        ),
        "bert-large-cased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"
        ),
        "bert-base-cased-finetuned-mrpc": (
            "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json"
        ),
        "bert-base-german-dbmdz-cased": (
            "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json"
        ),
        "bert-base-german-dbmdz-uncased": (
            "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json"
        ),
        "TurkuNLP/bert-base-finnish-cased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json"
        ),
        "TurkuNLP/bert-base-finnish-uncased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json"
        ),
        "wietsedv/bert-base-dutch-cased": (
            "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json"
        ),
    },
}

_A   =			{
    "bert-base-uncased": 5_12,
    "bert-large-uncased": 5_12,
    "bert-base-cased": 5_12,
    "bert-large-cased": 5_12,
    "bert-base-multilingual-uncased": 5_12,
    "bert-base-multilingual-cased": 5_12,
    "bert-base-chinese": 5_12,
    "bert-base-german-cased": 5_12,
    "bert-large-uncased-whole-word-masking": 5_12,
    "bert-large-cased-whole-word-masking": 5_12,
    "bert-large-uncased-whole-word-masking-finetuned-squad": 5_12,
    "bert-large-cased-whole-word-masking-finetuned-squad": 5_12,
    "bert-base-cased-finetuned-mrpc": 5_12,
    "bert-base-german-dbmdz-cased": 5_12,
    "bert-base-german-dbmdz-uncased": 5_12,
    "TurkuNLP/bert-base-finnish-cased-v1": 5_12,
    "TurkuNLP/bert-base-finnish-uncased-v1": 5_12,
    "wietsedv/bert-base-dutch-cased": 5_12,
}

_A   =			{
    "bert-base-uncased": {"do_lower_case": True},
    "bert-large-uncased": {"do_lower_case": True},
    "bert-base-cased": {"do_lower_case": False},
    "bert-large-cased": {"do_lower_case": False},
    "bert-base-multilingual-uncased": {"do_lower_case": True},
    "bert-base-multilingual-cased": {"do_lower_case": False},
    "bert-base-chinese": {"do_lower_case": False},
    "bert-base-german-cased": {"do_lower_case": False},
    "bert-large-uncased-whole-word-masking": {"do_lower_case": True},
    "bert-large-cased-whole-word-masking": {"do_lower_case": False},
    "bert-large-uncased-whole-word-masking-finetuned-squad": {"do_lower_case": True},
    "bert-large-cased-whole-word-masking-finetuned-squad": {"do_lower_case": False},
    "bert-base-cased-finetuned-mrpc": {"do_lower_case": False},
    "bert-base-german-dbmdz-cased": {"do_lower_case": False},
    "bert-base-german-dbmdz-uncased": {"do_lower_case": True},
    "TurkuNLP/bert-base-finnish-cased-v1": {"do_lower_case": False},
    "TurkuNLP/bert-base-finnish-uncased-v1": {"do_lower_case": True},
    "wietsedv/bert-base-dutch-cased": {"do_lower_case": False},
}







class 					lowerCamelCase		(  A_  ):
     UpperCAmelCase__    :							Optional[Any]									=		VOCAB_FILES_NAMES
     UpperCAmelCase__    :							Union[str, Any]									=		PRETRAINED_VOCAB_FILES_MAP
     UpperCAmelCase__    :							Union[str, Any]									=		PRETRAINED_INIT_CONFIGURATION
     UpperCAmelCase__    :							Any									=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     UpperCAmelCase__    :							int									=		BertTokenizer





     def __init__(self				:   Union[str, Any]    ,   _A				:   Union[str, Any]=None    ,   _A				:   Optional[int]=None    ,   _A				:   List[str]=True    ,   _A				:   List[Any]="[UNK]"    ,   _A				:   Union[str, Any]="[SEP]"    ,   _A				:   List[Any]="[PAD]"    ,   _A				:   List[Any]="[CLS]"    ,   _A				:   Union[str, Any]="[MASK]"    ,   _A				:   int=True    ,   _A				:   Tuple=None    ,   **_A				:   Optional[int]    ,   )						->						int:
         super().__init__(
             _A    ,   tokenizer_file=_A    ,   do_lower_case=_A    ,   unk_token=_A    ,   sep_token=_A    ,   pad_token=_A    ,   cls_token=_A    ,   mask_token=_A    ,   tokenize_chinese_chars=_A    ,   strip_accents=_A    ,   **_A    ,   )

         snake_case								= json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
         if (
             normalizer_state.get("lowercase"    ,   _A       ) != do_lower_case
             or normalizer_state.get("strip_accents"    ,   _A       ) != strip_accents
             or normalizer_state.get("handle_chinese_chars"    ,   _A       ) != tokenize_chinese_chars
         ):
             snake_case								= getattr(_A    ,   normalizer_state.pop("type"       )       )
             snake_case								= do_lower_case
             snake_case								= strip_accents
             snake_case								= tokenize_chinese_chars
             snake_case								= normalizer_class(**_A       )

         snake_case								= do_lower_case





     def 		UpperCAmelCase(self				:   str    ,   _A				:   Union[str, Any]    ,   _A				:   int=None       )						->						Any:
         snake_case								= [self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
             output += token_ids_a + [self.sep_token_id]

         return output





     def 		UpperCAmelCase(self				:   Optional[Any]    ,   _A				:   List[int]    ,   _A				:   Optional[List[int]] = None       )						->						List[int]:
         snake_case								= [self.sep_token_id]
         snake_case								= [self.cls_token_id]
         if token_ids_a is None:
             return len(cls + token_ids_a + sep       ) * [0]
         return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]





     def 		UpperCAmelCase(self				:   Union[str, Any]    ,   _A				:   str    ,   _A				:   Optional[str] = None       )						->						Tuple[str]:
         snake_case								= self._tokenizer.model.save(_A    ,   name=_A       )
         return tuple(_A       )

