

"""simple docstring"""




import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class   __lowercase    (			__lowerCAmelCase		,	unittest.TestCase     ):




			"""simple docstring"""







			_UpperCAmelCase       :							List[str]      							=					BertTokenizer
			_UpperCAmelCase       :							List[str]      							=					BertTokenizerFast
			_UpperCAmelCase       :							Union[str, Any]      							=					True
			_UpperCAmelCase       :							Dict      							=					True
			_UpperCAmelCase       :							str      							=					filter_non_english



			def 	_SCREAMING_SNAKE_CASE      (			self  :					int):
									super().setUp()

									SCREAMING_SNAKE_CASE_:			Optional[int]  		=	[
									    "[UNK]",
									    "[CLS]",
									    "[SEP]",
									    "[PAD]",
									    "[MASK]",
									    "want",
									    "##want",
									    "##ed",
									    "wa",
									    "un",
									    "runn",
									    "##ing",
									    ",",
									    "low",
									    "lowest",
									]
									SCREAMING_SNAKE_CASE_:			Optional[int]  		=	os.path.join(self.tmpdirname       ,						VOCAB_FILES_NAMES["vocab_file"])
									with open(self.vocab_file       ,						"w"       ,						encoding="utf-8") as vocab_writer:
															vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))



			def 	_SCREAMING_SNAKE_CASE      (			self  :					List[str]       ,						lowerCAmelCase__  :					Any):
									SCREAMING_SNAKE_CASE_:			Optional[int]  		=	"UNwant\u00E9d,running"
									SCREAMING_SNAKE_CASE_:			List[str]  		=	"unwanted, running"
									return input_text, output_text



			def 	_SCREAMING_SNAKE_CASE      (			self  :					str):
									SCREAMING_SNAKE_CASE_:			str  		=	self.tokenizer_class(self.vocab_file)

									SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	tokenizer.tokenize("UNwant\u00E9d,running")
									self.assertListEqual(lowerCAmelCase_       ,						["un", "##want", "##ed", ",", "runn", "##ing"])
									self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_)       ,						[9, 6, 7, 12, 10, 11])



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Optional[Any]):
									if not self.test_rust_tokenizer:
															return

									SCREAMING_SNAKE_CASE_:			Any  		=	self.get_tokenizer()
									SCREAMING_SNAKE_CASE_:			Dict  		=	self.get_rust_tokenizer()

									SCREAMING_SNAKE_CASE_:			Tuple  		=	"UNwant\u00E9d,running"

									SCREAMING_SNAKE_CASE_:			Any  		=	tokenizer.tokenize(lowerCAmelCase_)
									SCREAMING_SNAKE_CASE_:			Optional[int]  		=	rust_tokenizer.tokenize(lowerCAmelCase_)
									self.assertListEqual(lowerCAmelCase_       ,						lowerCAmelCase_)

									SCREAMING_SNAKE_CASE_:			Tuple  		=	tokenizer.encode(lowerCAmelCase_       ,						add_special_tokens=lowerCAmelCase_)
									SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	rust_tokenizer.encode(lowerCAmelCase_       ,						add_special_tokens=lowerCAmelCase_)
									self.assertListEqual(lowerCAmelCase_       ,						lowerCAmelCase_)

									SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	self.get_rust_tokenizer()
									SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	tokenizer.encode(lowerCAmelCase_)
									SCREAMING_SNAKE_CASE_:			int  		=	rust_tokenizer.encode(lowerCAmelCase_)
									self.assertListEqual(lowerCAmelCase_       ,						lowerCAmelCase_)

									# With lower casing
									SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	self.get_tokenizer(do_lower_case=lowerCAmelCase_)
									SCREAMING_SNAKE_CASE_:			int  		=	self.get_rust_tokenizer(do_lower_case=lowerCAmelCase_)

									SCREAMING_SNAKE_CASE_:			Optional[int]  		=	"UNwant\u00E9d,running"

									SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	tokenizer.tokenize(lowerCAmelCase_)
									SCREAMING_SNAKE_CASE_:			Dict  		=	rust_tokenizer.tokenize(lowerCAmelCase_)
									self.assertListEqual(lowerCAmelCase_       ,						lowerCAmelCase_)

									SCREAMING_SNAKE_CASE_:			Tuple  		=	tokenizer.encode(lowerCAmelCase_       ,						add_special_tokens=lowerCAmelCase_)
									SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	rust_tokenizer.encode(lowerCAmelCase_       ,						add_special_tokens=lowerCAmelCase_)
									self.assertListEqual(lowerCAmelCase_       ,						lowerCAmelCase_)

									SCREAMING_SNAKE_CASE_:			str  		=	self.get_rust_tokenizer()
									SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	tokenizer.encode(lowerCAmelCase_)
									SCREAMING_SNAKE_CASE_:			Tuple  		=	rust_tokenizer.encode(lowerCAmelCase_)
									self.assertListEqual(lowerCAmelCase_       ,						lowerCAmelCase_)



			def 	_SCREAMING_SNAKE_CASE      (			self  :					str):
									SCREAMING_SNAKE_CASE_:			int  		=	BasicTokenizer()

									self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz")       ,						["ah", "\u535A", "\u63A8", "zz"])



			def 	_SCREAMING_SNAKE_CASE      (			self  :					List[str]):
									SCREAMING_SNAKE_CASE_:			List[Any]  		=	BasicTokenizer(do_lower_case=lowerCAmelCase_)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  ")       ,						["hello", "!", "how", "are", "you", "?"])
									self.assertListEqual(tokenizer.tokenize("H\u00E9llo")       ,						["hello"])



			def 	_SCREAMING_SNAKE_CASE      (			self  :					str):
									SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	BasicTokenizer(do_lower_case=lowerCAmelCase_       ,						strip_accents=lowerCAmelCase_)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")       ,						["hällo", "!", "how", "are", "you", "?"])
									self.assertListEqual(tokenizer.tokenize("H\u00E9llo")       ,						["h\u00E9llo"])



			def 	_SCREAMING_SNAKE_CASE      (			self  :					List[Any]):
									SCREAMING_SNAKE_CASE_:			Optional[int]  		=	BasicTokenizer(do_lower_case=lowerCAmelCase_       ,						strip_accents=lowerCAmelCase_)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")       ,						["hallo", "!", "how", "are", "you", "?"])
									self.assertListEqual(tokenizer.tokenize("H\u00E9llo")       ,						["hello"])



			def 	_SCREAMING_SNAKE_CASE      (			self  :					int):
									SCREAMING_SNAKE_CASE_:			Dict  		=	BasicTokenizer(do_lower_case=lowerCAmelCase_)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")       ,						["hallo", "!", "how", "are", "you", "?"])
									self.assertListEqual(tokenizer.tokenize("H\u00E9llo")       ,						["hello"])



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Union[str, Any]):
									SCREAMING_SNAKE_CASE_:			List[Any]  		=	BasicTokenizer(do_lower_case=lowerCAmelCase_)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  ")       ,						["HeLLo", "!", "how", "Are", "yoU", "?"])



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Optional[Any]):
									SCREAMING_SNAKE_CASE_:			Dict  		=	BasicTokenizer(do_lower_case=lowerCAmelCase_       ,						strip_accents=lowerCAmelCase_)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")       ,						["HäLLo", "!", "how", "Are", "yoU", "?"])



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Dict):
									SCREAMING_SNAKE_CASE_:			List[str]  		=	BasicTokenizer(do_lower_case=lowerCAmelCase_       ,						strip_accents=lowerCAmelCase_)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")       ,						["HaLLo", "!", "how", "Are", "yoU", "?"])



			def 	_SCREAMING_SNAKE_CASE      (			self  :					int):
									SCREAMING_SNAKE_CASE_:			Tuple  		=	BasicTokenizer(do_lower_case=lowerCAmelCase_       ,						never_split=["[UNK]"])

									self.assertListEqual(
									    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]")       ,						["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"])



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Any):
									SCREAMING_SNAKE_CASE_:			int  		=	BasicTokenizer()
									SCREAMING_SNAKE_CASE_:			List[Any]  		=	"a\n'll !!to?'d of, can't."
									SCREAMING_SNAKE_CASE_:			Optional[int]  		=	["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
									self.assertListEqual(tokenizer.tokenize(lowerCAmelCase_)       ,						lowerCAmelCase_)



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Any):
									SCREAMING_SNAKE_CASE_:			Tuple  		=	["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

									SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	{}
									for i, token in enumerate(lowerCAmelCase_):
															SCREAMING_SNAKE_CASE_:			List[Any]  		=	i
									SCREAMING_SNAKE_CASE_:			int  		=	WordpieceTokenizer(vocab=lowerCAmelCase_       ,						unk_token="[UNK]")

									self.assertListEqual(tokenizer.tokenize("")       ,						[])

									self.assertListEqual(tokenizer.tokenize("unwanted running")       ,						["un", "##want", "##ed", "runn", "##ing"])

									self.assertListEqual(tokenizer.tokenize("unwantedX running")       ,						["[UNK]", "runn", "##ing"])



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Dict):
									self.assertTrue(_is_whitespace(" "))
									self.assertTrue(_is_whitespace("\t"))
									self.assertTrue(_is_whitespace("\r"))
									self.assertTrue(_is_whitespace("\n"))
									self.assertTrue(_is_whitespace("\u00A0"))

									self.assertFalse(_is_whitespace("A"))
									self.assertFalse(_is_whitespace("-"))



			def 	_SCREAMING_SNAKE_CASE      (			self  :					str):
									self.assertTrue(_is_control("\u0005"))

									self.assertFalse(_is_control("A"))
									self.assertFalse(_is_control(" "))
									self.assertFalse(_is_control("\t"))
									self.assertFalse(_is_control("\r"))



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Optional[Any]):
									self.assertTrue(_is_punctuation("-"))
									self.assertTrue(_is_punctuation("$"))
									self.assertTrue(_is_punctuation("`"))
									self.assertTrue(_is_punctuation("."))

									self.assertFalse(_is_punctuation("A"))
									self.assertFalse(_is_punctuation(" "))



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Any):
									SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	self.get_tokenizer()
									SCREAMING_SNAKE_CASE_:			Dict  		=	self.get_rust_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(lowerCAmelCase_) for t in ["Test", "\xad", "test"]]       ,						[["[UNK]"], [], ["[UNK]"]])

									self.assertListEqual(
									    [rust_tokenizer.tokenize(lowerCAmelCase_) for t in ["Test", "\xad", "test"]]       ,						[["[UNK]"], [], ["[UNK]"]])



			@slow
			def 	_SCREAMING_SNAKE_CASE      (			self  :					List[Any]):
									SCREAMING_SNAKE_CASE_:			List[str]  		=	self.tokenizer_class.from_pretrained("bert-base-uncased")

									SCREAMING_SNAKE_CASE_:			int  		=	tokenizer.encode("sequence builders"       ,						add_special_tokens=lowerCAmelCase_)
									SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	tokenizer.encode("multi-sequence build"       ,						add_special_tokens=lowerCAmelCase_)

									SCREAMING_SNAKE_CASE_:			Dict  		=	tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_)
									SCREAMING_SNAKE_CASE_:			Dict  		=	tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_       ,						lowerCAmelCase_)

									assert encoded_sentence == [101] + text + [102]
									assert encoded_pair == [101] + text + [102] + text_a + [102]



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Tuple):
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"):
																					SCREAMING_SNAKE_CASE_:			List[Any]  		=	self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_       ,						**lowerCAmelCase_)

																					SCREAMING_SNAKE_CASE_:			Dict  		=	F"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
																					SCREAMING_SNAKE_CASE_:			Dict  		=	tokenizer_r.encode_plus(
																					    lowerCAmelCase_       ,						return_attention_mask=lowerCAmelCase_       ,						return_token_type_ids=lowerCAmelCase_       ,						return_offsets_mapping=lowerCAmelCase_       ,						add_special_tokens=lowerCAmelCase_       ,						)

																					SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	tokenizer_r.do_lower_case if hasattr(lowerCAmelCase_       ,						"do_lower_case") else False
																					SCREAMING_SNAKE_CASE_:			List[str]  		=	(
																					    [
																					        ((0, 0), tokenizer_r.cls_token),
																					        ((0, 1), "A"),
																					        ((1, 2), ","),
																					        ((3, 5), "na"),
																					        ((5, 6), "##ï"),
																					        ((6, 8), "##ve"),
																					        ((9, 15), tokenizer_r.mask_token),
																					        ((16, 21), "Allen"),
																					        ((21, 23), "##NL"),
																					        ((23, 24), "##P"),
																					        ((25, 33), "sentence"),
																					        ((33, 34), "."),
																					        ((0, 0), tokenizer_r.sep_token),
																					    ]
																					    if not do_lower_case
																					    else [
																					        ((0, 0), tokenizer_r.cls_token),
																					        ((0, 1), "a"),
																					        ((1, 2), ","),
																					        ((3, 8), "naive"),
																					        ((9, 15), tokenizer_r.mask_token),
																					        ((16, 21), "allen"),
																					        ((21, 23), "##nl"),
																					        ((23, 24), "##p"),
																					        ((25, 33), "sentence"),
																					        ((33, 34), "."),
																					        ((0, 0), tokenizer_r.sep_token),
																					    ]
																					)

																					self.assertEqual(
																					    [e[1] for e in expected_results]       ,						tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]))
																					self.assertEqual([e[0] for e in expected_results]       ,						tokens["offset_mapping"])




			def 	_SCREAMING_SNAKE_CASE      (			self  :					Dict):
									SCREAMING_SNAKE_CASE_:			Any  		=	["的", "人", "有"]
									SCREAMING_SNAKE_CASE_:			Tuple  		=	"".join(lowerCAmelCase_)
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"):
																					SCREAMING_SNAKE_CASE_:			Optional[int]  		=	True
																					SCREAMING_SNAKE_CASE_:			Optional[int]  		=	self.tokenizer_class.from_pretrained(lowerCAmelCase_       ,						**lowerCAmelCase_)
																					SCREAMING_SNAKE_CASE_:			Dict  		=	self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_       ,						**lowerCAmelCase_)

																					SCREAMING_SNAKE_CASE_:			str  		=	tokenizer_p.encode(lowerCAmelCase_       ,						add_special_tokens=lowerCAmelCase_)
																					SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	tokenizer_r.encode(lowerCAmelCase_       ,						add_special_tokens=lowerCAmelCase_)

																					SCREAMING_SNAKE_CASE_:			Tuple  		=	tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_)
																					SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_)

																					# it is expected that each Chinese character is not preceded by "##"
																					self.assertListEqual(lowerCAmelCase_       ,						lowerCAmelCase_)
																					self.assertListEqual(lowerCAmelCase_       ,						lowerCAmelCase_)

																					SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	False
																					SCREAMING_SNAKE_CASE_:			Tuple  		=	self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_       ,						**lowerCAmelCase_)
																					SCREAMING_SNAKE_CASE_:			List[Any]  		=	self.tokenizer_class.from_pretrained(lowerCAmelCase_       ,						**lowerCAmelCase_)

																					SCREAMING_SNAKE_CASE_:			List[str]  		=	tokenizer_r.encode(lowerCAmelCase_       ,						add_special_tokens=lowerCAmelCase_)
																					SCREAMING_SNAKE_CASE_:			Any  		=	tokenizer_p.encode(lowerCAmelCase_       ,						add_special_tokens=lowerCAmelCase_)

																					SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_)
																					SCREAMING_SNAKE_CASE_:			Optional[int]  		=	tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_)

																					# it is expected that only the first Chinese character is not preceded by "##".
																					SCREAMING_SNAKE_CASE_:			str  		=	[
																					    F"##{token}" if idx != 0 else token for idx, token in enumerate(lowerCAmelCase_)
																					]
																					self.assertListEqual(lowerCAmelCase_       ,						lowerCAmelCase_)
																					self.assertListEqual(lowerCAmelCase_       ,						lowerCAmelCase_)









def       A_				(   _UpperCAmelCase = 10**9   ):
						SCREAMING_SNAKE_CASE_:			List[str]  		=	1
						SCREAMING_SNAKE_CASE_:			Optional[int]  		=	2

						SCREAMING_SNAKE_CASE_:			int  		=	0
						SCREAMING_SNAKE_CASE_:			Dict  		=	0
						SCREAMING_SNAKE_CASE_:			List[str]  		=	0
						while perimeter <= max_perimeter:
												perimeters_sum += perimeter

												prev_value += 2 * value
												value += prev_value

												SCREAMING_SNAKE_CASE_:			Any  		=	2 * value + 2 if i % 2 == 0 else 2 * value - 2
												i += 1

						return perimeters_sum


if __name__ == "__main__":
						print(f'''{solution() = }''')






