





import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class     __SCREAMING_SNAKE_CASE   (       _lowerCAmelCase				,						unittest.TestCase    ):
							snake_case       : Union[str, Any]         =    BertTokenizer
							snake_case       : List[Any]         =    BertTokenizerFast
							snake_case       : List[Any]         =    True
							snake_case       : str         =    True
							snake_case       : Dict         =    filter_non_english






							def 			_lowerCamelCase						(  self					):
														super().setUp()

														UpperCamelCase__			       =     [
														    """[UNK]""",
														    """[CLS]""",
														    """[SEP]""",
														    """[PAD]""",
														    """[MASK]""",
														    """want""",
														    """##want""",
														    """##ed""",
														    """wa""",
														    """un""",
														    """runn""",
														    """##ing""",
														    """,""",
														    """low""",
														    """lowest""",
														]
														UpperCamelCase__			       =     os.path.join(self.tmpdirname					,	VOCAB_FILES_NAMES["""vocab_file"""]					)
														with open(self.vocab_file					,	"""w"""					,	encoding="""utf-8"""					) as vocab_writer:
																					vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]					)					)






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					):
														UpperCamelCase__			       =     """UNwant\u00E9d,running"""
														UpperCamelCase__			       =     """unwanted, running"""
														return input_text, output_text






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     self.tokenizer_class(self.vocab_file					)

														UpperCamelCase__			       =     tokenizer.tokenize("""UNwant\u00E9d,running"""					)
														self.assertListEqual(SCREAMING_SNAKE_CASE_					,	["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]					)
														self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_					)					,	[9, 6, 7, 12, 10, 11]					)






							def 			_lowerCamelCase						(  self					):
														if not self.test_rust_tokenizer:
																					return

														UpperCamelCase__			       =     self.get_tokenizer()
														UpperCamelCase__			       =     self.get_rust_tokenizer()

														UpperCamelCase__			       =     """UNwant\u00E9d,running"""

														UpperCamelCase__			       =     tokenizer.tokenize(SCREAMING_SNAKE_CASE_					)
														UpperCamelCase__			       =     rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE_					)
														self.assertListEqual(SCREAMING_SNAKE_CASE_					,	SCREAMING_SNAKE_CASE_					)

														UpperCamelCase__			       =     tokenizer.encode(SCREAMING_SNAKE_CASE_					,	add_special_tokens=SCREAMING_SNAKE_CASE_					)
														UpperCamelCase__			       =     rust_tokenizer.encode(SCREAMING_SNAKE_CASE_					,	add_special_tokens=SCREAMING_SNAKE_CASE_					)
														self.assertListEqual(SCREAMING_SNAKE_CASE_					,	SCREAMING_SNAKE_CASE_					)

														UpperCamelCase__			       =     self.get_rust_tokenizer()
														UpperCamelCase__			       =     tokenizer.encode(SCREAMING_SNAKE_CASE_					)
														UpperCamelCase__			       =     rust_tokenizer.encode(SCREAMING_SNAKE_CASE_					)
														self.assertListEqual(SCREAMING_SNAKE_CASE_					,	SCREAMING_SNAKE_CASE_					)

														# With lower casing
														UpperCamelCase__			       =     self.get_tokenizer(do_lower_case=SCREAMING_SNAKE_CASE_					)
														UpperCamelCase__			       =     self.get_rust_tokenizer(do_lower_case=SCREAMING_SNAKE_CASE_					)

														UpperCamelCase__			       =     """UNwant\u00E9d,running"""

														UpperCamelCase__			       =     tokenizer.tokenize(SCREAMING_SNAKE_CASE_					)
														UpperCamelCase__			       =     rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE_					)
														self.assertListEqual(SCREAMING_SNAKE_CASE_					,	SCREAMING_SNAKE_CASE_					)

														UpperCamelCase__			       =     tokenizer.encode(SCREAMING_SNAKE_CASE_					,	add_special_tokens=SCREAMING_SNAKE_CASE_					)
														UpperCamelCase__			       =     rust_tokenizer.encode(SCREAMING_SNAKE_CASE_					,	add_special_tokens=SCREAMING_SNAKE_CASE_					)
														self.assertListEqual(SCREAMING_SNAKE_CASE_					,	SCREAMING_SNAKE_CASE_					)

														UpperCamelCase__			       =     self.get_rust_tokenizer()
														UpperCamelCase__			       =     tokenizer.encode(SCREAMING_SNAKE_CASE_					)
														UpperCamelCase__			       =     rust_tokenizer.encode(SCREAMING_SNAKE_CASE_					)
														self.assertListEqual(SCREAMING_SNAKE_CASE_					,	SCREAMING_SNAKE_CASE_					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     BasicTokenizer()

														self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""					)					,	["""ah""", """\u535A""", """\u63A8""", """zz"""]					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_					)

														self.assertListEqual(
														    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """					)					,	["""hello""", """!""", """how""", """are""", """you""", """?"""]					)
														self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""					)					,	["""hello"""]					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_					,	strip_accents=SCREAMING_SNAKE_CASE_					)

														self.assertListEqual(
														    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """					)					,	["""hällo""", """!""", """how""", """are""", """you""", """?"""]					)
														self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""					)					,	["""h\u00E9llo"""]					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_					,	strip_accents=SCREAMING_SNAKE_CASE_					)

														self.assertListEqual(
														    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """					)					,	["""hallo""", """!""", """how""", """are""", """you""", """?"""]					)
														self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""					)					,	["""hello"""]					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_					)

														self.assertListEqual(
														    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """					)					,	["""hallo""", """!""", """how""", """are""", """you""", """?"""]					)
														self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""					)					,	["""hello"""]					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_					)

														self.assertListEqual(
														    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """					)					,	["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_					,	strip_accents=SCREAMING_SNAKE_CASE_					)

														self.assertListEqual(
														    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """					)					,	["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_					,	strip_accents=SCREAMING_SNAKE_CASE_					)

														self.assertListEqual(
														    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """					)					,	["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_					,	never_split=["""[UNK]"""]					)

														self.assertListEqual(
														    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""					)					,	["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     BasicTokenizer()
														UpperCamelCase__			       =     """a\n'll !!to?'d of, can't."""
														UpperCamelCase__			       =     ["""a""", """'""", """ll""", """!""", """!""", """to""", """?""", """'""", """d""", """of""", """,""", """can""", """'""", """t""", """."""]
														self.assertListEqual(tokenizer.tokenize(SCREAMING_SNAKE_CASE_					)					,	SCREAMING_SNAKE_CASE_					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

														UpperCamelCase__			       =     {}
														for i, token in enumerate(SCREAMING_SNAKE_CASE_					):
																					UpperCamelCase__			       =     i
														UpperCamelCase__			       =     WordpieceTokenizer(vocab=SCREAMING_SNAKE_CASE_					,	unk_token="""[UNK]"""					)

														self.assertListEqual(tokenizer.tokenize(""""""					)					,	[]					)

														self.assertListEqual(tokenizer.tokenize("""unwanted running"""					)					,	["""un""", """##want""", """##ed""", """runn""", """##ing"""]					)

														self.assertListEqual(tokenizer.tokenize("""unwantedX running"""					)					,	["""[UNK]""", """runn""", """##ing"""]					)






							def 			_lowerCamelCase						(  self					):
														self.assertTrue(_is_whitespace(""" """					)					)
														self.assertTrue(_is_whitespace("""\t"""					)					)
														self.assertTrue(_is_whitespace("""\r"""					)					)
														self.assertTrue(_is_whitespace("""\n"""					)					)
														self.assertTrue(_is_whitespace("""\u00A0"""					)					)

														self.assertFalse(_is_whitespace("""A"""					)					)
														self.assertFalse(_is_whitespace("""-"""					)					)






							def 			_lowerCamelCase						(  self					):
														self.assertTrue(_is_control("""\u0005"""					)					)

														self.assertFalse(_is_control("""A"""					)					)
														self.assertFalse(_is_control(""" """					)					)
														self.assertFalse(_is_control("""\t"""					)					)
														self.assertFalse(_is_control("""\r"""					)					)






							def 			_lowerCamelCase						(  self					):
														self.assertTrue(_is_punctuation("""-"""					)					)
														self.assertTrue(_is_punctuation("""$"""					)					)
														self.assertTrue(_is_punctuation("""`"""					)					)
														self.assertTrue(_is_punctuation("""."""					)					)

														self.assertFalse(_is_punctuation("""A"""					)					)
														self.assertFalse(_is_punctuation(""" """					)					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     self.get_tokenizer()
														UpperCamelCase__			       =     self.get_rust_tokenizer()

														# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
														self.assertListEqual([tokenizer.tokenize(SCREAMING_SNAKE_CASE_					) for t in ["""Test""", """\xad""", """test"""]]					,	[["""[UNK]"""], [], ["""[UNK]"""]]					)

														self.assertListEqual(
														    [rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE_					) for t in ["""Test""", """\xad""", """test"""]]					,	[["""[UNK]"""], [], ["""[UNK]"""]]					)






							@slow
							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     self.tokenizer_class.from_pretrained("""bert-base-uncased"""					)

														UpperCamelCase__			       =     tokenizer.encode("""sequence builders"""					,	add_special_tokens=SCREAMING_SNAKE_CASE_					)
														UpperCamelCase__			       =     tokenizer.encode("""multi-sequence build"""					,	add_special_tokens=SCREAMING_SNAKE_CASE_					)

														UpperCamelCase__			       =     tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_					)
														UpperCamelCase__			       =     tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_					,	SCREAMING_SNAKE_CASE_					)

														assert encoded_sentence == [101] + text + [102]
														assert encoded_pair == [101] + text + [102] + text_a + [102]






							def 			_lowerCamelCase						(  self					):
														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""					):
																												UpperCamelCase__			       =     self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_					,	**SCREAMING_SNAKE_CASE_					)

																												UpperCamelCase__			       =     f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																												UpperCamelCase__			       =     tokenizer_r.encode_plus(
																												    SCREAMING_SNAKE_CASE_					,	return_attention_mask=SCREAMING_SNAKE_CASE_					,	return_token_type_ids=SCREAMING_SNAKE_CASE_					,	return_offsets_mapping=SCREAMING_SNAKE_CASE_					,	add_special_tokens=SCREAMING_SNAKE_CASE_					,	)

																												UpperCamelCase__			       =     tokenizer_r.do_lower_case if hasattr(SCREAMING_SNAKE_CASE_					,	"""do_lower_case"""					) else False
																												UpperCamelCase__			       =     (
																												    [
																												        ((0, 0), tokenizer_r.cls_token),
																												        ((0, 1), """A"""),
																												        ((1, 2), ""","""),
																												        ((3, 5), """na"""),
																												        ((5, 6), """##ï"""),
																												        ((6, 8), """##ve"""),
																												        ((9, 15), tokenizer_r.mask_token),
																												        ((16, 21), """Allen"""),
																												        ((21, 23), """##NL"""),
																												        ((23, 24), """##P"""),
																												        ((25, 33), """sentence"""),
																												        ((33, 34), """."""),
																												        ((0, 0), tokenizer_r.sep_token),
																												    ]
																												    if not do_lower_case
																												    else [
																												        ((0, 0), tokenizer_r.cls_token),
																												        ((0, 1), """a"""),
																												        ((1, 2), ""","""),
																												        ((3, 8), """naive"""),
																												        ((9, 15), tokenizer_r.mask_token),
																												        ((16, 21), """allen"""),
																												        ((21, 23), """##nl"""),
																												        ((23, 24), """##p"""),
																												        ((25, 33), """sentence"""),
																												        ((33, 34), """."""),
																												        ((0, 0), tokenizer_r.sep_token),
																												    ]
																												)

																												self.assertEqual(
																												    [e[1] for e in expected_results]					,	tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]					)					)
																												self.assertEqual([e[0] for e in expected_results]					,	tokens["""offset_mapping"""]					)







							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     ["""的""", """人""", """有"""]
														UpperCamelCase__			       =     """""".join(SCREAMING_SNAKE_CASE_					)
														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""					):
																												UpperCamelCase__			       =     True
																												UpperCamelCase__			       =     self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_					,	**SCREAMING_SNAKE_CASE_					)
																												UpperCamelCase__			       =     self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_					,	**SCREAMING_SNAKE_CASE_					)

																												UpperCamelCase__			       =     tokenizer_p.encode(SCREAMING_SNAKE_CASE_					,	add_special_tokens=SCREAMING_SNAKE_CASE_					)
																												UpperCamelCase__			       =     tokenizer_r.encode(SCREAMING_SNAKE_CASE_					,	add_special_tokens=SCREAMING_SNAKE_CASE_					)

																												UpperCamelCase__			       =     tokenizer_r.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_					)
																												UpperCamelCase__			       =     tokenizer_p.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_					)

																												# it is expected that each Chinese character is not preceded by "##"
																												self.assertListEqual(SCREAMING_SNAKE_CASE_					,	SCREAMING_SNAKE_CASE_					)
																												self.assertListEqual(SCREAMING_SNAKE_CASE_					,	SCREAMING_SNAKE_CASE_					)

																												UpperCamelCase__			       =     False
																												UpperCamelCase__			       =     self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_					,	**SCREAMING_SNAKE_CASE_					)
																												UpperCamelCase__			       =     self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_					,	**SCREAMING_SNAKE_CASE_					)

																												UpperCamelCase__			       =     tokenizer_r.encode(SCREAMING_SNAKE_CASE_					,	add_special_tokens=SCREAMING_SNAKE_CASE_					)
																												UpperCamelCase__			       =     tokenizer_p.encode(SCREAMING_SNAKE_CASE_					,	add_special_tokens=SCREAMING_SNAKE_CASE_					)

																												UpperCamelCase__			       =     tokenizer_r.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_					)
																												UpperCamelCase__			       =     tokenizer_p.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_					)

																												# it is expected that only the first Chinese character is not preceded by "##".
																												UpperCamelCase__			       =     [
																												    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(SCREAMING_SNAKE_CASE_					)
																												]
																												self.assertListEqual(SCREAMING_SNAKE_CASE_					,	SCREAMING_SNAKE_CASE_					)
																												self.assertListEqual(SCREAMING_SNAKE_CASE_					,	SCREAMING_SNAKE_CASE_					)











from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase__     =	logging.get_logger(__name__)

UpperCamelCase__     =	{
    "microsoft/cvt-13": "https://huggingface.co/microsoft/cvt-13/resolve/main/config.json",
    # See all Cvt models at https://huggingface.co/models?filter=cvt
}

class     __SCREAMING_SNAKE_CASE   (       _a    ):
							snake_case       : int         =    """cvt"""






							def __init__(  self					,	__lowerCAmelCase=3					,	__lowerCAmelCase=[7, 3, 3]					,	__lowerCAmelCase=[4, 2, 2]					,	__lowerCAmelCase=[2, 1, 1]					,	__lowerCAmelCase=[64, 192, 384]					,	__lowerCAmelCase=[1, 3, 6]					,	__lowerCAmelCase=[1, 2, 10]					,	__lowerCAmelCase=[4.0, 4.0, 4.0]					,	__lowerCAmelCase=[0.0, 0.0, 0.0]					,	__lowerCAmelCase=[0.0, 0.0, 0.0]					,	__lowerCAmelCase=[0.0, 0.0, 0.1]					,	__lowerCAmelCase=[True, True, True]					,	__lowerCAmelCase=[False, False, True]					,	__lowerCAmelCase=["dw_bn", "dw_bn", "dw_bn"]					,	__lowerCAmelCase=[3, 3, 3]					,	__lowerCAmelCase=[1, 1, 1]					,	__lowerCAmelCase=[2, 2, 2]					,	__lowerCAmelCase=[1, 1, 1]					,	__lowerCAmelCase=[1, 1, 1]					,	__lowerCAmelCase=0.02					,	__lowerCAmelCase=1E-12					,	**__lowerCAmelCase					,	):
														super().__init__(**__lowerCAmelCase					)
														UpperCamelCase__			       =     num_channels
														UpperCamelCase__			       =     patch_sizes
														UpperCamelCase__			       =     patch_stride
														UpperCamelCase__			       =     patch_padding
														UpperCamelCase__			       =     embed_dim
														UpperCamelCase__			       =     num_heads
														UpperCamelCase__			       =     depth
														UpperCamelCase__			       =     mlp_ratio
														UpperCamelCase__			       =     attention_drop_rate
														UpperCamelCase__			       =     drop_rate
														UpperCamelCase__			       =     drop_path_rate
														UpperCamelCase__			       =     qkv_bias
														UpperCamelCase__			       =     cls_token
														UpperCamelCase__			       =     qkv_projection_method
														UpperCamelCase__			       =     kernel_qkv
														UpperCamelCase__			       =     padding_kv
														UpperCamelCase__			       =     stride_kv
														UpperCamelCase__			       =     padding_q
														UpperCamelCase__			       =     stride_q
														UpperCamelCase__			       =     initializer_range
														UpperCamelCase__			       =     layer_norm_eps




