



import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class 	lowerCamelCase__      (							_SCREAMING_SNAKE_CASE				,    unittest.TestCase):




		'''simple docstring'''



		snake_case_     =RoCBertTokenizer
		snake_case_     =None
		snake_case_     =False
		snake_case_     =True
		snake_case_     =filter_non_english



		def 		lowerCAmelCase__				(self			)					->      Tuple:

						"""simple docstring"""






						super().setUp()

						lowerCAmelCase__					:    List[Any]												=					['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''你''', '''好''', '''是''', '''谁''', '''a''', '''b''', '''c''', '''d''']
						lowerCAmelCase__					:    Optional[int]												=					{}
						lowerCAmelCase__					:    Optional[Any]												=					{}
						for i, value in enumerate(A_			):
										lowerCAmelCase__					:    Optional[int]												=					i
										lowerCAmelCase__					:    int												=					i
						lowerCAmelCase__					:    Dict												=					os.path.join(self.tmpdirname		,VOCAB_FILES_NAMES['''vocab_file''']			)
						lowerCAmelCase__					:    List[str]												=					os.path.join(self.tmpdirname		,VOCAB_FILES_NAMES['''word_shape_file''']			)
						lowerCAmelCase__					:    Any												=					os.path.join(self.tmpdirname		,VOCAB_FILES_NAMES['''word_pronunciation_file''']			)
						with open(self.vocab_file		,'''w'''		,encoding='''utf-8'''			) as vocab_writer:
										vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]			)			)
						with open(self.word_shape_file		,'''w'''		,encoding='''utf-8'''			) as word_shape_writer:
										json.dump(A_		,A_		,ensure_ascii=A_			)
						with open(self.word_pronunciation_file		,'''w'''		,encoding='''utf-8'''			) as word_pronunciation_writer:
										json.dump(A_		,A_		,ensure_ascii=A_			)



		def 		lowerCAmelCase__				(self			)					->      Any:

						"""simple docstring"""






						lowerCAmelCase__					:    Union[str, Any]												=					self.tokenizer_class(self.vocab_file		,self.word_shape_file		,self.word_pronunciation_file			)

						lowerCAmelCase__					:    Optional[int]												=					tokenizer.tokenize('''你好[SEP]你是谁'''			)
						self.assertListEqual(A_		,['''你''', '''好''', '''[SEP]''', '''你''', '''是''', '''谁''']			)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(A_			)		,[5, 6, 2, 5, 7, 8]			)
						self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(A_			)		,[5, 6, 2, 5, 7, 8]			)
						self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(A_			)		,[5, 6, 2, 5, 7, 8]			)



		def 		lowerCAmelCase__				(self			)					->      Union[str, Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    List[Any]												=					RoCBertBasicTokenizer()

						self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''			)		,['''ah''', '''\u535A''', '''\u63A8''', '''zz''']			)



		def 		lowerCAmelCase__				(self			)					->      List[str]:

						"""simple docstring"""






						lowerCAmelCase__					:    Optional[int]												=					RoCBertBasicTokenizer(do_lower_case=A_			)

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''			)		,['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']			)
						self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''			)		,['''hello''']			)



		def 		lowerCAmelCase__				(self			)					->      str:

						"""simple docstring"""






						lowerCAmelCase__					:    int												=					RoCBertBasicTokenizer(do_lower_case=A_		,strip_accents=A_			)

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			)		,['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']			)
						self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''			)		,['''h\u00E9llo''']			)



		def 		lowerCAmelCase__				(self			)					->      int:

						"""simple docstring"""






						lowerCAmelCase__					:    int												=					RoCBertBasicTokenizer(do_lower_case=A_		,strip_accents=A_			)

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			)		,['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']			)
						self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''			)		,['''hello''']			)



		def 		lowerCAmelCase__				(self			)					->      Dict:

						"""simple docstring"""






						lowerCAmelCase__					:    Optional[Any]												=					RoCBertBasicTokenizer(do_lower_case=A_			)

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			)		,['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']			)
						self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''			)		,['''hello''']			)



		def 		lowerCAmelCase__				(self			)					->      str:

						"""simple docstring"""






						lowerCAmelCase__					:    List[Any]												=					RoCBertBasicTokenizer(do_lower_case=A_			)

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''			)		,['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']			)



		def 		lowerCAmelCase__				(self			)					->      Union[str, Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    List[str]												=					RoCBertBasicTokenizer(do_lower_case=A_		,strip_accents=A_			)

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			)		,['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']			)



		def 		lowerCAmelCase__				(self			)					->      Union[str, Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    Any												=					RoCBertBasicTokenizer(do_lower_case=A_		,strip_accents=A_			)

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			)		,['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']			)



		def 		lowerCAmelCase__				(self			)					->      Optional[Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    Any												=					RoCBertBasicTokenizer(do_lower_case=A_		,never_split=['''[UNK]''']			)

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''			)		,['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']			)



		def 		lowerCAmelCase__				(self			)					->      Optional[int]:

						"""simple docstring"""






						lowerCAmelCase__					:    List[str]												=					['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

						lowerCAmelCase__					:    Tuple												=					{}
						for i, token in enumerate(A_			):
										lowerCAmelCase__					:    List[Any]												=					i
						lowerCAmelCase__					:    Dict												=					RoCBertWordpieceTokenizer(vocab=A_		,unk_token='''[UNK]'''			)

						self.assertListEqual(tokenizer.tokenize(''''''			)		,[]			)

						self.assertListEqual(tokenizer.tokenize('''unwanted running'''			)		,['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']			)

						self.assertListEqual(tokenizer.tokenize('''unwantedX running'''			)		,['''[UNK]''', '''runn''', '''##ing''']			)



		def 		lowerCAmelCase__				(self			)					->      Optional[int]:

						"""simple docstring"""






						self.assertTrue(_is_whitespace(''' '''			)			)
						self.assertTrue(_is_whitespace('''\t'''			)			)
						self.assertTrue(_is_whitespace('''\r'''			)			)
						self.assertTrue(_is_whitespace('''\n'''			)			)
						self.assertTrue(_is_whitespace('''\u00A0'''			)			)

						self.assertFalse(_is_whitespace('''A'''			)			)
						self.assertFalse(_is_whitespace('''-'''			)			)



		def 		lowerCAmelCase__				(self			)					->      int:

						"""simple docstring"""






						self.assertTrue(_is_control('''\u0005'''			)			)

						self.assertFalse(_is_control('''A'''			)			)
						self.assertFalse(_is_control(''' '''			)			)
						self.assertFalse(_is_control('''\t'''			)			)
						self.assertFalse(_is_control('''\r'''			)			)



		def 		lowerCAmelCase__				(self			)					->      Optional[int]:

						"""simple docstring"""






						self.assertTrue(_is_punctuation('''-'''			)			)
						self.assertTrue(_is_punctuation('''$'''			)			)
						self.assertTrue(_is_punctuation('''`'''			)			)
						self.assertTrue(_is_punctuation('''.'''			)			)

						self.assertFalse(_is_punctuation('''A'''			)			)
						self.assertFalse(_is_punctuation(''' '''			)			)



		def 		lowerCAmelCase__				(self			)					->      Optional[Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    List[Any]												=					self.get_tokenizer()

						# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
						self.assertListEqual([tokenizer.tokenize(A_			) for t in ['''Test''', '''\xad''', '''test''']]		,[['''[UNK]'''], [], ['''[UNK]''']]			)

						if self.test_rust_tokenizer:
										lowerCAmelCase__					:    Union[str, Any]												=					self.get_rust_tokenizer()
										self.assertListEqual(
										    [rust_tokenizer.tokenize(A_			) for t in ['''Test''', '''\xad''', '''test''']]		,[['''[UNK]'''], [], ['''[UNK]''']]			)



		def 		lowerCAmelCase__				(self			)					->      Dict:

						"""simple docstring"""






						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
														lowerCAmelCase__					:    Union[str, Any]												=					self.rust_tokenizer_class.from_pretrained(A_		,**A_			)

														lowerCAmelCase__					:    Optional[int]												=					f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
														lowerCAmelCase__					:    Optional[int]												=					tokenizer_r.encode_plus(
														    A_		,return_attention_mask=A_		,return_token_type_ids=A_		,return_offsets_mapping=A_		,add_special_tokens=A_		,)

														lowerCAmelCase__					:    Any												=					tokenizer_r.do_lower_case if hasattr(A_		,'''do_lower_case'''			) else False
														lowerCAmelCase__					:    Optional[Any]												=					(
														    [
														        ((0, 0), tokenizer_r.cls_token),
														        ((0, 1), '''A'''),
														        ((1, 2), ''','''),
														        ((3, 5), '''na'''),
														        ((5, 6), '''##ï'''),
														        ((6, 8), '''##ve'''),
														        ((9, 15), tokenizer_r.mask_token),
														        ((16, 21), '''Allen'''),
														        ((21, 23), '''##NL'''),
														        ((23, 24), '''##P'''),
														        ((25, 33), '''sentence'''),
														        ((33, 34), '''.'''),
														        ((0, 0), tokenizer_r.sep_token),
														    ]
														    if not do_lower_case
														    else [
														        ((0, 0), tokenizer_r.cls_token),
														        ((0, 1), '''a'''),
														        ((1, 2), ''','''),
														        ((3, 8), '''naive'''),
														        ((9, 15), tokenizer_r.mask_token),
														        ((16, 21), '''allen'''),
														        ((21, 23), '''##nl'''),
														        ((23, 24), '''##p'''),
														        ((25, 33), '''sentence'''),
														        ((33, 34), '''.'''),
														        ((0, 0), tokenizer_r.sep_token),
														    ]
														)

														self.assertEqual(
														    [e[1] for e in expected_results]		,tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']			)			)
														self.assertEqual([e[0] for e in expected_results]		,tokens['''offset_mapping''']			)



		def 		lowerCAmelCase__				(self			)					->      List[str]:

						"""simple docstring"""






						lowerCAmelCase__					:    Optional[int]												=					['''的''', '''人''', '''有''']
						lowerCAmelCase__					:    List[Any]												=					''''''.join(A_			)
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
														lowerCAmelCase__					:    Tuple												=					True
														lowerCAmelCase__					:    Union[str, Any]												=					self.tokenizer_class.from_pretrained(A_		,**A_			)
														lowerCAmelCase__					:    Any												=					self.rust_tokenizer_class.from_pretrained(A_		,**A_			)

														lowerCAmelCase__					:    int												=					tokenizer_p.encode(A_		,add_special_tokens=A_			)
														lowerCAmelCase__					:    Optional[Any]												=					tokenizer_r.encode(A_		,add_special_tokens=A_			)

														lowerCAmelCase__					:    Any												=					tokenizer_r.convert_ids_to_tokens(A_			)
														lowerCAmelCase__					:    List[str]												=					tokenizer_p.convert_ids_to_tokens(A_			)

														# it is expected that each Chinese character is not preceded by "##"
														self.assertListEqual(A_		,A_			)
														self.assertListEqual(A_		,A_			)

														lowerCAmelCase__					:    Dict												=					False
														lowerCAmelCase__					:    Union[str, Any]												=					self.rust_tokenizer_class.from_pretrained(A_		,**A_			)
														lowerCAmelCase__					:    Optional[int]												=					self.tokenizer_class.from_pretrained(A_		,**A_			)

														lowerCAmelCase__					:    Tuple												=					tokenizer_r.encode(A_		,add_special_tokens=A_			)
														lowerCAmelCase__					:    int												=					tokenizer_p.encode(A_		,add_special_tokens=A_			)

														lowerCAmelCase__					:    str												=					tokenizer_r.convert_ids_to_tokens(A_			)
														lowerCAmelCase__					:    Dict												=					tokenizer_p.convert_ids_to_tokens(A_			)

														# it is expected that only the first Chinese character is not preceded by "##".
														lowerCAmelCase__					:    Optional[int]												=					[
														    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(A_			)
														]
														self.assertListEqual(A_		,A_			)
														self.assertListEqual(A_		,A_			)



		@slow
		def 		lowerCAmelCase__				(self			)					->      Optional[Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    int												=					self.tokenizer_class(self.vocab_file		,self.word_shape_file		,self.word_pronunciation_file			)

						lowerCAmelCase__					:    int												=					tokenizer.encode('''你好'''		,add_special_tokens=A_			)
						lowerCAmelCase__					:    int												=					tokenizer.encode('''你是谁'''		,add_special_tokens=A_			)

						lowerCAmelCase__					:    str												=					tokenizer.build_inputs_with_special_tokens(A_			)
						lowerCAmelCase__					:    int												=					tokenizer.build_inputs_with_special_tokens(A_		,A_			)

						assert encoded_sentence == [1] + text + [2]
						assert encoded_pair == [1] + text + [2] + text_a + [2]



		def 		lowerCAmelCase__				(self			)					->      List[str]:

						"""simple docstring"""






						lowerCAmelCase__					:    Tuple												=					self.get_tokenizers(do_lower_case=A_			)
						for tokenizer in tokenizers:
										with self.subTest(f"""{tokenizer.__class__.__name__}"""			):
														lowerCAmelCase__					:    Optional[Any]												=					'''你好，你是谁'''
														lowerCAmelCase__					:    Tuple												=					tokenizer.tokenize(A_			)
														lowerCAmelCase__					:    Dict												=					tokenizer.convert_tokens_to_ids(A_			)
														lowerCAmelCase__					:    Optional[int]												=					tokenizer.convert_tokens_to_shape_ids(A_			)
														lowerCAmelCase__					:    List[str]												=					tokenizer.convert_tokens_to_pronunciation_ids(A_			)
														lowerCAmelCase__					:    Any												=					tokenizer.prepare_for_model(
														    A_		,A_		,A_		,add_special_tokens=A_			)

														lowerCAmelCase__					:    int												=					tokenizer.encode_plus(A_		,add_special_tokens=A_			)

														self.assertEqual(A_		,A_			)






def      lowerCAmelCase__		(   lowerCamelCase_		:		str = "The quick brown fox jumps over the lazy dog"	,):





				'''simple docstring'''

				lowerCAmelCase__					:    Any												=					set()

				# Replace all the whitespace in our sentence
				lowerCAmelCase__					:    List[Any]												=					input_str.replace(''' '''	,'''''')
				for alpha in input_str:
								if "a" <= alpha.lower() <= "z":
												frequency.add(alpha.lower())
				return len(lowerCamelCase_) == 26






def      lowerCAmelCase__		(   lowerCamelCase_		:		str = "The quick brown fox jumps over the lazy dog"	,):





				'''simple docstring'''

				lowerCAmelCase__					:    List[str]												=					[False] * 26
				for char in input_str:
								if char.islower():
												lowerCAmelCase__					:    Union[str, Any]												=					True
								elif char.isupper():
												lowerCAmelCase__					:    str												=					True
				return all(lowerCamelCase_)






def      lowerCAmelCase__		(   lowerCamelCase_		:		str = "The quick brown fox jumps over the lazy dog"	,):





				'''simple docstring'''

				return len({char for char in input_str.lower() if char.isalpha()}) == 26






def      lowerCAmelCase__		(   ):





				'''simple docstring'''

				from timeit import timeit

				lowerCAmelCase__					:    Optional[Any]												=					'''from __main__ import is_pangram, is_pangram_faster, is_pangram_fastest'''
				print(timeit('''is_pangram()'''	,setup=lowerCamelCase_))
				print(timeit('''is_pangram_faster()'''	,setup=lowerCamelCase_))
				print(timeit('''is_pangram_fastest()'''	,setup=lowerCamelCase_))
				# 5.348480500048026, 2.6477354579837993, 1.8470395830227062
				# 5.036091582966037, 2.644472333951853,  1.8869528750656173


if __name__ == "__main__":
							import doctest

							doctest.testmod()
							benchmark()

