


'''simple docstring'''



import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_lowercase      :      Dict            =					get_tests_dir("""fixtures/test_sentencepiece.model""")






@require_sentencepiece
@require_tokenizers
class 	UpperCamelCase__(       lowerCAmelCase						,    unittest.TestCase	):
				__magic_name__    :				Tuple  		=	XLNetTokenizer
				__magic_name__    :				Any  		=	XLNetTokenizerFast
				__magic_name__    :				Optional[int]  		=	True
				__magic_name__    :				Union[str, Any]  		=	True




				def 					a__(		self      :			List[Any]				)->					List[Any]:

										"""simple docstring"""




										super().setUp()

										# We have a SentencePiece fixture for testing
										UpperCAmelCase					    =     XLNetTokenizer(lowerCAmelCase				,   keep_accents=lowerCAmelCase				)
										tokenizer.sanitize_special_tokens()
										tokenizer.save_pretrained(self.tmpdirname				)




				def 					a__(		self      :			List[str]				)->					Optional[Any]:

										"""simple docstring"""




										UpperCAmelCase					    =     '''<s>'''
										UpperCAmelCase					    =     1

										self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCAmelCase				)				,   lowerCAmelCase				)
										self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCAmelCase				)				,   lowerCAmelCase				)




				def 					a__(		self      :			Optional[int]				)->					Union[str, Any]:

										"""simple docstring"""




										UpperCAmelCase					    =     list(self.get_tokenizer().get_vocab().keys()				)

										self.assertEqual(vocab_keys[0]				,   '''<unk>'''				)
										self.assertEqual(vocab_keys[1]				,   '''<s>'''				)
										self.assertEqual(vocab_keys[-1]				,   '''<eod>'''				)
										self.assertEqual(len(lowerCAmelCase				)				,   1006				)




				def 					a__(		self      :			int				)->					str:

										"""simple docstring"""




										self.assertEqual(self.get_tokenizer().vocab_size				,   1000				)




				def 					a__(		self      :			int				)->					Tuple:

										"""simple docstring"""




										UpperCAmelCase					    =     XLNetTokenizer(lowerCAmelCase				,   keep_accents=lowerCAmelCase				)

										UpperCAmelCase					    =     tokenizer.tokenize('''This is a test'''				)
										self.assertListEqual(lowerCAmelCase				,   ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']				)

										self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase				)				,   [285, 46, 10, 170, 382]				)

										UpperCAmelCase					    =     tokenizer.tokenize('''I was born in 92000, and this is falsé.'''				)
										self.assertListEqual(
										    lowerCAmelCase				,   [
										        SPIECE_UNDERLINE + '''I''',
										        SPIECE_UNDERLINE + '''was''',
										        SPIECE_UNDERLINE + '''b''',
										        '''or''',
										        '''n''',
										        SPIECE_UNDERLINE + '''in''',
										        SPIECE_UNDERLINE + '''''',
										        '''9''',
										        '''2''',
										        '''0''',
										        '''0''',
										        '''0''',
										        ''',''',
										        SPIECE_UNDERLINE + '''and''',
										        SPIECE_UNDERLINE + '''this''',
										        SPIECE_UNDERLINE + '''is''',
										        SPIECE_UNDERLINE + '''f''',
										        '''al''',
										        '''s''',
										        '''é''',
										        '''.''',
										    ]				,   )
										UpperCAmelCase					    =     tokenizer.convert_tokens_to_ids(lowerCAmelCase				)
										self.assertListEqual(lowerCAmelCase				,   [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]				)

										UpperCAmelCase					    =     tokenizer.convert_ids_to_tokens(lowerCAmelCase				)
										self.assertListEqual(
										    lowerCAmelCase				,   [
										        SPIECE_UNDERLINE + '''I''',
										        SPIECE_UNDERLINE + '''was''',
										        SPIECE_UNDERLINE + '''b''',
										        '''or''',
										        '''n''',
										        SPIECE_UNDERLINE + '''in''',
										        SPIECE_UNDERLINE + '''''',
										        '''<unk>''',
										        '''2''',
										        '''0''',
										        '''0''',
										        '''0''',
										        ''',''',
										        SPIECE_UNDERLINE + '''and''',
										        SPIECE_UNDERLINE + '''this''',
										        SPIECE_UNDERLINE + '''is''',
										        SPIECE_UNDERLINE + '''f''',
										        '''al''',
										        '''s''',
										        '''<unk>''',
										        '''.''',
										    ]				,   )




				def 					a__(		self      :			Optional[Any]				)->					Optional[int]:

										"""simple docstring"""




										UpperCAmelCase					    =     XLNetTokenizer(lowerCAmelCase				,   do_lower_case=lowerCAmelCase				)
										UpperCAmelCase					    =     tokenizer.tokenize('''I was born in 92000, and this is falsé.'''				)
										self.assertListEqual(
										    lowerCAmelCase				,   [
										        SPIECE_UNDERLINE + '''''',
										        '''i''',
										        SPIECE_UNDERLINE + '''was''',
										        SPIECE_UNDERLINE + '''b''',
										        '''or''',
										        '''n''',
										        SPIECE_UNDERLINE + '''in''',
										        SPIECE_UNDERLINE + '''''',
										        '''9''',
										        '''2''',
										        '''0''',
										        '''0''',
										        '''0''',
										        ''',''',
										        SPIECE_UNDERLINE + '''and''',
										        SPIECE_UNDERLINE + '''this''',
										        SPIECE_UNDERLINE + '''is''',
										        SPIECE_UNDERLINE + '''f''',
										        '''al''',
										        '''se''',
										        '''.''',
										    ]				,   )
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''				)				,   ['''▁he''', '''ll''', '''o''']				)




				def 					a__(		self      :			Any				)->					Dict:

										"""simple docstring"""




										UpperCAmelCase					    =     XLNetTokenizer(lowerCAmelCase				,   do_lower_case=lowerCAmelCase				)
										UpperCAmelCase					    =     tokenizer.tokenize('''I was born in 92000, and this is falsé.'''				)
										self.assertListEqual(
										    lowerCAmelCase				,   [
										        SPIECE_UNDERLINE + '''I''',
										        SPIECE_UNDERLINE + '''was''',
										        SPIECE_UNDERLINE + '''b''',
										        '''or''',
										        '''n''',
										        SPIECE_UNDERLINE + '''in''',
										        SPIECE_UNDERLINE + '''''',
										        '''9''',
										        '''2''',
										        '''0''',
										        '''0''',
										        '''0''',
										        ''',''',
										        SPIECE_UNDERLINE + '''and''',
										        SPIECE_UNDERLINE + '''this''',
										        SPIECE_UNDERLINE + '''is''',
										        SPIECE_UNDERLINE + '''f''',
										        '''al''',
										        '''se''',
										        '''.''',
										    ]				,   )




				@slow
				def 					a__(		self      :			Optional[Any]				)->					str:

										"""simple docstring"""




										UpperCAmelCase					    =     XLNetTokenizer.from_pretrained('''xlnet-base-cased'''				)

										UpperCAmelCase					    =     tokenizer.encode('''sequence builders'''				,   add_special_tokens=lowerCAmelCase				)
										UpperCAmelCase					    =     tokenizer.encode('''multi-sequence build'''				,   add_special_tokens=lowerCAmelCase				)

										UpperCAmelCase					    =     tokenizer.build_inputs_with_special_tokens(lowerCAmelCase				)
										UpperCAmelCase					    =     tokenizer.build_inputs_with_special_tokens(lowerCAmelCase				,   lowerCAmelCase				)

										assert encoded_sentence == text + [4, 3]
										assert encoded_pair == text + [4] + text_a + [4, 3]





				@slow
				def 					a__(		self      :			List[str]				)->					Tuple:

										"""simple docstring"""




										UpperCAmelCase					    =     {'''input_ids''': [[17, 21442, 270, 17, 10, 14645, 318, 34, 17, 4546, 3145, 787, 13, 7752, 22018, 23, 21, 17, 4546, 3145, 787, 13, 3352, 14431, 13, 5500, 11, 1176, 580, 13, 16819, 4797, 23, 17, 10, 17135, 658, 19, 457, 7932, 13, 184, 19, 3154, 17135, 6468, 19, 1404, 12269, 19, 4229, 5356, 16264, 46, 19, 17, 20545, 10395, 9, 9, 9, 11, 28, 6421, 9531, 20729, 17, 10, 353, 17022, 11, 21, 6421, 9531, 16949, 17, 10, 11509, 753, 11, 33, 95, 2421, 7385, 956, 14431, 2626, 25, 842, 7385, 4836, 21, 1429, 2272, 9855, 3120, 161, 24738, 19, 13203, 658, 218, 787, 21, 430, 18482, 847, 2637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 22178, 27, 1064, 22, 956, 13, 11101, 1429, 5854, 24313, 18953, 40, 422, 24366, 68, 1758, 37, 10483, 14257, 31, 207, 263, 21, 203, 3773, 25, 71, 9735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2049, 3442, 17, 13894, 3380, 23, 95, 18, 17634, 2288, 9, 4, 3]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
										# fmt: on

										self.tokenizer_integration_test_util(
										    expected_encoding=lowerCAmelCase				,   model_name='''xlnet-base-cased'''				,   revision='''c841166438c31ec7ca9a106dee7bb312b73ae511'''				,   )









'''simple docstring'''



import numpy as np
from cva import COLOR_BGR2GRAY, CV_8UC3, cvtColor, filteraD, imread, imshow, waitKey


def 				lowerCamelCase__				(	A       :      int				,       A       :      int				,       A       :      int				,       A       :      int				,       A       :      int				,       A       :      int					):




						'''simple docstring'''






						if (ksize % 2) == 0:
												UpperCAmelCase					    =     ksize + 1
						UpperCAmelCase					    =     np.zeros((ksize, ksize)				,       dtype=np.floataa					)

						# each value
						for y in range(A					):
												for x in range(A					):
																		# distance from center
																		UpperCAmelCase					    =     x - ksize // 2
																		UpperCAmelCase					    =     y - ksize // 2

																		# degree to radiant
																		UpperCAmelCase					    =     theta / 1_80 * np.pi
																		UpperCAmelCase					    =     np.cos(_theta					)
																		UpperCAmelCase					    =     np.sin(_theta					)

																		# get kernel x
																		UpperCAmelCase					    =     cos_theta * px + sin_theta * py

																		# get kernel y
																		UpperCAmelCase					    =     -sin_theta * px + cos_theta * py

																		# fill kernel
																		UpperCAmelCase					    =     np.exp(
																		    -(_x**2 + gamma**2 * _y**2) / (2 * sigma**2)					) * np.cos(2 * np.pi * _x / lambd + psi					)

						return gabor


if __name__ == "__main__":
	import doctest

	doctest.testmod()
	# read original image
	_lowercase      :      Tuple            =					imread("""../image_data/lena.jpg""")
	# turn image in gray scale value
	_lowercase      :      int            =					cvtColor(img, COLOR_BGR2GRAY)

	# Apply multiple Kernel to detect edges
	_lowercase      :      List[str]            =					np.zeros(gray.shape[:2])
	for theta in [0, 30, 60, 90, 120, 150]:
		_lowercase      :      List[Any]            =					gabor_filter_kernel(10, 8, theta, 10, 0, 0)
		out += filteraD(gray, CV_8UC3, kernel_aa)
	_lowercase      :      Optional[int]            =					out / out.max() * 255
	_lowercase      :      Optional[int]            =					out.astype(np.uinta)

	imshow("""Original""", gray)
	imshow("""Gabor filter with 20x20 mask and 6 directions""", out)

	waitKey(0)





