


'''simple docstring'''



import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
	from test_module.custom_tokenization_fast import CustomTokenizerFast






class 	UpperCamelCase__(       unittest.TestCase	):




				def 					a__(		self      :			Dict				)->					List[str]:

										"""simple docstring"""




										UpperCAmelCase					    =     mock.Mock()
										UpperCAmelCase					    =     500
										UpperCAmelCase					    =     {}
										UpperCAmelCase					    =     HTTPError
										UpperCAmelCase					    =     {}

										# Download this model to make sure it's in the cache.
										UpperCAmelCase					    =     BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''				)

										# Under the mock environment we get a 500 error when trying to reach the tokenizer.
										with mock.patch('''requests.Session.request'''				,   return_value=lowerCAmelCase				) as mock_head:
																UpperCAmelCase					    =     BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''				)
																# This check we did call the fake head request
																mock_head.assert_called()




				@require_tokenizers
				def 					a__(		self      :			int				)->					Optional[int]:

										"""simple docstring"""




										UpperCAmelCase					    =     mock.Mock()
										UpperCAmelCase					    =     500
										UpperCAmelCase					    =     {}
										UpperCAmelCase					    =     HTTPError
										UpperCAmelCase					    =     {}

										# Download this model to make sure it's in the cache.
										UpperCAmelCase					    =     GPTaTokenizerFast.from_pretrained('''gpt2'''				)

										# Under the mock environment we get a 500 error when trying to reach the tokenizer.
										with mock.patch('''requests.Session.request'''				,   return_value=lowerCAmelCase				) as mock_head:
																UpperCAmelCase					    =     GPTaTokenizerFast.from_pretrained('''gpt2'''				)
																# This check we did call the fake head request
																mock_head.assert_called()




				def 					a__(		self      :			Dict				)->					List[Any]:

										"""simple docstring"""




										try:
																UpperCAmelCase					    =     tempfile.mktemp()
																with open(lowerCAmelCase				,   '''wb'''				) as f:
																						http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''				,   lowerCAmelCase				)

																UpperCAmelCase					    =     AlbertTokenizer.from_pretrained(lowerCAmelCase				)
										finally:
																os.remove(lowerCAmelCase				)

										# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
										# the current folder and have the right name.
										if os.path.isfile('''tokenizer.json'''				):
																# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
																return
										try:
																with open('''tokenizer.json'''				,   '''wb'''				) as f:
																						http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'''				,   lowerCAmelCase				)
																UpperCAmelCase					    =     AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''				)
																# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
																self.assertEqual(tokenizer.vocab_size				,   1000				)
																# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

										finally:
																os.remove('''tokenizer.json'''				)





				def 					a__(		self      :			Tuple				)->					List[Any]:

										"""simple docstring"""




										UpperCAmelCase					    =     AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''				)







@is_staging_test
class 	UpperCamelCase__(       unittest.TestCase	):
				__magic_name__    :				List[str]  		=	["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]




				@classmethod
				def 					a__(		cls      :			str				)->					Any:

										"""simple docstring"""




										UpperCAmelCase					    =     TOKEN
										HfFolder.save_token(lowerCAmelCase				)




				@classmethod
				def 					a__(		cls      :			Dict				)->					Optional[Any]:

										"""simple docstring"""




										try:
																delete_repo(token=cls._token				,   repo_id='''test-tokenizer'''				)
										except HTTPError:
																pass

										try:
																delete_repo(token=cls._token				,   repo_id='''valid_org/test-tokenizer-org'''				)
										except HTTPError:
																pass

										try:
																delete_repo(token=cls._token				,   repo_id='''test-dynamic-tokenizer'''				)
										except HTTPError:
																pass




				def 					a__(		self      :			List[str]				)->					Any:

										"""simple docstring"""




										with tempfile.TemporaryDirectory() as tmp_dir:
																UpperCAmelCase					    =     os.path.join(lowerCAmelCase				,   '''vocab.txt'''				)
																with open(lowerCAmelCase				,   '''w'''				,   encoding='''utf-8'''				) as vocab_writer:
																						vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]				)				)
																UpperCAmelCase					    =     BertTokenizer(lowerCAmelCase				)

										tokenizer.push_to_hub('''test-tokenizer'''				,   use_auth_token=self._token				)
										UpperCAmelCase					    =     BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""				)
										self.assertDictEqual(new_tokenizer.vocab				,   tokenizer.vocab				)

										# Reset repo
										delete_repo(token=self._token				,   repo_id='''test-tokenizer'''				)

										# Push to hub via save_pretrained
										with tempfile.TemporaryDirectory() as tmp_dir:
																tokenizer.save_pretrained(lowerCAmelCase				,   repo_id='''test-tokenizer'''				,   push_to_hub=lowerCAmelCase				,   use_auth_token=self._token				)

										UpperCAmelCase					    =     BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""				)
										self.assertDictEqual(new_tokenizer.vocab				,   tokenizer.vocab				)




				def 					a__(		self      :			Any				)->					Any:

										"""simple docstring"""




										with tempfile.TemporaryDirectory() as tmp_dir:
																UpperCAmelCase					    =     os.path.join(lowerCAmelCase				,   '''vocab.txt'''				)
																with open(lowerCAmelCase				,   '''w'''				,   encoding='''utf-8'''				) as vocab_writer:
																						vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]				)				)
																UpperCAmelCase					    =     BertTokenizer(lowerCAmelCase				)

										tokenizer.push_to_hub('''valid_org/test-tokenizer-org'''				,   use_auth_token=self._token				)
										UpperCAmelCase					    =     BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''				)
										self.assertDictEqual(new_tokenizer.vocab				,   tokenizer.vocab				)

										# Reset repo
										delete_repo(token=self._token				,   repo_id='''valid_org/test-tokenizer-org'''				)

										# Push to hub via save_pretrained
										with tempfile.TemporaryDirectory() as tmp_dir:
																tokenizer.save_pretrained(
																    lowerCAmelCase				,   repo_id='''valid_org/test-tokenizer-org'''				,   push_to_hub=lowerCAmelCase				,   use_auth_token=self._token				)

										UpperCAmelCase					    =     BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''				)
										self.assertDictEqual(new_tokenizer.vocab				,   tokenizer.vocab				)





				@require_tokenizers
				def 					a__(		self      :			Optional[Any]				)->					Union[str, Any]:

										"""simple docstring"""




										CustomTokenizer.register_for_auto_class()
										with tempfile.TemporaryDirectory() as tmp_dir:
																UpperCAmelCase					    =     os.path.join(lowerCAmelCase				,   '''vocab.txt'''				)
																with open(lowerCAmelCase				,   '''w'''				,   encoding='''utf-8'''				) as vocab_writer:
																						vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]				)				)
																UpperCAmelCase					    =     CustomTokenizer(lowerCAmelCase				)

										# No fast custom tokenizer
										tokenizer.push_to_hub('''test-dynamic-tokenizer'''				,   use_auth_token=self._token				)

										UpperCAmelCase					    =     AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""				,   trust_remote_code=lowerCAmelCase				)
										# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
										self.assertEqual(tokenizer.__class__.__name__				,   '''CustomTokenizer'''				)

										# Fast and slow custom tokenizer
										CustomTokenizerFast.register_for_auto_class()
										with tempfile.TemporaryDirectory() as tmp_dir:
																UpperCAmelCase					    =     os.path.join(lowerCAmelCase				,   '''vocab.txt'''				)
																with open(lowerCAmelCase				,   '''w'''				,   encoding='''utf-8'''				) as vocab_writer:
																						vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]				)				)

																UpperCAmelCase					    =     BertTokenizerFast.from_pretrained(lowerCAmelCase				)
																bert_tokenizer.save_pretrained(lowerCAmelCase				)
																UpperCAmelCase					    =     CustomTokenizerFast.from_pretrained(lowerCAmelCase				)

										tokenizer.push_to_hub('''test-dynamic-tokenizer'''				,   use_auth_token=self._token				)

										UpperCAmelCase					    =     AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""				,   trust_remote_code=lowerCAmelCase				)
										# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
										self.assertEqual(tokenizer.__class__.__name__				,   '''CustomTokenizerFast'''				)
										UpperCAmelCase					    =     AutoTokenizer.from_pretrained(
										    F"""{USER}/test-dynamic-tokenizer"""				,   use_fast=lowerCAmelCase				,   trust_remote_code=lowerCAmelCase				)
										# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
										self.assertEqual(tokenizer.__class__.__name__				,   '''CustomTokenizer'''				)







class 	UpperCamelCase__(       unittest.TestCase	):




				def 					a__(		self      :			Optional[Any]				)->					Dict:

										"""simple docstring"""




										UpperCAmelCase					    =     Trie()
										trie.add('''Hello 友達'''				)
										self.assertEqual(trie.data				,   {'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}				)
										trie.add('''Hello'''				)
										trie.data
										self.assertEqual(trie.data				,   {'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}				)




				def 					a__(		self      :			str				)->					Any:

										"""simple docstring"""




										UpperCAmelCase					    =     Trie()
										self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''				)				,   ['''[CLS] This is a extra_id_100''']				)
										trie.add('''[CLS]'''				)
										trie.add('''extra_id_1'''				)
										trie.add('''extra_id_100'''				)
										self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''				)				,   ['''[CLS]''', ''' This is a ''', '''extra_id_100''']				)




				def 					a__(		self      :			str				)->					Any:

										"""simple docstring"""




										UpperCAmelCase					    =     Trie()
										trie.add('''A'''				)
										self.assertEqual(trie.split('''ABC'''				)				,   ['''A''', '''BC''']				)
										self.assertEqual(trie.split('''BCA'''				)				,   ['''BC''', '''A''']				)




				def 					a__(		self      :			List[Any]				)->					Any:

										"""simple docstring"""




										UpperCAmelCase					    =     Trie()
										trie.add('''TOKEN]'''				)
										trie.add('''[SPECIAL_TOKEN]'''				)
										self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''				)				,   ['''This is something ''', '''[SPECIAL_TOKEN]''']				)




				def 					a__(		self      :			List[str]				)->					Tuple:

										"""simple docstring"""




										UpperCAmelCase					    =     Trie()
										trie.add('''A'''				)
										trie.add('''P'''				)
										trie.add('''[SPECIAL_TOKEN]'''				)
										self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''				)				,   ['''This is something ''', '''[SPECIAL_TOKEN]''']				)




				def 					a__(		self      :			str				)->					int:

										"""simple docstring"""




										UpperCAmelCase					    =     Trie()
										trie.add('''AB'''				)
										trie.add('''B'''				)
										trie.add('''C'''				)
										self.assertEqual(trie.split('''ABC'''				)				,   ['''AB''', '''C''']				)




				def 					a__(		self      :			Dict				)->					str:

										"""simple docstring"""




										UpperCAmelCase					    =     Trie()
										trie.add('''ABC'''				)
										trie.add('''B'''				)
										trie.add('''CD'''				)
										self.assertEqual(trie.split('''ABCD'''				)				,   ['''ABC''', '''D''']				)





				def 					a__(		self      :			int				)->					Optional[int]:

										"""simple docstring"""




										UpperCAmelCase					    =     Trie()
										UpperCAmelCase					    =     trie.cut_text('''ABC'''				,   [0, 0, 2, 1, 2, 3]				)
										self.assertEqual(lowerCAmelCase				,   ['''AB''', '''C''']				)









'''simple docstring'''



from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_lowercase      :      List[str]            =					{
    """configuration_time_series_transformer""": [
        """TIME_SERIES_TRANSFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """TimeSeriesTransformerConfig""",
    ],
}

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_lowercase      :      Dict            =					[
	    """TIME_SERIES_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST""",
	    """TimeSeriesTransformerForPrediction""",
	    """TimeSeriesTransformerModel""",
	    """TimeSeriesTransformerPreTrainedModel""",
	]


if TYPE_CHECKING:
	from .configuration_time_series_transformer import (
	    TIME_SERIES_TRANSFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP,
	    TimeSeriesTransformerConfig,
	)

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_time_series_transformer import (
		    TIME_SERIES_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
		    TimeSeriesTransformerForPrediction,
		    TimeSeriesTransformerModel,
		    TimeSeriesTransformerPreTrainedModel,
		)

else:
	import sys

	_lowercase      :      Tuple            =					_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)





