
"""simple docstring"""





import os
import re
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_a														=		logging.get_logger(__name__)

_a														=		{
    """vocab_file""": """vocab.txt""",
    """merges_file""": """bpe.codes""",
}

_a														=		{
    """vocab_file""": {
        """vinai/phobert-base""": """https://huggingface.co/vinai/phobert-base/resolve/main/vocab.txt""",
        """vinai/phobert-large""": """https://huggingface.co/vinai/phobert-large/resolve/main/vocab.txt""",
    },
    """merges_file""": {
        """vinai/phobert-base""": """https://huggingface.co/vinai/phobert-base/resolve/main/bpe.codes""",
        """vinai/phobert-large""": """https://huggingface.co/vinai/phobert-large/resolve/main/bpe.codes""",
    },
}

_a														=		{
    """vinai/phobert-base""": 256,
    """vinai/phobert-large""": 256,
}



def 				lowerCamelCase__		(      __snake_case     )							->		Tuple:


					"""simple docstring"""





					_UpperCamelCase										=  set()
					_UpperCamelCase										=  word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)     )
										_UpperCamelCase										=  char

					_UpperCamelCase										=  set(__snake_case     )
					return pairs






class    _UpperCAmelCase(   lowerCamelCase     ):
		lowercase__      =				VOCAB_FILES_NAMES
		lowercase__      =				PRETRAINED_VOCAB_FILES_MAP
		lowercase__      =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES



		def __init__( self    ,		__a    ,		__a    ,		__a="<s>"    ,		__a="</s>"    ,		__a="</s>"    ,		__a="<s>"    ,		__a="<unk>"    ,		__a="<pad>"    ,		__a="<mask>"    ,		**__a    ,		)	->	Any:






							'''simple docstring'''
							super().__init__(
							    bos_token=__a    ,		eos_token=__a    ,		unk_token=__a    ,		sep_token=__a    ,		cls_token=__a    ,		pad_token=__a    ,		mask_token=__a    ,		**__a    ,		)

							_UpperCamelCase										=  vocab_file
							_UpperCamelCase										=  merges_file

							_UpperCamelCase										=  {}
							_UpperCamelCase										=  0
							_UpperCamelCase										=  1
							_UpperCamelCase										=  2
							_UpperCamelCase										=  3

							self.add_from_file(__a)

							_UpperCamelCase										=  {v: k for k, v in self.encoder.items()}

							with open(__a    ,		encoding='''utf-8''') as merges_handle:
												_UpperCamelCase										=  merges_handle.read().split('''\n''')[:-1]
							_UpperCamelCase										=  [tuple(merge.split()[:-1]) for merge in merges]
							_UpperCamelCase										=  dict(zip(__a    ,		range(len(__a))))
							_UpperCamelCase										=  {}



		def 						UpperCAmelCase							( self    ,		__a    ,		__a = None)	->	List[int]:






							'''simple docstring'''

							if token_ids_a is None:
												return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
							_UpperCamelCase										=  [self.cls_token_id]
							_UpperCamelCase										=  [self.sep_token_id]
							return cls + token_ids_a + sep + sep + token_ids_a + sep



		def 						UpperCAmelCase							( self    ,		__a    ,		__a = None    ,		__a = False)	->	List[int]:






							'''simple docstring'''

							if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=__a    ,		token_ids_a=__a    ,		already_has_special_tokens=__a)

							if token_ids_a is None:
												return [1] + ([0] * len(__a)) + [1]
							return [1] + ([0] * len(__a)) + [1, 1] + ([0] * len(__a)) + [1]



		def 						UpperCAmelCase							( self    ,		__a    ,		__a = None)	->	List[int]:






							'''simple docstring'''
							_UpperCamelCase										=  [self.sep_token_id]
							_UpperCamelCase										=  [self.cls_token_id]

							if token_ids_a is None:
												return len(cls + token_ids_a + sep) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]



		@property
		def 						UpperCAmelCase							( self)	->	List[Any]:






							'''simple docstring'''
							return len(self.encoder)



		def 						UpperCAmelCase							( self)	->	Optional[Any]:






							'''simple docstring'''
							return dict(self.encoder    ,		**self.added_tokens_encoder)



		def 						UpperCAmelCase							( self    ,		__a)	->	int:






							'''simple docstring'''
							if token in self.cache:
												return self.cache[token]
							_UpperCamelCase										=  tuple(__a)
							_UpperCamelCase										=  tuple(list(word[:-1]) + [word[-1] + '''</w>'''])
							_UpperCamelCase										=  get_pairs(__a)

							if not pairs:
												return token

							while True:
												_UpperCamelCase										=  min(__a    ,		key=lambda __a: self.bpe_ranks.get(__a    ,		float('''inf''')))
												if bigram not in self.bpe_ranks:
																	break
												_UpperCamelCase			,  _UpperCamelCase										=  bigram
												_UpperCamelCase										=  []
												_UpperCamelCase										=  0
												while i < len(__a):
																	try:
																						_UpperCamelCase										=  word.index(__a    ,		__a)
																	except ValueError:
																						new_word.extend(word[i:])
																						break
																	else:
																						new_word.extend(word[i:j])
																						_UpperCamelCase										=  j

																	if word[i] == first and i < len(__a) - 1 and word[i + 1] == second:
																						new_word.append(first + second)
																						i += 2
																	else:
																						new_word.append(word[i])
																						i += 1
												_UpperCamelCase										=  tuple(__a)
												_UpperCamelCase										=  new_word
												if len(__a) == 1:
																	break
												else:
																	_UpperCamelCase										=  get_pairs(__a)
							_UpperCamelCase										=  '''@@ '''.join(__a)
							_UpperCamelCase										=  word[:-4]
							_UpperCamelCase										=  word
							return word



		def 						UpperCAmelCase							( self    ,		__a)	->	Dict:






							'''simple docstring'''
							_UpperCamelCase										=  []

							_UpperCamelCase										=  re.findall(R'''\S+\n?'''    ,		__a)

							for token in words:
												split_tokens.extend(list(self.bpe(__a).split(''' ''')))
							return split_tokens



		def 						UpperCAmelCase							( self    ,		__a)	->	Any:






							'''simple docstring'''
							return self.encoder.get(__a    ,		self.encoder.get(self.unk_token))



		def 						UpperCAmelCase							( self    ,		__a)	->	List[Any]:






							'''simple docstring'''
							return self.decoder.get(__a    ,		self.unk_token)



		def 						UpperCAmelCase							( self    ,		__a)	->	Optional[Any]:






							'''simple docstring'''
							_UpperCamelCase										=  ''' '''.join(__a).replace('''@@ '''    ,		'''''').strip()
							return out_string



		def 						UpperCAmelCase							( self    ,		__a    ,		__a = None)	->	Tuple[str]:






							'''simple docstring'''
							if not os.path.isdir(__a):
												logger.error(F'''Vocabulary path ({save_directory}) should be a directory''')
												return
							_UpperCamelCase										=  os.path.join(
							    __a    ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])
							_UpperCamelCase										=  os.path.join(
							    __a    ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file'''])

							if os.path.abspath(self.vocab_file) != os.path.abspath(__a):
												copyfile(self.vocab_file    ,		__a)

							if os.path.abspath(self.merges_file) != os.path.abspath(__a):
												copyfile(self.merges_file    ,		__a)

							return out_vocab_file, out_merge_file




		def 						UpperCAmelCase							( self    ,		__a)	->	Optional[int]:






							'''simple docstring'''
							if isinstance(__a    ,		__a):
												try:
																	with open(__a    ,		'''r'''    ,		encoding='''utf-8''') as fd:
																						self.add_from_file(__a)
												except FileNotFoundError as fnfe:
																	raise fnfe
												except UnicodeError:
																	raise Exception(F'''Incorrect encoding detected in {f}, please rebuild the dataset''')
												return

							_UpperCamelCase										=  f.readlines()
							for lineTmp in lines:
												_UpperCamelCase										=  lineTmp.strip()
												_UpperCamelCase										=  line.rfind(''' ''')
												if idx == -1:
																	raise ValueError('''Incorrect dictionary format, expected \'<token> <cnt>\'''')
												_UpperCamelCase										=  line[:idx]
												_UpperCamelCase										=  len(self.encoder)








"""simple docstring"""





import importlib
import json
import os
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
import transformers.models.auto
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.bert.configuration_bert import BertConfig
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, get_tests_dir


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402


_a														=		get_tests_dir("""fixtures/dummy-config.json""")






class    _UpperCAmelCase(   unittest.TestCase     ):



		def 						UpperCAmelCase							( self)	->	str:






							'''simple docstring'''
							_UpperCamelCase										=  0



		def 						UpperCAmelCase							( self)	->	List[str]:






							'''simple docstring'''
							self.assertIsNotNone(transformers.models.auto.__spec__)
							self.assertIsNotNone(importlib.util.find_spec('''transformers.models.auto'''))



		def 						UpperCAmelCase							( self)	->	str:






							'''simple docstring'''
							_UpperCamelCase										=  AutoConfig.from_pretrained('''bert-base-uncased''')
							self.assertIsInstance(__a    ,		__a)



		def 						UpperCAmelCase							( self)	->	Tuple:






							'''simple docstring'''
							_UpperCamelCase										=  AutoConfig.from_pretrained(__a)
							self.assertIsInstance(__a    ,		__a)



		def 						UpperCAmelCase							( self)	->	Optional[Any]:






							'''simple docstring'''
							_UpperCamelCase										=  AutoConfig.from_pretrained(__a)
							self.assertIsInstance(__a    ,		__a)



		def 						UpperCAmelCase							( self)	->	Union[str, Any]:






							'''simple docstring'''
							_UpperCamelCase										=  AutoConfig.for_model('''roberta''')
							self.assertIsInstance(__a    ,		__a)



		def 						UpperCAmelCase							( self)	->	Dict:






							'''simple docstring'''
							with tempfile.TemporaryDirectory() as tmp_dir:
												# This model name contains bert and roberta, but roberta ends up being picked.
												_UpperCamelCase										=  os.path.join(__a    ,		'''fake-roberta''')
												os.makedirs(__a    ,		exist_ok=__a)
												with open(os.path.join(__a    ,		'''config.json''')    ,		'''w''') as f:
																	f.write(json.dumps({}))
												_UpperCamelCase										=  AutoConfig.from_pretrained(__a)
												self.assertEqual(type(__a)    ,		__a)



		def 						UpperCAmelCase							( self)	->	Tuple:






							'''simple docstring'''
							try:
												AutoConfig.register('''custom'''    ,		__a)
												# Wrong model type will raise an error
												with self.assertRaises(__a):
																	AutoConfig.register('''model'''    ,		__a)
												# Trying to register something existing in the Transformers library will raise an error
												with self.assertRaises(__a):
																	AutoConfig.register('''bert'''    ,		__a)

												# Now that the config is registered, it can be used as any other config with the auto-API
												_UpperCamelCase										=  CustomConfig()
												with tempfile.TemporaryDirectory() as tmp_dir:
																	config.save_pretrained(__a)
																	_UpperCamelCase										=  AutoConfig.from_pretrained(__a)
																	self.assertIsInstance(__a    ,		__a)

							finally:
												if "custom" in CONFIG_MAPPING._extra_content:
																	del CONFIG_MAPPING._extra_content["custom"]



		def 						UpperCAmelCase							( self)	->	List[Any]:






							'''simple docstring'''
							with self.assertRaisesRegex(
							    __a    ,		'''bert-base is not a local folder and is not a valid model identifier'''):
												_UpperCamelCase										=  AutoConfig.from_pretrained('''bert-base''')



		def 						UpperCAmelCase							( self)	->	List[Any]:






							'''simple docstring'''
							with self.assertRaisesRegex(
							    __a    ,		R'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'''):
												_UpperCamelCase										=  AutoConfig.from_pretrained(__a    ,		revision='''aaaaaa''')



		def 						UpperCAmelCase							( self)	->	Optional[Any]:






							'''simple docstring'''
							with self.assertRaisesRegex(
							    __a    ,		'''hf-internal-testing/no-config-test-repo does not appear to have a file named config.json.'''    ,		):
												_UpperCamelCase										=  AutoConfig.from_pretrained('''hf-internal-testing/no-config-test-repo''')



		def 						UpperCAmelCase							( self)	->	int:






							'''simple docstring'''
							# If remote code is not set, we will time out when asking whether to load the model.
							with self.assertRaises(__a):
												_UpperCamelCase										=  AutoConfig.from_pretrained('''hf-internal-testing/test_dynamic_model''')
							# If remote code is disabled, we can't load this config.
							with self.assertRaises(__a):
												_UpperCamelCase										=  AutoConfig.from_pretrained('''hf-internal-testing/test_dynamic_model'''    ,		trust_remote_code=__a)

							_UpperCamelCase										=  AutoConfig.from_pretrained('''hf-internal-testing/test_dynamic_model'''    ,		trust_remote_code=__a)
							self.assertEqual(config.__class__.__name__    ,		'''NewModelConfig''')

							# Test config can be reloaded.
							with tempfile.TemporaryDirectory() as tmp_dir:
												config.save_pretrained(__a)
												_UpperCamelCase										=  AutoConfig.from_pretrained(__a    ,		trust_remote_code=__a)
							self.assertEqual(reloaded_config.__class__.__name__    ,		'''NewModelConfig''')




		def 						UpperCAmelCase							( self)	->	Any:






							'''simple docstring'''






							class    _UpperCAmelCase(   lowerCamelCase     ):
									lowercase__      =				'new-model'

							try:
												AutoConfig.register('''new-model'''    ,		__a)
												# If remote code is not set, the default is to use local
												_UpperCamelCase										=  AutoConfig.from_pretrained('''hf-internal-testing/test_dynamic_model''')
												self.assertEqual(config.__class__.__name__    ,		'''NewModelConfigLocal''')

												# If remote code is disabled, we load the local one.
												_UpperCamelCase										=  AutoConfig.from_pretrained('''hf-internal-testing/test_dynamic_model'''    ,		trust_remote_code=__a)
												self.assertEqual(config.__class__.__name__    ,		'''NewModelConfigLocal''')

												# If remote is enabled, we load from the Hub
												_UpperCamelCase										=  AutoConfig.from_pretrained('''hf-internal-testing/test_dynamic_model'''    ,		trust_remote_code=__a)
												self.assertEqual(config.__class__.__name__    ,		'''NewModelConfig''')

							finally:
												if "new-model" in CONFIG_MAPPING._extra_content:
																	del CONFIG_MAPPING._extra_content["new-model"]






