
"""simple docstring"""





import os
import re
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_a														=		logging.get_logger(__name__)

_a														=		{
    """vocab_file""": """vocab.txt""",
    """merges_file""": """bpe.codes""",
}

_a														=		{
    """vocab_file""": {
        """vinai/phobert-base""": """https://huggingface.co/vinai/phobert-base/resolve/main/vocab.txt""",
        """vinai/phobert-large""": """https://huggingface.co/vinai/phobert-large/resolve/main/vocab.txt""",
    },
    """merges_file""": {
        """vinai/phobert-base""": """https://huggingface.co/vinai/phobert-base/resolve/main/bpe.codes""",
        """vinai/phobert-large""": """https://huggingface.co/vinai/phobert-large/resolve/main/bpe.codes""",
    },
}

_a														=		{
    """vinai/phobert-base""": 256,
    """vinai/phobert-large""": 256,
}



def 				lowerCamelCase__		(      __snake_case     )							->		Tuple:


					"""simple docstring"""





					_UpperCamelCase										=  set()
					_UpperCamelCase										=  word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)     )
										_UpperCamelCase										=  char

					_UpperCamelCase										=  set(__snake_case     )
					return pairs






class    _UpperCAmelCase(   lowerCamelCase     ):
		lowercase__      =				VOCAB_FILES_NAMES
		lowercase__      =				PRETRAINED_VOCAB_FILES_MAP
		lowercase__      =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES



		def __init__( self    ,		__a    ,		__a    ,		__a="<s>"    ,		__a="</s>"    ,		__a="</s>"    ,		__a="<s>"    ,		__a="<unk>"    ,		__a="<pad>"    ,		__a="<mask>"    ,		**__a    ,		)	->	Any:






							'''simple docstring'''
							super().__init__(
							    bos_token=__a    ,		eos_token=__a    ,		unk_token=__a    ,		sep_token=__a    ,		cls_token=__a    ,		pad_token=__a    ,		mask_token=__a    ,		**__a    ,		)

							_UpperCamelCase										=  vocab_file
							_UpperCamelCase										=  merges_file

							_UpperCamelCase										=  {}
							_UpperCamelCase										=  0
							_UpperCamelCase										=  1
							_UpperCamelCase										=  2
							_UpperCamelCase										=  3

							self.add_from_file(__a)

							_UpperCamelCase										=  {v: k for k, v in self.encoder.items()}

							with open(__a    ,		encoding='''utf-8''') as merges_handle:
												_UpperCamelCase										=  merges_handle.read().split('''\n''')[:-1]
							_UpperCamelCase										=  [tuple(merge.split()[:-1]) for merge in merges]
							_UpperCamelCase										=  dict(zip(__a    ,		range(len(__a))))
							_UpperCamelCase										=  {}



		def 						UpperCAmelCase							( self    ,		__a    ,		__a = None)	->	List[int]:






							'''simple docstring'''

							if token_ids_a is None:
												return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
							_UpperCamelCase										=  [self.cls_token_id]
							_UpperCamelCase										=  [self.sep_token_id]
							return cls + token_ids_a + sep + sep + token_ids_a + sep



		def 						UpperCAmelCase							( self    ,		__a    ,		__a = None    ,		__a = False)	->	List[int]:






							'''simple docstring'''

							if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=__a    ,		token_ids_a=__a    ,		already_has_special_tokens=__a)

							if token_ids_a is None:
												return [1] + ([0] * len(__a)) + [1]
							return [1] + ([0] * len(__a)) + [1, 1] + ([0] * len(__a)) + [1]



		def 						UpperCAmelCase							( self    ,		__a    ,		__a = None)	->	List[int]:






							'''simple docstring'''
							_UpperCamelCase										=  [self.sep_token_id]
							_UpperCamelCase										=  [self.cls_token_id]

							if token_ids_a is None:
												return len(cls + token_ids_a + sep) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]



		@property
		def 						UpperCAmelCase							( self)	->	List[Any]:






							'''simple docstring'''
							return len(self.encoder)



		def 						UpperCAmelCase							( self)	->	Optional[Any]:






							'''simple docstring'''
							return dict(self.encoder    ,		**self.added_tokens_encoder)



		def 						UpperCAmelCase							( self    ,		__a)	->	int:






							'''simple docstring'''
							if token in self.cache:
												return self.cache[token]
							_UpperCamelCase										=  tuple(__a)
							_UpperCamelCase										=  tuple(list(word[:-1]) + [word[-1] + '''</w>'''])
							_UpperCamelCase										=  get_pairs(__a)

							if not pairs:
												return token

							while True:
												_UpperCamelCase										=  min(__a    ,		key=lambda __a: self.bpe_ranks.get(__a    ,		float('''inf''')))
												if bigram not in self.bpe_ranks:
																	break
												_UpperCamelCase			,  _UpperCamelCase										=  bigram
												_UpperCamelCase										=  []
												_UpperCamelCase										=  0
												while i < len(__a):
																	try:
																						_UpperCamelCase										=  word.index(__a    ,		__a)
																	except ValueError:
																						new_word.extend(word[i:])
																						break
																	else:
																						new_word.extend(word[i:j])
																						_UpperCamelCase										=  j

																	if word[i] == first and i < len(__a) - 1 and word[i + 1] == second:
																						new_word.append(first + second)
																						i += 2
																	else:
																						new_word.append(word[i])
																						i += 1
												_UpperCamelCase										=  tuple(__a)
												_UpperCamelCase										=  new_word
												if len(__a) == 1:
																	break
												else:
																	_UpperCamelCase										=  get_pairs(__a)
							_UpperCamelCase										=  '''@@ '''.join(__a)
							_UpperCamelCase										=  word[:-4]
							_UpperCamelCase										=  word
							return word



		def 						UpperCAmelCase							( self    ,		__a)	->	Dict:






							'''simple docstring'''
							_UpperCamelCase										=  []

							_UpperCamelCase										=  re.findall(R'''\S+\n?'''    ,		__a)

							for token in words:
												split_tokens.extend(list(self.bpe(__a).split(''' ''')))
							return split_tokens



		def 						UpperCAmelCase							( self    ,		__a)	->	Any:






							'''simple docstring'''
							return self.encoder.get(__a    ,		self.encoder.get(self.unk_token))



		def 						UpperCAmelCase							( self    ,		__a)	->	List[Any]:






							'''simple docstring'''
							return self.decoder.get(__a    ,		self.unk_token)



		def 						UpperCAmelCase							( self    ,		__a)	->	Optional[Any]:






							'''simple docstring'''
							_UpperCamelCase										=  ''' '''.join(__a).replace('''@@ '''    ,		'''''').strip()
							return out_string



		def 						UpperCAmelCase							( self    ,		__a    ,		__a = None)	->	Tuple[str]:






							'''simple docstring'''
							if not os.path.isdir(__a):
												logger.error(F'''Vocabulary path ({save_directory}) should be a directory''')
												return
							_UpperCamelCase										=  os.path.join(
							    __a    ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])
							_UpperCamelCase										=  os.path.join(
							    __a    ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file'''])

							if os.path.abspath(self.vocab_file) != os.path.abspath(__a):
												copyfile(self.vocab_file    ,		__a)

							if os.path.abspath(self.merges_file) != os.path.abspath(__a):
												copyfile(self.merges_file    ,		__a)

							return out_vocab_file, out_merge_file




		def 						UpperCAmelCase							( self    ,		__a)	->	Optional[int]:






							'''simple docstring'''
							if isinstance(__a    ,		__a):
												try:
																	with open(__a    ,		'''r'''    ,		encoding='''utf-8''') as fd:
																						self.add_from_file(__a)
												except FileNotFoundError as fnfe:
																	raise fnfe
												except UnicodeError:
																	raise Exception(F'''Incorrect encoding detected in {f}, please rebuild the dataset''')
												return

							_UpperCamelCase										=  f.readlines()
							for lineTmp in lines:
												_UpperCamelCase										=  lineTmp.strip()
												_UpperCamelCase										=  line.rfind(''' ''')
												if idx == -1:
																	raise ValueError('''Incorrect dictionary format, expected \'<token> <cnt>\'''')
												_UpperCamelCase										=  line[:idx]
												_UpperCamelCase										=  len(self.encoder)








"""simple docstring"""





import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch

				from transformers import (
				    MODEL_FOR_PRETRAINING_MAPPING,
				    AlbertForMaskedLM,
				    AlbertForMultipleChoice,
				    AlbertForPreTraining,
				    AlbertForQuestionAnswering,
				    AlbertForSequenceClassification,
				    AlbertForTokenClassification,
				    AlbertModel,
				)
				from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST






class    _UpperCAmelCase:



		def __init__( self    ,		__a    ,		__a=13    ,		__a=7    ,		__a=True    ,		__a=True    ,		__a=True    ,		__a=True    ,		__a=99    ,		__a=16    ,		__a=36    ,		__a=6    ,		__a=6    ,		__a=6    ,		__a=37    ,		__a="gelu"    ,		__a=0.1    ,		__a=0.1    ,		__a=5_12    ,		__a=16    ,		__a=2    ,		__a=0.02    ,		__a=3    ,		__a=4    ,		__a=None    ,		)	->	List[str]:






							'''simple docstring'''
							_UpperCamelCase										=  parent
							_UpperCamelCase										=  batch_size
							_UpperCamelCase										=  seq_length
							_UpperCamelCase										=  is_training
							_UpperCamelCase										=  use_input_mask
							_UpperCamelCase										=  use_token_type_ids
							_UpperCamelCase										=  use_labels
							_UpperCamelCase										=  vocab_size
							_UpperCamelCase										=  embedding_size
							_UpperCamelCase										=  hidden_size
							_UpperCamelCase										=  num_hidden_layers
							_UpperCamelCase										=  num_hidden_groups
							_UpperCamelCase										=  num_attention_heads
							_UpperCamelCase										=  intermediate_size
							_UpperCamelCase										=  hidden_act
							_UpperCamelCase										=  hidden_dropout_prob
							_UpperCamelCase										=  attention_probs_dropout_prob
							_UpperCamelCase										=  max_position_embeddings
							_UpperCamelCase										=  type_vocab_size
							_UpperCamelCase										=  type_sequence_label_size
							_UpperCamelCase										=  initializer_range
							_UpperCamelCase										=  num_labels
							_UpperCamelCase										=  num_choices
							_UpperCamelCase										=  scope



		def 						UpperCAmelCase							( self)	->	List[str]:






							'''simple docstring'''
							_UpperCamelCase										=  ids_tensor([self.batch_size, self.seq_length]    ,		self.vocab_size)

							_UpperCamelCase										=  None
							if self.use_input_mask:
												_UpperCamelCase										=  random_attention_mask([self.batch_size, self.seq_length])

							_UpperCamelCase										=  None
							if self.use_token_type_ids:
												_UpperCamelCase										=  ids_tensor([self.batch_size, self.seq_length]    ,		self.type_vocab_size)

							_UpperCamelCase										=  None
							_UpperCamelCase										=  None
							_UpperCamelCase										=  None
							if self.use_labels:
												_UpperCamelCase										=  ids_tensor([self.batch_size]    ,		self.type_sequence_label_size)
												_UpperCamelCase										=  ids_tensor([self.batch_size, self.seq_length]    ,		self.num_labels)
												_UpperCamelCase										=  ids_tensor([self.batch_size]    ,		self.num_choices)

							_UpperCamelCase										=  self.get_config()

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



		def 						UpperCAmelCase							( self)	->	List[str]:






							'''simple docstring'''
							return AlbertConfig(
							    vocab_size=self.vocab_size    ,		hidden_size=self.hidden_size    ,		num_hidden_layers=self.num_hidden_layers    ,		num_attention_heads=self.num_attention_heads    ,		intermediate_size=self.intermediate_size    ,		hidden_act=self.hidden_act    ,		hidden_dropout_prob=self.hidden_dropout_prob    ,		attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,		max_position_embeddings=self.max_position_embeddings    ,		type_vocab_size=self.type_vocab_size    ,		initializer_range=self.initializer_range    ,		num_hidden_groups=self.num_hidden_groups    ,		)



		def 						UpperCAmelCase							( self    ,		__a    ,		__a    ,		__a    ,		__a    ,		__a    ,		__a    ,		__a)	->	Optional[Any]:






							'''simple docstring'''
							_UpperCamelCase										=  AlbertModel(config=__a)
							model.to(__a)
							model.eval()
							_UpperCamelCase										=  model(__a    ,		attention_mask=__a    ,		token_type_ids=__a)
							_UpperCamelCase										=  model(__a    ,		token_type_ids=__a)
							_UpperCamelCase										=  model(__a)
							self.parent.assertEqual(result.last_hidden_state.shape    ,		(self.batch_size, self.seq_length, self.hidden_size))
							self.parent.assertEqual(result.pooler_output.shape    ,		(self.batch_size, self.hidden_size))



		def 						UpperCAmelCase							( self    ,		__a    ,		__a    ,		__a    ,		__a    ,		__a    ,		__a    ,		__a)	->	Union[str, Any]:






							'''simple docstring'''
							_UpperCamelCase										=  AlbertForPreTraining(config=__a)
							model.to(__a)
							model.eval()
							_UpperCamelCase										=  model(
							    __a    ,		attention_mask=__a    ,		token_type_ids=__a    ,		labels=__a    ,		sentence_order_label=__a    ,		)
							self.parent.assertEqual(result.prediction_logits.shape    ,		(self.batch_size, self.seq_length, self.vocab_size))
							self.parent.assertEqual(result.sop_logits.shape    ,		(self.batch_size, config.num_labels))



		def 						UpperCAmelCase							( self    ,		__a    ,		__a    ,		__a    ,		__a    ,		__a    ,		__a    ,		__a)	->	List[Any]:






							'''simple docstring'''
							_UpperCamelCase										=  AlbertForMaskedLM(config=__a)
							model.to(__a)
							model.eval()
							_UpperCamelCase										=  model(__a    ,		attention_mask=__a    ,		token_type_ids=__a    ,		labels=__a)
							self.parent.assertEqual(result.logits.shape    ,		(self.batch_size, self.seq_length, self.vocab_size))



		def 						UpperCAmelCase							( self    ,		__a    ,		__a    ,		__a    ,		__a    ,		__a    ,		__a    ,		__a)	->	Any:






							'''simple docstring'''
							_UpperCamelCase										=  AlbertForQuestionAnswering(config=__a)
							model.to(__a)
							model.eval()
							_UpperCamelCase										=  model(
							    __a    ,		attention_mask=__a    ,		token_type_ids=__a    ,		start_positions=__a    ,		end_positions=__a    ,		)
							self.parent.assertEqual(result.start_logits.shape    ,		(self.batch_size, self.seq_length))
							self.parent.assertEqual(result.end_logits.shape    ,		(self.batch_size, self.seq_length))



		def 						UpperCAmelCase							( self    ,		__a    ,		__a    ,		__a    ,		__a    ,		__a    ,		__a    ,		__a)	->	List[str]:






							'''simple docstring'''
							_UpperCamelCase										=  self.num_labels
							_UpperCamelCase										=  AlbertForSequenceClassification(__a)
							model.to(__a)
							model.eval()
							_UpperCamelCase										=  model(__a    ,		attention_mask=__a    ,		token_type_ids=__a    ,		labels=__a)
							self.parent.assertEqual(result.logits.shape    ,		(self.batch_size, self.num_labels))



		def 						UpperCAmelCase							( self    ,		__a    ,		__a    ,		__a    ,		__a    ,		__a    ,		__a    ,		__a)	->	Any:






							'''simple docstring'''
							_UpperCamelCase										=  self.num_labels
							_UpperCamelCase										=  AlbertForTokenClassification(config=__a)
							model.to(__a)
							model.eval()
							_UpperCamelCase										=  model(__a    ,		attention_mask=__a    ,		token_type_ids=__a    ,		labels=__a)
							self.parent.assertEqual(result.logits.shape    ,		(self.batch_size, self.seq_length, self.num_labels))



		def 						UpperCAmelCase							( self    ,		__a    ,		__a    ,		__a    ,		__a    ,		__a    ,		__a    ,		__a)	->	Any:






							'''simple docstring'''
							_UpperCamelCase										=  self.num_choices
							_UpperCamelCase										=  AlbertForMultipleChoice(config=__a)
							model.to(__a)
							model.eval()
							_UpperCamelCase										=  input_ids.unsqueeze(1).expand(-1    ,		self.num_choices    ,		-1).contiguous()
							_UpperCamelCase										=  token_type_ids.unsqueeze(1).expand(-1    ,		self.num_choices    ,		-1).contiguous()
							_UpperCamelCase										=  input_mask.unsqueeze(1).expand(-1    ,		self.num_choices    ,		-1).contiguous()
							_UpperCamelCase										=  model(
							    __a    ,		attention_mask=__a    ,		token_type_ids=__a    ,		labels=__a    ,		)
							self.parent.assertEqual(result.logits.shape    ,		(self.batch_size, self.num_choices))




		def 						UpperCAmelCase							( self)	->	List[str]:






							'''simple docstring'''
							_UpperCamelCase										=  self.prepare_config_and_inputs()
							(
							    (
							    _UpperCamelCase
							)			,  (
							    _UpperCamelCase
							)			,  (
							    _UpperCamelCase
							)			,  (
							    _UpperCamelCase
							)			,  (
							    _UpperCamelCase
							)			,  (
							    _UpperCamelCase
							)			,  (
							    _UpperCamelCase
							)			,  
							)										=  config_and_inputs
							_UpperCamelCase										=  {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
							return config, inputs_dict






@require_torch
class    _UpperCAmelCase(   lowerCamelCase				,					lowerCamelCase				,					unittest.TestCase     ):
		lowercase__      =				(
		    (
		        AlbertModel,
		        AlbertForPreTraining,
		        AlbertForMaskedLM,
		        AlbertForMultipleChoice,
		        AlbertForSequenceClassification,
		        AlbertForTokenClassification,
		        AlbertForQuestionAnswering,
		    )
		    if is_torch_available()
		    else ()
		)
		lowercase__      =				(
		    {
		        'feature-extraction': AlbertModel,
		        'fill-mask': AlbertForMaskedLM,
		        'question-answering': AlbertForQuestionAnswering,
		        'text-classification': AlbertForSequenceClassification,
		        'token-classification': AlbertForTokenClassification,
		        'zero-shot': AlbertForSequenceClassification,
		    }
		    if is_torch_available()
		    else {}
		)
		lowercase__      =				True



		def 						UpperCAmelCase							( self    ,		__a    ,		__a    ,		__a=False)	->	Tuple:






							'''simple docstring'''
							_UpperCamelCase										=  super()._prepare_for_class(__a    ,		__a    ,		return_labels=__a)

							if return_labels:
												if model_class in get_values(__a):
																	_UpperCamelCase										=  torch.zeros(
																	    (self.model_tester.batch_size, self.model_tester.seq_length)    ,		dtype=torch.long    ,		device=__a)
																	_UpperCamelCase										=  torch.zeros(
																	    self.model_tester.batch_size    ,		dtype=torch.long    ,		device=__a)
							return inputs_dict



		def 						UpperCAmelCase							( self)	->	Dict:






							'''simple docstring'''
							_UpperCamelCase										=  AlbertModelTester(self)
							_UpperCamelCase										=  ConfigTester(self    ,		config_class=__a    ,		hidden_size=37)



		def 						UpperCAmelCase							( self)	->	Optional[int]:






							'''simple docstring'''
							self.config_tester.run_common_tests()



		def 						UpperCAmelCase							( self)	->	List[Any]:






							'''simple docstring'''
							_UpperCamelCase										=  self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*__a)



		def 						UpperCAmelCase							( self)	->	int:






							'''simple docstring'''
							_UpperCamelCase										=  self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_pretraining(*__a)



		def 						UpperCAmelCase							( self)	->	Optional[int]:






							'''simple docstring'''
							_UpperCamelCase										=  self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_masked_lm(*__a)



		def 						UpperCAmelCase							( self)	->	Any:






							'''simple docstring'''
							_UpperCamelCase										=  self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_multiple_choice(*__a)



		def 						UpperCAmelCase							( self)	->	Any:






							'''simple docstring'''
							_UpperCamelCase										=  self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_question_answering(*__a)



		def 						UpperCAmelCase							( self)	->	List[Any]:






							'''simple docstring'''
							_UpperCamelCase										=  self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_sequence_classification(*__a)



		def 						UpperCAmelCase							( self)	->	Dict:






							'''simple docstring'''
							_UpperCamelCase										=  self.model_tester.prepare_config_and_inputs()
							for type in ["absolute", "relative_key", "relative_key_query"]:
												_UpperCamelCase										=  type
												self.model_tester.create_and_check_model(*__a)




		@slow
		def 						UpperCAmelCase							( self)	->	int:






							'''simple docstring'''
							for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
												_UpperCamelCase										=  AlbertModel.from_pretrained(__a)
												self.assertIsNotNone(__a)






@require_torch
class    _UpperCAmelCase(   unittest.TestCase     ):



		@slow
		def 						UpperCAmelCase							( self)	->	Tuple:






							'''simple docstring'''
							_UpperCamelCase										=  AlbertModel.from_pretrained('''albert-base-v2''')
							_UpperCamelCase										=  torch.tensor([[0, 3_45, 2_32, 3_28, 7_40, 1_40, 16_95, 69, 60_78, 15_88, 2]])
							_UpperCamelCase										=  torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])
							with torch.no_grad():
												_UpperCamelCase										=  model(__a    ,		attention_mask=__a)[0]
							_UpperCamelCase										=  torch.Size((1, 11, 7_68))
							self.assertEqual(output.shape    ,		__a)
							_UpperCamelCase										=  torch.tensor(
							    [[[-0.6513, 1.5035, -0.2766], [-0.6515, 1.5046, -0.2780], [-0.6512, 1.5049, -0.2784]]])

							self.assertTrue(torch.allclose(output[:, 1:4, 1:4]    ,		__a    ,		atol=1e-4))






