






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCamelCase			    =     logging.get_logger(__name__)

UpperCamelCase			    =     '''▁'''

UpperCamelCase			    =     {'''vocab_file''': '''sentencepiece.bpe.model''', '''monolingual_vocab_file''': '''dict.txt'''}

UpperCamelCase			    =     {
    '''vocab_file''': {
        '''vinai/bartpho-syllable''': '''https://huggingface.co/vinai/bartpho-syllable/resolve/main/sentencepiece.bpe.model''',
    },
    '''monolingual_vocab_file''': {
        '''vinai/bartpho-syllable''': '''https://huggingface.co/vinai/bartpho-syllable/resolve/main/dict.txt''',
    },
}

UpperCamelCase			    =     {'''vinai/bartpho-syllable''': 1_024}



class 				__UpperCAmelCase		(__SCREAMING_SNAKE_CASE   ):
			__snake_case    :				Tuple									=							VOCAB_FILES_NAMES
			__snake_case    :				Dict									=							PRETRAINED_VOCAB_FILES_MAP
			__snake_case    :				Dict									=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			__snake_case    :				Union[str, Any]									=							['input_ids', 'attention_mask']







			def __init__(     self:  int				,     UpperCAmelCase_:  int				,     UpperCAmelCase_:  List[str]				,     UpperCAmelCase_:  Optional[Any]="<s>"				,     UpperCAmelCase_:  str="</s>"				,     UpperCAmelCase_:  List[str]="</s>"				,     UpperCAmelCase_:  Dict="<s>"				,     UpperCAmelCase_:  Any="<unk>"				,     UpperCAmelCase_:  Optional[int]="<pad>"				,     UpperCAmelCase_:  List[str]="<mask>"				,     UpperCAmelCase_:  Tuple = None				,     **UpperCAmelCase_:  Any				,     ) ->    Tuple:







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  AddedToken(_a				,     lstrip=_a				,     rstrip=_a					) if isinstance(_a				,     _a					) else mask_token

									_SCREAMING_SNAKE_CASE     				=  {} if sp_model_kwargs is None else sp_model_kwargs

									super().__init__(
									    bos_token=_a				,     eos_token=_a				,     unk_token=_a				,     sep_token=_a				,     cls_token=_a				,     pad_token=_a				,     mask_token=_a				,     sp_model_kwargs=self.sp_model_kwargs				,     **_a				,     )

									_SCREAMING_SNAKE_CASE     				=  vocab_file
									_SCREAMING_SNAKE_CASE     				=  monolingual_vocab_file
									_SCREAMING_SNAKE_CASE     				=  spm.SentencePieceProcessor(**self.sp_model_kwargs					)
									self.sp_model.Load(str(_a					)					)

									# Load the reduced vocab

									# Keep order of special tokens for backward compatibility
									_SCREAMING_SNAKE_CASE     				=  {}
									_SCREAMING_SNAKE_CASE     				=  0
									for token in [bos_token, pad_token, eos_token, unk_token, sep_token, cls_token]:
															if str(_a					) not in self.fairseq_tokens_to_ids:
																					_SCREAMING_SNAKE_CASE     				=  cnt
																					cnt += 1
									with open(_a				,     """r"""				,     encoding="""utf-8"""					) as f:
															for line in f.readlines():
																					_SCREAMING_SNAKE_CASE     				=  line.strip().split()[0]
																					_SCREAMING_SNAKE_CASE     				=  len(self.fairseq_tokens_to_ids					)
									if str(_a					) not in self.fairseq_tokens_to_ids:
															_SCREAMING_SNAKE_CASE     				=  len(self.fairseq_tokens_to_ids					)

									_SCREAMING_SNAKE_CASE     				=  {v: k for k, v in self.fairseq_tokens_to_ids.items()}







			def __getstate__(     self:  str					) ->    Optional[Any]:







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  self.__dict__.copy()
									_SCREAMING_SNAKE_CASE     				=  None
									_SCREAMING_SNAKE_CASE     				=  self.sp_model.serialized_model_proto()
									return state







			def __setstate__(     self:  Tuple				,     UpperCAmelCase_:  List[str]					) ->    int:







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  d

									# for backward compatibility
									if not hasattr(self				,     """sp_model_kwargs"""					):
															_SCREAMING_SNAKE_CASE     				=  {}

									_SCREAMING_SNAKE_CASE     				=  spm.SentencePieceProcessor(**self.sp_model_kwargs					)
									self.sp_model.LoadFromSerializedProto(self.sp_model_proto					)







			def 							UpperCamelCase				(     self:  Optional[int]				,     UpperCAmelCase_:  Union[str, Any]				,     UpperCAmelCase_:  Optional[int] = None					) ->    List[Any]:







									'''simple docstring'''




									if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
									_SCREAMING_SNAKE_CASE     				=  [self.cls_token_id]
									_SCREAMING_SNAKE_CASE     				=  [self.sep_token_id]
									return cls + token_ids_a + sep + sep + token_ids_a + sep







			def 							UpperCamelCase				(     self:  Any				,     UpperCAmelCase_:  str				,     UpperCAmelCase_:  List[str] = None				,     UpperCAmelCase_:  Tuple = False					) ->    Dict:







									'''simple docstring'''




									if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=_a				,     token_ids_a=_a				,     already_has_special_tokens=_a					)

									if token_ids_a is None:
															return [1] + ([0] * len(_a					)) + [1]
									return [1] + ([0] * len(_a					)) + [1, 1] + ([0] * len(_a					)) + [1]







			def 							UpperCamelCase				(     self:  Optional[int]				,     UpperCAmelCase_:  str				,     UpperCAmelCase_:  List[Any] = None					) ->    Optional[int]:







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  [self.sep_token_id]
									_SCREAMING_SNAKE_CASE     				=  [self.cls_token_id]

									if token_ids_a is None:
															return len(cls + token_ids_a + sep					) * [0]
									return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]







			@property
			def 							UpperCamelCase				(     self:  List[Any]					) ->    int:







									'''simple docstring'''




									return len(self.fairseq_ids_to_tokens					)







			def 							UpperCamelCase				(     self:  Any					) ->    int:







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  {self.convert_ids_to_tokens(_a					): i for i in range(self.vocab_size					)}
									vocab.update(self.added_tokens_encoder					)
									return vocab







			def 							UpperCamelCase				(     self:  Optional[Any]				,     UpperCAmelCase_:  Dict					) ->    Dict:







									'''simple docstring'''




									return self.sp_model.encode(_a				,     out_type=_a					)







			def 							UpperCamelCase				(     self:  int				,     UpperCAmelCase_:  Dict					) ->    Any:







									'''simple docstring'''




									if token in self.fairseq_tokens_to_ids:
															return self.fairseq_tokens_to_ids[token]
									else:
															return self.unk_token_id







			def 							UpperCamelCase				(     self:  Any				,     UpperCAmelCase_:  Dict					) ->    List[Any]:







									'''simple docstring'''




									return self.fairseq_ids_to_tokens[index]







			def 							UpperCamelCase				(     self:  Dict				,     UpperCAmelCase_:  Union[str, Any]					) ->    str:







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  """""".join(_a					).replace(_a				,     """ """					).strip()
									return out_string







			def 							UpperCamelCase				(     self:  Dict				,     UpperCAmelCase_:  List[Any]				,     UpperCAmelCase_:  Optional[int] = None					) ->    Optional[int]:







									'''simple docstring'''




									if not os.path.isdir(_a					):
															logger.error(F'Vocabulary path ({save_directory}) should be a directory'					)
															return
									_SCREAMING_SNAKE_CASE     				=  os.path.join(
									    _a				,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]					)
									_SCREAMING_SNAKE_CASE     				=  os.path.join(
									    _a				,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""monolingual_vocab_file"""]				,     )

									if os.path.abspath(self.vocab_file					) != os.path.abspath(_a					) and os.path.isfile(self.vocab_file					):
															copyfile(self.vocab_file				,     _a					)
									elif not os.path.isfile(self.vocab_file					):
															with open(_a				,     """wb"""					) as fi:
																					_SCREAMING_SNAKE_CASE     				=  self.sp_model.serialized_model_proto()
																					fi.write(_a					)

									if os.path.abspath(self.monolingual_vocab_file					) != os.path.abspath(
									    _a					) and os.path.isfile(self.monolingual_vocab_file					):
															copyfile(self.monolingual_vocab_file				,     _a					)
									elif not os.path.isfile(self.monolingual_vocab_file					):
															with open(_a				,     """w"""				,     encoding="""utf-8"""					) as fp:
																					for token in self.fairseq_tokens_to_ids:
																											if token not in self.all_special_tokens:
																																	fp.write(F'{str(_a					)} \n'					)

									return out_vocab_file, out_monolingual_vocab_file









from typing import TYPE_CHECKING

from ....utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


UpperCamelCase			    =     {'''configuration_mmbt''': ['''MMBTConfig''']}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		UpperCamelCase			    =     ['''MMBTForClassification''', '''MMBTModel''', '''ModalEmbeddings''']


if TYPE_CHECKING:
		from .configuration_mmbt import MMBTConfig

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_mmbt import MMBTForClassification, MMBTModel, ModalEmbeddings

else:
		import sys

		UpperCamelCase			    =     _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)

