



import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase__		:					Optional[int]							=logging.get_logger(__name__)

lowerCAmelCase__		:					Dict							='''▁'''

lowerCAmelCase__		:					Union[str, Any]							={
    '''vocab_file''': '''vocab.json''',
    '''spm_file''': '''sentencepiece.bpe.model''',
    '''tokenizer_config_file''': '''tokenizer_config.json''',
}

lowerCAmelCase__		:					Optional[int]							={
    '''vocab_file''': {
        '''facebook/m2m100_418M''': '''https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json''',
        '''facebook/m2m100_1.2B''': '''https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json''',
    },
    '''spm_file''': {
        '''facebook/m2m100_418M''': '''https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model''',
        '''facebook/m2m100_1.2B''': '''https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model''',
    },
    '''tokenizer_config_file''': {
        '''facebook/m2m100_418M''': '''https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json''',
        '''facebook/m2m100_1.2B''': '''https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json''',
    },
}

lowerCAmelCase__		:					int							={
    '''facebook/m2m100_418M''': 1024,
}

# fmt: off
lowerCAmelCase__		:					List[str]							={
    '''m2m100''': ['''af''', '''am''', '''ar''', '''ast''', '''az''', '''ba''', '''be''', '''bg''', '''bn''', '''br''', '''bs''', '''ca''', '''ceb''', '''cs''', '''cy''', '''da''', '''de''', '''el''', '''en''', '''es''', '''et''', '''fa''', '''ff''', '''fi''', '''fr''', '''fy''', '''ga''', '''gd''', '''gl''', '''gu''', '''ha''', '''he''', '''hi''', '''hr''', '''ht''', '''hu''', '''hy''', '''id''', '''ig''', '''ilo''', '''is''', '''it''', '''ja''', '''jv''', '''ka''', '''kk''', '''km''', '''kn''', '''ko''', '''lb''', '''lg''', '''ln''', '''lo''', '''lt''', '''lv''', '''mg''', '''mk''', '''ml''', '''mn''', '''mr''', '''ms''', '''my''', '''ne''', '''nl''', '''no''', '''ns''', '''oc''', '''or''', '''pa''', '''pl''', '''ps''', '''pt''', '''ro''', '''ru''', '''sd''', '''si''', '''sk''', '''sl''', '''so''', '''sq''', '''sr''', '''ss''', '''su''', '''sv''', '''sw''', '''ta''', '''th''', '''tl''', '''tn''', '''tr''', '''uk''', '''ur''', '''uz''', '''vi''', '''wo''', '''xh''', '''yi''', '''yo''', '''zh''', '''zu'''],
    '''wmt21''': ['''en''', '''ha''', '''is''', '''ja''', '''cs''', '''ru''', '''zh''', '''de''']
}

class        UpperCAmelCase_  (       UpperCamelCase_	):

						'''simple docstring'''




						UpperCamelCase__      :			Optional[int]      =  VOCAB_FILES_NAMES
						UpperCamelCase__      :			str      =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						UpperCamelCase__      :			Union[str, Any]      =  PRETRAINED_VOCAB_FILES_MAP
						UpperCamelCase__      :			List[Any]      =  ['''input_ids''', '''attention_mask''']

						UpperCamelCase__      :			List[int]      =  []
						UpperCamelCase__      :			List[int]      =  []






						def __init__(   self				,							_A				,							_A				,							_A=None				,							_A=None				,							_A="<s>"				,							_A="</s>"				,							_A="</s>"				,							_A="<pad>"				,							_A="<unk>"				,							_A="m2m100"				,							_A = None				,							_A=8				,							**_A				,							):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= {} if sp_model_kwargs is None else sp_model_kwargs

													__SCREAMING_SNAKE_CASE									= language_codes
													__SCREAMING_SNAKE_CASE									= FAIRSEQ_LANGUAGE_CODES[language_codes]
													__SCREAMING_SNAKE_CASE									= {lang_code: f"""__{lang_code}__""" for lang_code in fairseq_language_code}

													__SCREAMING_SNAKE_CASE									= kwargs.get('additional_special_tokens'				,							[]		)
													kwargs["additional_special_tokens"] += [
													    self.get_lang_token(_A		)
													    for lang_code in fairseq_language_code
													    if self.get_lang_token(_A		) not in kwargs["additional_special_tokens"]
													]

													super().__init__(
													    src_lang=_A				,							tgt_lang=_A				,							bos_token=_A				,							eos_token=_A				,							sep_token=_A				,							unk_token=_A				,							pad_token=_A				,							language_codes=_A				,							sp_model_kwargs=self.sp_model_kwargs				,							num_madeup_words=_A				,							**_A				,							)

													__SCREAMING_SNAKE_CASE									= vocab_file
													__SCREAMING_SNAKE_CASE									= load_json(_A		)
													__SCREAMING_SNAKE_CASE									= {v: k for k, v in self.encoder.items()}
													__SCREAMING_SNAKE_CASE									= spm_file
													__SCREAMING_SNAKE_CASE									= load_spm(_A				,							self.sp_model_kwargs		)

													__SCREAMING_SNAKE_CASE									= len(self.encoder		)

													__SCREAMING_SNAKE_CASE									= {
													    self.get_lang_token(_A		): self.encoder_size + i for i, lang_code in enumerate(_A		)
													}
													__SCREAMING_SNAKE_CASE									= {lang_code: self.encoder_size + i for i, lang_code in enumerate(_A		)}
													__SCREAMING_SNAKE_CASE									= {v: k for k, v in self.lang_token_to_id.items()}

													__SCREAMING_SNAKE_CASE									= src_lang if src_lang is not None else 'en'
													__SCREAMING_SNAKE_CASE									= tgt_lang
													__SCREAMING_SNAKE_CASE									= self.get_lang_id(self._src_lang		)
													self.set_src_lang_special_tokens(self._src_lang		)

													__SCREAMING_SNAKE_CASE									= num_madeup_words






						@property
						def 	_A     (   self		):







													'''simple docstring'''

													return len(self.encoder		) + len(self.lang_token_to_id		)






						@property
						def 	_A     (   self		):







													'''simple docstring'''

													return self._src_lang






						@src_lang.setter
						def 	_A     (   self				,							_A		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= new_src_lang
													self.set_src_lang_special_tokens(self._src_lang		)






						def 	_A     (   self				,							_A		):







													'''simple docstring'''

													return self.sp_model.encode(_A				,							out_type=_A		)






						def 	_A     (   self				,							_A		):







													'''simple docstring'''

													if token in self.lang_token_to_id:
																				return self.lang_token_to_id[token]
													return self.encoder.get(_A				,							self.encoder[self.unk_token]		)






						def 	_A     (   self				,							_A		):







													'''simple docstring'''

													if index in self.id_to_lang_token:
																				return self.id_to_lang_token[index]
													return self.decoder.get(_A				,							self.unk_token		)






						def 	_A     (   self				,							_A		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= []
													__SCREAMING_SNAKE_CASE									= ''
													for token in tokens:
																				# make sure that special tokens are not decoded using sentencepiece model
																				if token in self.all_special_tokens:
																											out_string += self.sp_model.decode(_A		) + token
																											__SCREAMING_SNAKE_CASE									= []
																				else:
																											current_sub_tokens.append(_A		)
													out_string += self.sp_model.decode(_A		)
													return out_string.strip()






						def 	_A     (   self				,							_A				,							_A = None				,							_A = False		):







													'''simple docstring'''

													if already_has_special_tokens:
																				return super().get_special_tokens_mask(
																				    token_ids_a=_A				,							token_ids_a=_A				,							already_has_special_tokens=_A		)

													__SCREAMING_SNAKE_CASE									= [1] * len(self.prefix_tokens		)
													__SCREAMING_SNAKE_CASE									= [1] * len(self.suffix_tokens		)
													if token_ids_a is None:
																				return prefix_ones + ([0] * len(_A		)) + suffix_ones
													return prefix_ones + ([0] * len(_A		)) + ([0] * len(_A		)) + suffix_ones






						def 	_A     (   self				,							_A				,							_A = None		):







													'''simple docstring'''

													if token_ids_a is None:
																				return self.prefix_tokens + token_ids_a + self.suffix_tokens
													# We don't expect to process pairs, but leave the pair logic for API consistency
													return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






						def 	_A     (   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= {self.convert_ids_to_tokens(_A		): i for i in range(self.vocab_size		)}
													vocab.update(self.added_tokens_encoder		)
													return vocab






						def __getstate__(   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= self.__dict__.copy()
													__SCREAMING_SNAKE_CASE									= None
													return state






						def __setstate__(   self				,							_A		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= d

													# for backward compatibility
													if not hasattr(self				,							'sp_model_kwargs'		):
																				__SCREAMING_SNAKE_CASE									= {}

													__SCREAMING_SNAKE_CASE									= load_spm(self.spm_file				,							self.sp_model_kwargs		)






						def 	_A     (   self				,							_A				,							_A = None		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= Path(_A		)
													if not save_dir.is_dir():
																				raise OSError(f"""{save_directory} should be a directory"""		)
													__SCREAMING_SNAKE_CASE									= save_dir / (
													    (filename_prefix + '-' if filename_prefix else '') + self.vocab_files_names['vocab_file']
													)
													__SCREAMING_SNAKE_CASE									= save_dir / (
													    (filename_prefix + '-' if filename_prefix else '') + self.vocab_files_names['spm_file']
													)

													save_json(self.encoder				,							_A		)

													if os.path.abspath(self.spm_file		) != os.path.abspath(_A		) and os.path.isfile(self.spm_file		):
																				copyfile(self.spm_file				,							_A		)
													elif not os.path.isfile(self.spm_file		):
																				with open(_A				,							'wb'		) as fi:
																											__SCREAMING_SNAKE_CASE									= self.sp_model.serialized_model_proto()
																											fi.write(_A		)

													return (str(_A		), str(_A		))






						def 	_A     (   self				,							_A				,							_A = "en"				,							_A = None				,							_A = "ro"				,							**_A				,							):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= src_lang
													__SCREAMING_SNAKE_CASE									= tgt_lang
													self.set_src_lang_special_tokens(self.src_lang		)
													return super().prepare_seqaseq_batch(_A				,							_A				,							**_A		)






						def 	_A     (   self				,							_A				,							_A				,							_A				,							**_A		):







													'''simple docstring'''

													if src_lang is None or tgt_lang is None:
																				raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'		)
													__SCREAMING_SNAKE_CASE									= src_lang
													__SCREAMING_SNAKE_CASE									= self(_A				,							add_special_tokens=_A				,							**_A		)
													__SCREAMING_SNAKE_CASE									= self.get_lang_id(_A		)
													__SCREAMING_SNAKE_CASE									= tgt_lang_id
													return inputs






						def 	_A     (   self		):







													'''simple docstring'''

													self.set_src_lang_special_tokens(self.src_lang		)






						def 	_A     (   self		):







													'''simple docstring'''

													self.set_tgt_lang_special_tokens(self.tgt_lang		)






						def 	_A     (   self				,							_A		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= self.get_lang_token(_A		)
													__SCREAMING_SNAKE_CASE									= self.lang_token_to_id[lang_token]
													__SCREAMING_SNAKE_CASE									= [self.cur_lang_id]
													__SCREAMING_SNAKE_CASE									= [self.eos_token_id]






						def 	_A     (   self				,							_A		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= self.get_lang_token(_A		)
													__SCREAMING_SNAKE_CASE									= self.lang_token_to_id[lang_token]
													__SCREAMING_SNAKE_CASE									= [self.cur_lang_id]
													__SCREAMING_SNAKE_CASE									= [self.eos_token_id]






						def 	_A     (   self				,							_A		):







													'''simple docstring'''

													return self.lang_code_to_token[lang]






						def 	_A     (   self				,							_A		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= self.get_lang_token(_A		)
													return self.lang_token_to_id[lang_token]







def      __lowercase       (			a__							,       a__       )    ->			sentencepiece.SentencePieceProcessor:
							__SCREAMING_SNAKE_CASE									= sentencepiece.SentencePieceProcessor(**a__       )
							spm.Load(str(a__       )       )
							return spm






def      __lowercase       (			a__       )    ->			Union[Dict, List]:
							with open(a__							,       'r'       ) as f:
														return json.load(a__       )






def      __lowercase       (			a__							,       a__       )    ->			None:
							with open(a__							,       'w'       ) as f:
														json.dump(a__							,       a__							,       indent=2       )











from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


lowerCAmelCase__		:					str							={'''configuration_vit_mae''': ['''VIT_MAE_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''ViTMAEConfig''']}

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			lowerCAmelCase__		:					Optional[int]							=[
			    '''VIT_MAE_PRETRAINED_MODEL_ARCHIVE_LIST''',
			    '''ViTMAEForPreTraining''',
			    '''ViTMAELayer''',
			    '''ViTMAEModel''',
			    '''ViTMAEPreTrainedModel''',
			]

try:
			if not is_tf_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			lowerCAmelCase__		:					List[str]							=[
			    '''TFViTMAEForPreTraining''',
			    '''TFViTMAEModel''',
			    '''TFViTMAEPreTrainedModel''',
			]

if TYPE_CHECKING:
			from .configuration_vit_mae import VIT_MAE_PRETRAINED_CONFIG_ARCHIVE_MAP, ViTMAEConfig

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_vit_mae import (
						    VIT_MAE_PRETRAINED_MODEL_ARCHIVE_LIST,
						    ViTMAEForPreTraining,
						    ViTMAELayer,
						    ViTMAEModel,
						    ViTMAEPreTrainedModel,
						)

			try:
						if not is_tf_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_tf_vit_mae import TFViTMAEForPreTraining, TFViTMAEModel, TFViTMAEPreTrainedModel


else:
			import sys

			lowerCAmelCase__		:					Optional[int]							=_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)






