



"""simple docstring"""




import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase__		:					int							=logging.get_logger(__name__)

lowerCAmelCase__		:					Union[str, Any]							='▁'

lowerCAmelCase__		:					Dict							={'vocab_file': 'sentencepiece.bpe.model'}

lowerCAmelCase__		:					Union[str, Any]							={
    'vocab_file': {
        'facebook/xglm-564M': 'https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model',
    }
}

lowerCAmelCase__		:					str							={
    'facebook/xglm-564M': 2048,
}

class        UpperCAmelCase_  (       SCREAMING_SNAKE_CASE__	):

						'''simple docstring'''




						UpperCamelCase__      :			Optional[int]      =  VOCAB_FILES_NAMES
						UpperCamelCase__      :			Any      =  PRETRAINED_VOCAB_FILES_MAP
						UpperCamelCase__      :			List[str]      =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						UpperCamelCase__      :			Union[str, Any]      =  ['''input_ids''', '''attention_mask''']






						def __init__(   self				,							_A				,							_A="<s>"				,							_A="</s>"				,							_A="</s>"				,							_A="<s>"				,							_A="<unk>"				,							_A="<pad>"				,							_A = None				,							**_A				,							):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= {} if sp_model_kwargs is None else sp_model_kwargs

													# Compatibility with the original tokenizer
													__SCREAMING_SNAKE_CASE									= 7
													__SCREAMING_SNAKE_CASE									= [f"""<madeupword{i}>""" for i in range(self.num_madeup_words		)]

													__SCREAMING_SNAKE_CASE									= kwargs.get('additional_special_tokens'				,							[]		)
													kwargs["additional_special_tokens"] += [
													    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
													]

													super().__init__(
													    bos_token=_A				,							eos_token=_A				,							unk_token=_A				,							sep_token=_A				,							cls_token=_A				,							pad_token=_A				,							sp_model_kwargs=self.sp_model_kwargs				,							**_A				,							)

													__SCREAMING_SNAKE_CASE									= spm.SentencePieceProcessor(**self.sp_model_kwargs		)
													self.sp_model.Load(str(_A		)		)
													__SCREAMING_SNAKE_CASE									= vocab_file

													# Original fairseq vocab and spm vocab must be "aligned":
													# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
													# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
													# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
													# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

													# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
													__SCREAMING_SNAKE_CASE									= 1

													# Mimic fairseq token-to-id alignment for the first 4 token
													__SCREAMING_SNAKE_CASE									= {'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

													__SCREAMING_SNAKE_CASE									= len(self.sp_model		)
													__SCREAMING_SNAKE_CASE									= {f"""<madeupword{i}>""": sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words		)}
													self.fairseq_tokens_to_ids.update(_A		)

													__SCREAMING_SNAKE_CASE									= {v: k for k, v in self.fairseq_tokens_to_ids.items()}






						def __getstate__(   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= self.__dict__.copy()
													__SCREAMING_SNAKE_CASE									= None
													__SCREAMING_SNAKE_CASE									= self.sp_model.serialized_model_proto()
													return state






						def __setstate__(   self				,							_A		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= d

													# for backward compatibility
													if not hasattr(self				,							'sp_model_kwargs'		):
																				__SCREAMING_SNAKE_CASE									= {}

													__SCREAMING_SNAKE_CASE									= spm.SentencePieceProcessor(**self.sp_model_kwargs		)
													self.sp_model.LoadFromSerializedProto(self.sp_model_proto		)






						def 	_A     (   self				,							_A				,							_A = None		):







													'''simple docstring'''

													if token_ids_a is None:
																				return [self.sep_token_id] + token_ids_a
													__SCREAMING_SNAKE_CASE									= [self.sep_token_id]
													return sep + token_ids_a + sep + sep + token_ids_a






						def 	_A     (   self				,							_A				,							_A = None				,							_A = False		):







													'''simple docstring'''

													if already_has_special_tokens:
																				return super().get_special_tokens_mask(
																				    token_ids_a=_A				,							token_ids_a=_A				,							already_has_special_tokens=_A		)

													if token_ids_a is None:
																				return [1] + ([0] * len(_A		))
													return [1] + ([0] * len(_A		)) + [1, 1] + ([0] * len(_A		))






						def 	_A     (   self				,							_A				,							_A = None		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= [self.sep_token_id]

													if token_ids_a is None:
																				return len(sep + token_ids_a		) * [0]
													return len(sep + token_ids_a + sep + sep + token_ids_a		) * [0]






						@property
						def 	_A     (   self		):







													'''simple docstring'''

													return len(self.sp_model		) + self.fairseq_offset + self.num_madeup_words






						def 	_A     (   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= {self.convert_ids_to_tokens(_A		): i for i in range(self.vocab_size		)}
													vocab.update(self.added_tokens_encoder		)
													return vocab






						def 	_A     (   self				,							_A		):







													'''simple docstring'''

													return self.sp_model.encode(_A				,							out_type=_A		)






						def 	_A     (   self				,							_A		):







													'''simple docstring'''

													if token in self.fairseq_tokens_to_ids:
																				return self.fairseq_tokens_to_ids[token]
													__SCREAMING_SNAKE_CASE									= self.sp_model.PieceToId(_A		)

													# Need to return unknown token if the SP model returned 0
													return spm_id + self.fairseq_offset if spm_id else self.unk_token_id






						def 	_A     (   self				,							_A		):







													'''simple docstring'''

													if index in self.fairseq_ids_to_tokens:
																				return self.fairseq_ids_to_tokens[index]
													return self.sp_model.IdToPiece(index - self.fairseq_offset		)






						def 	_A     (   self				,							_A		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= ''.join(_A		).replace(_A				,							' '		).strip()
													return out_string






						def 	_A     (   self				,							_A				,							_A = None		):







													'''simple docstring'''

													if not os.path.isdir(_A		):
																				logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""		)
																				return
													__SCREAMING_SNAKE_CASE									= os.path.join(
													    _A				,							(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']		)

													if os.path.abspath(self.vocab_file		) != os.path.abspath(_A		) and os.path.isfile(self.vocab_file		):
																				copyfile(self.vocab_file				,							_A		)
													elif not os.path.isfile(self.vocab_file		):
																				with open(_A				,							'wb'		) as fi:
																											__SCREAMING_SNAKE_CASE									= self.sp_model.serialized_model_proto()
																											fi.write(_A		)

													return (out_vocab_file,)











from ..utils import DummyObject, requires_backends

class        UpperCAmelCase_  (       metaclass=UpperCamelCase_	):

						'''simple docstring'''




						UpperCamelCase__      :			Optional[int]      =  ['''speech''']






						def __init__(   self				,							*_A				,							**_A		):







													'''simple docstring'''

													requires_backends(self				,							['speech']		)







class        UpperCAmelCase_  (       metaclass=UpperCamelCase_	):

						'''simple docstring'''




						UpperCamelCase__      :			Optional[int]      =  ['''speech''']






						def __init__(   self				,							*_A				,							**_A		):







													'''simple docstring'''

													requires_backends(self				,							['speech']		)






