






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCamelCase			    =     logging.get_logger(__name__)

UpperCamelCase			    =     '''▁'''

UpperCamelCase			    =     {'''vocab_file''': '''sentencepiece.bpe.model'''}

UpperCamelCase			    =     {
    '''vocab_file''': {
        '''facebook/xglm-564M''': '''https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model''',
    }
}

UpperCamelCase			    =     {
    '''facebook/xglm-564M''': 2_048,
}



class 				__UpperCAmelCase		(_UpperCAmelCase   ):
			__snake_case    :				Optional[Any]									=							VOCAB_FILES_NAMES
			__snake_case    :				List[str]									=							PRETRAINED_VOCAB_FILES_MAP
			__snake_case    :				List[str]									=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			__snake_case    :				Any									=							["input_ids", "attention_mask"]







			def __init__(     self:  Optional[int]				,     UpperCAmelCase_:  Union[str, Any]				,     UpperCAmelCase_:  Any="<s>"				,     UpperCAmelCase_:  Optional[Any]="</s>"				,     UpperCAmelCase_:  List[str]="</s>"				,     UpperCAmelCase_:  List[str]="<s>"				,     UpperCAmelCase_:  List[Any]="<unk>"				,     UpperCAmelCase_:  List[Any]="<pad>"				,     UpperCAmelCase_:  Optional[Dict[str, Any]] = None				,     **UpperCAmelCase_:  Optional[int]				,     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  {} if sp_model_kwargs is None else sp_model_kwargs

									# Compatibility with the original tokenizer
									_SCREAMING_SNAKE_CASE     				=  7
									_SCREAMING_SNAKE_CASE     				=  [F'<madeupword{i}>' for i in range(self.num_madeup_words					)]

									_SCREAMING_SNAKE_CASE     				=  kwargs.get("""additional_special_tokens"""				,     []					)
									kwargs["additional_special_tokens"] += [
									    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
									]

									super().__init__(
									    bos_token=UpperCAmelCase_				,     eos_token=UpperCAmelCase_				,     unk_token=UpperCAmelCase_				,     sep_token=UpperCAmelCase_				,     cls_token=UpperCAmelCase_				,     pad_token=UpperCAmelCase_				,     sp_model_kwargs=self.sp_model_kwargs				,     **UpperCAmelCase_				,     )

									_SCREAMING_SNAKE_CASE     				=  spm.SentencePieceProcessor(**self.sp_model_kwargs					)
									self.sp_model.Load(str(UpperCAmelCase_					)					)
									_SCREAMING_SNAKE_CASE     				=  vocab_file

									# Original fairseq vocab and spm vocab must be "aligned":
									# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
									# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
									# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
									# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

									# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
									_SCREAMING_SNAKE_CASE     				=  1

									# Mimic fairseq token-to-id alignment for the first 4 token
									_SCREAMING_SNAKE_CASE     				=  {"""<s>""": 0, """<pad>""": 1, """</s>""": 2, """<unk>""": 3}

									_SCREAMING_SNAKE_CASE     				=  len(self.sp_model					)
									_SCREAMING_SNAKE_CASE     				=  {F'<madeupword{i}>': sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words					)}
									self.fairseq_tokens_to_ids.update(UpperCAmelCase_					)

									_SCREAMING_SNAKE_CASE     				=  {v: k for k, v in self.fairseq_tokens_to_ids.items()}







			def __getstate__(     self:  Union[str, Any]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  self.__dict__.copy()
									_SCREAMING_SNAKE_CASE     				=  None
									_SCREAMING_SNAKE_CASE     				=  self.sp_model.serialized_model_proto()
									return state







			def __setstate__(     self:  str				,     UpperCAmelCase_:  Dict					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  d

									# for backward compatibility
									if not hasattr(self				,     """sp_model_kwargs"""					):
															_SCREAMING_SNAKE_CASE     				=  {}

									_SCREAMING_SNAKE_CASE     				=  spm.SentencePieceProcessor(**self.sp_model_kwargs					)
									self.sp_model.LoadFromSerializedProto(self.sp_model_proto					)







			def 							UpperCamelCase				(     self:  Optional[int]				,     UpperCAmelCase_:  List[int]				,     UpperCAmelCase_:  Optional[List[int]] = None					):







									'''simple docstring'''




									if token_ids_a is None:
															return [self.sep_token_id] + token_ids_a
									_SCREAMING_SNAKE_CASE     				=  [self.sep_token_id]
									return sep + token_ids_a + sep + sep + token_ids_a







			def 							UpperCamelCase				(     self:  List[str]				,     UpperCAmelCase_:  List[int]				,     UpperCAmelCase_:  Optional[List[int]] = None				,     UpperCAmelCase_:  bool = False					):







									'''simple docstring'''




									if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=UpperCAmelCase_				,     token_ids_a=UpperCAmelCase_				,     already_has_special_tokens=UpperCAmelCase_					)

									if token_ids_a is None:
															return [1] + ([0] * len(UpperCAmelCase_					))
									return [1] + ([0] * len(UpperCAmelCase_					)) + [1, 1] + ([0] * len(UpperCAmelCase_					))







			def 							UpperCamelCase				(     self:  List[Any]				,     UpperCAmelCase_:  List[int]				,     UpperCAmelCase_:  Optional[List[int]] = None					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  [self.sep_token_id]

									if token_ids_a is None:
															return len(sep + token_ids_a					) * [0]
									return len(sep + token_ids_a + sep + sep + token_ids_a					) * [0]







			@property
			def 							UpperCamelCase				(     self:  Union[str, Any]					):







									'''simple docstring'''




									return len(self.sp_model					) + self.fairseq_offset + self.num_madeup_words







			def 							UpperCamelCase				(     self:  Optional[Any]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  {self.convert_ids_to_tokens(UpperCAmelCase_					): i for i in range(self.vocab_size					)}
									vocab.update(self.added_tokens_encoder					)
									return vocab







			def 							UpperCamelCase				(     self:  int				,     UpperCAmelCase_:  str					):







									'''simple docstring'''




									return self.sp_model.encode(UpperCAmelCase_				,     out_type=UpperCAmelCase_					)







			def 							UpperCamelCase				(     self:  List[str]				,     UpperCAmelCase_:  List[str]					):







									'''simple docstring'''




									if token in self.fairseq_tokens_to_ids:
															return self.fairseq_tokens_to_ids[token]
									_SCREAMING_SNAKE_CASE     				=  self.sp_model.PieceToId(UpperCAmelCase_					)

									# Need to return unknown token if the SP model returned 0
									return spm_id + self.fairseq_offset if spm_id else self.unk_token_id







			def 							UpperCamelCase				(     self:  List[str]				,     UpperCAmelCase_:  Optional[int]					):







									'''simple docstring'''




									if index in self.fairseq_ids_to_tokens:
															return self.fairseq_ids_to_tokens[index]
									return self.sp_model.IdToPiece(index - self.fairseq_offset					)







			def 							UpperCamelCase				(     self:  Dict				,     UpperCAmelCase_:  Optional[int]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  """""".join(UpperCAmelCase_					).replace(UpperCAmelCase_				,     """ """					).strip()
									return out_string







			def 							UpperCamelCase				(     self:  Dict				,     UpperCAmelCase_:  str				,     UpperCAmelCase_:  Optional[str] = None					):







									'''simple docstring'''




									if not os.path.isdir(UpperCAmelCase_					):
															logger.error(F'Vocabulary path ({save_directory}) should be a directory'					)
															return
									_SCREAMING_SNAKE_CASE     				=  os.path.join(
									    UpperCAmelCase_				,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]					)

									if os.path.abspath(self.vocab_file					) != os.path.abspath(UpperCAmelCase_					) and os.path.isfile(self.vocab_file					):
															copyfile(self.vocab_file				,     UpperCAmelCase_					)
									elif not os.path.isfile(self.vocab_file					):
															with open(UpperCAmelCase_				,     """wb"""					) as fi:
																					_SCREAMING_SNAKE_CASE     				=  self.sp_model.serialized_model_proto()
																					fi.write(UpperCAmelCase_					)

									return (out_vocab_file,)









UpperCamelCase			    =     {0: [2, 3], 1: [0], 2: [1], 3: [4], 4: []}

UpperCamelCase			    =     {0: [1, 2, 3], 1: [2], 2: [0], 3: [4], 4: [5], 5: [3]}

def 	__lowerCamelCase      (		snake_case__	,snake_case__	,snake_case__		)							->  list[int]:






						"""simple docstring"""





						_SCREAMING_SNAKE_CASE     				=  True
						_SCREAMING_SNAKE_CASE     				=  []

						for neighbour in graph[vert]:
												if not visited[neighbour]:
																		order += topology_sort(snake_case__	,snake_case__	,snake_case__		)

						order.append(snake_case__		)

						return order

def 	__lowerCamelCase      (		snake_case__	,snake_case__	,snake_case__		)							->  list[int]:






						"""simple docstring"""





						_SCREAMING_SNAKE_CASE     				=  True
						_SCREAMING_SNAKE_CASE     				=  [vert]

						for neighbour in reversed_graph[vert]:
												if not visited[neighbour]:
																		component += find_components(snake_case__	,snake_case__	,snake_case__		)

						return component




def 	__lowerCamelCase      (		snake_case__		)							->  list[list[int]]:






						"""simple docstring"""





						_SCREAMING_SNAKE_CASE     				=  len(snake_case__		) * [False]
						_SCREAMING_SNAKE_CASE     				=  {vert: [] for vert in range(len(snake_case__		)		)}

						for vert, neighbours in graph.items():
												for neighbour in neighbours:
																		reversed_graph[neighbour].append(snake_case__		)

						_SCREAMING_SNAKE_CASE     				=  []
						for i, was_visited in enumerate(snake_case__		):
												if not was_visited:
																		order += topology_sort(snake_case__	,snake_case__	,snake_case__		)

						_SCREAMING_SNAKE_CASE     				=  []
						_SCREAMING_SNAKE_CASE     				=  len(snake_case__		) * [False]

						for i in range(len(snake_case__		)		):
												_SCREAMING_SNAKE_CASE     				=  order[len(snake_case__		) - i - 1]
												if not visited[vert]:
																		_SCREAMING_SNAKE_CASE     				=  find_components(snake_case__	,snake_case__	,snake_case__		)
																		components_list.append(snake_case__		)

						return components_list

