

'''simple docstring'''



import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


__snake_case			:     Optional[Any]										=      logging.get_logger(__name__)

__snake_case			:     Union[str, Any]										=      {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

__snake_case			:     Tuple										=      {
    'vocab_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/vocab.json',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/vocab.json',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/vocab.json',
        'roberta-base-openai-detector': 'https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json',
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/merges.txt',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/merges.txt',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/merges.txt',
        'roberta-base-openai-detector': 'https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt',
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt'
        ),
    },
    'tokenizer_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/tokenizer.json',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/tokenizer.json',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json',
        'roberta-base-openai-detector': (
            'https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json'
        ),
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json'
        ),
    },
}

__snake_case			:     Optional[Any]										=      {
    'roberta-base': 512,
    'roberta-large': 512,
    'roberta-large-mnli': 512,
    'distilroberta-base': 512,
    'roberta-base-openai-detector': 512,
    'roberta-large-openai-detector': 512,
}



class lowerCamelCase			(  lowercase_						):



			'''simple docstring'''






			__snake_case        =	VOCAB_FILES_NAMES
			__snake_case        =	PRETRAINED_VOCAB_FILES_MAP
			__snake_case        =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			__snake_case        =	['input_ids', 'attention_mask']
			__snake_case        =	RobertaTokenizer


			def __init__(				self	:						Union[str, Any]		,      lowerCAmelCase_	:						Union[str, Any]=None		,      lowerCAmelCase_	:						List[str]=None		,      lowerCAmelCase_	:						List[Any]=None		,      lowerCAmelCase_	:						Any="replace"		,      lowerCAmelCase_	:						Union[str, Any]="<s>"		,      lowerCAmelCase_	:						Tuple="</s>"		,      lowerCAmelCase_	:						Optional[Any]="</s>"		,      lowerCAmelCase_	:						Any="<s>"		,      lowerCAmelCase_	:						List[str]="<unk>"		,      lowerCAmelCase_	:						Optional[int]="<pad>"		,      lowerCAmelCase_	:						str="<mask>"		,      lowerCAmelCase_	:						Tuple=False		,      lowerCAmelCase_	:						Any=True		,      **lowerCAmelCase_	:						Tuple		,      )      ->		Optional[Any]:





									'''simple docstring'''
									super().__init__(
									    lowerCAmelCase_		,      lowerCAmelCase_		,      tokenizer_file=lowerCAmelCase_		,      errors=lowerCAmelCase_		,      bos_token=lowerCAmelCase_		,      eos_token=lowerCAmelCase_		,      sep_token=lowerCAmelCase_		,      cls_token=lowerCAmelCase_		,      unk_token=lowerCAmelCase_		,      pad_token=lowerCAmelCase_		,      mask_token=lowerCAmelCase_		,      add_prefix_space=lowerCAmelCase_		,      trim_offsets=lowerCAmelCase_		,      **lowerCAmelCase_		,      )

									A__					:       Optional[Any]     =json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()			)
									if pre_tok_state.get("""add_prefix_space"""		,      lowerCAmelCase_			) != add_prefix_space:
															A__					:       Union[str, Any]     =getattr(lowerCAmelCase_		,      pre_tok_state.pop("""type"""			)			)
															A__					:       int     =add_prefix_space
															A__					:       str     =pre_tok_class(**lowerCAmelCase_			)

									A__					:       int     =add_prefix_space

									A__					:       Optional[int]     ="""post_processor"""
									A__					:       Any     =getattr(self.backend_tokenizer		,      lowerCAmelCase_		,      lowerCAmelCase_			)
									if tokenizer_component_instance:
															A__					:       Tuple     =json.loads(tokenizer_component_instance.__getstate__()			)

															# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
															if "sep" in state:
																					A__					:       Any     =tuple(state["""sep"""]			)
															if "cls" in state:
																					A__					:       str     =tuple(state["""cls"""]			)

															A__					:       Any     =False

															if state.get("""add_prefix_space"""		,      lowerCAmelCase_			) != add_prefix_space:
																					A__					:       Union[str, Any]     =add_prefix_space
																					A__					:       int     =True

															if state.get("""trim_offsets"""		,      lowerCAmelCase_			) != trim_offsets:
																					A__					:       Dict     =trim_offsets
																					A__					:       int     =True

															if changes_to_apply:
																					A__					:       List[str]     =getattr(lowerCAmelCase_		,      state.pop("""type"""			)			)
																					A__					:       Any     =component_class(**lowerCAmelCase_			)
																					setattr(self.backend_tokenizer		,      lowerCAmelCase_		,      lowerCAmelCase_			)


			@property
			def lowercase__						(				self	:						Optional[Any]			)      ->		str:





									'''simple docstring'''
									if self._mask_token is None:
															if self.verbose:
																					logger.error("""Using mask_token, but it is not set yet."""			)
															return None
									return str(self._mask_token			)


			@mask_token.setter
			def lowercase__						(				self	:						int		,      lowerCAmelCase_	:						Optional[Any]			)      ->		Union[str, Any]:





									'''simple docstring'''
									A__					:       int     =AddedToken(lowerCAmelCase_		,      lstrip=lowerCAmelCase_		,      rstrip=lowerCAmelCase_			) if isinstance(lowerCAmelCase_		,      lowerCAmelCase_			) else value
									A__					:       int     =value


			def lowercase__						(				self	:						str		,      *lowerCAmelCase_	:						List[str]		,      **lowerCAmelCase_	:						int			)      ->		BatchEncoding:





									'''simple docstring'''
									A__					:       int     =kwargs.get("""is_split_into_words"""		,      lowerCAmelCase_			)
									assert self.add_prefix_space or not is_split_into_words, (
									    f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
									    "to use it with pretokenized inputs."
									)

									return super()._batch_encode_plus(*lowerCAmelCase_		,      **lowerCAmelCase_			)


			def lowercase__						(				self	:						Optional[int]		,      *lowerCAmelCase_	:						Tuple		,      **lowerCAmelCase_	:						Optional[int]			)      ->		BatchEncoding:





									'''simple docstring'''
									A__					:       Dict     =kwargs.get("""is_split_into_words"""		,      lowerCAmelCase_			)

									assert self.add_prefix_space or not is_split_into_words, (
									    f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
									    "to use it with pretokenized inputs."
									)

									return super()._encode_plus(*lowerCAmelCase_		,      **lowerCAmelCase_			)


			def lowercase__						(				self	:						List[Any]		,      lowerCAmelCase_	:						str		,      lowerCAmelCase_	:						Optional[str] = None			)      ->		Tuple[str]:





									'''simple docstring'''
									A__					:       Optional[int]     =self._tokenizer.model.save(lowerCAmelCase_		,      name=lowerCAmelCase_			)
									return tuple(lowerCAmelCase_			)


			def lowercase__						(				self	:						Any		,      lowerCAmelCase_	:						List[str]		,      lowerCAmelCase_	:						Tuple=None			)      ->		Dict:





									'''simple docstring'''
									A__					:       Any     =[self.bos_token_id] + token_ids_a + [self.eos_token_id]
									if token_ids_a is None:
															return output

									return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]



			def lowercase__						(				self	:						int		,      lowerCAmelCase_	:						List[int]		,      lowerCAmelCase_	:						Optional[List[int]] = None			)      ->		List[int]:





									'''simple docstring'''
									A__					:       Any     =[self.sep_token_id]
									A__					:       Union[str, Any]     =[self.cls_token_id]

									if token_ids_a is None:
															return len(cls + token_ids_a + sep			) * [0]
									return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]




'''simple docstring'''



import torch
from torch import nn



class lowerCamelCase			(  nn.Module						):



			'''simple docstring'''






			def __init__(				self	:						Any		,      lowerCAmelCase_	:						List[str]		,      lowerCAmelCase_	:						List[Any]		,      lowerCAmelCase_	:						Dict		,      lowerCAmelCase_	:						str		,      lowerCAmelCase_	:						Optional[int]=1		,      lowerCAmelCase_	:						str=False			)      ->		List[str]:





									'''simple docstring'''
									super().__init__()

									A__					:       Any     =n_token
									A__					:       int     =d_embed
									A__					:       Any     =d_proj

									A__					:       Tuple     =cutoffs + [n_token]
									A__					:       Optional[Any]     =[0] + self.cutoffs
									A__					:       Dict     =div_val

									A__					:       str     =self.cutoffs[0]
									A__					:       Optional[Any]     =len(self.cutoffs			) - 1
									A__					:       List[Any]     =self.shortlist_size + self.n_clusters

									if self.n_clusters > 0:
															A__					:       Any     =nn.Parameter(torch.zeros(self.n_clusters		,      self.d_embed			)			)
															A__					:       str     =nn.Parameter(torch.zeros(self.n_clusters			)			)

									A__					:       Union[str, Any]     =nn.ModuleList()
									A__					:       Optional[int]     =nn.ParameterList()

									if div_val == 1:
															for i in range(len(self.cutoffs			)			):
																					if d_proj != d_embed:
																											self.out_projs.append(nn.Parameter(torch.FloatTensor(lowerCAmelCase_		,      lowerCAmelCase_			)			)			)
																					else:
																											self.out_projs.append(lowerCAmelCase_			)

															self.out_layers.append(nn.Linear(lowerCAmelCase_		,      lowerCAmelCase_			)			)
									else:
															for i in range(len(self.cutoffs			)			):
																					A__   ,						A__					:       Optional[int]     =self.cutoff_ends[i], self.cutoff_ends[i + 1]
																					A__					:       Tuple     =d_embed // (div_val**i)

																					self.out_projs.append(nn.Parameter(torch.FloatTensor(lowerCAmelCase_		,      lowerCAmelCase_			)			)			)

																					self.out_layers.append(nn.Linear(lowerCAmelCase_		,      r_idx - l_idx			)			)

									A__					:       Optional[int]     =keep_order


			def lowercase__						(				self	:						Dict		,      lowerCAmelCase_	:						int		,      lowerCAmelCase_	:						Tuple		,      lowerCAmelCase_	:						Union[str, Any]		,      lowerCAmelCase_	:						Any			)      ->		Union[str, Any]:





									'''simple docstring'''
									if proj is None:
															A__					:       Optional[int]     =nn.functional.linear(lowerCAmelCase_		,      lowerCAmelCase_		,      bias=lowerCAmelCase_			)
									else:
															# if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
															A__					:       Optional[int]     =nn.functional.linear(lowerCAmelCase_		,      proj.t().contiguous()			)
															A__					:       Union[str, Any]     =nn.functional.linear(lowerCAmelCase_		,      lowerCAmelCase_		,      bias=lowerCAmelCase_			)
															# else:
															#     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
															#     if bias is not None:
															#         logit = logit + bias

									return logit


			def lowercase__						(				self	:						Tuple		,      lowerCAmelCase_	:						List[str]		,      lowerCAmelCase_	:						List[Any]=None		,      lowerCAmelCase_	:						Dict=False			)      ->		Optional[int]:





									'''simple docstring'''

									if labels is not None:
															# Shift so that tokens < n predict n
															A__					:       Optional[Any]     =hidden[..., :-1, :].contiguous()
															A__					:       List[Any]     =labels[..., 1:].contiguous()
															A__					:       Optional[int]     =hidden.view(-1		,      hidden.size(-1			)			)
															A__					:       str     =labels.view(-1			)
															if hidden.size(0			) != labels.size(0			):
																					raise RuntimeError("""Input and labels should have the same size in the batch dimension."""			)
									else:
															A__					:       Optional[int]     =hidden.view(-1		,      hidden.size(-1			)			)

									if self.n_clusters == 0:
															A__					:       Optional[Any]     =self._compute_logit(lowerCAmelCase_		,      self.out_layers[0].weight		,      self.out_layers[0].bias		,      self.out_projs[0]			)
															if labels is not None:
																					A__					:       Tuple     =labels != -1_00
																					A__					:       int     =torch.zeros_like(lowerCAmelCase_		,      dtype=hidden.dtype		,      device=hidden.device			)
																					A__					:       Union[str, Any]     =(
																					    -nn.functional.log_softmax(lowerCAmelCase_		,      dim=-1			)[mask].gather(1		,      labels[mask].unsqueeze(1			)			).squeeze(1			)
																					)
															else:
																					A__					:       List[Any]     =nn.functional.log_softmax(lowerCAmelCase_		,      dim=-1			)
									else:
															# construct weights and biases
															A__   ,						A__					:       Any     =[], []
															for i in range(len(self.cutoffs			)			):
																					if self.div_val == 1:
																											A__   ,						A__					:       Optional[int]     =self.cutoff_ends[i], self.cutoff_ends[i + 1]
																											A__					:       int     =self.out_layers[0].weight[l_idx:r_idx]
																											A__					:       List[str]     =self.out_layers[0].bias[l_idx:r_idx]
																					else:
																											A__					:       List[str]     =self.out_layers[i].weight
																											A__					:       Union[str, Any]     =self.out_layers[i].bias

																					if i == 0:
																											A__					:       Tuple     =torch.cat([weight_i, self.cluster_weight]		,      dim=0			)
																											A__					:       List[str]     =torch.cat([bias_i, self.cluster_bias]		,      dim=0			)

																					weights.append(lowerCAmelCase_			)
																					biases.append(lowerCAmelCase_			)

															A__   ,						A__   ,						A__					:       Tuple     =weights[0], biases[0], self.out_projs[0]

															A__					:       List[Any]     =self._compute_logit(lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_			)
															A__					:       int     =nn.functional.log_softmax(lowerCAmelCase_		,      dim=1			)

															if labels is None:
																					A__					:       Union[str, Any]     =hidden.new_empty((head_logit.size(0			), self.n_token)			)
															else:
																					A__					:       Union[str, Any]     =torch.zeros_like(lowerCAmelCase_		,      dtype=hidden.dtype		,      device=hidden.device			)

															A__					:       Any     =0
															A__					:       Tuple     =[0] + self.cutoffs
															for i in range(len(lowerCAmelCase_			) - 1			):
																					A__   ,						A__					:       Tuple     =cutoff_values[i], cutoff_values[i + 1]

																					if labels is not None:
																											A__					:       Tuple     =(labels >= l_idx) & (labels < r_idx)
																											A__					:       Any     =mask_i.nonzero().squeeze()

																											if indices_i.numel() == 0:
																																	continue

																											A__					:       int     =labels.index_select(0		,      lowerCAmelCase_			) - l_idx
																											A__					:       List[str]     =head_logprob.index_select(0		,      lowerCAmelCase_			)
																											A__					:       str     =hidden.index_select(0		,      lowerCAmelCase_			)
																					else:
																											A__					:       Optional[Any]     =hidden

																					if i == 0:
																											if labels is not None:
																																	A__					:       Optional[Any]     =head_logprob_i.gather(1		,      target_i[:, None]			).squeeze(1			)
																											else:
																																	A__					:       Union[str, Any]     =head_logprob[:, : self.cutoffs[0]]
																					else:
																											A__   ,						A__   ,						A__					:       Dict     =weights[i], biases[i], self.out_projs[i]

																											A__					:       List[Any]     =self._compute_logit(lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_			)
																											A__					:       List[Any]     =nn.functional.log_softmax(lowerCAmelCase_		,      dim=1			)
																											A__					:       Optional[Any]     =self.cutoffs[0] + i - 1  # No probability for the head cluster
																											if labels is not None:
																																	A__					:       Union[str, Any]     =head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
																																	    1		,      target_i[:, None]			).squeeze(1			)
																											else:
																																	A__					:       List[str]     =head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
																																	A__					:       Tuple     =logprob_i

																					if labels is not None:
																											if (hasattr(self		,      """keep_order"""			) and self.keep_order) or keep_order:
																																	out.index_copy_(0		,      lowerCAmelCase_		,      -logprob_i			)
																											else:
																																	out[offset : offset + logprob_i.size(0			)].copy_(-logprob_i			)
																											offset += logprob_i.size(0			)

									return out



			def lowercase__						(				self	:						List[str]		,      lowerCAmelCase_	:						Optional[Any]			)      ->		Any:





									'''simple docstring'''
									if self.n_clusters == 0:
															A__					:       List[str]     =self._compute_logit(lowerCAmelCase_		,      self.out_layers[0].weight		,      self.out_layers[0].bias		,      self.out_projs[0]			)
															return nn.functional.log_softmax(lowerCAmelCase_		,      dim=-1			)
									else:
															# construct weights and biases
															A__   ,						A__					:       List[str]     =[], []
															for i in range(len(self.cutoffs			)			):
																					if self.div_val == 1:
																											A__   ,						A__					:       int     =self.cutoff_ends[i], self.cutoff_ends[i + 1]
																											A__					:       List[str]     =self.out_layers[0].weight[l_idx:r_idx]
																											A__					:       List[Any]     =self.out_layers[0].bias[l_idx:r_idx]
																					else:
																											A__					:       Dict     =self.out_layers[i].weight
																											A__					:       Any     =self.out_layers[i].bias

																					if i == 0:
																											A__					:       List[str]     =torch.cat([weight_i, self.cluster_weight]		,      dim=0			)
																											A__					:       Tuple     =torch.cat([bias_i, self.cluster_bias]		,      dim=0			)

																					weights.append(lowerCAmelCase_			)
																					biases.append(lowerCAmelCase_			)

															A__   ,						A__   ,						A__					:       Optional[int]     =weights[0], biases[0], self.out_projs[0]
															A__					:       Any     =self._compute_logit(lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_			)

															A__					:       Dict     =hidden.new_empty((head_logit.size(0			), self.n_token)			)
															A__					:       Dict     =nn.functional.log_softmax(lowerCAmelCase_		,      dim=1			)

															A__					:       Tuple     =[0] + self.cutoffs
															for i in range(len(lowerCAmelCase_			) - 1			):
																					A__   ,						A__					:       List[Any]     =cutoff_values[i], cutoff_values[i + 1]

																					if i == 0:
																											A__					:       Tuple     =head_logprob[:, : self.cutoffs[0]]
																					else:
																											A__   ,						A__   ,						A__					:       Any     =weights[i], biases[i], self.out_projs[i]

																											A__					:       Dict     =self._compute_logit(lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_			)
																											A__					:       str     =nn.functional.log_softmax(lowerCAmelCase_		,      dim=1			)

																											A__					:       str     =head_logprob[:, -i] + tail_logprob_i
																											A__					:       List[Any]     =logprob_i

															return out

