

'''simple docstring'''



import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
		from .tokenization_xlnet import XLNetTokenizer
else:
		__snake_case			:     Tuple										=      None


__snake_case			:     List[str]										=      logging.get_logger(__name__)

__snake_case			:     Union[str, Any]										=      {'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

__snake_case			:     List[str]										=      {
    'vocab_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model',
    },
    'tokenizer_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/tokenizer.json',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/tokenizer.json',
    },
}

__snake_case			:     Optional[int]										=      {
    'xlnet-base-cased': None,
    'xlnet-large-cased': None,
}

__snake_case			:     int										=      '▁'

# Segments (not really needed)
__snake_case			:     Union[str, Any]										=      0
__snake_case			:     Union[str, Any]										=      1
__snake_case			:     Optional[Any]										=      2
__snake_case			:     Dict										=      3
__snake_case			:     Any										=      4



class lowerCamelCase			(  lowercase_						):



			'''simple docstring'''






			__snake_case        =	VOCAB_FILES_NAMES
			__snake_case        =	PRETRAINED_VOCAB_FILES_MAP
			__snake_case        =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			__snake_case        =	'left'
			__snake_case        =	XLNetTokenizer


			def __init__(				self	:						List[str]		,      lowerCAmelCase_	:						Tuple=None		,      lowerCAmelCase_	:						Any=None		,      lowerCAmelCase_	:						str=False		,      lowerCAmelCase_	:						Tuple=True		,      lowerCAmelCase_	:						str=False		,      lowerCAmelCase_	:						List[Any]="<s>"		,      lowerCAmelCase_	:						Optional[Any]="</s>"		,      lowerCAmelCase_	:						int="<unk>"		,      lowerCAmelCase_	:						Optional[Any]="<sep>"		,      lowerCAmelCase_	:						Any="<pad>"		,      lowerCAmelCase_	:						int="<cls>"		,      lowerCAmelCase_	:						List[str]="<mask>"		,      lowerCAmelCase_	:						Optional[Any]=["<eop>", "<eod>"]		,      **lowerCAmelCase_	:						Any		,      )      ->		Union[str, Any]:





									'''simple docstring'''
									A__					:       List[Any]     =AddedToken(lowerCAmelCase_		,      lstrip=lowerCAmelCase_		,      rstrip=lowerCAmelCase_			) if isinstance(lowerCAmelCase_		,      lowerCAmelCase_			) else mask_token

									super().__init__(
									    vocab_file=lowerCAmelCase_		,      tokenizer_file=lowerCAmelCase_		,      do_lower_case=lowerCAmelCase_		,      remove_space=lowerCAmelCase_		,      keep_accents=lowerCAmelCase_		,      bos_token=lowerCAmelCase_		,      eos_token=lowerCAmelCase_		,      unk_token=lowerCAmelCase_		,      sep_token=lowerCAmelCase_		,      pad_token=lowerCAmelCase_		,      cls_token=lowerCAmelCase_		,      mask_token=lowerCAmelCase_		,      additional_special_tokens=lowerCAmelCase_		,      **lowerCAmelCase_		,      )

									A__					:       str     =3
									A__					:       Optional[Any]     =do_lower_case
									A__					:       Union[str, Any]     =remove_space
									A__					:       str     =keep_accents
									A__					:       Dict     =vocab_file
									A__					:       Any     =False if not self.vocab_file else True


			def lowercase__						(				self	:						Union[str, Any]		,      lowerCAmelCase_	:						List[int]		,      lowerCAmelCase_	:						Optional[List[int]] = None			)      ->		List[int]:





									'''simple docstring'''
									A__					:       List[Any]     =[self.sep_token_id]
									A__					:       Tuple     =[self.cls_token_id]
									if token_ids_a is None:
															return token_ids_a + sep + cls
									return token_ids_a + sep + token_ids_a + sep + cls


			def lowercase__						(				self	:						Union[str, Any]		,      lowerCAmelCase_	:						List[int]		,      lowerCAmelCase_	:						Optional[List[int]] = None			)      ->		List[int]:





									'''simple docstring'''
									A__					:       int     =[self.sep_token_id]
									A__					:       Union[str, Any]     =[2]

									if token_ids_a is None:
															return len(token_ids_a + sep			) * [0] + cls_segment_id
									return len(token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1] + cls_segment_id



			def lowercase__						(				self	:						List[str]		,      lowerCAmelCase_	:						str		,      lowerCAmelCase_	:						Optional[str] = None			)      ->		Tuple[str]:





									'''simple docstring'''
									if not self.can_save_slow_tokenizer:
															raise ValueError(
															    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
															    """tokenizer."""			)

									if not os.path.isdir(lowerCAmelCase_			):
															logger.error(f"Vocabulary path ({save_directory}) should be a directory"			)
															return
									A__					:       Any     =os.path.join(
									    lowerCAmelCase_		,      (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]			)

									if os.path.abspath(self.vocab_file			) != os.path.abspath(lowerCAmelCase_			):
															copyfile(self.vocab_file		,      lowerCAmelCase_			)

									return (out_vocab_file,)




'''simple docstring'''



from ...configuration_utils import PretrainedConfig
from ...utils import logging


__snake_case			:     Union[str, Any]										=      logging.get_logger(__name__)

__snake_case			:     Optional[int]										=      {
    'bigcode/gpt_bigcode-santacoder': 'https://huggingface.co/bigcode/gpt_bigcode-santacoder/resolve/main/config.json',
}



class lowerCamelCase			(  lowercase_						):



			'''simple docstring'''






			__snake_case        =	'gpt_bigcode'
			__snake_case        =	['past_key_values']
			__snake_case        =	{
			    'hidden_size': 'n_embd',
			    'max_position_embeddings': 'n_positions',
			    'num_attention_heads': 'n_head',
			    'num_hidden_layers': 'n_layer',
			}


			def __init__(				self	:						Optional[Any]		,      lowerCAmelCase_	:						List[str]=5_02_57		,      lowerCAmelCase_	:						str=10_24		,      lowerCAmelCase_	:						str=7_68		,      lowerCAmelCase_	:						str=12		,      lowerCAmelCase_	:						int=12		,      lowerCAmelCase_	:						Dict=None		,      lowerCAmelCase_	:						List[Any]="gelu_pytorch_tanh"		,      lowerCAmelCase_	:						Union[str, Any]=0.1		,      lowerCAmelCase_	:						Optional[Any]=0.1		,      lowerCAmelCase_	:						Optional[int]=0.1		,      lowerCAmelCase_	:						Dict=1e-5		,      lowerCAmelCase_	:						str=0.02		,      lowerCAmelCase_	:						List[Any]=True		,      lowerCAmelCase_	:						str=True		,      lowerCAmelCase_	:						str=5_02_56		,      lowerCAmelCase_	:						Dict=5_02_56		,      lowerCAmelCase_	:						Union[str, Any]=True		,      lowerCAmelCase_	:						str=True		,      lowerCAmelCase_	:						Optional[Any]=True		,      **lowerCAmelCase_	:						Optional[Any]		,      )      ->		Tuple:





									'''simple docstring'''
									A__					:       Optional[Any]     =vocab_size
									A__					:       Optional[Any]     =n_positions
									A__					:       List[str]     =n_embd
									A__					:       str     =n_layer
									A__					:       Optional[int]     =n_head
									A__					:       Optional[int]     =n_inner
									A__					:       int     =activation_function
									A__					:       int     =resid_pdrop
									A__					:       int     =embd_pdrop
									A__					:       Dict     =attn_pdrop
									A__					:       Any     =layer_norm_epsilon
									A__					:       List[Any]     =initializer_range
									A__					:       Dict     =scale_attn_weights
									A__					:       Any     =use_cache
									A__					:       List[Any]     =attention_softmax_in_fpaa
									A__					:       Optional[int]     =scale_attention_softmax_in_fpaa
									A__					:       Dict     =multi_query

									A__					:       List[str]     =bos_token_id
									A__					:       Any     =eos_token_id

									super().__init__(bos_token_id=lowerCAmelCase_		,      eos_token_id=lowerCAmelCase_		,      **lowerCAmelCase_			)

