



import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
							if is_torch_available():
														import torch
							if is_tf_available():
														import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


__lowerCAmelCase						      =       logging.get_logger(__name__)

__lowerCAmelCase						      =       {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__lowerCAmelCase						      =       {
    '''vocab_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''Salesforce/codegen-350M-mono''': (
            '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json'''
        ),
    },
}

__lowerCAmelCase						      =       {
    '''Salesforce/codegen-350M-mono''': 2_048,
}

class      __magic_name__						(   _UpperCamelCase       ):
							lowerCAmelCase	:   Tuple           = VOCAB_FILES_NAMES
							lowerCAmelCase	:   int           = PRETRAINED_VOCAB_FILES_MAP
							lowerCAmelCase	:   Dict           = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowerCAmelCase	:   Optional[Any]           = ['input_ids', 'attention_mask']
							lowerCAmelCase	:   str           = CodeGenTokenizer







							def __init__(							self :	Dict     ,_UpperCAmelCase :	Any=None     ,_UpperCAmelCase :	List[Any]=None     ,_UpperCAmelCase :	Union[str, Any]=None     ,_UpperCAmelCase :	Dict="<|endoftext|>"     ,_UpperCAmelCase :	Dict="<|endoftext|>"     ,_UpperCAmelCase :	List[Any]="<|endoftext|>"     ,_UpperCAmelCase :	Tuple=False     ,**_UpperCAmelCase :	Union[str, Any]     ,):
									super().__init__(
									    _UpperCAmelCase     ,_UpperCAmelCase     ,tokenizer_file=_UpperCAmelCase     ,unk_token=_UpperCAmelCase     ,bos_token=_UpperCAmelCase     ,eos_token=_UpperCAmelCase     ,add_prefix_space=_UpperCAmelCase     ,**_UpperCAmelCase     ,)

									if kwargs.pop('add_bos_token'     ,_UpperCAmelCase		):
											_a					:				Any				=							kwargs.pop('name_or_path'     ,''		)
											raise ValueError(
											    'Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token.'
											    'Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n'
											    F"""`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"""
											    F"""`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"""
											    'This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005.'
											    ' so that the fast tokenizer works correctly.'		)

									_a					:				List[str]				=							json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
									if pre_tok_state.get('add_prefix_space'     ,_UpperCAmelCase		) != add_prefix_space:
											_a					:				List[str]				=							getattr(_UpperCAmelCase     ,pre_tok_state.pop('type'		)		)
											_a					:				int				=							add_prefix_space
											_a					:				str				=							pre_tok_class(**_UpperCAmelCase		)

									_a					:				Optional[Any]				=							add_prefix_space







							def 							__lowercase      (							self :	List[str]     ,*_UpperCAmelCase :	Any     ,**_UpperCAmelCase :	str		):
									_a					:				Optional[Any]				=							kwargs.get('is_split_into_words'     ,_UpperCAmelCase		)
									assert self.add_prefix_space or not is_split_into_words, (
									    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
									    "to use it with pretokenized inputs."
									)

									return super()._batch_encode_plus(*_UpperCAmelCase     ,**_UpperCAmelCase		)







							def 							__lowercase      (							self :	int     ,*_UpperCAmelCase :	int     ,**_UpperCAmelCase :	Union[str, Any]		):
									_a					:				Union[str, Any]				=							kwargs.get('is_split_into_words'     ,_UpperCAmelCase		)

									assert self.add_prefix_space or not is_split_into_words, (
									    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
									    "to use it with pretokenized inputs."
									)

									return super()._encode_plus(*_UpperCAmelCase     ,**_UpperCAmelCase		)







							def 							__lowercase      (							self :	Union[str, Any]     ,_UpperCAmelCase :	str     ,_UpperCAmelCase :	Optional[str] = None		):
									_a					:				Tuple				=							self._tokenizer.model.save(_UpperCAmelCase     ,name=_UpperCAmelCase		)
									return tuple(_UpperCAmelCase		)







							def 							__lowercase      (							self :	Tuple     ,_UpperCAmelCase :	Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]     ,_UpperCAmelCase :	bool = False     ,_UpperCAmelCase :	bool = None     ,_UpperCAmelCase :	Optional[List[str]] = None     ,**_UpperCAmelCase :	List[str]     ,):
									_a					:				List[Any]				=							super().decode(
									    token_ids=_UpperCAmelCase     ,skip_special_tokens=_UpperCAmelCase     ,clean_up_tokenization_spaces=_UpperCAmelCase     ,**_UpperCAmelCase     ,)

									if truncate_before_pattern is not None and len(_UpperCAmelCase		) > 0:
											_a					:				Optional[int]				=							self.truncate(_UpperCAmelCase     ,_UpperCAmelCase		)

									return decoded_text







							def 							__lowercase      (							self :	int     ,_UpperCAmelCase :	Tuple     ,_UpperCAmelCase :	List[Any]		):
									def find_re(_UpperCAmelCase :	int     ,_UpperCAmelCase :	Union[str, Any]     ,_UpperCAmelCase :	int		):
											_a					:				int				=							pattern.search(_UpperCAmelCase     ,_UpperCAmelCase		)
											return m.start() if m else -1

									_a					:				Union[str, Any]				=							[re.compile(_UpperCAmelCase     ,re.MULTILINE		) for pattern in truncate_before_pattern]

									_a					:				Tuple				=							list(re.finditer('^print'     ,_UpperCAmelCase     ,re.MULTILINE		)		)

									if len(_UpperCAmelCase		) > 1:
											_a					:				str				=							completion[: prints[1].start()]

									_a					:				Tuple				=							list(re.finditer('^def'     ,_UpperCAmelCase     ,re.MULTILINE		)		)

									if len(_UpperCAmelCase		) > 1:
											_a					:				Union[str, Any]				=							completion[: defs[1].start()]

									_a					:				Union[str, Any]				=							0

									_a					:				Optional[Any]				=							[
									    pos for pos in [find_re(_UpperCAmelCase     ,_UpperCAmelCase     ,_UpperCAmelCase		) for terminal in terminals] if pos != -1
									]

									if len(_UpperCAmelCase		) > 0:
											return completion[: min(_UpperCAmelCase		)]
									else:
											return completion









'''simple docstring'''

from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__lowerCAmelCase						      =       logging.get_logger(__name__)

__lowerCAmelCase						      =       {
    '''distilbert-base-uncased''': '''https://huggingface.co/distilbert-base-uncased/resolve/main/config.json''',
    '''distilbert-base-uncased-distilled-squad''': (
        '''https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/config.json'''
    ),
    '''distilbert-base-cased''': '''https://huggingface.co/distilbert-base-cased/resolve/main/config.json''',
    '''distilbert-base-cased-distilled-squad''': (
        '''https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/config.json'''
    ),
    '''distilbert-base-german-cased''': '''https://huggingface.co/distilbert-base-german-cased/resolve/main/config.json''',
    '''distilbert-base-multilingual-cased''': (
        '''https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/config.json'''
    ),
    '''distilbert-base-uncased-finetuned-sst-2-english''': (
        '''https://huggingface.co/distilbert-base-uncased-finetuned-sst-2-english/resolve/main/config.json'''
    ),
}

class      __magic_name__						(   _UpperCamelCase       ):
							lowerCAmelCase	:   Optional[int]           = 'distilbert'
							lowerCAmelCase	:   Optional[int]           = {
							    'hidden_size': 'dim',
							    'num_attention_heads': 'n_heads',
							    'num_hidden_layers': 'n_layers',
							}







							def __init__(							self :	str     ,_UpperCAmelCase :	Optional[Any]=30522     ,_UpperCAmelCase :	int=512     ,_UpperCAmelCase :	int=False     ,_UpperCAmelCase :	str=6     ,_UpperCAmelCase :	int=12     ,_UpperCAmelCase :	Union[str, Any]=768     ,_UpperCAmelCase :	Optional[Any]=4 * 768     ,_UpperCAmelCase :	int=0.1     ,_UpperCAmelCase :	Union[str, Any]=0.1     ,_UpperCAmelCase :	Tuple="gelu"     ,_UpperCAmelCase :	Any=0.02     ,_UpperCAmelCase :	List[Any]=0.1     ,_UpperCAmelCase :	Dict=0.2     ,_UpperCAmelCase :	Union[str, Any]=0     ,**_UpperCAmelCase :	List[Any]     ,):
									_a					:				Optional[Any]				=							vocab_size
									_a					:				Dict				=							max_position_embeddings
									_a					:				Optional[int]				=							sinusoidal_pos_embds
									_a					:				List[Any]				=							n_layers
									_a					:				Any				=							n_heads
									_a					:				int				=							dim
									_a					:				Optional[Any]				=							hidden_dim
									_a					:				str				=							dropout
									_a					:				Optional[Any]				=							attention_dropout
									_a					:				Any				=							activation
									_a					:				Tuple				=							initializer_range
									_a					:				Union[str, Any]				=							qa_dropout
									_a					:				List[Any]				=							seq_classif_dropout
									super().__init__(**_UpperCAmelCase     ,pad_token_id=_UpperCAmelCase		)

class      __magic_name__						(   _UpperCamelCase       ):







							@property
							def 							__lowercase      (							self :	Union[str, Any]		):
									if self.task == "multiple-choice":
											_a					:				Union[str, Any]				=							{0: 'batch', 1: 'choice', 2: 'sequence'}
									else:
											_a					:				List[str]				=							{0: 'batch', 1: 'sequence'}
									return OrderedDict(
									    [
									        ('input_ids', dynamic_axis),
									        ('attention_mask', dynamic_axis),
									    ]		)




