

'''simple docstring'''




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


_lowerCAmelCase         =						logging.get_logger(__name__)

_lowerCAmelCase         =						{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_lowerCAmelCase         =						{
    "vocab_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-german-cased": "https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt",
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-german-cased": (
            "https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json"
        ),
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
    },
}

_lowerCAmelCase         =						{
    "distilbert-base-uncased": 512,
    "distilbert-base-uncased-distilled-squad": 512,
    "distilbert-base-cased": 512,
    "distilbert-base-cased-distilled-squad": 512,
    "distilbert-base-german-cased": 512,
    "distilbert-base-multilingual-cased": 512,
}


_lowerCAmelCase         =						{
    "distilbert-base-uncased": {"do_lower_case": True},
    "distilbert-base-uncased-distilled-squad": {"do_lower_case": True},
    "distilbert-base-cased": {"do_lower_case": False},
    "distilbert-base-cased-distilled-squad": {"do_lower_case": False},
    "distilbert-base-german-cased": {"do_lower_case": False},
    "distilbert-base-multilingual-cased": {"do_lower_case": False},
}






class   _SCREAMING_SNAKE_CASE   (   __a     ):
 __SCREAMING_SNAKE_CASE					:Dict      	=					VOCAB_FILES_NAMES
 __SCREAMING_SNAKE_CASE					:Dict      	=					PRETRAINED_VOCAB_FILES_MAP
 __SCREAMING_SNAKE_CASE					:Dict      	=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __SCREAMING_SNAKE_CASE					:List[str]      	=					PRETRAINED_INIT_CONFIGURATION
 __SCREAMING_SNAKE_CASE					:List[str]      	=					["""input_ids""", """attention_mask"""]
 __SCREAMING_SNAKE_CASE					:int      	=					DistilBertTokenizer



 def __init__(   self		:   List[Any]			,				a__		:   List[str]=None			,				a__		:   List[Any]=None			,				a__		:   Any=True			,				a__		:   Optional[Any]="[UNK]"			,				a__		:   int="[SEP]"			,				a__		:   List[Any]="[PAD]"			,				a__		:   Optional[Any]="[CLS]"			,				a__		:   int="[MASK]"			,				a__		:   List[Any]=True			,				a__		:   Optional[int]=None			,				**a__		:   List[Any]			,				):
      super().__init__(
          a__			,				tokenizer_file=a__			,				do_lower_case=a__			,				unk_token=a__			,				sep_token=a__			,				pad_token=a__			,				cls_token=a__			,				mask_token=a__			,				tokenize_chinese_chars=a__			,				strip_accents=a__			,				**a__			,				)

      __magic_name__						       =    json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
      if (
          normalizer_state.get('''lowercase'''			,				a__      ) != do_lower_case
          or normalizer_state.get('''strip_accents'''			,				a__      ) != strip_accents
          or normalizer_state.get('''handle_chinese_chars'''			,				a__      ) != tokenize_chinese_chars
      ):
           __magic_name__						       =    getattr(a__			,				normalizer_state.pop('''type'''      )      )
           __magic_name__						       =    do_lower_case
           __magic_name__						       =    strip_accents
           __magic_name__						       =    tokenize_chinese_chars
           __magic_name__						       =    normalizer_class(**a__      )

      __magic_name__						       =    do_lower_case



 def 			snake_case__  (   self		:   Dict			,				a__		:   Tuple			,				a__		:   List[Any]=None      ):
      __magic_name__						       =    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
           output += token_ids_a + [self.sep_token_id]

      return output



 def 			snake_case__  (   self		:   int			,				a__		:   List[int]			,				a__		:   Optional[List[int]] = None      ):
      __magic_name__						       =    [self.sep_token_id]
      __magic_name__						       =    [self.cls_token_id]
      if token_ids_a is None:
           return len(cls + token_ids_a + sep      ) * [0]
      return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]



 def 			snake_case__  (   self		:   List[str]			,				a__		:   str			,				a__		:   Optional[str] = None      ):
      __magic_name__						       =    self._tokenizer.model.save(a__			,				name=a__      )
      return tuple(a__      )









'''simple docstring'''




from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxSeqaSeqConfigWithPast
from ...utils import logging


if TYPE_CHECKING:
      from ...feature_extraction_utils import FeatureExtractionMixin
      from ...tokenization_utils_base import PreTrainedTokenizerBase
      from ...utils import TensorType

_lowerCAmelCase         =						logging.get_logger(__name__)

_lowerCAmelCase         =						{
    "openai/whisper-base": "https://huggingface.co/openai/whisper-base/resolve/main/config.json",
}

# fmt: off
_lowerCAmelCase         =						[
    1, 2, 7, 8, 9, 10, 14, 25,
    26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
    63, 90, 91, 92, 93, 357, 366, 438, 532, 685,
    705, 796, 930, 1058, 1220, 1267, 1279, 1303, 1343, 1377,
    1391, 1635, 1782, 1875, 2162, 2361, 2488, 3467, 4008, 4211,
    4600, 4808, 5299, 5855, 6329, 7203, 9609, 9959, 10563, 10786,
    11420, 11709, 11907, 13163, 13697, 13700, 14808, 15306, 16410, 16791,
    17992, 19203, 19510, 20724, 22305, 22935, 27007, 30109, 30420, 33409,
    34949, 40283, 40493, 40549, 47282, 49146, 50257, 50359, 50360, 50361
]
_lowerCAmelCase         =						[
    1, 2, 7, 8, 9, 10, 14, 25,
    26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
    63, 90, 91, 92, 93, 359, 503, 522, 542, 873,
    893, 902, 918, 922, 931, 1350, 1853, 1982, 2460, 2627,
    3246, 3253, 3268, 3536, 3846, 3961, 4183, 4667, 6585, 6647,
    7273, 9061, 9383, 10428, 10929, 11938, 12033, 12331, 12562, 13793,
    14157, 14635, 15265, 15618, 16553, 16604, 18362, 18956, 20075, 21675,
    22520, 26130, 26161, 26435, 28279, 29464, 31650, 32302, 32470, 36865,
    42863, 47425, 49870, 50254, 50258, 50360, 50361, 50362
]






class   _SCREAMING_SNAKE_CASE   (   __a     ):
 __SCREAMING_SNAKE_CASE					:str      	=					"""whisper"""
 __SCREAMING_SNAKE_CASE					:str      	=					["""past_key_values"""]
 __SCREAMING_SNAKE_CASE					:Tuple      	=					{"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}



 def __init__(   self		:   Dict			,				a__		:   Optional[int]=5_1865			,				a__		:   str=80			,				a__		:   List[str]=6			,				a__		:   List[str]=4			,				a__		:   List[Any]=6			,				a__		:   Union[str, Any]=4			,				a__		:   Tuple=1536			,				a__		:   Optional[int]=1536			,				a__		:   List[str]=0.0			,				a__		:   Union[str, Any]=0.0			,				a__		:   Union[str, Any]=5_0257			,				a__		:   Dict=True			,				a__		:   Optional[Any]=True			,				a__		:   Union[str, Any]="gelu"			,				a__		:   Tuple=256			,				a__		:   Dict=0.0			,				a__		:   str=0.0			,				a__		:   Optional[Any]=0.0			,				a__		:   int=0.02			,				a__		:   Any=False			,				a__		:   List[Any]=1500			,				a__		:   Optional[int]=448			,				a__		:   Dict=5_0256			,				a__		:   str=5_0256			,				a__		:   Tuple=5_0256			,				a__		:   List[str]=None			,				a__		:   List[Any]=[220, 5_0256]			,				a__		:   Any=False			,				a__		:   Dict=256			,				a__		:   Optional[Any]=False			,				a__		:   str=0.05			,				a__		:   List[Any]=10			,				a__		:   List[Any]=2			,				a__		:   Optional[int]=0.0			,				a__		:   List[Any]=10			,				a__		:   Union[str, Any]=0			,				a__		:   int=7			,				**a__		:   Any			,				):
      __magic_name__						       =    vocab_size
      __magic_name__						       =    num_mel_bins
      __magic_name__						       =    d_model
      __magic_name__						       =    encoder_layers
      __magic_name__						       =    encoder_attention_heads
      __magic_name__						       =    decoder_layers
      __magic_name__						       =    decoder_attention_heads
      __magic_name__						       =    decoder_ffn_dim
      __magic_name__						       =    encoder_ffn_dim
      __magic_name__						       =    dropout
      __magic_name__						       =    attention_dropout
      __magic_name__						       =    activation_dropout
      __magic_name__						       =    activation_function
      __magic_name__						       =    init_std
      __magic_name__						       =    encoder_layerdrop
      __magic_name__						       =    decoder_layerdrop
      __magic_name__						       =    use_cache
      __magic_name__						       =    encoder_layers
      __magic_name__						       =    scale_embedding  # scale factor will be sqrt(d_model) if True
      __magic_name__						       =    max_source_positions
      __magic_name__						       =    max_target_positions

      # Audio Classification-specific parameters. Feel free to ignore for other classes.
      __magic_name__						       =    classifier_proj_size
      __magic_name__						       =    use_weighted_layer_sum

      # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
      __magic_name__						       =    apply_spec_augment
      __magic_name__						       =    mask_time_prob
      __magic_name__						       =    mask_time_length
      __magic_name__						       =    mask_time_min_masks
      __magic_name__						       =    mask_feature_prob
      __magic_name__						       =    mask_feature_length
      __magic_name__						       =    mask_feature_min_masks

      __magic_name__						       =    median_filter_width

      super().__init__(
          pad_token_id=a__			,				bos_token_id=a__			,				eos_token_id=a__			,				is_encoder_decoder=a__			,				decoder_start_token_id=a__			,				suppress_tokens=a__			,				begin_suppress_tokens=a__			,				**a__			,				)






class   _SCREAMING_SNAKE_CASE   (   __a     ):



 @property
 def 			snake_case__  (   self		:   List[str]      ):
      __magic_name__						       =    OrderedDict(
          [
              ('''input_features''', {0: '''batch''', 1: '''feature_size''', 2: '''encoder_sequence'''}),
          ]      )
      if self.use_past:
           __magic_name__						       =    {0: '''batch'''}
      else:
           __magic_name__						       =    {0: '''batch''', 1: '''decoder_sequence'''}

      if self.use_past:
           self.fill_with_past_key_values_(a__			,				direction='''inputs'''      )

      return common_inputs



 def 			snake_case__  (   self		:   Optional[int]			,				a__		:   Union["PreTrainedTokenizerBase", "FeatureExtractionMixin"]			,				a__		:   int = -1			,				a__		:   int = -1			,				a__		:   bool = False			,				a__		:   Optional["TensorType"] = None			,				a__		:   int = 2_2050			,				a__		:   float = 5.0			,				a__		:   int = 220			,				):
      __magic_name__						       =    OrderedDict()
      __magic_name__						       =    OnnxConfig.generate_dummy_inputs(
          self			,				preprocessor=preprocessor.feature_extractor			,				batch_size=a__			,				framework=a__			,				sampling_rate=a__			,				time_duration=a__			,				frequency=a__			,				)
      __magic_name__						       =    encoder_inputs['''input_features'''].shape[2]
      __magic_name__						       =    encoder_sequence_length // 2 if self.use_past else seq_length

      __magic_name__						       =    super().generate_dummy_inputs(
          preprocessor.tokenizer			,				a__			,				a__			,				a__			,				a__      )

      __magic_name__						       =    encoder_inputs.pop('''input_features'''      )
      __magic_name__						       =    decoder_inputs.pop('''decoder_input_ids'''      )

      if "past_key_values" in decoder_inputs:
           __magic_name__						       =    decoder_inputs.pop('''past_key_values'''      )

      return dummy_inputs



 @property
 def 			snake_case__  (   self		:   Dict      ):
      return 1E-3






