


import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_snake_case					:		Optional[Any]      =    logging.get_logger(__name__)

_snake_case					:		Union[str, Any]      =    "▁"

_snake_case					:		Dict      =    {"vocab_file": "vocab.txt", "sentencepiece_model_ckpt": "sentencepiece.bpe.model"}

_snake_case					:		Optional[int]      =    {
    "sentencepiece_model_file": "sentencepiece.bpe.model",
    "vocab_file": "vocab.txt",
}

_snake_case					:		Union[str, Any]      =    {
    "vocab_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
    },
    "sentencepiece_model_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
    },
}

_snake_case					:		List[Any]      =    {
    "ernie-m-base": 514,
    "ernie-m-large": 514,
}

_snake_case					:		List[Any]      =    {
    "ernie-m-base": {"do_lower_case": False},
    "ernie-m-large": {"do_lower_case": False},
}

class     a		(_lowerCAmelCase				):




							"""simple docstring"""


							__UpperCAmelCase   :							int					  =  ["input_ids"]

							__UpperCAmelCase   :							Optional[Any]					  =  VOCAB_FILES_NAMES
							__UpperCAmelCase   :							int					  =  PRETRAINED_INIT_CONFIGURATION
							__UpperCAmelCase   :							Dict					  =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__UpperCAmelCase   :							Optional[Any]					  =  PRETRAINED_VOCAB_FILES_MAP
							__UpperCAmelCase   :							Optional[Any]					  =  RESOURCE_FILES_NAMES

							def __init__(							self	:  List[Any]   ,     lowerCamelCase	:  Dict   ,     lowerCamelCase	:  Tuple=None   ,     lowerCamelCase	:  List[Any]=False   ,     lowerCamelCase	:  Optional[int]="utf8"   ,     lowerCamelCase	:  Optional[Any]="[UNK]"   ,     lowerCamelCase	:  Optional[Any]="[SEP]"   ,     lowerCamelCase	:  Optional[Any]="[PAD]"   ,     lowerCamelCase	:  Dict="[CLS]"   ,     lowerCamelCase	:  int="[MASK]"   ,     lowerCamelCase	:  Optional[Dict[str, Any]] = None   ,     **lowerCamelCase	:  List[Any]   ,     )     ->   Any:
														__snake_case		:      Tuple										=       {} if sp_model_kwargs is None else sp_model_kwargs
														super().__init__(
														    do_lower_case=_lowerCAmelCase   ,     unk_token=_lowerCAmelCase   ,     sep_token=_lowerCAmelCase   ,     pad_token=_lowerCAmelCase   ,     cls_token=_lowerCAmelCase   ,     mask_token=_lowerCAmelCase   ,     vocab_file=_lowerCAmelCase   ,     encoding=_lowerCAmelCase   ,     sp_model_kwargs=self.sp_model_kwargs   ,     **_lowerCAmelCase   ,     )
														__snake_case		:      Dict										=       do_lower_case
														__snake_case		:      str										=       sentencepiece_model_ckpt
														__snake_case		:      Dict										=       spm.SentencePieceProcessor(**self.sp_model_kwargs    )
														self.sp_model.Load(_lowerCAmelCase    )

														# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
														if vocab_file is not None:
																					__snake_case		:      int										=       self.load_vocab(filepath=_lowerCAmelCase    )
														else:
																					__snake_case		:      Union[str, Any]										=       {self.sp_model.id_to_piece(_lowerCAmelCase    ): id for id in range(self.sp_model.get_piece_size()    )}
														__snake_case		:      Optional[Any]										=       {v: k for k, v in self.vocab.items()}

							def    __snake_case (							self	:  Union[str, Any]   ,     lowerCamelCase	:  Any    )     ->   Dict:

														if text is None:
																					return None

														__snake_case		:      Tuple										=       self.tokenize(_lowerCAmelCase    )
														__snake_case			,      __snake_case		:      Dict										=       "", []

														for i, ch in enumerate(_lowerCAmelCase    ):
																					if ch in self.SP_CHAR_MAPPING:
																												__snake_case		:      str										=       self.SP_CHAR_MAPPING.get(_lowerCAmelCase    )
																					else:
																												__snake_case		:      Dict										=       unicodedata.normalize("NFKC"   ,     _lowerCAmelCase    )
																					if self.is_whitespace(_lowerCAmelCase    ):
																												continue
																					normalized_text += ch
																					char_mapping.extend([i] * len(_lowerCAmelCase    )    )

														__snake_case			,      __snake_case			,      __snake_case		:      List[str]										=       normalized_text, [], 0

														if self.do_lower_case:
																					__snake_case		:      Union[str, Any]										=       text.lower()

														for token in split_tokens:
																					if token[:1] == "▁":
																												__snake_case		:      Dict										=       token[1:]
																					__snake_case		:      List[str]										=       text[offset:].index(_lowerCAmelCase    ) + offset
																					__snake_case		:      int										=       start + len(_lowerCAmelCase    )

																					token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)    )
																					__snake_case		:      Tuple										=       end
														return token_mapping

							@property
							def    __snake_case (							self	:  Optional[Any]    )     ->   str:
														return len(self.vocab    )

							def    __snake_case (							self	:  str    )     ->   Any:
														return dict(self.vocab   ,     **self.added_tokens_encoder    )

							def __getstate__(							self	:  Optional[int]    )     ->   Union[str, Any]:
														__snake_case		:      Tuple										=       self.__dict__.copy()
														__snake_case		:      str										=       None
														return state

							def __setstate__(							self	:  Optional[Any]   ,     lowerCamelCase	:  Any    )     ->   Optional[int]:
														__snake_case		:      Dict										=       d

														# for backward compatibility
														if not hasattr(self   ,     "sp_model_kwargs"    ):
																					__snake_case		:      Dict										=       {}

														__snake_case		:      str										=       spm.SentencePieceProcessor(**self.sp_model_kwargs    )
														self.sp_model.Load(self.sentencepiece_model_ckpt    )

							def    __snake_case (							self	:  int   ,     lowerCamelCase	:  Optional[int]    )     ->   Optional[Any]:
														return "".join((self.SP_CHAR_MAPPING.get(_lowerCAmelCase   ,     _lowerCAmelCase    ) for c in text)    )

							def    __snake_case (							self	:  Dict   ,     lowerCamelCase	:  int   ,     lowerCamelCase	:  str=False   ,     lowerCamelCase	:  Union[str, Any]=64   ,     lowerCamelCase	:  Optional[Any]=0.1    )     ->   Any:

														if self.sp_model_kwargs.get("enable_sampling"    ) is True:
																					__snake_case		:      int										=       True
														if self.sp_model_kwargs.get("alpha"    ) is not None:
																					__snake_case		:      Union[str, Any]										=       self.sp_model_kwargs.get("alpha"    )
														if self.sp_model_kwargs.get("nbest_size"    ) is not None:
																					__snake_case		:      str										=       self.sp_model_kwargs.get("nbest_size"    )

														if not enable_sampling:
																					__snake_case		:      List[Any]										=       self.sp_model.EncodeAsPieces(_lowerCAmelCase    )
														else:
																					__snake_case		:      Optional[Any]										=       self.sp_model.SampleEncodeAsPieces(_lowerCAmelCase   ,     _lowerCAmelCase   ,     _lowerCAmelCase    )
														__snake_case		:      int										=       []
														for pi, piece in enumerate(_lowerCAmelCase    ):
																					if piece == SPIECE_UNDERLINE:
																												if not pieces[pi + 1].startswith(_lowerCAmelCase    ) and pi != 0:
																																			new_pieces.append(_lowerCAmelCase    )
																																			continue
																												else:
																																			continue
																					__snake_case		:      Union[str, Any]										=       0
																					for i, chunk in enumerate(_lowerCAmelCase    ):
																												if chunk == SPIECE_UNDERLINE:
																																			continue
																												if self.is_ch_char(_lowerCAmelCase    ) or self.is_punct(_lowerCAmelCase    ):
																																			if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																										new_pieces.append(piece[lst_i:i]    )
																																			new_pieces.append(_lowerCAmelCase    )
																																			__snake_case		:      str										=       i + 1
																												elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
																																			if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																										new_pieces.append(piece[lst_i:i]    )
																																			__snake_case		:      Dict										=       i
																												elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
																																			if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																										new_pieces.append(piece[lst_i:i]    )
																																			__snake_case		:      Optional[int]										=       i
																					if len(_lowerCAmelCase    ) > lst_i:
																												new_pieces.append(piece[lst_i:]    )
														return new_pieces

							def    __snake_case (							self	:  List[Any]   ,     lowerCamelCase	:  Union[str, Any]    )     ->   List[str]:
														__snake_case		:      Any										=       "".join(_lowerCAmelCase    ).replace(_lowerCAmelCase   ,     " "    ).strip()
														return out_string

							def    __snake_case (							self	:  Tuple   ,     lowerCamelCase	:  int    )     ->   int:
														__snake_case		:      Tuple										=       self.convert_ids_to_tokens(_lowerCAmelCase    )
														__snake_case		:      Union[str, Any]										=       "".join(_lowerCAmelCase    ).replace(_lowerCAmelCase   ,     " "    ).strip()
														return out_string

							def    __snake_case (							self	:  List[Any]   ,     lowerCamelCase	:  Union[str, Any]    )     ->   str:
														return self.vocab.get(_lowerCAmelCase   ,     self.vocab.get(self.unk_token    )    )

							def    __snake_case (							self	:  Any   ,     lowerCamelCase	:  List[Any]    )     ->   Any:
														return self.reverse_vocab.get(_lowerCAmelCase   ,     self.unk_token    )

							def    __snake_case (							self	:  Union[str, Any]   ,     lowerCamelCase	:  Tuple   ,     lowerCamelCase	:  Dict=None    )     ->   Tuple:

														if token_ids_a is None:
																					return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
														__snake_case		:      Any										=       [self.cls_token_id]
														__snake_case		:      List[Any]										=       [self.sep_token_id]
														return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep

							def    __snake_case (							self	:  Tuple   ,     lowerCamelCase	:  str   ,     lowerCamelCase	:  List[str]=None    )     ->   List[Any]:

														if offset_mapping_a is None:
																					return [(0, 0)] + offset_mapping_a + [(0, 0)]

														return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]

							def    __snake_case (							self	:  Dict   ,     lowerCamelCase	:  Any   ,     lowerCamelCase	:  Optional[Any]=None   ,     lowerCamelCase	:  Any=False    )     ->   int:

														if already_has_special_tokens:
																					if token_ids_a is not None:
																												raise ValueError(
																												    "You should not supply a second sequence if the provided sequence of "
																												    "ids is already formatted with special tokens for the model."    )
																					return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

														if token_ids_a is not None:
																					return [1] + ([0] * len(_lowerCAmelCase    )) + [1, 1] + ([0] * len(_lowerCAmelCase    )) + [1]
														return [1] + ([0] * len(_lowerCAmelCase    )) + [1]

							def    __snake_case (							self	:  Optional[Any]   ,     lowerCamelCase	:  List[int]   ,     lowerCamelCase	:  Optional[List[int]] = None    )     ->   List[str]:

														if token_ids_a is None:
																					# [CLS] X [SEP]
																					return (len(_lowerCAmelCase    ) + 2) * [0]

														# [CLS] A [SEP] [SEP] B [SEP]
														return [0] * (len(_lowerCAmelCase    ) + 1) + [1] * (len(_lowerCAmelCase    ) + 3)

							def    __snake_case (							self	:  List[str]   ,     lowerCamelCase	:  List[Any]    )     ->   Dict:

														if "\u4e00" <= char <= "\u9fff":
																					return True
														return False

							def    __snake_case (							self	:  List[str]   ,     lowerCamelCase	:  Tuple    )     ->   str:

														if ("a" <= char <= "z") or ("A" <= char <= "Z"):
																					return True
														return False

							def    __snake_case (							self	:  str   ,     lowerCamelCase	:  str    )     ->   List[Any]:

														if char in ",;:.?!~，；：。？！《》【】":
																					return True
														return False

							def    __snake_case (							self	:  int   ,     lowerCamelCase	:  Any    )     ->   Tuple:

														if char == " " or char == "\t" or char == "\n" or char == "\r":
																					return True
														if len(_lowerCAmelCase    ) == 1:
																					__snake_case		:      List[Any]										=       unicodedata.category(_lowerCAmelCase    )
																					if cat == "Zs":
																												return True
														return False

							def    __snake_case (							self	:  Optional[Any]   ,     lowerCamelCase	:  Dict    )     ->   List[str]:
														__snake_case		:      Optional[Any]										=       {}
														with io.open(_lowerCAmelCase   ,     "r"   ,     encoding="utf-8"    ) as f:
																					for index, line in enumerate(_lowerCAmelCase    ):
																												__snake_case		:      str										=       line.rstrip("\n"    )
																												__snake_case		:      Dict										=       int(_lowerCAmelCase    )

														return token_to_idx






							def    __snake_case (							self	:  Optional[Any]   ,     lowerCamelCase	:  str   ,     lowerCamelCase	:  Optional[str] = None    )     ->   int:
														__snake_case		:      Optional[Any]										=       0
														if os.path.isdir(_lowerCAmelCase    ):
																					__snake_case		:      Union[str, Any]										=       os.path.join(
																					    _lowerCAmelCase   ,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]    )
														else:
																					__snake_case		:      Optional[Any]										=       (filename_prefix + "-" if filename_prefix else "") + save_directory
														with open(_lowerCAmelCase   ,     "w"   ,     encoding="utf-8"    ) as writer:
																					for token, token_index in sorted(self.vocab.items()   ,     key=lambda lowerCamelCase    : kv[1]    ):
																												if index != token_index:
																																			logger.warning(
																																			    F'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
																																			    " Please check that the vocabulary is not corrupted!"    )
																																			__snake_case		:      List[str]										=       token_index
																												writer.write(token + "\n"    )
																												index += 1

														__snake_case		:      Dict										=       os.path.join(_lowerCAmelCase   ,     "sentencepiece.bpe.model"    )
														with open(_lowerCAmelCase   ,     "wb"    ) as fi:
																					__snake_case		:      str										=       self.sp_model.serialized_model_proto()
																					fi.write(_lowerCAmelCase    )

														return (vocab_file,)





from ...configuration_utils import PretrainedConfig
from ...utils import logging


_snake_case					:		Any      =    logging.get_logger(__name__)

_snake_case					:		Dict      =    {
    "facebook/s2t-wav2vec2-large-en-de": (
        "https://huggingface.co/facebook/s2t-wav2vec2-large-en-de/resolve/main/config.json"
    ),
    # See all Speech2Text models at https://huggingface.co/models?filter=speech2text2
}

class     a		(_lowerCAmelCase				):




							"""simple docstring"""


							__UpperCAmelCase   :							Union[str, Any]					  =  "speech_to_text_2"
							__UpperCAmelCase   :							Dict					  =  ["past_key_values"]
							__UpperCAmelCase   :							str					  =  {"num_attention_heads": "decoder_attention_heads", "hidden_size": "d_model"}

							def __init__(							self	:  str   ,     lowerCamelCase	:  List[Any]=10000   ,     lowerCamelCase	:  Tuple=6   ,     lowerCamelCase	:  Optional[Any]=2048   ,     lowerCamelCase	:  Optional[int]=4   ,     lowerCamelCase	:  Optional[int]=0.0   ,     lowerCamelCase	:  str=True   ,     lowerCamelCase	:  str="relu"   ,     lowerCamelCase	:  Tuple=256   ,     lowerCamelCase	:  Union[str, Any]=0.1   ,     lowerCamelCase	:  Optional[int]=0.0   ,     lowerCamelCase	:  Any=0.0   ,     lowerCamelCase	:  Optional[int]=0.02   ,     lowerCamelCase	:  Dict=2   ,     lowerCamelCase	:  List[str]=True   ,     lowerCamelCase	:  str=1   ,     lowerCamelCase	:  Any=0   ,     lowerCamelCase	:  List[Any]=2   ,     lowerCamelCase	:  List[Any]=1024   ,     **lowerCamelCase	:  List[str]   ,     )     ->   Any:
														__snake_case		:      Tuple										=       vocab_size
														__snake_case		:      int										=       d_model
														__snake_case		:      List[str]										=       decoder_ffn_dim
														__snake_case		:      Union[str, Any]										=       decoder_layers
														__snake_case		:      Dict										=       decoder_attention_heads
														__snake_case		:      Any										=       dropout
														__snake_case		:      List[str]										=       attention_dropout
														__snake_case		:      Optional[Any]										=       activation_dropout
														__snake_case		:      Union[str, Any]										=       activation_function
														__snake_case		:      Union[str, Any]										=       init_std
														__snake_case		:      Union[str, Any]										=       decoder_layerdrop
														__snake_case		:      Optional[int]										=       use_cache
														__snake_case		:      Optional[int]										=       decoder_layers
														__snake_case		:      str										=       scale_embedding  # scale factor will be sqrt(d_model) if True
														__snake_case		:      Dict										=       max_target_positions

														super().__init__(
														    pad_token_id=lowerCamelCase   ,     bos_token_id=lowerCamelCase   ,     eos_token_id=lowerCamelCase   ,     decoder_start_token_id=lowerCamelCase   ,     **lowerCamelCase   ,     )

