"""simple docstring"""







import copy
from typing import Any, Dict, List, Optional, Union

import numpy as np

from ...audio_utils import mel_filter_bank, spectrogram, window_function
from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import TensorType, logging


_a					:					Optional[int]=			logging.get_logger(__name__)

class 		UpperCamelCase      (   lowercase						):
				UpperCAmelCase    :				Any   =		["""input_features"""]



				def __init__(self    :				int     ,  _A    :				Any=80     ,  _A    :				Any=1_60_00     ,  _A    :				Union[str, Any]=1_60     ,  _A    :				int=30     ,  _A    :				str=4_00     ,  _A    :				Any=0.0     ,  _A    :				Optional[int]=False     ,  **_A    :				List[str]     ,  )   ->     Optional[int]:
					super().__init__(
					    feature_size=_A     ,  sampling_rate=_A     ,  padding_value=_A     ,  return_attention_mask=_A     ,  **_A     ,  )
					__snake_case      :							List[str]						=			n_fft
					__snake_case      :							str						=			hop_length
					__snake_case      :							Optional[Any]						=			chunk_length
					__snake_case      :							int						=			chunk_length * sampling_rate
					__snake_case      :							List[str]						=			self.n_samples // hop_length
					__snake_case      :							Any						=			sampling_rate
					__snake_case      :							Optional[Any]						=			mel_filter_bank(
					    num_frequency_bins=1 + n_fft // 2     ,  num_mel_filters=_A     ,  min_frequency=0.0     ,  max_frequency=80_00.0     ,  sampling_rate=_A     ,  norm='slaney'     ,  mel_scale='slaney'     ,  )



				def  _lowercase					(self    :				str     ,  _A    :				np.array)   ->     np.ndarray:
					__snake_case      :							Optional[int]						=			spectrogram(
					    _A     ,  window_function(self.n_fft     ,  'hann')     ,  frame_length=self.n_fft     ,  hop_length=self.hop_length     ,  power=2.0     ,  mel_filters=self.mel_filters     ,  log_mel='log10'     ,  )
					__snake_case      :							List[str]						=			log_spec[:, :-1]
					__snake_case      :							str						=			np.maximum(_A     ,  log_spec.max() - 8.0)
					__snake_case      :							Any						=			(log_spec + 4.0) / 4.0
					return log_spec



				@staticmethod
				# Copied from transformers.models.wav2vec2.feature_extraction_wav2vec2.Wav2Vec2FeatureExtractor.zero_mean_unit_var_norm
				def  _lowercase					(_A    :				List[np.ndarray]     ,  _A    :				List[np.ndarray]     ,  _A    :				float = 0.0)   ->     List[np.ndarray]:
					if attention_mask is not None:
						__snake_case      :							Optional[Any]						=			np.array(_A     ,  np.intaa)
						__snake_case      :							Tuple						=			[]

						for vector, length in zip(_A     ,  attention_mask.sum(-1)):
							__snake_case      :							Dict						=			(vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1E-7)
							if length < normed_slice.shape[0]:
								__snake_case      :							List[Any]						=			padding_value

							normed_input_values.append(_A)
					else:
						__snake_case      :							int						=			[(x - x.mean()) / np.sqrt(x.var() + 1E-7) for x in input_values]

					return normed_input_values



				def __call__(self    :				Any     ,  _A    :				Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]     ,  _A    :				bool = True     ,  _A    :				Optional[int] = None     ,  _A    :				Optional[Union[str, TensorType]] = None     ,  _A    :				Optional[bool] = None     ,  _A    :				Optional[str] = "max_length"     ,  _A    :				Optional[int] = None     ,  _A    :				Optional[int] = None     ,  _A    :				Optional[bool] = None     ,  **_A    :				Any     ,  )   ->     BatchFeature:

					if sampling_rate is not None:
						if sampling_rate != self.sampling_rate:
							raise ValueError(
							    f"The model corresponding to this feature extractor: {self.__class__.__name__} was trained using a"
							    f" sampling rate of {self.sampling_rate}. Please make sure that the provided `raw_speech` input"
							    f" was sampled with {self.sampling_rate} and not {sampling_rate}.")
					else:
						logger.warning(
						    'It is strongly recommended to pass the `sampling_rate` argument to this function. '
						    'Failing to do so can result in silent errors that might be hard to debug.')

					__snake_case      :							Dict						=			isinstance(_A     ,  np.ndarray) and len(raw_speech.shape) > 1
					if is_batched_numpy and len(raw_speech.shape) > 2:
						raise ValueError(f"Only mono-channel audio is supported for input to {self}")
					__snake_case      :							Dict						=			is_batched_numpy or (
					    isinstance(_A     ,  (list, tuple)) and (isinstance(raw_speech[0]     ,  (np.ndarray, tuple, list)))
					)

					if is_batched:
						__snake_case      :							List[str]						=			[np.asarray([speech]     ,  dtype=np.floataa).T for speech in raw_speech]
					elif not is_batched and not isinstance(_A     ,  np.ndarray):
						__snake_case      :							Tuple						=			np.asarray(_A     ,  dtype=np.floataa)
					elif isinstance(_A     ,  np.ndarray) and raw_speech.dtype is np.dtype(np.floataa):
						__snake_case      :							str						=			raw_speech.astype(np.floataa)

					# always return batch
					if not is_batched:
						__snake_case      :							int						=			[np.asarray([raw_speech]).T]

					__snake_case      :							Any						=			BatchFeature({'input_features': raw_speech})

					# convert into correct format for padding

					__snake_case      :							Any						=			self.pad(
					    _A     ,  padding=_A     ,  max_length=max_length if max_length else self.n_samples     ,  truncation=_A     ,  pad_to_multiple_of=_A     ,  return_attention_mask=return_attention_mask or do_normalize     ,  )

					# zero-mean and unit-variance normalization
					if do_normalize:
						__snake_case      :							Optional[Any]						=			self.zero_mean_unit_var_norm(
						    padded_inputs['input_features']     ,  attention_mask=padded_inputs['attention_mask']     ,  padding_value=self.padding_value     ,  )
						__snake_case      :							Optional[Any]						=			np.stack(padded_inputs['input_features']     ,  axis=0)

					# make sure list is in array format
					__snake_case      :							Union[str, Any]						=			padded_inputs.get('input_features').transpose(2     ,  0     ,  1)

					__snake_case      :							List[Any]						=			[self._np_extract_fbank_features(_A) for waveform in input_features[0]]

					if isinstance(input_features[0]     ,  _A):
						__snake_case      :							Union[str, Any]						=			[np.asarray(_A     ,  dtype=np.floataa) for feature in input_features]
					else:
						__snake_case      :							Optional[int]						=			input_features

					if return_attention_mask:
						# rescale from sample (48000) to feature (3000)
						__snake_case      :							Union[str, Any]						=			padded_inputs['attention_mask'][:, :: self.hop_length]

					if return_tensors is not None:
						__snake_case      :							Dict						=			padded_inputs.convert_to_tensors(_A)

					return padded_inputs






				def  _lowercase					(self    :				Optional[int])   ->     Dict[str, Any]:
					__snake_case      :							List[Any]						=			copy.deepcopy(self.__dict__)
					__snake_case      :							Union[str, Any]						=			self.__class__.__name__
					if "mel_filters" in output:
						del output["mel_filters"]
					return output





"""simple docstring"""







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


_a					:					Any=			logging.get_logger(__name__)

_a					:					str=			{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_a					:					Optional[Any]=			[
    "small",
    "small-base",
    "medium",
    "medium-base",
    "intermediate",
    "intermediate-base",
    "large",
    "large-base",
    "xlarge",
    "xlarge-base",
]

_a					:					List[Any]=			{
    "vocab_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt",
        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt",
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt",
        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt",
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json",
        "funnel-transformer/small-base": (
            "https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json",
        "funnel-transformer/large-base": (
            "https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"
        ),
    },
}
_a					:					str=			{f'''funnel-transformer/{name}''': 512 for name in _model_names}
_a					:					List[Any]=			{f'''funnel-transformer/{name}''': {"do_lower_case": True} for name in _model_names}

class 		UpperCamelCase      (   lowercase						):
				UpperCAmelCase    :				List[str]   =		VOCAB_FILES_NAMES
				UpperCAmelCase    :				Tuple   =		PRETRAINED_VOCAB_FILES_MAP
				UpperCAmelCase    :				int   =		PRETRAINED_INIT_CONFIGURATION
				UpperCAmelCase    :				Tuple   =		FunnelTokenizer
				UpperCAmelCase    :				Tuple   =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				UpperCAmelCase    :				int   =		2



				def __init__(self    :				int     ,  _A    :				Any=None     ,  _A    :				Union[str, Any]=None     ,  _A    :				Union[str, Any]=True     ,  _A    :				List[str]="<unk>"     ,  _A    :				Any="<sep>"     ,  _A    :				Dict="<pad>"     ,  _A    :				Tuple="<cls>"     ,  _A    :				Dict="<mask>"     ,  _A    :				Optional[Any]="<s>"     ,  _A    :				List[Any]="</s>"     ,  _A    :				Optional[int]=True     ,  _A    :				Dict=True     ,  _A    :				Tuple=None     ,  _A    :				int="##"     ,  **_A    :				Any     ,  )   ->     str:
					super().__init__(
					    _A     ,  tokenizer_file=_A     ,  do_lower_case=_A     ,  unk_token=_A     ,  sep_token=_A     ,  pad_token=_A     ,  cls_token=_A     ,  mask_token=_A     ,  bos_token=_A     ,  eos_token=_A     ,  clean_text=_A     ,  tokenize_chinese_chars=_A     ,  strip_accents=_A     ,  wordpieces_prefix=_A     ,  **_A     ,  )

					__snake_case      :							Optional[int]						=			json.loads(self.backend_tokenizer.normalizer.__getstate__())
					if (
					    normalizer_state.get('lowercase'     ,  _A) != do_lower_case
					    or normalizer_state.get('strip_accents'     ,  _A) != strip_accents
					    or normalizer_state.get('handle_chinese_chars'     ,  _A) != tokenize_chinese_chars
					):
						__snake_case      :							List[str]						=			getattr(_A     ,  normalizer_state.pop('type'))
						__snake_case      :							int						=			do_lower_case
						__snake_case      :							Optional[int]						=			strip_accents
						__snake_case      :							str						=			tokenize_chinese_chars
						__snake_case      :							Optional[int]						=			normalizer_class(**_A)

					__snake_case      :							str						=			do_lower_case



				def  _lowercase					(self    :				Optional[Any]     ,  _A    :				Dict     ,  _A    :				Tuple=None)   ->     Any:
					__snake_case      :							List[Any]						=			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

					if token_ids_a:
						output += token_ids_a + [self.sep_token_id]

					return output



				def  _lowercase					(self    :				str     ,  _A    :				List[int]     ,  _A    :				Optional[List[int]] = None)   ->     List[int]:
					__snake_case      :							Union[str, Any]						=			[self.sep_token_id]
					__snake_case      :							Union[str, Any]						=			[self.cls_token_id]
					if token_ids_a is None:
						return len(cls) * [self.cls_token_type_id] + len(token_ids_a + sep) * [0]
					return len(cls) * [self.cls_token_type_id] + len(token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]






				def  _lowercase					(self    :				Tuple     ,  _A    :				str     ,  _A    :				Optional[str] = None)   ->     Tuple[str]:
					__snake_case      :							int						=			self._tokenizer.model.save(_A     ,  name=_A)
					return tuple(_A)





