"""simple docstring"""







import itertools
import random
import unittest

import numpy as np

from transformers import is_speech_available
from transformers.testing_utils import require_torch, require_torchaudio

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_speech_available():
	from transformers import SpeechaTextFeatureExtractor

_a					:					Optional[Any]=			random.Random()
def   __UpperCAmelCase    (   UpperCAmelCase_   :    Optional[Any]      ,							UpperCAmelCase_   :    str=1.0      ,							UpperCAmelCase_   :    Optional[Any]=None      ,							UpperCAmelCase_   :    Tuple=None  )							->      Dict:

	'''simple docstring'''





	if rng is None:
		__snake_case      :							Dict						=			global_rng

	__snake_case      :							Union[str, Any]						=			[]
	for batch_idx in range(shape[0]  ):
		values.append([]  )
		for _ in range(shape[1]  ):
			values[-1].append(rng.random() * scale  )

	return values

@require_torch
@require_torchaudio
class 		UpperCamelCase      (   unittest.TestCase						):



				def __init__(self    :				Dict     ,  _A    :				str     ,  _A    :				List[Any]=7     ,  _A    :				Optional[Any]=4_00     ,  _A    :				int=20_00     ,  _A    :				Optional[int]=24     ,  _A    :				int=24     ,  _A    :				Dict=0.0     ,  _A    :				List[Any]=1_60_00     ,  _A    :				str=True     ,  _A    :				Tuple=True     ,  )   ->     Optional[Any]:
					__snake_case      :							List[Any]						=			parent
					__snake_case      :							int						=			batch_size
					__snake_case      :							Optional[Any]						=			min_seq_length
					__snake_case      :							Tuple						=			max_seq_length
					__snake_case      :							Dict						=			(self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
					__snake_case      :							Dict						=			feature_size
					__snake_case      :							str						=			num_mel_bins
					__snake_case      :							Union[str, Any]						=			padding_value
					__snake_case      :							Optional[int]						=			sampling_rate
					__snake_case      :							Optional[Any]						=			return_attention_mask
					__snake_case      :							List[str]						=			do_normalize



				def  _lowercase					(self    :				Dict)   ->     Union[str, Any]:
					return {
					    "feature_size": self.feature_size,
					    "num_mel_bins": self.num_mel_bins,
					    "padding_value": self.padding_value,
					    "sampling_rate": self.sampling_rate,
					    "return_attention_mask": self.return_attention_mask,
					    "do_normalize": self.do_normalize,
					}






				def  _lowercase					(self    :				Optional[Any]     ,  _A    :				int=False     ,  _A    :				Union[str, Any]=False)   ->     str:
					def _flatten(_A    :				int):
						return list(itertools.chain(*_A))

					if equal_length:
						__snake_case      :							Optional[Any]						=			[floats_list((self.max_seq_length, self.feature_size)) for _ in range(self.batch_size)]
					else:
						# make sure that inputs increase in size
						__snake_case      :							Union[str, Any]						=			[
						    floats_list((x, self.feature_size))
						    for x in range(self.min_seq_length     ,  self.max_seq_length     ,  self.seq_length_diff)
						]
					if numpify:
						__snake_case      :							Optional[int]						=			[np.asarray(_A) for x in speech_inputs]
					return speech_inputs







@require_torch
@require_torchaudio
class 		UpperCamelCase      (   lowercase				,     unittest.TestCase						):
				UpperCAmelCase    :				Dict   =		SpeechaTextFeatureExtractor if is_speech_available() else None



				def  _lowercase					(self    :				str)   ->     Any:
					__snake_case      :							Dict						=			SpeechaTextFeatureExtractionTester(self)



				def  _lowercase					(self    :				Any     ,  _A    :				List[str])   ->     Any:
					self.assertTrue(np.all(np.mean(_A     ,  axis=0) < 1E-3))
					self.assertTrue(np.all(np.abs(np.var(_A     ,  axis=0) - 1) < 1E-3))



				def  _lowercase					(self    :				Tuple)   ->     Optional[int]:
					# Tests that all call wrap to encode_plus and batch_encode_plus
					__snake_case      :							Union[str, Any]						=			self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
					# create three inputs of length 800, 1000, and 1200
					__snake_case      :							List[Any]						=			[floats_list((1, x))[0] for x in range(8_00     ,  14_00     ,  2_00)]
					__snake_case      :							Union[str, Any]						=			[np.asarray(_A) for speech_input in speech_inputs]

					# Test feature size
					__snake_case      :							Optional[int]						=			feature_extractor(_A     ,  padding=_A     ,  return_tensors='np').input_features
					self.assertTrue(input_features.ndim == 3)
					self.assertTrue(input_features.shape[-1] == feature_extractor.feature_size)

					# Test not batched input
					__snake_case      :							int						=			feature_extractor(speech_inputs[0]     ,  return_tensors='np').input_features
					__snake_case      :							Any						=			feature_extractor(np_speech_inputs[0]     ,  return_tensors='np').input_features
					self.assertTrue(np.allclose(_A     ,  _A     ,  atol=1E-3))

					# Test batched
					__snake_case      :							Optional[int]						=			feature_extractor(_A     ,  return_tensors='np').input_features
					__snake_case      :							int						=			feature_extractor(_A     ,  return_tensors='np').input_features
					for enc_seq_a, enc_seq_a in zip(_A     ,  _A):
						self.assertTrue(np.allclose(_A     ,  _A     ,  atol=1E-3))

					# Test 2-D numpy arrays are batched.
					__snake_case      :							Dict						=			[floats_list((1, x))[0] for x in (8_00, 8_00, 8_00)]
					__snake_case      :							Union[str, Any]						=			np.asarray(_A)
					__snake_case      :							Optional[Any]						=			feature_extractor(_A     ,  return_tensors='np').input_features
					__snake_case      :							Tuple						=			feature_extractor(_A     ,  return_tensors='np').input_features
					for enc_seq_a, enc_seq_a in zip(_A     ,  _A):
						self.assertTrue(np.allclose(_A     ,  _A     ,  atol=1E-3))



				def  _lowercase					(self    :				Tuple)   ->     Union[str, Any]:
					__snake_case      :							Optional[Any]						=			self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
					__snake_case      :							List[Any]						=			[floats_list((1, x))[0] for x in range(8_00     ,  14_00     ,  2_00)]

					__snake_case      :							Union[str, Any]						=			['longest', 'max_length', 'do_not_pad']
					__snake_case      :							Union[str, Any]						=			[None, 16, None]
					for max_length, padding in zip(_A     ,  _A):
						__snake_case      :							str						=			feature_extractor(
						    _A     ,  padding=_A     ,  max_length=_A     ,  return_attention_mask=_A)
						__snake_case      :							Dict						=			inputs.input_features
						__snake_case      :							Dict						=			inputs.attention_mask
						__snake_case      :							List[Any]						=			[np.sum(_A) for x in attention_mask]

						self._check_zero_mean_unit_variance(input_features[0][: fbank_feat_lengths[0]])
						self._check_zero_mean_unit_variance(input_features[1][: fbank_feat_lengths[1]])
						self._check_zero_mean_unit_variance(input_features[2][: fbank_feat_lengths[2]])



				def  _lowercase					(self    :				List[Any])   ->     Optional[int]:
					__snake_case      :							Union[str, Any]						=			self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
					__snake_case      :							Optional[Any]						=			[floats_list((1, x))[0] for x in range(8_00     ,  14_00     ,  2_00)]

					__snake_case      :							List[Any]						=			['longest', 'max_length', 'do_not_pad']
					__snake_case      :							List[str]						=			[None, 16, None]
					for max_length, padding in zip(_A     ,  _A):
						__snake_case      :							Dict						=			feature_extractor(
						    _A     ,  max_length=_A     ,  padding=_A     ,  return_tensors='np'     ,  return_attention_mask=_A)
						__snake_case      :							Dict						=			inputs.input_features
						__snake_case      :							Optional[Any]						=			inputs.attention_mask
						__snake_case      :							List[str]						=			[np.sum(_A) for x in attention_mask]

						self._check_zero_mean_unit_variance(input_features[0][: fbank_feat_lengths[0]])
						self.assertTrue(input_features[0][fbank_feat_lengths[0] :].sum() < 1E-6)
						self._check_zero_mean_unit_variance(input_features[1][: fbank_feat_lengths[1]])
						self.assertTrue(input_features[0][fbank_feat_lengths[1] :].sum() < 1E-6)
						self._check_zero_mean_unit_variance(input_features[2][: fbank_feat_lengths[2]])



				def  _lowercase					(self    :				Dict)   ->     Optional[int]:
					__snake_case      :							Tuple						=			self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
					__snake_case      :							Any						=			[floats_list((1, x))[0] for x in range(8_00     ,  14_00     ,  2_00)]
					__snake_case      :							str						=			feature_extractor(
					    _A     ,  padding='max_length'     ,  max_length=4     ,  truncation=_A     ,  return_tensors='np'     ,  return_attention_mask=_A     ,  )
					__snake_case      :							Union[str, Any]						=			inputs.input_features
					__snake_case      :							str						=			inputs.attention_mask
					__snake_case      :							str						=			np.sum(attention_mask == 1     ,  axis=1)

					self._check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]])
					self._check_zero_mean_unit_variance(input_features[1])
					self._check_zero_mean_unit_variance(input_features[2])



				def  _lowercase					(self    :				int)   ->     Tuple:
					__snake_case      :							Union[str, Any]						=			self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
					__snake_case      :							List[Any]						=			[floats_list((1, x))[0] for x in range(8_00     ,  14_00     ,  2_00)]
					__snake_case      :							Optional[Any]						=			feature_extractor(
					    _A     ,  padding='longest'     ,  max_length=4     ,  truncation=_A     ,  return_tensors='np'     ,  return_attention_mask=_A     ,  )
					__snake_case      :							Union[str, Any]						=			inputs.input_features
					__snake_case      :							Optional[Any]						=			inputs.attention_mask
					__snake_case      :							Union[str, Any]						=			np.sum(attention_mask == 1     ,  axis=1)

					self._check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]])
					self._check_zero_mean_unit_variance(input_features[1, : fbank_feat_lengths[1]])
					self._check_zero_mean_unit_variance(input_features[2])

					# make sure that if max_length < longest -> then pad to max_length
					self.assertEqual(input_features.shape     ,  (3, 4, 24))

					__snake_case      :							Optional[Any]						=			[floats_list((1, x))[0] for x in range(8_00     ,  14_00     ,  2_00)]
					__snake_case      :							Tuple						=			feature_extractor(
					    _A     ,  padding='longest'     ,  max_length=16     ,  truncation=_A     ,  return_tensors='np'     ,  return_attention_mask=_A     ,  )
					__snake_case      :							Tuple						=			inputs.input_features
					__snake_case      :							Union[str, Any]						=			inputs.attention_mask
					__snake_case      :							int						=			np.sum(attention_mask == 1     ,  axis=1)

					self._check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]])
					self._check_zero_mean_unit_variance(input_features[1, : fbank_feat_lengths[1]])
					self._check_zero_mean_unit_variance(input_features[2])

					# make sure that if max_length < longest -> then pad to max_length
					self.assertEqual(input_features.shape     ,  (3, 6, 24))



				def  _lowercase					(self    :				int)   ->     Dict:
					import torch

					__snake_case      :							int						=			self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
					__snake_case      :							Any						=			np.random.rand(1_00     ,  32).astype(np.floataa)
					__snake_case      :							List[Any]						=			np_speech_inputs.tolist()

					for inputs in [py_speech_inputs, np_speech_inputs]:
						__snake_case      :							str						=			feature_extractor.pad([{'input_features': inputs}]     ,  return_tensors='np')
						self.assertTrue(np_processed.input_features.dtype == np.floataa)
						__snake_case      :							Optional[Any]						=			feature_extractor.pad([{'input_features': inputs}]     ,  return_tensors='pt')
						self.assertTrue(pt_processed.input_features.dtype == torch.floataa)



				def  _lowercase					(self    :				List[str]     ,  _A    :				Union[str, Any])   ->     Union[str, Any]:
					from datasets import load_dataset

					__snake_case      :							Tuple						=			load_dataset('hf-internal-testing/librispeech_asr_dummy'     ,  'clean'     ,  split='validation')
					# automatic decoding with librispeech
					__snake_case      :							int						=			ds.sort('id').select(range(_A))[:num_samples]['audio']

					return [x["array"] for x in speech_samples]






				def  _lowercase					(self    :				Union[str, Any])   ->     List[Any]:
					# fmt: off
					__snake_case      :							Any						=			np.array([
					    -1.5_745, -1.7_713, -1.7_020, -1.6_069, -1.2_250, -1.1_105, -0.9_072, -0.8_241,
					    -1.2_310, -0.8_098, -0.3_320, -0.4_101, -0.7_985, -0.4_996, -0.8_213, -0.9_128,
					    -1.0_420, -1.1_286, -1.0_440, -0.7_999, -0.8_405, -1.2_275, -1.5_443, -1.4_625,
					])
					# fmt: on

					__snake_case      :							Optional[int]						=			self._load_datasamples(1)
					__snake_case      :							List[str]						=			self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
					__snake_case      :							str						=			feature_extractor(_A     ,  return_tensors='pt').input_features
					self.assertEquals(input_features.shape     ,  (1, 5_84, 24))
					self.assertTrue(np.allclose(input_features[0, 0, :30]     ,  _A     ,  atol=1E-4))






"""simple docstring"""







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


_a					:					List[Any]=			logging.get_logger(__name__)

_a					:					Any=			{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_a					:					int=			{
    "vocab_file": {
        "junnyu/roformer_chinese_small": "https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt",
        "junnyu/roformer_chinese_base": "https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt",
        "junnyu/roformer_chinese_char_small": (
            "https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_chinese_char_base": (
            "https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_small_discriminator": (
            "https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_small_generator": (
            "https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt"
        ),
    }
}

_a					:					Optional[Any]=			{
    "junnyu/roformer_chinese_small": 1_536,
    "junnyu/roformer_chinese_base": 1_536,
    "junnyu/roformer_chinese_char_small": 512,
    "junnyu/roformer_chinese_char_base": 512,
    "junnyu/roformer_small_discriminator": 128,
    "junnyu/roformer_small_generator": 128,
}


_a					:					str=			{
    "junnyu/roformer_chinese_small": {"do_lower_case": True},
    "junnyu/roformer_chinese_base": {"do_lower_case": True},
    "junnyu/roformer_chinese_char_small": {"do_lower_case": True},
    "junnyu/roformer_chinese_char_base": {"do_lower_case": True},
    "junnyu/roformer_small_discriminator": {"do_lower_case": True},
    "junnyu/roformer_small_generator": {"do_lower_case": True},
}

class 		UpperCamelCase      (   lowercase						):
				UpperCAmelCase    :				Union[str, Any]   =		VOCAB_FILES_NAMES
				UpperCAmelCase    :				Optional[int]   =		PRETRAINED_VOCAB_FILES_MAP
				UpperCAmelCase    :				List[str]   =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				UpperCAmelCase    :				int   =		PRETRAINED_INIT_CONFIGURATION
				UpperCAmelCase    :				Dict   =		RoFormerTokenizer



				def __init__(self    :				List[Any]     ,  _A    :				Any=None     ,  _A    :				int=None     ,  _A    :				Dict=True     ,  _A    :				List[Any]="[UNK]"     ,  _A    :				Tuple="[SEP]"     ,  _A    :				List[Any]="[PAD]"     ,  _A    :				str="[CLS]"     ,  _A    :				int="[MASK]"     ,  _A    :				Optional[int]=True     ,  _A    :				List[str]=None     ,  **_A    :				int     ,  )   ->     Dict:
					super().__init__(
					    _A     ,  tokenizer_file=_A     ,  do_lower_case=_A     ,  unk_token=_A     ,  sep_token=_A     ,  pad_token=_A     ,  cls_token=_A     ,  mask_token=_A     ,  tokenize_chinese_chars=_A     ,  strip_accents=_A     ,  **_A     ,  )

					__snake_case      :							List[Any]						=			json.loads(self.backend_tokenizer.normalizer.__getstate__())
					if (
					    pre_tok_state.get('lowercase'     ,  _A) != do_lower_case
					    or pre_tok_state.get('strip_accents'     ,  _A) != strip_accents
					):
						__snake_case      :							Union[str, Any]						=			getattr(_A     ,  pre_tok_state.pop('type'))
						__snake_case      :							Union[str, Any]						=			do_lower_case
						__snake_case      :							str						=			strip_accents
						__snake_case      :							Optional[int]						=			pre_tok_class(**_A)

					__snake_case      :							int						=			do_lower_case



				def __getstate__(self    :				Optional[Any])   ->     Dict:
					__snake_case      :							Optional[int]						=			self.__dict__.copy()
					__snake_case      :							int						=			BertPreTokenizer()
					return state



				def __setstate__(self    :				Optional[Any]     ,  _A    :				Optional[Any])   ->     Dict:
					__snake_case      :							List[str]						=			d
					__snake_case      :							str						=			self.__dict__['_tokenizer'].get_vocab()
					__snake_case      :							int						=			PreTokenizer.custom(JiebaPreTokenizer(_A))



				def  _lowercase					(self    :				int     ,  _A    :				Tuple     ,  _A    :				Any=None)   ->     str:
					__snake_case      :							Dict						=			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

					if token_ids_a:
						output += token_ids_a + [self.sep_token_id]

					return output



				def  _lowercase					(self    :				List[str]     ,  _A    :				List[int]     ,  _A    :				Optional[List[int]] = None)   ->     List[int]:
					__snake_case      :							Tuple						=			[self.sep_token_id]
					__snake_case      :							Optional[Any]						=			[self.cls_token_id]
					if token_ids_a is None:
						return len(cls + token_ids_a + sep) * [0]
					return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]



				def  _lowercase					(self    :				List[Any]     ,  _A    :				str     ,  _A    :				Optional[str] = None)   ->     Tuple[str]:
					__snake_case      :							List[Any]						=			self._tokenizer.model.save(_A     ,  name=_A)
					return tuple(_A)






				def  _lowercase					(self    :				int     ,  _A    :				Optional[int]     ,  _A    :				Tuple=None     ,  _A    :				Tuple=None     ,  _A    :				Dict=False     ,  **_A    :				Optional[int]     ,  )   ->     Optional[Any]:
					__snake_case      :							Optional[Any]						=			BertPreTokenizer()
					return super().save_pretrained(_A     ,  _A     ,  _A     ,  _A     ,  **_A)





