"""simple docstring"""







import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
		from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse("1.6"):
		_a					:					Optional[Any]=			True
		from torch.cuda.amp import autocast


_a					:					Optional[Any]=			logging.getLogger(__name__)

@dataclass
class 		UpperCamelCase      :
				UpperCAmelCase    :				str   =		field(
				    metadata={"""help""": """Path to pretrained model or model identifier from huggingface.co/models"""}						)
				UpperCAmelCase    :				Optional[str]   =		field(
				    default=_SCREAMING_SNAKE_CASE				,     metadata={"""help""": """Where do you want to store the pretrained models downloaded from huggingface.co"""}				,     )
				UpperCAmelCase    :				Optional[bool]   =		field(
				    default=_SCREAMING_SNAKE_CASE				,     metadata={"""help""": """Whether to freeze the feature extractor layers of the model."""}						)
				UpperCAmelCase    :				Optional[bool]   =		field(
				    default=_SCREAMING_SNAKE_CASE				,     metadata={"""help""": """Whether to log verbose messages or not."""}				,     )
				UpperCAmelCase    :				Optional[float]   =		field(
				    default=2.0				,     metadata={"""help""": """Maximum temperature for gumbel softmax."""}						)
				UpperCAmelCase    :				Optional[float]   =		field(
				    default=0.5				,     metadata={"""help""": """Minimum temperature for gumbel softmax."""}						)
				UpperCAmelCase    :				Optional[float]   =		field(
				    default=0.999995				,     metadata={"""help""": """Decay of gumbel temperature during training."""}						)







def   __UpperCAmelCase    (   UpperCAmelCase_   :    str      ,							UpperCAmelCase_   :    Tuple  )							->      str:

	'''simple docstring'''





	logging.basicConfig(
	    format='%(asctime)s - %(levelname)s - %(name)s - %(message)s'      ,							datefmt='%m/%d/%Y %H:%M:%S'      ,							handlers=[logging.StreamHandler(sys.stdout  )]      ,							)
	__snake_case      :							List[Any]						=			logging.WARNING
	if model_args.verbose_logging:
		__snake_case      :							Optional[int]						=			logging.DEBUG
	elif trainer_utils.is_main_process(training_args.local_rank  ):
		__snake_case      :							Union[str, Any]						=			logging.INFO
	logger.setLevel(UpperCAmelCase_  )

@dataclass
class 		UpperCamelCase      :
				UpperCAmelCase    :				str   =		field(
				    default=_SCREAMING_SNAKE_CASE				,     metadata={"""help""": """The name of the dataset to use (via the datasets library)."""}						)
				UpperCAmelCase    :				Optional[str]   =		field(
				    default=_SCREAMING_SNAKE_CASE				,     metadata={"""help""": """The configuration name of the dataset to use (via the datasets library)."""}						)
				UpperCAmelCase    :				Optional[str]   =		field(
				    default="""train"""				,     metadata={
				        """help""": """The name of the training data set split to use (via the datasets library). Defaults to 'train'"""
				    }				,     )
				UpperCAmelCase    :				Optional[str]   =		field(
				    default="""validation"""				,     metadata={
				        """help""": (
				            """The name of the validation data set split to use (via the datasets library). Defaults to 'validation'"""
				        )
				    }				,     )
				UpperCAmelCase    :				Optional[str]   =		field(
				    default="""file"""				,     metadata={"""help""": """Column in the dataset that contains speech file path. Defaults to 'file'"""}				,     )
				UpperCAmelCase    :				bool   =		field(
				    default=_SCREAMING_SNAKE_CASE				,     metadata={"""help""": """Overwrite the cached preprocessed datasets or not."""}						)
				UpperCAmelCase    :				Optional[int]   =		field(
				    default=1				,     metadata={
				        """help""": """The percentage of the train set used as validation set in case there's no validation split"""
				    }				,     )
				UpperCAmelCase    :				Optional[int]   =		field(
				    default=_SCREAMING_SNAKE_CASE				,     metadata={"""help""": """The number of processes to use for the preprocessing."""}				,     )
				UpperCAmelCase    :				Optional[float]   =		field(
				    default=20.0				,     metadata={"""help""": """Filter audio files that are longer than `max_duration_in_seconds` seconds"""}						)







@dataclass
class 		UpperCamelCase      :
				UpperCAmelCase    :				WavaVecaForPreTraining
				UpperCAmelCase    :				WavaVecaFeatureExtractor
				UpperCAmelCase    :				Union[bool, str]   =		"longest"
				UpperCAmelCase    :				Optional[int]   =		None
				UpperCAmelCase    :				Optional[int]   =		None



				def __call__(self    :				Union[str, Any]     ,  _A    :				Any)   ->     Dict[str, torch.Tensor]:
					__snake_case      :							Optional[Any]						=			self.feature_extractor.pad(
					    A_     ,  max_length=self.max_length     ,  padding=self.padding     ,  pad_to_multiple_of=self.pad_to_multiple_of     ,  return_tensors='pt'     ,  )
					__snake_case      :							Tuple						=			self.model._get_feat_extract_output_lengths(batch['input_values'].shape[-1])

					__snake_case      :							str						=			batch['input_values'].shape[0]

					# make sure that no loss is computed on padded inputs
					if batch["attention_mask"] is not None:
						# compute real output lengths according to convolution formula
						__snake_case      :							Dict						=			self.model._get_feat_extract_output_lengths(batch['attention_mask'].sum(-1)).to(
						    torch.long)

						__snake_case      :							Dict						=			torch.zeros(
						    (batch_size, mask_indices_seq_length)     ,  dtype=torch.long     ,  device=batch['input_values'].device)

						# these two operations makes sure that all values
						# before the output lengths indices are attended to
						__snake_case      :							List[str]						=			1
						__snake_case      :							Tuple						=			attention_mask.flip([-1]).cumsum(-1).flip([-1]).bool()

					# sample randomly masked indices
					__snake_case      :							int						=			_compute_mask_indices(
					    (batch_size, mask_indices_seq_length)     ,  self.model.config.mask_time_prob     ,  self.model.config.mask_time_length     ,  attention_mask=A_     ,  min_masks=2     ,  )

					return batch







class 		UpperCamelCase      (   _SCREAMING_SNAKE_CASE						):



				def __init__(self    :				int     ,  *_A    :				Any     ,  _A    :				Any=1     ,  _A    :				Optional[Any]=0     ,  _A    :				Dict=1.0     ,  **_A    :				Tuple)   ->     List[Any]:
					super().__init__(*A_     ,  **A_)
					__snake_case      :							List[str]						=			0
					__snake_case      :							Union[str, Any]						=			max_gumbel_temp
					__snake_case      :							Dict						=			min_gumbel_temp
					__snake_case      :							Any						=			gumbel_temp_decay






				def  _lowercase					(self    :				Optional[Any]     ,  _A    :				Tuple     ,  _A    :				Union[str, Any])   ->     torch.Tensor:
					model.train()
					__snake_case      :							Optional[Any]						=			self._prepare_inputs(A_)

					if self.use_amp:
						with autocast():
							__snake_case      :							Any						=			self.compute_loss(A_     ,  A_)
					else:
						__snake_case      :							str						=			self.compute_loss(A_     ,  A_)

					if self.args.n_gpu > 1 or self.deepspeed:
						if model.module.config.ctc_loss_reduction == "mean":
							__snake_case      :							int						=			loss.mean()
						elif model.module.config.ctc_loss_reduction == "sum":
							__snake_case      :							Any						=			loss.sum() / (inputs['mask_time_indices']).sum()
						else:
							raise ValueError(f"{model.config.ctc_loss_reduction} is not valid. Choose one of [\'mean\', \'sum\']")

					if self.args.gradient_accumulation_steps > 1:
						__snake_case      :							Tuple						=			loss / self.args.gradient_accumulation_steps

					if self.use_amp:
						self.scaler.scale(A_).backward()
					elif self.use_apex:
						with amp.scale_loss(A_     ,  self.optimizer) as scaled_loss:
							scaled_loss.backward()
					elif self.deepspeed:
						self.deepspeed.backward(A_)
					else:
						loss.backward()

					self.num_update_step += 1
					# make sure gumbel softmax temperature is decayed
					if self.args.n_gpu > 1 or self.deepspeed:
						model.module.set_gumbel_temperature(
						    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step     ,  self.min_gumbel_temp))
					else:
						model.set_gumbel_temperature(
						    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step     ,  self.min_gumbel_temp))

					return loss.detach()







def   __UpperCAmelCase    (   )							->      List[Any]:

	'''simple docstring'''





	__snake_case      :							int						=			HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)  )

	__snake_case		,    __snake_case		,    __snake_case      :							List[str]						=			parser.parse_args_into_dataclasses()
	configure_logger(UpperCAmelCase_      ,							UpperCAmelCase_  )

	# Downloading and loading a dataset from the hub.
	__snake_case      :							Optional[int]						=			load_dataset(data_args.dataset_name      ,							data_args.dataset_config_name      ,							cache_dir=model_args.cache_dir  )

	if "validation" not in datasets.keys():
		# make sure only "validation" and "train" keys remain"
		__snake_case      :							Union[str, Any]						=			DatasetDict()
		__snake_case      :							List[str]						=			load_dataset(
		    data_args.dataset_name      ,							data_args.dataset_config_name      ,							split=F"{data_args.train_split_name}[:{data_args.validation_split_percentage}%]"      ,							cache_dir=model_args.cache_dir      ,							)
		__snake_case      :							int						=			load_dataset(
		    data_args.dataset_name      ,							data_args.dataset_config_name      ,							split=F"{data_args.train_split_name}[{data_args.validation_split_percentage}%:]"      ,							cache_dir=model_args.cache_dir      ,							)
	else:
		# make sure only "validation" and "train" keys remain"
		__snake_case      :							List[Any]						=			DatasetDict()
		__snake_case      :							str						=			load_dataset(
		    data_args.dataset_name      ,							data_args.dataset_config_name      ,							split='validation'      ,							cache_dir=model_args.cache_dir      ,							)
		__snake_case      :							int						=			load_dataset(
		    data_args.dataset_name      ,							data_args.dataset_config_name      ,							split=F"{data_args.train_split_name}"      ,							cache_dir=model_args.cache_dir      ,							)

	# only normalized-inputs-training is supported
	__snake_case      :							Optional[Any]						=			WavaVecaFeatureExtractor.from_pretrained(
	    model_args.model_name_or_path      ,							cache_dir=model_args.cache_dir      ,							do_normalize=UpperCAmelCase_  )

	def prepare_dataset(UpperCAmelCase_   :    List[str]  ):
		# check that all files have the correct sampling rate
		__snake_case		,    __snake_case      :							Optional[Any]						=			librosa.load(batch[data_args.speech_file_column]      ,							sr=feature_extractor.sampling_rate  )
		return batch

	# load audio files into numpy arrays
	__snake_case      :							str						=			datasets.map(
	    UpperCAmelCase_      ,							num_proc=data_args.preprocessing_num_workers      ,							remove_columns=datasets['train'].column_names  )

	# filter audio files that are too long
	__snake_case      :							Any						=			vectorized_datasets.filter(
	    lambda UpperCAmelCase_  : len(data['speech']  ) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate  )  )

	def normalize(UpperCAmelCase_   :    Optional[int]  ):
		return feature_extractor(batch['speech']      ,							sampling_rate=feature_extractor.sampling_rate  )

	# normalize and transform to `BatchFeatures`
	__snake_case      :							Any						=			vectorized_datasets.map(
	    UpperCAmelCase_      ,							batched=UpperCAmelCase_      ,							num_proc=data_args.preprocessing_num_workers      ,							load_from_cache_file=not data_args.overwrite_cache      ,							remove_columns=vectorized_datasets['train'].column_names      ,							)

	# pretraining is only supported for "newer" stable layer norm architecture
	# apply_spec_augment has to be True, mask_feature_prob has to be 0.0
	__snake_case      :							Union[str, Any]						=			WavaVecaConfig.from_pretrained(
	    model_args.model_name_or_path      ,							cache_dir=model_args.cache_dir      ,							gradient_checkpointing=training_args.gradient_checkpointing      ,							)

	if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
		raise ValueError(
		    'PreTraining is only supported for ``config.do_stable_layer_norm=True`` and'
		    ' ``config.feat_extract_norm=\'layer\''  )

	__snake_case      :							Tuple						=			WavaVecaForPreTraining(UpperCAmelCase_  )

	__snake_case      :							Optional[Any]						=			DataCollatorForWavaVecaPretraining(model=UpperCAmelCase_      ,							feature_extractor=UpperCAmelCase_  )

	__snake_case      :							str						=			WavaVecaPreTrainer(
	    model=UpperCAmelCase_      ,							data_collator=UpperCAmelCase_      ,							args=UpperCAmelCase_      ,							train_dataset=vectorized_datasets['train']      ,							eval_dataset=vectorized_datasets['validation']      ,							tokenizer=UpperCAmelCase_      ,							max_gumbel_temp=model_args.max_gumbel_temperature      ,							min_gumbel_temp=model_args.min_gumbel_temperature      ,							gumbel_temp_decay=model_args.gumbel_temperature_decay      ,							)
	trainer.train()


if __name__ == "__main__":
		main()






"""simple docstring"""







from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig


_a					:					Tuple=			{
    "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/config.json",
    "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/config.json",
    "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/config.json",
    "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/config.json",
    "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/config.json",
    "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/config.json",
    "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/config.json",
    "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/config.json",
}

class 		UpperCamelCase      (   lowercase						):
				UpperCAmelCase    :				str   =		"""albert"""



				def __init__(self    :				Union[str, Any]     ,  _A    :				int=3_00_00     ,  _A    :				Any=1_28     ,  _A    :				Tuple=40_96     ,  _A    :				int=12     ,  _A    :				Tuple=1     ,  _A    :				int=64     ,  _A    :				Optional[Any]=1_63_84     ,  _A    :				Optional[Any]=1     ,  _A    :				List[str]="gelu_new"     ,  _A    :				Any=0     ,  _A    :				Optional[Any]=0     ,  _A    :				List[Any]=5_12     ,  _A    :				List[Any]=2     ,  _A    :				Dict=0.02     ,  _A    :				Union[str, Any]=1E-12     ,  _A    :				Tuple=0.1     ,  _A    :				int="absolute"     ,  _A    :				List[Any]=0     ,  _A    :				str=2     ,  _A    :				int=3     ,  **_A    :				Tuple     ,  )   ->     Union[str, Any]:
					super().__init__(pad_token_id=_A     ,  bos_token_id=_A     ,  eos_token_id=_A     ,  **_A)

					__snake_case      :							str						=			vocab_size
					__snake_case      :							List[Any]						=			embedding_size
					__snake_case      :							Tuple						=			hidden_size
					__snake_case      :							Optional[Any]						=			num_hidden_layers
					__snake_case      :							List[str]						=			num_hidden_groups
					__snake_case      :							Dict						=			num_attention_heads
					__snake_case      :							Any						=			inner_group_num
					__snake_case      :							Union[str, Any]						=			hidden_act
					__snake_case      :							List[Any]						=			intermediate_size
					__snake_case      :							List[str]						=			hidden_dropout_prob
					__snake_case      :							int						=			attention_probs_dropout_prob
					__snake_case      :							Any						=			max_position_embeddings
					__snake_case      :							Optional[int]						=			type_vocab_size
					__snake_case      :							Optional[Any]						=			initializer_range
					__snake_case      :							List[Any]						=			layer_norm_eps
					__snake_case      :							Optional[Any]						=			classifier_dropout_prob
					__snake_case      :							List[str]						=			position_embedding_type







class 		UpperCamelCase      (   lowercase						):



				@property
				def  _lowercase					(self    :				Tuple)   ->     Mapping[str, Mapping[int, str]]:
					if self.task == "multiple-choice":
						__snake_case      :							int						=			{0: 'batch', 1: 'choice', 2: 'sequence'}
					else:
						__snake_case      :							Optional[Any]						=			{0: 'batch', 1: 'sequence'}
					return OrderedDict(
					    [
					        ('input_ids', dynamic_axis),
					        ('attention_mask', dynamic_axis),
					        ('token_type_ids', dynamic_axis),
					    ])





