




"""simple docstring"""


import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
     from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse('''1.6'''):
     A_      =    True
     from torch.cuda.amp import autocast


A_      =    logging.getLogger(__name__)






@dataclass
class  lowercase:







  '''simple docstring'''





  lowercase__			 = field(
      metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}			)
  lowercase__			 = field(
      default=lowerCamelCase__	,      metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"}	,      )
  lowercase__			 = field(
      default=lowerCamelCase__	,      metadata={"help": "Whether to freeze the feature extractor layers of the model."}			)
  lowercase__			 = field(
      default=lowerCamelCase__	,      metadata={"help": "Whether to log verbose messages or not."}	,      )
  lowercase__			 = field(
      default=2.0	,      metadata={"help": "Maximum temperature for gumbel softmax."}			)
  lowercase__			 = field(
      default=0.5	,      metadata={"help": "Minimum temperature for gumbel softmax."}			)
  lowercase__			 = field(
      default=0.99_9995	,      metadata={"help": "Decay of gumbel temperature during training."}			)







def  UpperCAmelCase__	(snake_case__							: ModelArguments		,       snake_case__							: TrainingArguments   ):
  """simple docstring"""

  logging.basicConfig(
      format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s"""		,       datefmt="""%m/%d/%Y %H:%M:%S"""		,       handlers=[logging.StreamHandler(sys.stdout   )]		,       )
  _snake_case							:		Any								=  logging.WARNING
  if model_args.verbose_logging:
    _snake_case							:		Any								=  logging.DEBUG
  elif trainer_utils.is_main_process(training_args.local_rank   ):
    _snake_case							:		Any								=  logging.INFO
  logger.setLevel(snake_case__   )






@dataclass
class  lowercase:







  '''simple docstring'''





  lowercase__			 = field(
      default=lowerCamelCase__	,      metadata={"help": "The name of the dataset to use (via the datasets library)."}			)
  lowercase__			 = field(
      default=lowerCamelCase__	,      metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}			)
  lowercase__			 = field(
      default="train"	,      metadata={
          "help": "The name of the training data set split to use (via the datasets library). Defaults to 'train'"
      }	,      )
  lowercase__			 = field(
      default="validation"	,      metadata={
          "help": (
              "The name of the validation data set split to use (via the datasets library). Defaults to 'validation'"
          )
      }	,      )
  lowercase__			 = field(
      default="file"	,      metadata={"help": "Column in the dataset that contains speech file path. Defaults to 'file'"}	,      )
  lowercase__			 = field(
      default=lowerCamelCase__	,      metadata={"help": "Overwrite the cached preprocessed datasets or not."}			)
  lowercase__			 = field(
      default=1	,      metadata={
          "help": "The percentage of the train set used as validation set in case there's no validation split"
      }	,      )
  lowercase__			 = field(
      default=lowerCamelCase__	,      metadata={"help": "The number of processes to use for the preprocessing."}	,      )
  lowercase__			 = field(
      default=20.0	,      metadata={"help": "Filter audio files that are longer than `max_duration_in_seconds` seconds"}			)







@dataclass
class  lowercase:







  '''simple docstring'''





  lowercase__			 = 42
  lowercase__			 = 42
  lowercase__			 = "longest"
  lowercase__			 = None
  lowercase__			 = None


  def __call__(    self:		int,				a_:		List[Dict[str, Union[List[int], torch.Tensor]]]		):




    '''simple docstring'''



    _snake_case							:		List[Any]								=  self.feature_extractor.pad(
        __A,				max_length=self.max_length,				padding=self.padding,				pad_to_multiple_of=self.pad_to_multiple_of,				return_tensors="""pt""",				)
    _snake_case							:		int								=  self.model._get_feat_extract_output_lengths(batch["""input_values"""].shape[-1]		)

    _snake_case							:		Tuple								=  batch['''input_values'''].shape[0]

    # make sure that no loss is computed on padded inputs
    if batch["attention_mask"] is not None:
      # compute real output lengths according to convolution formula
      _snake_case							:		Tuple								=  self.model._get_feat_extract_output_lengths(batch["""attention_mask"""].sum(-1		)		).to(
          torch.long		)

      _snake_case							:		Dict								=  torch.zeros(
          (batch_size, mask_indices_seq_length),				dtype=torch.long,				device=batch["""input_values"""].device		)

      # these two operations makes sure that all values
      # before the output lengths indices are attended to
      _snake_case							:		Tuple								=  1
      _snake_case							:		int								=  attention_mask.flip([-1]		).cumsum(-1		).flip([-1]		).bool()

    # sample randomly masked indices
    _snake_case							:		Dict								=  _compute_mask_indices(
        (batch_size, mask_indices_seq_length),				self.model.config.mask_time_prob,				self.model.config.mask_time_length,				attention_mask=__A,				min_masks=2,				)

    return batch







class  lowercase(					lowerCamelCase__			):







  '''simple docstring'''





  def __init__(    self:		Union[str, Any],				*a_:		int,				a_:		Dict=1,				a_:		Any=0,				a_:		Optional[Any]=1.0,				**a_:		Any		):




    '''simple docstring'''



    super().__init__(*__A,				**__A		)
    _snake_case							:		Any								=  0
    _snake_case							:		Any								=  max_gumbel_temp
    _snake_case							:		Optional[Any]								=  min_gumbel_temp
    _snake_case							:		str								=  gumbel_temp_decay


  def 			UpperCamelCase_       (    self:		Dict,				a_:		nn.Module,				a_:		Dict[str, Union[torch.Tensor, Any]]		):




    '''simple docstring'''



    model.train()
    _snake_case							:		List[Any]								=  self._prepare_inputs(__A		)

    if self.use_amp:
      with autocast():
        _snake_case							:		Optional[Any]								=  self.compute_loss(__A,				__A		)
    else:
      _snake_case							:		Optional[int]								=  self.compute_loss(__A,				__A		)

    if self.args.n_gpu > 1 or self.deepspeed:
      if model.module.config.ctc_loss_reduction == "mean":
        _snake_case							:		Optional[Any]								=  loss.mean()
      elif model.module.config.ctc_loss_reduction == "sum":
        _snake_case							:		str								=  loss.sum() / (inputs['''mask_time_indices''']).sum()
      else:
        raise ValueError(f"{model.config.ctc_loss_reduction} is not valid. Choose one of [\'mean\', \'sum\']"		)

    if self.args.gradient_accumulation_steps > 1:
      _snake_case							:		Any								=  loss / self.args.gradient_accumulation_steps

    if self.use_amp:
      self.scaler.scale(__A		).backward()
    elif self.use_apex:
      with amp.scale_loss(__A,				self.optimizer		) as scaled_loss:
        scaled_loss.backward()
    elif self.deepspeed:
      self.deepspeed.backward(__A		)
    else:
      loss.backward()

    self.num_update_step += 1
    # make sure gumbel softmax temperature is decayed
    if self.args.n_gpu > 1 or self.deepspeed:
      model.module.set_gumbel_temperature(
          max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step,				self.min_gumbel_temp		)		)
    else:
      model.set_gumbel_temperature(
          max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step,				self.min_gumbel_temp		)		)

    return loss.detach()







def  UpperCAmelCase__	():
  """simple docstring"""

  _snake_case							:		Optional[Any]								=  HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)   )

  _snake_case							:		List[Any]								=  parser.parse_args_into_dataclasses()
  configure_logger(snake_case__		,       snake_case__   )

  # Downloading and loading a dataset from the hub.
  _snake_case							:		int								=  load_dataset(data_args.dataset_name		,       data_args.dataset_config_name		,       cache_dir=model_args.cache_dir   )

  if "validation" not in datasets.keys():
    # make sure only "validation" and "train" keys remain"
    _snake_case							:		Union[str, Any]								=  DatasetDict()
    _snake_case							:		Any								=  load_dataset(
        data_args.dataset_name		,       data_args.dataset_config_name		,       split=F"{data_args.train_split_name}[:{data_args.validation_split_percentage}%]"		,       cache_dir=model_args.cache_dir		,       )
    _snake_case							:		Tuple								=  load_dataset(
        data_args.dataset_name		,       data_args.dataset_config_name		,       split=F"{data_args.train_split_name}[{data_args.validation_split_percentage}%:]"		,       cache_dir=model_args.cache_dir		,       )
  else:
    # make sure only "validation" and "train" keys remain"
    _snake_case							:		Optional[Any]								=  DatasetDict()
    _snake_case							:		List[Any]								=  load_dataset(
        data_args.dataset_name		,       data_args.dataset_config_name		,       split="""validation"""		,       cache_dir=model_args.cache_dir		,       )
    _snake_case							:		int								=  load_dataset(
        data_args.dataset_name		,       data_args.dataset_config_name		,       split=F"{data_args.train_split_name}"		,       cache_dir=model_args.cache_dir		,       )

  # only normalized-inputs-training is supported
  _snake_case							:		Optional[Any]								=  WavaVecaFeatureExtractor.from_pretrained(
      model_args.model_name_or_path		,       cache_dir=model_args.cache_dir		,       do_normalize=snake_case__   )

  def prepare_dataset(snake_case__							: Dict   ):
    # check that all files have the correct sampling rate
    _snake_case							:		Optional[Any]								=  librosa.load(batch[data_args.speech_file_column]		,       sr=feature_extractor.sampling_rate   )
    return batch

  # load audio files into numpy arrays
  _snake_case							:		str								=  datasets.map(
      snake_case__		,       num_proc=data_args.preprocessing_num_workers		,       remove_columns=datasets["""train"""].column_names   )

  # filter audio files that are too long
  _snake_case							:		int								=  vectorized_datasets.filter(
      lambda snake_case__   : len(data["""speech"""]   ) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate   )   )

  def normalize(snake_case__							: Dict   ):
    return feature_extractor(batch["""speech"""]		,       sampling_rate=feature_extractor.sampling_rate   )

  # normalize and transform to `BatchFeatures`
  _snake_case							:		Any								=  vectorized_datasets.map(
      snake_case__		,       batched=snake_case__		,       num_proc=data_args.preprocessing_num_workers		,       load_from_cache_file=not data_args.overwrite_cache		,       remove_columns=vectorized_datasets["""train"""].column_names		,       )

  # pretraining is only supported for "newer" stable layer norm architecture
  # apply_spec_augment has to be True, mask_feature_prob has to be 0.0
  _snake_case							:		Optional[int]								=  WavaVecaConfig.from_pretrained(
      model_args.model_name_or_path		,       cache_dir=model_args.cache_dir		,       gradient_checkpointing=training_args.gradient_checkpointing		,       )

  if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
    raise ValueError(
        """PreTraining is only supported for ``config.do_stable_layer_norm=True`` and"""
        """ ``config.feat_extract_norm=\'layer\'"""   )

  _snake_case							:		Any								=  WavaVecaForPreTraining(snake_case__   )

  _snake_case							:		int								=  DataCollatorForWavaVecaPretraining(model=snake_case__		,       feature_extractor=snake_case__   )

  _snake_case							:		Any								=  WavaVecaPreTrainer(
      model=snake_case__		,       data_collator=snake_case__		,       args=snake_case__		,       train_dataset=vectorized_datasets["""train"""]		,       eval_dataset=vectorized_datasets["""validation"""]		,       tokenizer=snake_case__		,       max_gumbel_temp=model_args.max_gumbel_temperature		,       min_gumbel_temp=model_args.min_gumbel_temperature		,       gumbel_temp_decay=model_args.gumbel_temperature_decay		,       )
  trainer.train()


if __name__ == "__main__":
     main()








"""simple docstring"""


def  UpperCAmelCase__	(snake_case__							: int = 10_00   ):
  """simple docstring"""

  _snake_case		, _snake_case							:		List[Any]								=  1, 1
  _snake_case							:		str								=  []
  for i in range(1		,       n + 1   ):
    _snake_case							:		Any								=  prev_numerator + 2 * prev_denominator
    _snake_case							:		Optional[Any]								=  prev_numerator + prev_denominator
    if len(str(snake_case__   )   ) > len(str(snake_case__   )   ):
      result.append(snake_case__   )
    _snake_case							:		int								=  numerator
    _snake_case							:		Any								=  denominator

  return len(snake_case__   )


if __name__ == "__main__":
     print(F'''{solution() = }''')


