

import inspect
import unittest
import warnings

from transformers import DeiTConfig
from transformers.models.auto import get_values
from transformers.testing_utils import (
    require_accelerate,
    require_torch,
    require_torch_gpu,
    require_vision,
    slow,
    torch_device,
)
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch
     from torch import nn

     from transformers import (
         MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
         MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
         MODEL_MAPPING,
         DeiTForImageClassification,
         DeiTForImageClassificationWithTeacher,
         DeiTForMaskedImageModeling,
         DeiTModel,
     )
     from transformers.models.deit.modeling_deit import DEIT_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
     from PIL import Image

     from transformers import DeiTImageProcessor





class  __UpperCAmelCase					:



   def __init__(							self							:					int,		__A							:					Dict,		__A							:					Any=1_3,		__A							:					List[str]=3_0,		__A							:					int=2,		__A							:					Union[str, Any]=3,		__A							:					List[str]=True,		__A							:					List[Any]=True,		__A							:					List[str]=3_2,		__A							:					int=5,		__A							:					List[str]=4,		__A							:					Tuple=3_7,		__A							:					Tuple="gelu",		__A							:					List[str]=0.1,		__A							:					Dict=0.1,		__A							:					Tuple=1_0,		__A							:					str=0.0_2,		__A							:					Dict=3,		__A							:					Any=None,		__A							:					Union[str, Any]=2,		):
     UpperCAmelCase	:							Tuple						=			parent
     UpperCAmelCase	:							Any						=			batch_size
     UpperCAmelCase	:							Optional[Any]						=			image_size
     UpperCAmelCase	:							Any						=			patch_size
     UpperCAmelCase	:							Any						=			num_channels
     UpperCAmelCase	:							Dict						=			is_training
     UpperCAmelCase	:							List[Any]						=			use_labels
     UpperCAmelCase	:							int						=			hidden_size
     UpperCAmelCase	:							Optional[int]						=			num_hidden_layers
     UpperCAmelCase	:							Tuple						=			num_attention_heads
     UpperCAmelCase	:							Optional[Any]						=			intermediate_size
     UpperCAmelCase	:							List[Any]						=			hidden_act
     UpperCAmelCase	:							Tuple						=			hidden_dropout_prob
     UpperCAmelCase	:							Optional[int]						=			attention_probs_dropout_prob
     UpperCAmelCase	:							str						=			type_sequence_label_size
     UpperCAmelCase	:							List[Any]						=			initializer_range
     UpperCAmelCase	:							int						=			scope
     UpperCAmelCase	:							Optional[Any]						=			encoder_stride

     # in DeiT, the seq length equals the number of patches + 2 (we add 2 for the [CLS] and distilation tokens)
     UpperCAmelCase	:							str						=			(image_size // patch_size) ** 2
     UpperCAmelCase	:							str						=			num_patches + 2



   def 	__magic_name__   (							self							:					Any						):
     UpperCAmelCase	:							int						=			floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]						)

     UpperCAmelCase	:							Tuple						=			None
     if self.use_labels:
       UpperCAmelCase	:							str						=			ids_tensor([self.batch_size],		self.type_sequence_label_size						)

     UpperCAmelCase	:							Union[str, Any]						=			self.get_config()

     return config, pixel_values, labels



   def 	__magic_name__   (							self							:					Any						):
     return DeiTConfig(
         image_size=self.image_size,		patch_size=self.patch_size,		num_channels=self.num_channels,		hidden_size=self.hidden_size,		num_hidden_layers=self.num_hidden_layers,		num_attention_heads=self.num_attention_heads,		intermediate_size=self.intermediate_size,		hidden_act=self.hidden_act,		hidden_dropout_prob=self.hidden_dropout_prob,		attention_probs_dropout_prob=self.attention_probs_dropout_prob,		is_decoder=__A,		initializer_range=self.initializer_range,		encoder_stride=self.encoder_stride,		)



   def 	__magic_name__   (							self							:					Dict,		__A							:					Union[str, Any],		__A							:					Union[str, Any],		__A							:					List[Any]						):
     UpperCAmelCase	:							Union[str, Any]						=			DeiTModel(config=__A						)
     model.to(__A						)
     model.eval()
     UpperCAmelCase	:							Optional[int]						=			model(__A						)
     self.parent.assertEqual(result.last_hidden_state.shape,		(self.batch_size, self.seq_length, self.hidden_size)						)



   def 	__magic_name__   (							self							:					List[str],		__A							:					List[Any],		__A							:					List[Any],		__A							:					Optional[Any]						):
     UpperCAmelCase	:							Optional[int]						=			DeiTForMaskedImageModeling(config=__A						)
     model.to(__A						)
     model.eval()
     UpperCAmelCase	:							Optional[Any]						=			model(__A						)
     self.parent.assertEqual(
         result.reconstruction.shape,		(self.batch_size, self.num_channels, self.image_size, self.image_size)						)

     # test greyscale images
     UpperCAmelCase	:							str						=			1
     UpperCAmelCase	:							Union[str, Any]						=			DeiTForMaskedImageModeling(__A						)
     model.to(__A						)
     model.eval()

     UpperCAmelCase	:							List[Any]						=			floats_tensor([self.batch_size, 1, self.image_size, self.image_size]						)
     UpperCAmelCase	:							str						=			model(__A						)
     self.parent.assertEqual(result.reconstruction.shape,		(self.batch_size, 1, self.image_size, self.image_size)						)



   def 	__magic_name__   (							self							:					Union[str, Any],		__A							:					Dict,		__A							:					int,		__A							:					Optional[int]						):
     UpperCAmelCase	:							str						=			self.type_sequence_label_size
     UpperCAmelCase	:							Tuple						=			DeiTForImageClassification(__A						)
     model.to(__A						)
     model.eval()
     UpperCAmelCase	:							Any						=			model(__A,		labels=__A						)
     self.parent.assertEqual(result.logits.shape,		(self.batch_size, self.type_sequence_label_size)						)

     # test greyscale images
     UpperCAmelCase	:							Optional[Any]						=			1
     UpperCAmelCase	:							int						=			DeiTForImageClassification(__A						)
     model.to(__A						)
     model.eval()

     UpperCAmelCase	:							int						=			floats_tensor([self.batch_size, 1, self.image_size, self.image_size]						)
     UpperCAmelCase	:							Union[str, Any]						=			model(__A,		labels=__A						)
     self.parent.assertEqual(result.logits.shape,		(self.batch_size, self.type_sequence_label_size)						)





   def 	__magic_name__   (							self							:					int						):
     UpperCAmelCase	:							List[Any]						=			self.prepare_config_and_inputs()
     (
         (
         UpperCAmelCase
     )			,						(
         UpperCAmelCase
     )			,						(
         UpperCAmelCase
     )			,						
     )	:							int						=			config_and_inputs
     UpperCAmelCase	:							Optional[int]						=			{'''pixel_values''': pixel_values}
     return config, inputs_dict






@require_torch
class  __UpperCAmelCase					(	lowerCamelCase__      ,  lowerCamelCase__      ,  unittest.TestCase						):
   UpperCamelCase       					=					(
       (
           DeiTModel,
           DeiTForImageClassification,
           DeiTForImageClassificationWithTeacher,
           DeiTForMaskedImageModeling,
       )
       if is_torch_available()
       else ()
   )
   UpperCamelCase       					=					(
       {
           """feature-extraction""": DeiTModel,
           """image-classification""": (DeiTForImageClassification, DeiTForImageClassificationWithTeacher),
       }
       if is_torch_available()
       else {}
   )

   UpperCamelCase       					=					False
   UpperCamelCase       					=					False
   UpperCamelCase       					=					False



   def 	__magic_name__   (							self							:					Dict						):
     UpperCAmelCase	:							Dict						=			DeiTModelTester(self						)
     UpperCAmelCase	:							Union[str, Any]						=			ConfigTester(self,		config_class=__A,		has_text_modality=__A,		hidden_size=3_7						)



   def 	__magic_name__   (							self							:					List[Any]						):
     self.config_tester.run_common_tests()



   @unittest.skip(reason='''DeiT does not use inputs_embeds'''						)
   def 	__magic_name__   (							self							:					List[str]						):
     pass



   def 	__magic_name__   (							self							:					List[Any]						):
     UpperCAmelCase			,						UpperCAmelCase	:							Dict						=			self.model_tester.prepare_config_and_inputs_for_common()

     for model_class in self.all_model_classes:
       UpperCAmelCase	:							Optional[Any]						=			model_class(__A						)
       self.assertIsInstance(model.get_input_embeddings(),		(nn.Module)						)
       UpperCAmelCase	:							str						=			model.get_output_embeddings()
       self.assertTrue(x is None or isinstance(__A,		nn.Linear						)						)



   def 	__magic_name__   (							self							:					List[str]						):
     UpperCAmelCase			,						UpperCAmelCase	:							Tuple						=			self.model_tester.prepare_config_and_inputs_for_common()

     for model_class in self.all_model_classes:
       UpperCAmelCase	:							List[str]						=			model_class(__A						)
       UpperCAmelCase	:							Optional[Any]						=			inspect.signature(model.forward						)
       # signature.parameters is an OrderedDict => so arg_names order is deterministic
       UpperCAmelCase	:							Optional[int]						=			[*signature.parameters.keys()]

       UpperCAmelCase	:							str						=			['''pixel_values''']
       self.assertListEqual(arg_names[:1],		__A						)



   def 	__magic_name__   (							self							:					List[str]						):
     UpperCAmelCase	:							Union[str, Any]						=			self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_model(*__A						)



   def 	__magic_name__   (							self							:					int						):
     UpperCAmelCase	:							int						=			self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_masked_image_modeling(*__A						)



   def 	__magic_name__   (							self							:					Union[str, Any]						):
     UpperCAmelCase	:							Optional[Any]						=			self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_image_classification(*__A						)



   def 	__magic_name__   (							self							:					List[str],		__A							:					List[Any],		__A							:					int,		__A							:					Optional[Any]=False						):
     UpperCAmelCase	:							Optional[Any]						=			super()._prepare_for_class(__A,		__A,		return_labels=__A						)

     if return_labels:
       if model_class.__name__ == "DeiTForImageClassificationWithTeacher":
         del inputs_dict["labels"]

     return inputs_dict



   def 	__magic_name__   (							self							:					List[Any]						):
     if not self.model_tester.is_training:
       return

     UpperCAmelCase			,						UpperCAmelCase	:							int						=			self.model_tester.prepare_config_and_inputs_for_common()
     UpperCAmelCase	:							int						=			True

     for model_class in self.all_model_classes:
       # DeiTForImageClassificationWithTeacher supports inference-only
       if (
           model_class in get_values(__A						)
           or model_class.__name__ == "DeiTForImageClassificationWithTeacher"
       ):
         continue
       UpperCAmelCase	:							Tuple						=			model_class(__A						)
       model.to(__A						)
       model.train()
       UpperCAmelCase	:							List[str]						=			self._prepare_for_class(__A,		__A,		return_labels=__A						)
       UpperCAmelCase	:							Union[str, Any]						=			model(**__A						).loss
       loss.backward()



   def 	__magic_name__   (							self							:					Dict						):
     UpperCAmelCase			,						UpperCAmelCase	:							Any						=			self.model_tester.prepare_config_and_inputs_for_common()
     if not self.model_tester.is_training:
       return

     UpperCAmelCase	:							List[str]						=			False
     UpperCAmelCase	:							List[Any]						=			True

     for model_class in self.all_model_classes:
       if model_class in get_values(__A						) or not model_class.supports_gradient_checkpointing:
         continue
       # DeiTForImageClassificationWithTeacher supports inference-only
       if model_class.__name__ == "DeiTForImageClassificationWithTeacher":
         continue
       UpperCAmelCase	:							Any						=			model_class(__A						)
       model.gradient_checkpointing_enable()
       model.to(__A						)
       model.train()
       UpperCAmelCase	:							Dict						=			self._prepare_for_class(__A,		__A,		return_labels=__A						)
       UpperCAmelCase	:							Tuple						=			model(**__A						).loss
       loss.backward()



   def 	__magic_name__   (							self							:					Optional[int]						):
     UpperCAmelCase			,						UpperCAmelCase	:							Optional[Any]						=			self.model_tester.prepare_config_and_inputs_for_common()

     UpperCAmelCase	:							Tuple						=			[
         {'''title''': '''multi_label_classification''', '''num_labels''': 2, '''dtype''': torch.float},
         {'''title''': '''single_label_classification''', '''num_labels''': 1, '''dtype''': torch.long},
         {'''title''': '''regression''', '''num_labels''': 1, '''dtype''': torch.float},
     ]

     for model_class in self.all_model_classes:
       if (
           model_class
           not in [
               *get_values(__A						),
               *get_values(__A						),
           ]
           or model_class.__name__ == "DeiTForImageClassificationWithTeacher"
       ):
         continue

       for problem_type in problem_types:
         with self.subTest(msg=F'''Testing {model_class} with {problem_type["title"]}'''						):
           UpperCAmelCase	:							Optional[int]						=			problem_type['''title''']
           UpperCAmelCase	:							Union[str, Any]						=			problem_type['''num_labels''']

           UpperCAmelCase	:							Optional[Any]						=			model_class(__A						)
           model.to(__A						)
           model.train()

           UpperCAmelCase	:							List[Any]						=			self._prepare_for_class(__A,		__A,		return_labels=__A						)

           if problem_type["num_labels"] > 1:
             UpperCAmelCase	:							int						=			inputs['''labels'''].unsqueeze(1						).repeat(1,		problem_type['''num_labels''']						)

           UpperCAmelCase	:							Optional[int]						=			inputs['''labels'''].to(problem_type['''dtype''']						)

           # This tests that we do not trigger the warning form PyTorch "Using a target size that is different
           # to the input size. This will likely lead to incorrect results due to broadcasting. Please ensure
           # they have the same size." which is a symptom something in wrong for the regression problem.
           # See https://github.com/huggingface/transformers/issues/11780
           with warnings.catch_warnings(record=__A						) as warning_list:
             UpperCAmelCase	:							List[Any]						=			model(**__A						).loss
           for w in warning_list:
             if "Using a target size that is different to the input size" in str(w.message						):
               raise ValueError(
                   F'''Something is going wrong in the regression problem: intercepted {w.message}'''						)

           loss.backward()





   @slow
   def 	__magic_name__   (							self							:					int						):
     for model_name in DEIT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
       UpperCAmelCase	:							Tuple						=			DeiTModel.from_pretrained(__A						)
       self.assertIsNotNone(__A						)






def       a__					(		)  ->						List[str]:
  UpperCAmelCase	:							List[str]						=			Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''      )
  return image






@require_torch
@require_vision
class  __UpperCAmelCase					(	unittest.TestCase						):



   @cached_property
   def 	__magic_name__   (							self							:					Optional[Any]						):
     return (
         DeiTImageProcessor.from_pretrained('''facebook/deit-base-distilled-patch16-224'''						)
         if is_vision_available()
         else None
     )



   @slow
   def 	__magic_name__   (							self							:					Tuple						):
     UpperCAmelCase	:							List[Any]						=			DeiTForImageClassificationWithTeacher.from_pretrained('''facebook/deit-base-distilled-patch16-224'''						).to(
         __A						)

     UpperCAmelCase	:							int						=			self.default_image_processor
     UpperCAmelCase	:							List[Any]						=			prepare_img()
     UpperCAmelCase	:							Any						=			image_processor(images=__A,		return_tensors='''pt'''						).to(__A						)

     # forward pass
     with torch.no_grad():
       UpperCAmelCase	:							List[Any]						=			model(**__A						)

     # verify the logits
     UpperCAmelCase	:							Union[str, Any]						=			torch.Size((1, 1_0_0_0)						)
     self.assertEqual(outputs.logits.shape,		__A						)

     UpperCAmelCase	:							List[str]						=			torch.tensor([-1.0_2_6_6, 0.1_9_1_2, -1.2_8_6_1]						).to(__A						)

     self.assertTrue(torch.allclose(outputs.logits[0, :3],		__A,		atol=1E-4						)						)





   @slow
   @require_accelerate
   @require_torch_gpu
   def 	__magic_name__   (							self							:					str						):
     UpperCAmelCase	:							Dict						=			DeiTModel.from_pretrained(
         '''facebook/deit-base-distilled-patch16-224''',		torch_dtype=torch.floataa,		device_map='''auto'''						)
     UpperCAmelCase	:							Tuple						=			self.default_image_processor

     UpperCAmelCase	:							Optional[Any]						=			prepare_img()
     UpperCAmelCase	:							int						=			image_processor(images=__A,		return_tensors='''pt'''						)
     UpperCAmelCase	:							Dict						=			inputs.pixel_values.to(__A						)

     # forward pass to make sure inference works in fp16
     with torch.no_grad():
       UpperCAmelCase	:							List[Any]						=			model(__A						)








import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
     from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse("1.6"):
     _lowerCamelCase    :       List[str]        =							True
     from torch.cuda.amp import autocast


_lowerCamelCase    :       Any        =							logging.getLogger(__name__)





@dataclass
class  __UpperCAmelCase					:
   UpperCamelCase       					=					field(
       metadata={"""help""": """Path to pretrained model or model identifier from huggingface.co/models"""}						)
   UpperCamelCase       					=					field(
       default=lowerCamelCase__      ,  metadata={"""help""": """Where do you want to store the pretrained models downloaded from huggingface.co"""}      ,  )
   UpperCamelCase       					=					field(
       default=lowerCamelCase__      ,  metadata={"""help""": """Whether to freeze the feature extractor layers of the model."""}						)
   UpperCamelCase       					=					field(
       default=lowerCamelCase__      ,  metadata={"""help""": """Whether to log verbose messages or not."""}      ,  )
   UpperCamelCase       					=					field(
       default=2.0      ,  metadata={"""help""": """Maximum temperature for gumbel softmax."""}						)
   UpperCamelCase       					=					field(
       default=0.5      ,  metadata={"""help""": """Minimum temperature for gumbel softmax."""}						)
   UpperCamelCase       					=					field(
       default=0.9_9_9_9_9_5      ,  metadata={"""help""": """Decay of gumbel temperature during training."""}						)






def       a__					(		UpperCAmelCase							:      ModelArguments    ,			UpperCAmelCase							:      TrainingArguments      )  ->						Any:
  logging.basicConfig(
      format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s'''    ,			datefmt='''%m/%d/%Y %H:%M:%S'''    ,			handlers=[logging.StreamHandler(sys.stdout      )]    ,			)
  UpperCAmelCase	:							Any						=			logging.WARNING
  if model_args.verbose_logging:
    UpperCAmelCase	:							Any						=			logging.DEBUG
  elif trainer_utils.is_main_process(training_args.local_rank      ):
    UpperCAmelCase	:							Any						=			logging.INFO
  logger.setLevel(UpperCAmelCase      )





@dataclass
class  __UpperCAmelCase					:
   UpperCamelCase       					=					field(
       default=lowerCamelCase__      ,  metadata={"""help""": """The name of the dataset to use (via the datasets library)."""}						)
   UpperCamelCase       					=					field(
       default=lowerCamelCase__      ,  metadata={"""help""": """The configuration name of the dataset to use (via the datasets library)."""}						)
   UpperCamelCase       					=					field(
       default="""train"""      ,  metadata={
           """help""": """The name of the training data set split to use (via the datasets library). Defaults to 'train'"""
       }      ,  )
   UpperCamelCase       					=					field(
       default="""validation"""      ,  metadata={
           """help""": (
               """The name of the validation data set split to use (via the datasets library). Defaults to 'validation'"""
           )
       }      ,  )
   UpperCamelCase       					=					field(
       default="""file"""      ,  metadata={"""help""": """Column in the dataset that contains speech file path. Defaults to 'file'"""}      ,  )
   UpperCamelCase       					=					field(
       default=lowerCamelCase__      ,  metadata={"""help""": """Overwrite the cached preprocessed datasets or not."""}						)
   UpperCamelCase       					=					field(
       default=1      ,  metadata={
           """help""": """The percentage of the train set used as validation set in case there's no validation split"""
       }      ,  )
   UpperCamelCase       					=					field(
       default=lowerCamelCase__      ,  metadata={"""help""": """The number of processes to use for the preprocessing."""}      ,  )
   UpperCamelCase       					=					field(
       default=2_0.0      ,  metadata={"""help""": """Filter audio files that are longer than `max_duration_in_seconds` seconds"""}						)






@dataclass
class  __UpperCAmelCase					:
   UpperCamelCase       					=					42
   UpperCamelCase       					=					42
   UpperCamelCase       					=					"longest"
   UpperCamelCase       					=					None
   UpperCamelCase       					=					None



   def __call__(							self							:					int,		__A							:					List[Dict[str, Union[List[int], torch.Tensor]]]						):
     # reformat list to dict and set to pytorch format
     UpperCAmelCase	:							List[Any]						=			self.feature_extractor.pad(
         __A,		max_length=self.max_length,		padding=self.padding,		pad_to_multiple_of=self.pad_to_multiple_of,		return_tensors='''pt''',		)
     UpperCAmelCase	:							int						=			self.model._get_feat_extract_output_lengths(batch['''input_values'''].shape[-1]						)

     UpperCAmelCase	:							Tuple						=			batch['''input_values'''].shape[0]

     # make sure that no loss is computed on padded inputs
     if batch["attention_mask"] is not None:
       # compute real output lengths according to convolution formula
       UpperCAmelCase	:							Tuple						=			self.model._get_feat_extract_output_lengths(batch['''attention_mask'''].sum(-1						)						).to(
           torch.long						)

       UpperCAmelCase	:							Dict						=			torch.zeros(
           (batch_size, mask_indices_seq_length),		dtype=torch.long,		device=batch['''input_values'''].device						)

       # these two operations makes sure that all values
       # before the output lengths indices are attended to
       UpperCAmelCase	:							Tuple						=			1
       UpperCAmelCase	:							int						=			attention_mask.flip([-1]						).cumsum(-1						).flip([-1]						).bool()

     # sample randomly masked indices
     UpperCAmelCase	:							Dict						=			_compute_mask_indices(
         (batch_size, mask_indices_seq_length),		self.model.config.mask_time_prob,		self.model.config.mask_time_length,		attention_mask=__A,		min_masks=2,		)

     return batch






class  __UpperCAmelCase					(	lowerCamelCase__						):



   def __init__(							self							:					Union[str, Any],		*__A							:					int,		__A							:					Dict=1,		__A							:					Any=0,		__A							:					Optional[Any]=1.0,		**__A							:					Any						):
     super().__init__(*__A,		**__A						)
     UpperCAmelCase	:							Any						=			0
     UpperCAmelCase	:							Any						=			max_gumbel_temp
     UpperCAmelCase	:							Optional[Any]						=			min_gumbel_temp
     UpperCAmelCase	:							str						=			gumbel_temp_decay





   def 	__magic_name__   (							self							:					Dict,		__A							:					nn.Module,		__A							:					Dict[str, Union[torch.Tensor, Any]]						):
     model.train()
     UpperCAmelCase	:							List[Any]						=			self._prepare_inputs(__A						)

     if self.use_amp:
       with autocast():
         UpperCAmelCase	:							Optional[Any]						=			self.compute_loss(__A,		__A						)
     else:
       UpperCAmelCase	:							Optional[int]						=			self.compute_loss(__A,		__A						)

     if self.args.n_gpu > 1 or self.deepspeed:
       if model.module.config.ctc_loss_reduction == "mean":
         UpperCAmelCase	:							Optional[Any]						=			loss.mean()
       elif model.module.config.ctc_loss_reduction == "sum":
         UpperCAmelCase	:							str						=			loss.sum() / (inputs['''mask_time_indices''']).sum()
       else:
         raise ValueError(F'''{model.config.ctc_loss_reduction} is not valid. Choose one of [\'mean\', \'sum\']'''						)

     if self.args.gradient_accumulation_steps > 1:
       UpperCAmelCase	:							Any						=			loss / self.args.gradient_accumulation_steps

     if self.use_amp:
       self.scaler.scale(__A						).backward()
     elif self.use_apex:
       with amp.scale_loss(__A,		self.optimizer						) as scaled_loss:
         scaled_loss.backward()
     elif self.deepspeed:
       self.deepspeed.backward(__A						)
     else:
       loss.backward()

     self.num_update_step += 1
     # make sure gumbel softmax temperature is decayed
     if self.args.n_gpu > 1 or self.deepspeed:
       model.module.set_gumbel_temperature(
           max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step,		self.min_gumbel_temp						)						)
     else:
       model.set_gumbel_temperature(
           max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step,		self.min_gumbel_temp						)						)

     return loss.detach()






def       a__					(		)  ->						Union[str, Any]:
  # See all possible arguments in src/transformers/training_args.py
  # or by passing the --help flag to this script.
  # We now keep distinct sets of args, for a cleaner separation of concerns.

  UpperCAmelCase	:							Optional[Any]						=			HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)      )

  UpperCAmelCase			,						UpperCAmelCase			,						UpperCAmelCase	:							List[Any]						=			parser.parse_args_into_dataclasses()
  configure_logger(UpperCAmelCase    ,			UpperCAmelCase      )

  # Downloading and loading a dataset from the hub.
  UpperCAmelCase	:							int						=			load_dataset(data_args.dataset_name    ,			data_args.dataset_config_name    ,			cache_dir=model_args.cache_dir      )

  if "validation" not in datasets.keys():
    # make sure only "validation" and "train" keys remain"
    UpperCAmelCase	:							Union[str, Any]						=			DatasetDict()
    UpperCAmelCase	:							Any						=			load_dataset(
        data_args.dataset_name    ,			data_args.dataset_config_name    ,			split=f'''{data_args.train_split_name}[:{data_args.validation_split_percentage}%]'''    ,			cache_dir=model_args.cache_dir    ,			)
    UpperCAmelCase	:							Tuple						=			load_dataset(
        data_args.dataset_name    ,			data_args.dataset_config_name    ,			split=f'''{data_args.train_split_name}[{data_args.validation_split_percentage}%:]'''    ,			cache_dir=model_args.cache_dir    ,			)
  else:
    # make sure only "validation" and "train" keys remain"
    UpperCAmelCase	:							Optional[Any]						=			DatasetDict()
    UpperCAmelCase	:							List[Any]						=			load_dataset(
        data_args.dataset_name    ,			data_args.dataset_config_name    ,			split='''validation'''    ,			cache_dir=model_args.cache_dir    ,			)
    UpperCAmelCase	:							int						=			load_dataset(
        data_args.dataset_name    ,			data_args.dataset_config_name    ,			split=f'''{data_args.train_split_name}'''    ,			cache_dir=model_args.cache_dir    ,			)

  # only normalized-inputs-training is supported
  UpperCAmelCase	:							Optional[Any]						=			WavaVecaFeatureExtractor.from_pretrained(
      model_args.model_name_or_path    ,			cache_dir=model_args.cache_dir    ,			do_normalize=UpperCAmelCase      )

  def prepare_dataset(UpperCAmelCase							:      Dict      ):
    # check that all files have the correct sampling rate
    UpperCAmelCase			,						UpperCAmelCase	:							Optional[Any]						=			librosa.load(batch[data_args.speech_file_column]    ,			sr=feature_extractor.sampling_rate      )
    return batch

  # load audio files into numpy arrays
  UpperCAmelCase	:							str						=			datasets.map(
      UpperCAmelCase    ,			num_proc=data_args.preprocessing_num_workers    ,			remove_columns=datasets['''train'''].column_names      )

  # filter audio files that are too long
  UpperCAmelCase	:							int						=			vectorized_datasets.filter(
      lambda UpperCAmelCase      : len(data['''speech''']      ) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate      )      )

  def normalize(UpperCAmelCase							:      Dict      ):
    return feature_extractor(batch['''speech''']    ,			sampling_rate=feature_extractor.sampling_rate      )

  # normalize and transform to `BatchFeatures`
  UpperCAmelCase	:							Any						=			vectorized_datasets.map(
      UpperCAmelCase    ,			batched=UpperCAmelCase    ,			num_proc=data_args.preprocessing_num_workers    ,			load_from_cache_file=not data_args.overwrite_cache    ,			remove_columns=vectorized_datasets['''train'''].column_names    ,			)

  # pretraining is only supported for "newer" stable layer norm architecture
  # apply_spec_augment has to be True, mask_feature_prob has to be 0.0
  UpperCAmelCase	:							Optional[int]						=			WavaVecaConfig.from_pretrained(
      model_args.model_name_or_path    ,			cache_dir=model_args.cache_dir    ,			gradient_checkpointing=training_args.gradient_checkpointing    ,			)

  if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
    raise ValueError(
        '''PreTraining is only supported for ``config.do_stable_layer_norm=True`` and'''
        ''' ``config.feat_extract_norm=\'layer\''''      )

  UpperCAmelCase	:							Any						=			WavaVecaForPreTraining(UpperCAmelCase      )

  UpperCAmelCase	:							int						=			DataCollatorForWavaVecaPretraining(model=UpperCAmelCase    ,			feature_extractor=UpperCAmelCase      )

  UpperCAmelCase	:							Any						=			WavaVecaPreTrainer(
      model=UpperCAmelCase    ,			data_collator=UpperCAmelCase    ,			args=UpperCAmelCase    ,			train_dataset=vectorized_datasets['''train''']    ,			eval_dataset=vectorized_datasets['''validation''']    ,			tokenizer=UpperCAmelCase    ,			max_gumbel_temp=model_args.max_gumbel_temperature    ,			min_gumbel_temp=model_args.min_gumbel_temperature    ,			gumbel_temp_decay=model_args.gumbel_temperature_decay    ,			)
  trainer.train()


if __name__ == "__main__":
     main()





