





'''simple docstring'''


import argparse
import json
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.utils.deepspeed import DummyOptim, DummyScheduler


lowerCAmelCase :					Any 							=16
lowerCAmelCase :					Tuple 							=32
def 							UpperCAmelCase_  (						__lowerCamelCase : Accelerator     ,__lowerCamelCase : int = 16     ,__lowerCamelCase : str = "bert-base-cased"			):
   lowercase_      :Tuple         =			AutoTokenizer.from_pretrained(A__			)
   lowercase_      :Tuple         =			load_dataset("glue"     ,"mrpc"			)

   def tokenize_function(__lowerCamelCase : Dict			):
      # max_length=None => use the model max length (it's actually the default)
      lowercase_      :Tuple         =			tokenizer(examples["sentence1"]     ,examples["sentence2"]     ,truncation=A__     ,max_length=A__			)
      return outputs

   # Apply the method we just defined to all the examples in all the splits of the dataset
   lowercase_      :Any         =			datasets.map(
       A__     ,batched=A__     ,remove_columns=["idx", "sentence1", "sentence2"]     ,load_from_cache_file=A__			)

   # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
   # transformers library
   lowercase_      :Tuple         =			tokenized_datasets.rename_column("label"     ,"labels"			)

   def collate_fn(__lowerCamelCase : List[str]			):
      # On TPU it's best to pad everything to the same length or training will be very slow.
      if accelerator.distributed_type == DistributedType.TPU:
         return tokenizer.pad(A__     ,padding="max_length"     ,max_length=1_28     ,return_tensors="pt"			)
      return tokenizer.pad(A__     ,padding="longest"     ,return_tensors="pt"			)

   # Instantiate dataloaders.
   lowercase_      :Tuple         =			DataLoader(
       tokenized_datasets["train"]     ,shuffle=A__     ,collate_fn=A__     ,batch_size=A__			)
   lowercase_      :int         =			DataLoader(
       tokenized_datasets["validation"]     ,shuffle=A__     ,collate_fn=A__     ,batch_size=A__			)

   return train_dataloader, eval_dataloader
def 							UpperCAmelCase_  (						__lowerCamelCase : List[str]     ,__lowerCamelCase : Tuple     ,__lowerCamelCase : int     ,__lowerCamelCase : Union[str, Any]			):
   model.eval()
   lowercase_      :int         =			0
   for step, batch in enumerate(A__			):
      # We could avoid this line since we set the accelerator with `device_placement=True`.
      batch.to(accelerator.device			)
      with torch.no_grad():
         lowercase_      :Any         =			model(**A__			)
      lowercase_      :Tuple         =			outputs.logits.argmax(dim=-1			)
      # It is slightly faster to call this once, than multiple times
      lowercase_      :Dict         =			accelerator.gather(
          (predictions, batch["labels"])			)  # If we are in a multiprocess environment, the last batch has duplicates
      if accelerator.use_distributed:
         if step == len(A__			) - 1:
            lowercase_      :List[str]         =			predictions[: len(eval_dataloader.dataset			) - samples_seen]
            lowercase_      :Union[str, Any]         =			references[: len(eval_dataloader.dataset			) - samples_seen]
         else:
            samples_seen += references.shape[0]
      metric.add_batch(
          predictions=A__     ,references=A__     ,)

   lowercase_      :Optional[int]         =			metric.compute()
   return eval_metric["accuracy"]
def 							UpperCAmelCase_  (						__lowerCamelCase : List[str]     ,__lowerCamelCase : Tuple			):
   lowercase_      :List[str]         =			Accelerator()

   # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
   lowercase_      :Union[str, Any]         =			config["""lr"""]
   lowercase_      :List[Any]         =			int(config["num_epochs"]			)
   lowercase_      :Tuple         =			int(config["seed"]			)
   lowercase_      :Union[str, Any]         =			int(config["batch_size"]			)
   lowercase_      :List[str]         =			args.model_name_or_path

   set_seed(A__			)
   lowercase_      :List[Any]         =			get_dataloaders(A__     ,A__     ,A__			)

   # Instantiate the model (we build the model here so that the seed also control new weights initialization)
   lowercase_      :int         =			AutoModelForSequenceClassification.from_pretrained(A__     ,return_dict=A__			)

   # Instantiate optimizer
   lowercase_      :str         =			(
       AdamW
       if accelerator.state.deepspeed_plugin is None
       or """optimizer""" not in accelerator.state.deepspeed_plugin.deepspeed_config
       else DummyOptim
   )
   lowercase_      :Union[str, Any]         =			optimizer_cls(params=model.parameters()     ,lr=A__			)

   if accelerator.state.deepspeed_plugin is not None:
      lowercase_      :Optional[Any]         =			accelerator.state.deepspeed_plugin.deepspeed_config[
          """gradient_accumulation_steps"""
      ]
   else:
      lowercase_      :Union[str, Any]         =			1
   lowercase_      :List[Any]         =			(len(A__			) * num_epochs) // gradient_accumulation_steps

   # Instantiate scheduler
   if (
       accelerator.state.deepspeed_plugin is None
       or "scheduler" not in accelerator.state.deepspeed_plugin.deepspeed_config
   ):
      lowercase_      :List[str]         =			get_linear_schedule_with_warmup(
          optimizer=A__     ,num_warmup_steps=0     ,num_training_steps=A__     ,)
   else:
      lowercase_      :List[Any]         =			DummyScheduler(A__     ,total_num_steps=A__     ,warmup_num_steps=0			)

   # Prepare everything
   # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
   # prepare method.
   lowercase_      :Optional[int]         =			accelerator.prepare(
       A__     ,A__     ,A__     ,A__     ,A__			)

   # We need to keep track of how many total steps we have iterated over
   lowercase_      :str         =			0
   # We also need to keep track of the stating epoch so files are named properly
   lowercase_      :List[str]         =			0
   lowercase_      :Any         =			evaluate.load("glue"     ,"mrpc"			)
   lowercase_      :List[Any]         =			num_epochs

   if args.partial_train_epoch is not None:
      lowercase_      :Dict         =			args.partial_train_epoch

   if args.resume_from_checkpoint:
      accelerator.load_state(args.resume_from_checkpoint			)
      lowercase_      :List[str]         =			args.resume_from_checkpoint.split("epoch_"			)[1]
      lowercase_      :List[str]         =			""""""
      for char in epoch_string:
         if char.isdigit():
            state_epoch_num += char
         else:
            break
      lowercase_      :Optional[int]         =			int(A__			) + 1
      lowercase_      :List[Any]         =			evaluation_loop(A__     ,A__     ,A__     ,A__			)
      accelerator.print("resumed checkpoint performance:"     ,A__			)
      accelerator.print("resumed checkpoint's scheduler's lr:"     ,lr_scheduler.get_lr()[0]			)
      accelerator.print("resumed optimizers's lr:"     ,optimizer.param_groups[0]["lr"]			)
      with open(os.path.join(args.output_dir     ,F'state_{starting_epoch-1}.json'			)     ,"r"			) as f:
         lowercase_      :Any         =			json.load(A__			)
         assert resumed_state["accuracy"] == accuracy, "Accuracy mismatch, loading from checkpoint failed"
         assert (
             resumed_state["lr"] == lr_scheduler.get_lr()[0]
         ), "Scheduler learning rate mismatch, loading from checkpoint failed"
         assert (
             resumed_state["optimizer_lr"] == optimizer.param_groups[0]["lr"]
         ), "Optimizer learning rate mismatch, loading from checkpoint failed"
         assert resumed_state["epoch"] == starting_epoch - 1, "Epoch mismatch, loading from checkpoint failed"
         return

      # Now we train the model
   lowercase_      :int         =			{}
   for epoch in range(A__     ,A__			):
      model.train()
      for step, batch in enumerate(A__			):
         lowercase_      :Optional[int]         =			model(**A__			)
         lowercase_      :Dict         =			outputs.loss
         lowercase_      :Union[str, Any]         =			loss / gradient_accumulation_steps
         accelerator.backward(A__			)
         if step % gradient_accumulation_steps == 0:
            optimizer.step()
            lr_scheduler.step()
            optimizer.zero_grad()

         overall_step += 1
      lowercase_      :str         =			F'epoch_{epoch}'
      lowercase_      :Union[str, Any]         =			os.path.join(args.output_dir     ,A__			)
      accelerator.save_state(A__			)
      lowercase_      :Any         =			evaluation_loop(A__     ,A__     ,A__     ,A__			)
      lowercase_      :Any         =			accuracy
      lowercase_      :Optional[Any]         =			lr_scheduler.get_lr()[0]
      lowercase_      :Tuple         =			optimizer.param_groups[0]["""lr"""]
      lowercase_      :Any         =			epoch
      lowercase_      :Union[str, Any]         =			overall_step
      accelerator.print(F'epoch {epoch}:'     ,A__			)

      accelerator.wait_for_everyone()
      if accelerator.is_main_process:
         with open(os.path.join(args.output_dir     ,F'state_{epoch}.json'			)     ,"w"			) as f:
            json.dump(A__     ,A__			)
def 							UpperCAmelCase_  (						):
   lowercase_      :Optional[Any]         =			argparse.ArgumentParser(description="Simple example of training script tracking peak GPU memory usage."			)
   parser.add_argument(
       "--model_name_or_path"     ,type=A__     ,default="bert-base-cased"     ,help="Path to pretrained model or model identifier from huggingface.co/models."     ,required=A__     ,)
   parser.add_argument(
       "--output_dir"     ,type=A__     ,default="."     ,help="Optional save directory where all checkpoint folders will be stored. Default is the current working directory."     ,)
   parser.add_argument(
       "--resume_from_checkpoint"     ,type=A__     ,default=A__     ,help="If the training should continue from a checkpoint folder."     ,)
   parser.add_argument(
       "--partial_train_epoch"     ,type=A__     ,default=A__     ,help="If passed, the training will stop after this number of epochs."     ,)
   parser.add_argument(
       "--num_epochs"     ,type=A__     ,default=2     ,help="Number of train epochs."     ,)
   lowercase_      :List[Any]         =			parser.parse_args()
   lowercase_      :Optional[int]         =			{"""lr""": 2e-5, """num_epochs""": args.num_epochs, """seed""": 42, """batch_size""": 16}

   training_function(A__     ,A__			)


if __name__ == "__main__":
 main()









'''simple docstring'''


from __future__ import annotations

import inspect
import unittest

import numpy as np

from transformers import DeiTConfig
from transformers.testing_utils import require_tf, require_vision, slow
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
 import tensorflow as tf

 from transformers import (
     TFDeiTForImageClassification,
     TFDeiTForImageClassificationWithTeacher,
     TFDeiTForMaskedImageModeling,
     TFDeiTModel,
 )
 from transformers.models.deit.modeling_tf_deit import TF_DEIT_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
 from PIL import Image

 from transformers import DeiTImageProcessor
class 				a_       :





     def __init__(			self		: int     ,				lowercase		: Optional[int]     ,				lowercase		: List[Any]=13     ,				lowercase		: Dict=30     ,				lowercase		: Dict=2     ,				lowercase		: Tuple=3     ,				lowercase		: Dict=True     ,				lowercase		: Dict=True     ,				lowercase		: Tuple=32     ,				lowercase		: List[Any]=2     ,				lowercase		: List[str]=4     ,				lowercase		: List[Any]=37     ,				lowercase		: Union[str, Any]="gelu"     ,				lowercase		: List[str]=0.1     ,				lowercase		: List[Any]=0.1     ,				lowercase		: List[str]=10     ,				lowercase		: Any=0.02     ,				lowercase		: Union[str, Any]=3     ,				lowercase		: Tuple=None     ,				lowercase		: List[str]=2     ,				):





        """simple docstring"""




        lowercase_      :Union[str, Any]         =			parent
        lowercase_      :Optional[int]         =			batch_size
        lowercase_      :Tuple         =			image_size
        lowercase_      :Any         =			patch_size
        lowercase_      :List[Any]         =			num_channels
        lowercase_      :Optional[Any]         =			is_training
        lowercase_      :str         =			use_labels
        lowercase_      :Any         =			hidden_size
        lowercase_      :Optional[int]         =			num_hidden_layers
        lowercase_      :List[Any]         =			num_attention_heads
        lowercase_      :str         =			intermediate_size
        lowercase_      :Optional[int]         =			hidden_act
        lowercase_      :List[Any]         =			hidden_dropout_prob
        lowercase_      :Optional[Any]         =			attention_probs_dropout_prob
        lowercase_      :Any         =			type_sequence_label_size
        lowercase_      :Dict         =			initializer_range
        lowercase_      :List[Any]         =			scope
        lowercase_      :List[str]         =			encoder_stride

        # in DeiT, the seq length equals the number of patches + 2 (we add 2 for the [CLS] and distilation tokens)
        lowercase_      :str         =			(image_size // patch_size) ** 2
        lowercase_      :Any         =			num_patches + 2





     def 							lowercase__				(			self		: List[Any]						):





        """simple docstring"""




        lowercase_      :List[str]         =			floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]						)

        lowercase_      :Optional[Any]         =			None
        if self.use_labels:
           lowercase_      :Optional[Any]         =			ids_tensor([self.batch_size]     ,				self.type_sequence_label_size						)

        lowercase_      :Any         =			self.get_config()

        return config, pixel_values, labels





     def 							lowercase__				(			self		: List[Any]						):





        """simple docstring"""




        return DeiTConfig(
            image_size=self.image_size     ,				patch_size=self.patch_size     ,				num_channels=self.num_channels     ,				hidden_size=self.hidden_size     ,				num_hidden_layers=self.num_hidden_layers     ,				num_attention_heads=self.num_attention_heads     ,				intermediate_size=self.intermediate_size     ,				hidden_act=self.hidden_act     ,				hidden_dropout_prob=self.hidden_dropout_prob     ,				attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,				is_decoder=lowercase     ,				initializer_range=self.initializer_range     ,				encoder_stride=self.encoder_stride     ,				)





     def 							lowercase__				(			self		: Tuple     ,				lowercase		: int     ,				lowercase		: str     ,				lowercase		: Tuple						):





        """simple docstring"""




        lowercase_      :Optional[int]         =			TFDeiTModel(config=lowercase						)
        lowercase_      :str         =			model(lowercase						)
        self.parent.assertEqual(result.last_hidden_state.shape     ,				(self.batch_size, self.seq_length, self.hidden_size)						)





     def 							lowercase__				(			self		: str     ,				lowercase		: str     ,				lowercase		: Optional[Any]     ,				lowercase		: str						):





        """simple docstring"""




        lowercase_      :List[str]         =			TFDeiTForMaskedImageModeling(config=lowercase						)
        lowercase_      :Optional[int]         =			model(lowercase						)
        self.parent.assertEqual(
            result.reconstruction.shape     ,				(self.batch_size, self.num_channels, self.image_size, self.image_size)						)

        # test greyscale images
        lowercase_      :List[str]         =			1
        lowercase_      :List[str]         =			TFDeiTForMaskedImageModeling(lowercase						)

        lowercase_      :Union[str, Any]         =			floats_tensor([self.batch_size, 1, self.image_size, self.image_size]						)
        lowercase_      :List[str]         =			model(lowercase						)
        self.parent.assertEqual(result.reconstruction.shape     ,				(self.batch_size, 1, self.image_size, self.image_size)						)





     def 							lowercase__				(			self		: List[str]     ,				lowercase		: Any     ,				lowercase		: Union[str, Any]     ,				lowercase		: Union[str, Any]						):





        """simple docstring"""




        lowercase_      :Tuple         =			self.type_sequence_label_size
        lowercase_      :str         =			TFDeiTForImageClassification(lowercase						)
        lowercase_      :str         =			model(lowercase     ,				labels=lowercase						)
        self.parent.assertEqual(result.logits.shape     ,				(self.batch_size, self.type_sequence_label_size)						)

        # test greyscale images
        lowercase_      :Optional[int]         =			1
        lowercase_      :Optional[int]         =			TFDeiTForImageClassification(lowercase						)

        lowercase_      :List[Any]         =			floats_tensor([self.batch_size, 1, self.image_size, self.image_size]						)
        lowercase_      :int         =			model(lowercase     ,				labels=lowercase						)
        self.parent.assertEqual(result.logits.shape     ,				(self.batch_size, self.type_sequence_label_size)						)







     def 							lowercase__				(			self		: Dict						):





        """simple docstring"""




        lowercase_      :Tuple         =			self.prepare_config_and_inputs()
        lowercase_      ,							lowercase_      ,							lowercase_      :Tuple         =			config_and_inputs
        lowercase_      :Tuple         =			{"pixel_values": pixel_values}
        return config, inputs_dict







@require_tf
class 				a_       (    _lowerCAmelCase     ,						_lowerCAmelCase     ,						unittest.TestCase							):
     __A								=      (
         (
             TFDeiTModel,
             TFDeiTForImageClassification,
             TFDeiTForImageClassificationWithTeacher,
             TFDeiTForMaskedImageModeling,
         )
         if is_tf_available()
         else ()
     )
     __A								=      (
         {
             "feature-extraction": TFDeiTModel,
             "image-classification": (TFDeiTForImageClassification, TFDeiTForImageClassificationWithTeacher),
         }
         if is_tf_available()
         else {}
     )

     __A								=      False
     __A								=      False
     __A								=      False
     __A								=      False





     def 							lowercase__				(			self		: Optional[int]						):





        """simple docstring"""




        lowercase_      :List[Any]         =			TFDeiTModelTester(self						)
        lowercase_      :List[Any]         =			ConfigTester(self     ,				config_class=lowercase     ,				has_text_modality=lowercase     ,				hidden_size=37						)





     def 							lowercase__				(			self		: Tuple						):





        """simple docstring"""




        self.config_tester.run_common_tests()





     @unittest.skip(reason="DeiT does not use inputs_embeds"						)
     def 							lowercase__				(			self		: Optional[int]						):





        """simple docstring"""




        pass





     def 							lowercase__				(			self		: Tuple						):





        """simple docstring"""




        lowercase_      ,							lowercase_      :List[str]         =			self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
           lowercase_      :Optional[Any]         =			model_class(lowercase						)
           self.assertIsInstance(model.get_input_embeddings()     ,				(tf.keras.layers.Layer)						)
           lowercase_      :Tuple         =			model.get_output_embeddings()
           self.assertTrue(x is None or isinstance(lowercase     ,				tf.keras.layers.Dense						)						)





     def 							lowercase__				(			self		: Any						):





        """simple docstring"""




        lowercase_      ,							lowercase_      :List[str]         =			self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
           lowercase_      :Tuple         =			model_class(lowercase						)
           lowercase_      :Optional[int]         =			inspect.signature(model.call						)
           # signature.parameters is an OrderedDict => so arg_names order is deterministic
           lowercase_      :Tuple         =			[*signature.parameters.keys()]

           lowercase_      :Any         =			["pixel_values"]
           self.assertListEqual(arg_names[:1]     ,				lowercase						)





     def 							lowercase__				(			self		: Optional[Any]						):





        """simple docstring"""




        lowercase_      :Optional[Any]         =			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*lowercase						)





     def 							lowercase__				(			self		: str						):





        """simple docstring"""




        lowercase_      :Optional[int]         =			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_masked_image_modeling(*lowercase						)





     def 							lowercase__				(			self		: List[str]						):





        """simple docstring"""




        lowercase_      :Optional[int]         =			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_image_classification(*lowercase						)





     def 							lowercase__				(			self		: List[Any]     ,				lowercase		: Optional[Any]     ,				lowercase		: List[str]     ,				lowercase		: Any=False						):





        """simple docstring"""




        lowercase_      :Dict         =			super()._prepare_for_class(lowercase     ,				lowercase     ,				return_labels=lowercase						)

        if return_labels:
           if "labels" in inputs_dict and "labels" not in inspect.signature(model_class.call						).parameters:
              del inputs_dict["labels"]

        return inputs_dict







     @slow
     def 							lowercase__				(			self		: List[Any]						):





        """simple docstring"""




        for model_name in TF_DEIT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
           lowercase_      :List[Any]         =			TFDeiTModel.from_pretrained(lowercase						)
           self.assertIsNotNone(lowercase						)







def 							UpperCAmelCase_  (						):
   lowercase_      :List[str]         =			Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"			)
   return image


@require_tf
@require_vision
class 				a_       (    unittest.TestCase							):





     @cached_property
     def 							lowercase__				(			self		: Dict						):





        """simple docstring"""




        return (
            DeiTImageProcessor.from_pretrained("facebook/deit-base-distilled-patch16-224"						)
            if is_vision_available()
            else None
        )







     @slow
     def 							lowercase__				(			self		: Optional[Any]						):





        """simple docstring"""




        lowercase_      :Optional[int]         =			TFDeiTForImageClassificationWithTeacher.from_pretrained("facebook/deit-base-distilled-patch16-224"						)

        lowercase_      :Optional[int]         =			self.default_image_processor
        lowercase_      :Union[str, Any]         =			prepare_img()
        lowercase_      :Dict         =			image_processor(images=lowercase     ,				return_tensors="tf"						)

        # forward pass
        lowercase_      :Union[str, Any]         =			model(**lowercase						)

        # verify the logits
        lowercase_      :Union[str, Any]         =			tf.TensorShape((1, 1_000)						)
        self.assertEqual(outputs.logits.shape     ,				lowercase						)

        lowercase_      :Dict         =			tf.constant([-1.02_66, 0.19_12, -1.28_61]						)

        self.assertTrue(np.allclose(outputs.logits[0, :3]     ,				lowercase     ,				atol=1e-4						)						)


