


import argparse
import logging
import os
from pathlib import Path
from typing import Any, Dict

import pytorch_lightning as pl
from pytorch_lightning.utilities import rank_zero_info

from transformers import (
    AdamW,
    AutoConfig,
    AutoModel,
    AutoModelForPreTraining,
    AutoModelForQuestionAnswering,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoModelForTokenClassification,
    AutoModelWithLMHead,
    AutoTokenizer,
    PretrainedConfig,
    PreTrainedTokenizer,
)
from transformers.optimization import (
    Adafactor,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.utils.versions import require_version


_A			=						logging.getLogger(__name__)

require_version('pytorch_lightning>=1.0.4')

_A			=						{
    'base': AutoModel,
    'sequence-classification': AutoModelForSequenceClassification,
    'question-answering': AutoModelForQuestionAnswering,
    'pretraining': AutoModelForPreTraining,
    'token-classification': AutoModelForTokenClassification,
    'language-modeling': AutoModelWithLMHead,
    'summarization': AutoModelForSeqaSeqLM,
    'translation': AutoModelForSeqaSeqLM,
}


# update this and the import above to support new schedulers from transformers.optimization
_A			=						{
    'linear': get_linear_schedule_with_warmup,
    'cosine': get_cosine_schedule_with_warmup,
    'cosine_w_restarts': get_cosine_with_hard_restarts_schedule_with_warmup,
    'polynomial': get_polynomial_decay_schedule_with_warmup,
    # '': get_constant_schedule,             # not supported for now
    # '': get_constant_schedule_with_warmup, # not supported for now
}
_A			=						sorted(arg_to_scheduler.keys())
_A			=						'{' + ', '.join(arg_to_scheduler_choices) + '}'






class    UpperCAmelCase__	( pl.LightningModule		):



    """simple docstring"""






    def __init__(      self ,       A_ ,       A_=None ,       A_="base" ,       A_=None ,       A_=None ,       A_=None ,       **A_ ,       )  ->	str:
         super().__init__()
         # TODO: move to self.save_hyperparameters()
         # self.save_hyperparameters()
         # can also expand arguments into trainer signature for easier reading

         self.save_hyperparameters(A_   )
         __UpperCamelCase   							=0
         __UpperCamelCase   							=Path(self.hparams.output_dir   )
         __UpperCamelCase   							=self.hparams.cache_dir if self.hparams.cache_dir else None
         if config is None:
              __UpperCamelCase   							=AutoConfig.from_pretrained(
                  self.hparams.config_name if self.hparams.config_name else self.hparams.model_name_or_path ,       **({'num_labels': num_labels} if num_labels is not None else {}) ,       cache_dir=A_ ,       **A_ ,       )
         else:
              __UpperCamelCase   							=config

         __UpperCamelCase   							=('encoder_layerdrop', 'decoder_layerdrop', 'dropout', 'attention_dropout')
         for p in extra_model_params:
              if getattr(self.hparams ,       A_ ,       A_   ):
                   assert hasattr(self.config ,       A_   ), f'model config doesn\'t have a `{p}` attribute'
                   setattr(self.config ,       A_ ,       getattr(self.hparams ,       A_   )   )

         if tokenizer is None:
              __UpperCamelCase   							=AutoTokenizer.from_pretrained(
                  self.hparams.tokenizer_name if self.hparams.tokenizer_name else self.hparams.model_name_or_path ,       cache_dir=A_ ,       )
         else:
              __UpperCamelCase   							=tokenizer
         __UpperCamelCase   							=MODEL_MODES[mode]
         if model is None:
              __UpperCamelCase   							=self.model_type.from_pretrained(
                  self.hparams.model_name_or_path ,       from_tf=bool('.ckpt' in self.hparams.model_name_or_path   ) ,       config=self.config ,       cache_dir=A_ ,       )
         else:
              __UpperCamelCase   							=model






    def _a (      self ,       *A_ ,       **A_   )  ->	Tuple:
         __UpperCamelCase   							=self.model_type.from_pretrained(*A_ ,       **A_   )






    def _a (      self   )  ->	List[Any]:
         __UpperCamelCase   							=arg_to_scheduler[self.hparams.lr_scheduler]
         __UpperCamelCase   							=get_schedule_func(
             self.opt ,       num_warmup_steps=self.hparams.warmup_steps ,       num_training_steps=self.total_steps()   )
         __UpperCamelCase   							={'scheduler': scheduler, 'interval': 'step', 'frequency': 1}
         return scheduler






    def _a (      self   )  ->	int:
         __UpperCamelCase   							=self.model
         __UpperCamelCase   							=['bias', 'LayerNorm.weight']
         __UpperCamelCase   							=[
             {
                 'params': [
                     p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay   )
                 ],  # check this named paramters
                 'weight_decay': self.hparams.weight_decay,
             },
             {
                 'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay   )],
                 'weight_decay': 0.0,
             },
         ]
         if self.hparams.adafactor:
              __UpperCamelCase   							=Adafactor(
                  A_ ,       lr=self.hparams.learning_rate ,       scale_parameter=A_ ,       relative_step=A_   )

         else:
              __UpperCamelCase   							=AdamW(
                  A_ ,       lr=self.hparams.learning_rate ,       eps=self.hparams.adam_epsilon   )
         __UpperCamelCase   							=optimizer

         __UpperCamelCase   							=self.get_lr_scheduler()

         return [optimizer], [scheduler]






    def _a (      self ,       A_ ,       A_   )  ->	Optional[int]:
         return self.validation_step(A_ ,       A_   )






    def _a (      self ,       A_   )  ->	Union[str, Any]:
         return self.validation_end(A_   )






    def _a (      self   )  ->	int:
         __UpperCamelCase   							=max(1 ,       self.hparams.gpus   )  # TODO: consider num_tpu_cores
         __UpperCamelCase   							=self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
         return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs






    def _a (      self ,       A_   )  ->	List[str]:
         if stage == "test":
              __UpperCamelCase   							=len(self.test_dataloader().dataset   )
         else:
              __UpperCamelCase   							=self.get_dataloader('train' ,       self.hparams.train_batch_size ,       shuffle=A_   )
              __UpperCamelCase   							=len(self.train_dataloader().dataset   )






    def _a (      self ,       A_ ,       A_ ,       A_ = False   )  ->	str:
         raise NotImplementedError('You must implement this for your task'   )






    def _a (      self   )  ->	str:
         return self.train_loader






    def _a (      self   )  ->	Optional[Any]:
         return self.get_dataloader('dev' ,       self.hparams.eval_batch_size ,       shuffle=A_   )






    def _a (      self   )  ->	Tuple:
         return self.get_dataloader('test' ,       self.hparams.eval_batch_size ,       shuffle=A_   )






    def _a (      self ,       A_   )  ->	int:
         return os.path.join(
             self.hparams.data_dir ,       'cached_{}_{}_{}'.format(
                 A_ ,       list(filter(A_ ,       self.hparams.model_name_or_path.split('/'   )   )   ).pop() ,       str(self.hparams.max_seq_length   ) ,       ) ,       )






    @pl.utilities.rank_zero_only
    def _a (      self ,       A_   )  ->	None:
         __UpperCamelCase   							=self.output_dir.joinpath('best_tfmr'   )
         __UpperCamelCase   							=self.step_count
         self.model.save_pretrained(A_   )
         self.tokenizer.save_pretrained(A_   )






    @staticmethod
    def _a (      A_ ,       A_   )  ->	Tuple:
         parser.add_argument(
             '--model_name_or_path' ,       default=A_ ,       type=A_ ,       required=A_ ,       help='Path to pretrained model or model identifier from huggingface.co/models' ,       )
         parser.add_argument(
             '--config_name' ,       default='' ,       type=A_ ,       help='Pretrained config name or path if not the same as model_name'   )
         parser.add_argument(
             '--tokenizer_name' ,       default=A_ ,       type=A_ ,       help='Pretrained tokenizer name or path if not the same as model_name' ,       )
         parser.add_argument(
             '--cache_dir' ,       default=str(Path(A_   ).parent / 'test_run' / 'cache'   ) ,       type=A_ ,       help='Where do you want to store the pre-trained models downloaded from huggingface.co' ,       )
         parser.add_argument(
             '--encoder_layerdrop' ,       type=A_ ,       help='Encoder layer dropout probability (Optional). Goes into model.config' ,       )
         parser.add_argument(
             '--decoder_layerdrop' ,       type=A_ ,       help='Decoder layer dropout probability (Optional). Goes into model.config' ,       )
         parser.add_argument(
             '--dropout' ,       type=A_ ,       help='Dropout probability (Optional). Goes into model.config' ,       )
         parser.add_argument(
             '--attention_dropout' ,       type=A_ ,       help='Attention dropout probability (Optional). Goes into model.config' ,       )
         parser.add_argument('--learning_rate' ,       default=5E-5 ,       type=A_ ,       help='The initial learning rate for Adam.'   )
         parser.add_argument(
             '--lr_scheduler' ,       default='linear' ,       choices=A_ ,       metavar=A_ ,       type=A_ ,       help='Learning rate scheduler' ,       )
         parser.add_argument('--weight_decay' ,       default=0.0 ,       type=A_ ,       help='Weight decay if we apply some.'   )
         parser.add_argument('--adam_epsilon' ,       default=1E-8 ,       type=A_ ,       help='Epsilon for Adam optimizer.'   )
         parser.add_argument('--warmup_steps' ,       default=0 ,       type=A_ ,       help='Linear warmup over warmup_steps.'   )
         parser.add_argument('--num_workers' ,       default=4 ,       type=A_ ,       help='kwarg passed to DataLoader'   )
         parser.add_argument('--num_train_epochs' ,       dest='max_epochs' ,       default=3 ,       type=A_   )
         parser.add_argument('--train_batch_size' ,       default=32 ,       type=A_   )
         parser.add_argument('--eval_batch_size' ,       default=32 ,       type=A_   )
         parser.add_argument('--adafactor' ,       action='store_true'   )






class    UpperCAmelCase__	( pl.Callback		):



    """simple docstring"""






    def _a (      self ,       A_ ,       A_   )  ->	int:
         if (
             trainer.is_global_zero and trainer.global_rank == 0
         ):  # we initialize the retriever only on master worker with RAY. In new pytorch-lightning accelorators are removed.
              pl_module.model.rag.retriever.init_retrieval()  # better to use hook functions.






class    UpperCAmelCase__	( pl.Callback		):



    """simple docstring"""






    def _a (      self ,       A_ ,       A_   )  ->	Any:
         # print(pl_module.model.rag)
         for name, param in pl_module.model.rag.named_parameters():
              if param.grad is None:
                   print(A_   )






class    UpperCAmelCase__	( pl.Callback		):



    """simple docstring"""






    def _a (      self ,       A_ ,       A_   )  ->	Union[str, Any]:
         __UpperCamelCase   							=trainer.lr_schedulers[0]['scheduler']
         __UpperCamelCase   							={f'lr_group_{i}': lr for i, lr in enumerate(lr_scheduler.get_lr()   )}
         pl_module.logger.log_metrics(A_   )






    def _a (      self ,       A_ ,       A_   )  ->	List[Any]:
         rank_zero_info('***** Validation results *****'   )
         __UpperCamelCase   							=trainer.callback_metrics
         # Log results
         for key in sorted(A_   ):
              if key not in ["log", "progress_bar"]:
                   rank_zero_info('{} = {}\n'.format(A_ ,       str(metrics[key]   )   )   )






    def _a (      self ,       A_ ,       A_   )  ->	str:
         rank_zero_info('***** Test results *****'   )
         __UpperCamelCase   							=trainer.callback_metrics
         # Log and save results to file
         __UpperCamelCase   							=os.path.join(pl_module.hparams.output_dir ,       'test_results.txt'   )
         with open(A_ ,       'w'   ) as writer:
              for key in sorted(A_   ):
                   if key not in ["log", "progress_bar"]:
                        rank_zero_info('{} = {}\n'.format(A_ ,       str(metrics[key]   )   )   )
                        writer.write('{} = {}\n'.format(A_ ,       str(metrics[key]   )   )   )

def 						_UpperCAmelCase    (      SCREAMING_SNAKE_CASE__      : str      ,					SCREAMING_SNAKE_CASE__      : Optional[int]    ):
     #  To allow all pl args uncomment the following line
     #  parser = pl.Trainer.add_argparse_args(parser)
     parser.add_argument(
         '--output_dir'      ,					default=str(Path(SCREAMING_SNAKE_CASE__    ).parent / 'test_run' / 'model_checkpoints'    )      ,					type=SCREAMING_SNAKE_CASE__      ,					help='The output directory where the model predictions and checkpoints will be written.'      ,					)
     parser.add_argument(
         '--fp16'      ,					action='store_true'      ,					help='Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit'      ,					)

     parser.add_argument(
         '--fp16_opt_level'      ,					type=SCREAMING_SNAKE_CASE__      ,					default='O2'      ,					help=(
             'For fp16: Apex AMP optimization level selected in [\'O0\', \'O1\', \'O2\', and \'O3\'].'
             'See details at https://nvidia.github.io/apex/amp.html'
         )      ,					)
     parser.add_argument('--n_tpu_cores'      ,					dest='tpu_cores'      ,					type=SCREAMING_SNAKE_CASE__    )
     parser.add_argument('--max_grad_norm'      ,					dest='gradient_clip_val'      ,					default=1.0      ,					type=SCREAMING_SNAKE_CASE__      ,					help='Max gradient norm'    )
     parser.add_argument('--do_train'      ,					action='store_true'      ,					help='Whether to run training.'    )
     parser.add_argument('--do_predict'      ,					action='store_true'      ,					help='Whether to run predictions on the test set.'    )
     parser.add_argument(
         '--gradient_accumulation_steps'      ,					dest='accumulate_grad_batches'      ,					type=SCREAMING_SNAKE_CASE__      ,					default=1      ,					help='Number of updates steps to accumulate before performing a backward/update pass.'      ,					)
     parser.add_argument('--seed'      ,					type=SCREAMING_SNAKE_CASE__      ,					default=42      ,					help='random seed for initialization'    )
     parser.add_argument(
         '--data_dir'      ,					default=str(Path(SCREAMING_SNAKE_CASE__    ).parent / 'test_run' / 'dummy-train-data'    )      ,					type=SCREAMING_SNAKE_CASE__      ,					help='The input data dir. Should contain the training files for the CoNLL-2003 NER task.'      ,					)





def 						_UpperCAmelCase    (      SCREAMING_SNAKE_CASE__      : BaseTransformer      ,					SCREAMING_SNAKE_CASE__      : argparse.Namespace      ,					SCREAMING_SNAKE_CASE__      : int=None      ,					SCREAMING_SNAKE_CASE__      : Dict=True      ,					SCREAMING_SNAKE_CASE__      : Tuple=[]      ,					SCREAMING_SNAKE_CASE__      : Optional[int]=None      ,					SCREAMING_SNAKE_CASE__      : Any=None      ,					**SCREAMING_SNAKE_CASE__      : Any      ,					):
     pl.seed_everything(args.seed    )

     # init model
     __UpperCamelCase   							=Path(model.hparams.output_dir    )
     odir.mkdir(exist_ok=SCREAMING_SNAKE_CASE__    )

     # add custom checkpoints
     if checkpoint_callback is None:
          __UpperCamelCase   							=pl.callbacks.ModelCheckpoint(
              filepath=args.output_dir      ,					prefix='checkpoint'      ,					monitor='val_loss'      ,					mode='min'      ,					save_top_k=1    )
     if early_stopping_callback:
          extra_callbacks.append(SCREAMING_SNAKE_CASE__    )
     if logging_callback is None:
          __UpperCamelCase   							=LoggingCallback()

     __UpperCamelCase   							={}

     if args.fpaa:
          __UpperCamelCase   							=16

     if args.gpus > 1:
          __UpperCamelCase   							='auto'
          __UpperCamelCase   							='ddp'

     __UpperCamelCase   							=args.accumulate_grad_batches
     __UpperCamelCase   							=None
     __UpperCamelCase   							='auto'

     __UpperCamelCase   							=pl.Trainer.from_argparse_args(
         SCREAMING_SNAKE_CASE__      ,					weights_summary=SCREAMING_SNAKE_CASE__      ,					callbacks=[logging_callback] + extra_callbacks + [InitCallback()] + [checkpoint_callback]      ,					logger=SCREAMING_SNAKE_CASE__      ,					val_check_interval=1      ,					num_sanity_val_steps=2      ,					**SCREAMING_SNAKE_CASE__      ,					)

     if args.do_train:
          trainer.fit(SCREAMING_SNAKE_CASE__    )

     else:
          print('RAG modeling tests with new set functions successfuly executed!'    )
     return trainer









from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    convert_to_rgb,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    OPENAI_CLIP_MEAN,
    OPENAI_CLIP_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


_A			=						logging.get_logger(__name__)


if is_vision_available():
   import PIL






class    UpperCAmelCase__	( A_		):



    """simple docstring"""






    UpperCAmelCase__						:		Optional[int]              =  ["pixel_values"]






    def __init__(      self ,       A_ = True ,       A_ = None ,       A_ = PILImageResampling.BICUBIC ,       A_ = True ,       A_ = None ,       A_ = True ,       A_ = 1 / 255 ,       A_ = True ,       A_ = None ,       A_ = None ,       A_ = True ,       **A_ ,       )  ->	None:
         super().__init__(**A_   )
         __UpperCamelCase   							=size if size is not None else {'shortest_edge': 224}
         __UpperCamelCase   							=get_size_dict(A_ ,       default_to_square=A_   )
         __UpperCamelCase   							=crop_size if crop_size is not None else {'height': 224, 'width': 224}
         __UpperCamelCase   							=get_size_dict(A_ ,       default_to_square=A_ ,       param_name='crop_size'   )

         __UpperCamelCase   							=do_resize
         __UpperCamelCase   							=size
         __UpperCamelCase   							=resample
         __UpperCamelCase   							=do_center_crop
         __UpperCamelCase   							=crop_size
         __UpperCamelCase   							=do_rescale
         __UpperCamelCase   							=rescale_factor
         __UpperCamelCase   							=do_normalize
         __UpperCamelCase   							=image_mean if image_mean is not None else OPENAI_CLIP_MEAN
         __UpperCamelCase   							=image_std if image_std is not None else OPENAI_CLIP_STD
         __UpperCamelCase   							=do_convert_rgb






    def _a (      self ,       A_ ,       A_ ,       A_ = PILImageResampling.BICUBIC ,       A_ = None ,       **A_ ,       )  ->	np.ndarray:
         __UpperCamelCase   							=get_size_dict(A_ ,       default_to_square=A_   )
         if "shortest_edge" not in size:
              raise ValueError(f'The `size` parameter must contain the key `shortest_edge`. Got {size.keys()}'   )
         __UpperCamelCase   							=get_resize_output_image_size(A_ ,       size=size['shortest_edge'] ,       default_to_square=A_   )
         return resize(A_ ,       size=A_ ,       resample=A_ ,       data_format=A_ ,       **A_   )






    def _a (      self ,       A_ ,       A_ ,       A_ = None ,       **A_ ,       )  ->	np.ndarray:
         __UpperCamelCase   							=get_size_dict(A_   )
         if "height" not in size or "width" not in size:
              raise ValueError(f'The `size` parameter must contain the keys (height, width). Got {size.keys()}'   )
         return center_crop(A_ ,       size=(size['height'], size['width']) ,       data_format=A_ ,       **A_   )






    def _a (      self ,       A_ ,       A_ ,       A_ = None ,       **A_ ,       )  ->	Union[str, Any]:
         return rescale(A_ ,       scale=A_ ,       data_format=A_ ,       **A_   )






    def _a (      self ,       A_ ,       A_ ,       A_ ,       A_ = None ,       **A_ ,       )  ->	np.ndarray:
         return normalize(A_ ,       mean=A_ ,       std=A_ ,       data_format=A_ ,       **A_   )






    def _a (      self ,       A_ ,       A_ = None ,       A_ = None ,       A_ = None ,       A_ = None ,       A_ = None ,       A_ = None ,       A_ = None ,       A_ = None ,       A_ = None ,       A_ = None ,       A_ = None ,       A_ = None ,       A_ = ChannelDimension.FIRST ,       **A_ ,       )  ->	PIL.Image.Image:
         __UpperCamelCase   							=do_resize if do_resize is not None else self.do_resize
         __UpperCamelCase   							=size if size is not None else self.size
         __UpperCamelCase   							=get_size_dict(A_ ,       param_name='size' ,       default_to_square=A_   )
         __UpperCamelCase   							=resample if resample is not None else self.resample
         __UpperCamelCase   							=do_center_crop if do_center_crop is not None else self.do_center_crop
         __UpperCamelCase   							=crop_size if crop_size is not None else self.crop_size
         __UpperCamelCase   							=get_size_dict(A_ ,       param_name='crop_size' ,       default_to_square=A_   )
         __UpperCamelCase   							=do_rescale if do_rescale is not None else self.do_rescale
         __UpperCamelCase   							=rescale_factor if rescale_factor is not None else self.rescale_factor
         __UpperCamelCase   							=do_normalize if do_normalize is not None else self.do_normalize
         __UpperCamelCase   							=image_mean if image_mean is not None else self.image_mean
         __UpperCamelCase   							=image_std if image_std is not None else self.image_std
         __UpperCamelCase   							=do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb

         __UpperCamelCase   							=make_list_of_images(A_   )

         if not valid_images(A_   ):
              raise ValueError(
                  'Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '
                  'torch.Tensor, tf.Tensor or jax.ndarray.'   )

         if do_resize and size is None:
              raise ValueError('Size must be specified if do_resize is True.'   )

         if do_center_crop and crop_size is None:
              raise ValueError('Crop size must be specified if do_center_crop is True.'   )

         if do_rescale and rescale_factor is None:
              raise ValueError('Rescale factor must be specified if do_rescale is True.'   )

         if do_normalize and (image_mean is None or image_std is None):
              raise ValueError('Image mean and std must be specified if do_normalize is True.'   )

         # PIL RGBA images are converted to RGB
         if do_convert_rgb:
              __UpperCamelCase   							=[convert_to_rgb(A_   ) for image in images]

         # All transformations expect numpy arrays.
         __UpperCamelCase   							=[to_numpy_array(A_   ) for image in images]

         if do_resize:
              __UpperCamelCase   							=[self.resize(image=A_ ,       size=A_ ,       resample=A_   ) for image in images]

         if do_center_crop:
              __UpperCamelCase   							=[self.center_crop(image=A_ ,       size=A_   ) for image in images]

         if do_rescale:
              __UpperCamelCase   							=[self.rescale(image=A_ ,       scale=A_   ) for image in images]

         if do_normalize:
              __UpperCamelCase   							=[self.normalize(image=A_ ,       mean=A_ ,       std=A_   ) for image in images]

         __UpperCamelCase   							=[to_channel_dimension_format(A_ ,       A_   ) for image in images]

         __UpperCamelCase   							={'pixel_values': images}
         return BatchFeature(data=A_ ,       tensor_type=A_   )





