



import argparse
import gc
import json
import os

import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.utils.deepspeed import DummyOptim, DummyScheduler


SCREAMING_SNAKE_CASE_:Any					=						16
SCREAMING_SNAKE_CASE_:int					=						32




def      __UpperCamelCase     (  _lowerCAmelCase		)	->  int:



    """simple docstring"""






    return int(x / 2**20		)






class SCREAMING_SNAKE_CASE__					:




    '''simple docstring'''






    def __enter__(	self		):
        gc.collect()
        torch.cuda.empty_cache()
        torch.cuda.reset_max_memory_allocated()  # reset the peak gauge to zero
        A			:	List[str]										=       torch.cuda.memory_allocated()
        return self






    def __exit__(	self,		*lowerCamelCase__		):
        gc.collect()
        torch.cuda.empty_cache()
        A			:	Any										=       torch.cuda.memory_allocated()
        A			:	List[Any]										=       torch.cuda.max_memory_allocated()
        A			:	Optional[int]										=       bamb(self.end - self.begin		)
        A			:	Union[str, Any]										=       bamb(self.peak - self.begin		)
        # print(f"delta used/peak {self.used:4d}/{self.peaked:4d}")




def      __UpperCamelCase     (  _lowerCAmelCase				,					_lowerCAmelCase = 16				,					_lowerCAmelCase = "bert-base-cased"				,					_lowerCAmelCase = 320				,					_lowerCAmelCase = 160				,					)	->  List[str]:



    """simple docstring"""






    A			:	int										=       AutoTokenizer.from_pretrained(_lowerCAmelCase		)
    A			:	Tuple										=       load_dataset(
        """glue"""				,					"""mrpc"""				,					split={"""train""": f'''train[:{n_train}]''', """validation""": f'''validation[:{n_val}]'''}		)

    def tokenize_function(_lowerCAmelCase		):
        # max_length=None => use the model max length (it's actually the default)
        A			:	str										=       tokenizer(examples["""sentence1"""]				,					examples["""sentence2"""]				,					truncation=_lowerCAmelCase				,					max_length=_lowerCAmelCase		)
        return outputs

    # Apply the method we just defined to all the examples in all the splits of the dataset
    A			:	int										=       datasets.map(
        _lowerCAmelCase				,					batched=_lowerCAmelCase				,					remove_columns=["""idx""", """sentence1""", """sentence2"""]				,					load_from_cache_file=_lowerCAmelCase		)

    # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
    # transformers library
    A			:	Tuple										=       tokenized_datasets.rename_column("""label"""				,					"""labels"""		)

    def collate_fn(_lowerCAmelCase		):
        # On TPU it's best to pad everything to the same length or training will be very slow.
        if accelerator.distributed_type == DistributedType.TPU:
            return tokenizer.pad(_lowerCAmelCase				,					padding="""max_length"""				,					max_length=128				,					return_tensors="""pt"""		)
        return tokenizer.pad(_lowerCAmelCase				,					padding="""longest"""				,					return_tensors="""pt"""		)

    # Instantiate dataloaders.
    A			:	Tuple										=       DataLoader(
        tokenized_datasets["""train"""]				,					shuffle=_lowerCAmelCase				,					collate_fn=_lowerCAmelCase				,					batch_size=_lowerCAmelCase		)
    A			:	Optional[int]										=       DataLoader(
        tokenized_datasets["""validation"""]				,					shuffle=_lowerCAmelCase				,					collate_fn=_lowerCAmelCase				,					batch_size=_lowerCAmelCase		)

    return train_dataloader, eval_dataloader




def      __UpperCamelCase     (  _lowerCAmelCase				,					_lowerCAmelCase		)	->  int:



    """simple docstring"""






    A			:	Dict										=       Accelerator()

    # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
    A			:	Tuple										=       config["""lr"""]
    A			:	Any										=       int(config["""num_epochs"""]		)
    A			:	List[str]										=       int(config["""seed"""]		)
    A			:	Union[str, Any]										=       int(config["""batch_size"""]		)
    A			:	int										=       args.model_name_or_path

    set_seed(_lowerCAmelCase		)
    A			:	Any										=       get_dataloaders(_lowerCAmelCase				,					_lowerCAmelCase				,					_lowerCAmelCase				,					args.n_train				,					args.n_val		)

    # Instantiate the model (we build the model here so that the seed also control new weights initialization)
    A			:	Optional[Any]										=       AutoModelForSequenceClassification.from_pretrained(_lowerCAmelCase				,					return_dict=_lowerCAmelCase		)

    # Instantiate optimizer
    A			:	Any										=       (
        AdamW
        if accelerator.state.deepspeed_plugin is None
        or """optimizer""" not in accelerator.state.deepspeed_plugin.deepspeed_config
        else DummyOptim
    )
    A			:	Any										=       optimizer_cls(params=model.parameters()				,					lr=_lowerCAmelCase		)

    if accelerator.state.deepspeed_plugin is not None:
        A			:	Dict										=       accelerator.state.deepspeed_plugin.deepspeed_config[
            """gradient_accumulation_steps"""
        ]
    else:
        A			:	str										=       1
    A			:	Any										=       (len(_lowerCAmelCase		) * num_epochs) // gradient_accumulation_steps

    # Instantiate scheduler
    if (
        accelerator.state.deepspeed_plugin is None
        or "scheduler" not in accelerator.state.deepspeed_plugin.deepspeed_config
    ):
        A			:	Union[str, Any]										=       get_linear_schedule_with_warmup(
            optimizer=_lowerCAmelCase				,					num_warmup_steps=0				,					num_training_steps=_lowerCAmelCase				,					)
    else:
        A			:	str										=       DummyScheduler(_lowerCAmelCase				,					total_num_steps=_lowerCAmelCase				,					warmup_num_steps=0		)

    # Prepare everything
    # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
    # prepare method.
    A			:	int										=       accelerator.prepare(
        _lowerCAmelCase				,					_lowerCAmelCase				,					_lowerCAmelCase				,					_lowerCAmelCase				,					_lowerCAmelCase		)

    # We need to keep track of how many total steps we have iterated over
    A			:	Optional[int]										=       0
    # We also need to keep track of the stating epoch so files are named properly
    A			:	List[Any]										=       0

    # Now we train the model
    A			:	Optional[Any]										=       {}
    for epoch in range(_lowerCAmelCase				,					_lowerCAmelCase		):
        with TorchTracemalloc() as tracemalloc:
            model.train()
            for step, batch in enumerate(_lowerCAmelCase		):
                A			:	int										=       model(**_lowerCAmelCase		)
                A			:	str										=       outputs.loss
                A			:	Tuple										=       loss / gradient_accumulation_steps
                accelerator.backward(_lowerCAmelCase		)
                if step % gradient_accumulation_steps == 0:
                    optimizer.step()
                    lr_scheduler.step()
                    optimizer.zero_grad()

                overall_step += 1

        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
        accelerator.print("""Memory before entering the train : {}""".format(bamb(tracemalloc.begin		)		)		)
        accelerator.print("""Memory consumed at the end of the train (end-begin): {}""".format(tracemalloc.used		)		)
        accelerator.print("""Peak Memory consumed during the train (max-begin): {}""".format(tracemalloc.peaked		)		)
        accelerator.print(
            """Total Peak Memory consumed during the train (max): {}""".format(
                tracemalloc.peaked + bamb(tracemalloc.begin		)		)		)
        A			:	str										=       tracemalloc.peaked + bamb(tracemalloc.begin		)
        if args.peak_memory_upper_bound is not None:
            assert (
                train_total_peak_memory[f'''epoch-{epoch}'''] <= args.peak_memory_upper_bound
            ), "Peak memory usage exceeded the upper bound"

    accelerator.wait_for_everyone()
    if accelerator.is_main_process:
        with open(os.path.join(args.output_dir				,					"""peak_memory_utilization.json"""		)				,					"""w"""		) as f:
            json.dump(_lowerCAmelCase				,					_lowerCAmelCase		)




def      __UpperCamelCase     (  )	->  Optional[int]:



    """simple docstring"""






    A			:	Dict										=       argparse.ArgumentParser(description="""Simple example of training script tracking peak GPU memory usage."""		)
    parser.add_argument(
        """--model_name_or_path"""				,					type=_lowerCAmelCase				,					default="""bert-base-cased"""				,					help="""Path to pretrained model or model identifier from huggingface.co/models."""				,					required=_lowerCAmelCase				,					)
    parser.add_argument(
        """--output_dir"""				,					type=_lowerCAmelCase				,					default="""."""				,					help="""Optional save directory where all checkpoint folders will be stored. Default is the current working directory."""				,					)
    parser.add_argument(
        """--peak_memory_upper_bound"""				,					type=_lowerCAmelCase				,					default=_lowerCAmelCase				,					help="""The upper bound of peak memory usage in MB. If set, the training will throw an error if the peak memory usage exceeds this value."""				,					)
    parser.add_argument(
        """--n_train"""				,					type=_lowerCAmelCase				,					default=320				,					help="""Number of training examples to use."""				,					)
    parser.add_argument(
        """--n_val"""				,					type=_lowerCAmelCase				,					default=160				,					help="""Number of validation examples to use."""				,					)
    parser.add_argument(
        """--num_epochs"""				,					type=_lowerCAmelCase				,					default=1				,					help="""Number of train epochs."""				,					)
    A			:	List[Any]										=       parser.parse_args()
    A			:	Tuple										=       {"""lr""": 2e-5, """num_epochs""": args.num_epochs, """seed""": 42, """batch_size""": 16}
    training_function(_lowerCAmelCase				,					_lowerCAmelCase		)


if __name__ == "__main__":
     main()







import copy
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE_:Optional[int]					=						logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_:Dict					=						{
    """BAAI/AltCLIP""": """https://huggingface.co/BAAI/AltCLIP/resolve/main/config.json""",
    # See all AltCLIP models at https://huggingface.co/models?filter=altclip
}






class SCREAMING_SNAKE_CASE__					(		SCREAMING_SNAKE_CASE__   ):




    '''simple docstring'''






    __lowerCamelCase      :			int =				"altclip_text_model"






    def __init__(	self,		lowerCamelCase__=25_0002,		lowerCamelCase__=1024,		lowerCamelCase__=24,		lowerCamelCase__=16,		lowerCamelCase__=4096,		lowerCamelCase__="gelu",		lowerCamelCase__=0.1,		lowerCamelCase__=0.1,		lowerCamelCase__=514,		lowerCamelCase__=1,		lowerCamelCase__=0.02,		lowerCamelCase__=0.02,		lowerCamelCase__=1e-05,		lowerCamelCase__=1,		lowerCamelCase__=0,		lowerCamelCase__=2,		lowerCamelCase__="absolute",		lowerCamelCase__=True,		lowerCamelCase__=768,		**lowerCamelCase__,		):
        super().__init__(pad_token_id=lowerCamelCase__,		bos_token_id=lowerCamelCase__,		eos_token_id=lowerCamelCase__,		**lowerCamelCase__		)

        A			:	Union[str, Any]										=       vocab_size
        A			:	Dict										=       hidden_size
        A			:	Union[str, Any]										=       num_hidden_layers
        A			:	List[str]										=       num_attention_heads
        A			:	str										=       hidden_act
        A			:	Dict										=       intermediate_size
        A			:	List[str]										=       hidden_dropout_prob
        A			:	Optional[Any]										=       attention_probs_dropout_prob
        A			:	Tuple										=       max_position_embeddings
        A			:	Optional[Any]										=       type_vocab_size
        A			:	Optional[Any]										=       initializer_range
        A			:	Optional[int]										=       initializer_factor
        A			:	Tuple										=       layer_norm_eps
        A			:	List[str]										=       position_embedding_type
        A			:	int										=       use_cache
        A			:	int										=       project_dim






class SCREAMING_SNAKE_CASE__					(		SCREAMING_SNAKE_CASE__   ):




    '''simple docstring'''






    __lowerCamelCase      :			Optional[Any] =				"altclip_vision_model"






    def __init__(	self,		lowerCamelCase__=768,		lowerCamelCase__=3072,		lowerCamelCase__=512,		lowerCamelCase__=12,		lowerCamelCase__=12,		lowerCamelCase__=3,		lowerCamelCase__=224,		lowerCamelCase__=32,		lowerCamelCase__="quick_gelu",		lowerCamelCase__=1e-5,		lowerCamelCase__=0.0,		lowerCamelCase__=0.02,		lowerCamelCase__=1.0,		**lowerCamelCase__,		):
        super().__init__(**lowerCamelCase__		)

        A			:	Optional[Any]										=       hidden_size
        A			:	Optional[int]										=       intermediate_size
        A			:	Union[str, Any]										=       projection_dim
        A			:	str										=       num_hidden_layers
        A			:	int										=       num_attention_heads
        A			:	Optional[Any]										=       num_channels
        A			:	Tuple										=       patch_size
        A			:	List[Any]										=       image_size
        A			:	Optional[int]										=       initializer_range
        A			:	Union[str, Any]										=       initializer_factor
        A			:	List[str]										=       attention_dropout
        A			:	int										=       layer_norm_eps
        A			:	str										=       hidden_act






    @classmethod
    def       _lowerCAmelCase (	cls,		lowerCamelCase__,		**lowerCamelCase__		):
        cls._set_token_in_kwargs(lowerCamelCase__		)

        A					,							A			:	Optional[Any]										=       cls.get_config_dict(lowerCamelCase__,		**lowerCamelCase__		)

        # get the vision config dict if we are loading from AltCLIPConfig
        if config_dict.get("""model_type"""		) == "altclip":
            A			:	Any										=       config_dict["""vision_config"""]

        if "model_type" in config_dict and hasattr(cls,		"""model_type"""		) and config_dict["model_type"] != cls.model_type:
            logger.warning(
                f'''You are using a model of type {config_dict["model_type"]} to instantiate a model of type '''
                f'''{cls.model_type}. This is not supported for all configurations of models and can yield errors.'''		)

        return cls.from_dict(lowerCamelCase__,		**lowerCamelCase__		)






class SCREAMING_SNAKE_CASE__					(		SCREAMING_SNAKE_CASE__   ):




    '''simple docstring'''






    __lowerCamelCase      :			List[Any] =				"altclip"
    __lowerCamelCase      :			List[Any] =				True






    def __init__(	self,		lowerCamelCase__=None,		lowerCamelCase__=None,		lowerCamelCase__=768,		lowerCamelCase__=2.6592,		**lowerCamelCase__		):
        # If `_config_dict` exist, we use them for the backward compatibility.
        # We pop out these 2 attributes before calling `super().__init__` to avoid them being saved (which causes a lot
        # of confusion!).
        A			:	Dict										=       kwargs.pop("""text_config_dict""",		lowerCamelCase__		)
        A			:	str										=       kwargs.pop("""vision_config_dict""",		lowerCamelCase__		)

        super().__init__(**lowerCamelCase__		)

        # Instead of simply assigning `[text|vision]_config_dict` to `[text|vision]_config`, we use the values in
        # `[text|vision]_config_dict` to update the values in `[text|vision]_config`. The values should be same in most
        # cases, but we don't want to break anything regarding `_config_dict` that existed before commit `8827e1b2`.
        if text_config_dict is not None:
            if text_config is None:
                A			:	Dict										=       {}

            # This is the complete result when using `text_config_dict`.
            A			:	str										=       AltCLIPTextConfig(**lowerCamelCase__		).to_dict()

            # Give a warning if the values exist in both `_text_config_dict` and `text_config` but being different.
            for key, value in _text_config_dict.items():
                if key in text_config and value != text_config[key] and key not in ["transformers_version"]:
                    # If specified in `text_config_dict`
                    if key in text_config_dict:
                        A			:	Optional[Any]										=       (
                            f'''`{key}` is found in both `text_config_dict` and `text_config` but with different values. '''
                            f'''The value `text_config_dict["{key}"]` will be used instead.'''
                        )
                    # If inferred from default argument values (just to be super careful)
                    else:
                        A			:	Optional[int]										=       (
                            f'''`text_config_dict` is provided which will be used to initialize `AltCLIPTextConfig`. The '''
                            f'''value `text_config["{key}"]` will be overriden.'''
                        )
                    logger.warning(lowerCamelCase__		)

            # Update all values in `text_config` with the ones in `_text_config_dict`.
            text_config.update(_text_config_dict		)

        if vision_config_dict is not None:
            if vision_config is None:
                A			:	int										=       {}

            # This is the complete result when using `vision_config_dict`.
            A			:	Union[str, Any]										=       AltCLIPVisionConfig(**lowerCamelCase__		).to_dict()
            # convert keys to string instead of integer
            if "id2label" in _vision_config_dict:
                A			:	Optional[int]										=       {
                    str(lowerCamelCase__		): value for key, value in _vision_config_dict["""id2label"""].items()
                }

            # Give a warning if the values exist in both `_vision_config_dict` and `vision_config` but being different.
            for key, value in _vision_config_dict.items():
                if key in vision_config and value != vision_config[key] and key not in ["transformers_version"]:
                    # If specified in `vision_config_dict`
                    if key in vision_config_dict:
                        A			:	Optional[int]										=       (
                            f'''`{key}` is found in both `vision_config_dict` and `vision_config` but with different '''
                            f'''values. The value `vision_config_dict["{key}"]` will be used instead.'''
                        )
                    # If inferred from default argument values (just to be super careful)
                    else:
                        A			:	Any										=       (
                            f'''`vision_config_dict` is provided which will be used to initialize `AltCLIPVisionConfig`. '''
                            f'''The value `vision_config["{key}"]` will be overriden.'''
                        )
                    logger.warning(lowerCamelCase__		)

            # Update all values in `vision_config` with the ones in `_vision_config_dict`.
            vision_config.update(_vision_config_dict		)

        if text_config is None:
            A			:	Tuple										=       {}
            logger.info("""`text_config` is `None`. Initializing the `AltCLIPTextConfig` with default values."""		)

        if vision_config is None:
            A			:	Union[str, Any]										=       {}
            logger.info("""`vision_config` is `None`. initializing the `AltCLIPVisionConfig` with default values."""		)

        A			:	Dict										=       AltCLIPTextConfig(**lowerCamelCase__		)
        A			:	Optional[int]										=       AltCLIPVisionConfig(**lowerCamelCase__		)

        A			:	List[str]										=       projection_dim
        A			:	Any										=       logit_scale_init_value
        A			:	Tuple										=       1.0






    @classmethod
    def       _lowerCAmelCase (	cls,		lowerCamelCase__,		lowerCamelCase__,		**lowerCamelCase__		):
        return cls(text_config=text_config.to_dict(),		vision_config=vision_config.to_dict(),		**lowerCamelCase__		)






    def       _lowerCAmelCase (	self		):
        A			:	str										=       copy.deepcopy(self.__dict__		)
        A			:	Any										=       self.text_config.to_dict()
        A			:	List[str]										=       self.vision_config.to_dict()
        A			:	Union[str, Any]										=       self.__class__.model_type
        return output


