


"""simple docstring"""



import random
import timeit
from functools import wraps
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_tf_auto import TF_MODEL_MAPPING, TF_MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_tf_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_tf_available():
  import tensorflow as tf
  from tensorflow.python.framework.errors_impl import ResourceExhaustedError

  from .benchmark_args_tf import TensorFlowBenchmarkArguments

if is_pyanvml_available():
  import pyanvml.pyanvml as nvml

__snake_case             =   logging.get_logger(__name__)







def   __lowerCAmelCase						(						lowercase  :  bool						,							lowercase  :  bool			)     ->						int:


    """simple docstring"""





    def run_func(lowercase  :  Union[str, Any]			):
        @wraps(lowercase			)
        def run_in_eager_mode(*lowercase  :  Tuple						,							**lowercase  :  List[Any]			):
            return func(*lowercase						,							**lowercase			)

        @wraps(lowercase			)
        @tf.function(experimental_compile=lowercase			)
        def run_in_graph_mode(*lowercase  :  List[str]						,							**lowercase  :  Optional[Any]			):
            return func(*lowercase						,							**lowercase			)

        if do_eager_mode is True:
            if use_xla is not False:
                raise ValueError(
                    "Cannot run model in XLA, if `args.eager_mode` is set to `True`. Please set `args.eager_mode=False`."			)
            return run_in_eager_mode
        else:
            return run_in_graph_mode

    return run_func







def   __lowerCAmelCase						(						lowercase  :  int						,							lowercase  :  int						,							lowercase  :  int			)     ->						["tf.Tensor"]:


    """simple docstring"""





    snake_case   :				int				 =   random.Random()
    snake_case   :				Dict				 =   [rng.randint(0						,							vocab_size - 1			) for i in range(batch_size * sequence_length			)]
    return tf.constant(lowercase						,							shape=(batch_size, sequence_length)						,							dtype=tf.intaa			)







class _lowerCAmelCase    (				snake_case_     ):
    __UpperCAmelCase			:	Optional[int]							  =  42
    __UpperCAmelCase			:	Optional[Any]							  =  42
    __UpperCAmelCase			:	Optional[Any]							  =  '''TensorFlow'''







    @property
    def 			lowerCamelCase		(     self						)     ->		Dict:

        '''simple docstring'''




        return tf.__version__







    def 			lowerCamelCase		(     self						,     UpperCamelCase__						,     UpperCamelCase__						,     UpperCamelCase__						)     ->		float:

        '''simple docstring'''




        snake_case   :				Optional[int]				 =   self.args.strategy
        if strategy is None:
            raise ValueError("A device strategy has to be initialized before using TensorFlow."						)
        snake_case   :				List[Any]				 =   self._prepare_inference_func(__UpperCAmelCase						,     __UpperCAmelCase						,     __UpperCAmelCase						)
        return self._measure_speed(_inference						)







    def 			lowerCamelCase		(     self						,     UpperCamelCase__						,     UpperCamelCase__						,     UpperCamelCase__						)     ->		float:

        '''simple docstring'''




        snake_case   :				Dict				 =   self.args.strategy
        if strategy is None:
            raise ValueError("A device strategy has to be initialized before using TensorFlow."						)
        snake_case   :				Optional[Any]				 =   self._prepare_train_func(__UpperCAmelCase						,     __UpperCAmelCase						,     __UpperCAmelCase						)
        return self._measure_speed(_train						)







    def 			lowerCamelCase		(     self						,     UpperCamelCase__						,     UpperCamelCase__						,     UpperCamelCase__						)     ->		[Memory, Optional[MemorySummary]]:

        '''simple docstring'''




        if self.args.is_gpu:
            tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]						,     __UpperCAmelCase						)
        snake_case   :				Any				 =   self.args.strategy
        if strategy is None:
            raise ValueError("A device strategy has to be initialized before using TensorFlow."						)
        snake_case   :				int				 =   self._prepare_inference_func(__UpperCAmelCase						,     __UpperCAmelCase						,     __UpperCAmelCase						)
        return self._measure_memory(_inference						)







    def 			lowerCamelCase		(     self						,     UpperCamelCase__						,     UpperCamelCase__						,     UpperCamelCase__						)     ->		[Memory, Optional[MemorySummary]]:

        '''simple docstring'''




        if self.args.is_gpu:
            tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]						,     __UpperCAmelCase						)
        snake_case   :				str				 =   self.args.strategy
        if strategy is None:
            raise ValueError("A device strategy has to be initialized before using TensorFlow."						)

        snake_case   :				Optional[int]				 =   self._prepare_train_func(__UpperCAmelCase						,     __UpperCAmelCase						,     __UpperCAmelCase						)
        return self._measure_memory(_train						)







    def 			lowerCamelCase		(     self						,     UpperCamelCase__						,     UpperCamelCase__						,     UpperCamelCase__						)     ->		Callable[[], None]:

        '''simple docstring'''




        snake_case   :				Union[str, Any]				 =   self.config_dict[model_name]

        if self.args.fpaa:
            raise NotImplementedError("Mixed precision is currently not supported."						)

        snake_case   :				List[Any]				 =   (
            hasattr(__UpperCAmelCase						,     "architectures"						)
            and isinstance(config.architectures						,     __UpperCAmelCase						)
            and len(config.architectures						) > 0
        )
        if not self.args.only_pretrain_model and has_model_class_in_config:
            try:
                snake_case   :				str				 =   "TF" + config.architectures[0]  # prepend 'TF' for tensorflow model
                snake_case   :				Optional[int]				 =   __import__("transformers"						,     fromlist=[model_class]						)
                snake_case   :				List[str]				 =   getattr(__UpperCAmelCase						,     __UpperCAmelCase						)
                snake_case   :				Tuple				 =   model_cls(__UpperCAmelCase						)
            except ImportError:
                raise ImportError(
                    F'{model_class} does not exist. If you just want to test the pretrained model, you might want to'
                    " set `--only_pretrain_model` or `args.only_pretrain_model=True`."						)
        else:
            snake_case   :				Optional[Any]				 =   TF_MODEL_MAPPING[config.__class__](__UpperCAmelCase						)

        # encoder-decoder has vocab size saved differently
        snake_case   :				Dict				 =   config.vocab_size if hasattr(__UpperCAmelCase						,     "vocab_size"						) else config.encoder.vocab_size
        snake_case   :				str				 =   random_input_ids(__UpperCAmelCase						,     __UpperCAmelCase						,     __UpperCAmelCase						)

        @run_with_tf_optimizations(self.args.eager_mode						,     self.args.use_xla						)
        def encoder_decoder_forward():
            return model(__UpperCAmelCase						,     decoder_input_ids=__UpperCAmelCase						,     training=__UpperCAmelCase						)

        @run_with_tf_optimizations(self.args.eager_mode						,     self.args.use_xla						)
        def encoder_forward():
            return model(__UpperCAmelCase						,     training=__UpperCAmelCase						)

        snake_case   :				List[Any]				 =   encoder_decoder_forward if config.is_encoder_decoder else encoder_forward

        return _inference







    def 			lowerCamelCase		(     self						,     UpperCamelCase__						,     UpperCamelCase__						,     UpperCamelCase__						)     ->		Callable[[], None]:

        '''simple docstring'''




        snake_case   :				str				 =   self.config_dict[model_name]

        if self.args.eager_mode is not False:
            raise ValueError("Training cannot be done in eager mode. Please make sure that `args.eager_mode = False`."						)

        if self.args.fpaa:
            raise NotImplementedError("Mixed precision is currently not supported."						)

        snake_case   :				List[str]				 =   (
            hasattr(__UpperCAmelCase						,     "architectures"						)
            and isinstance(config.architectures						,     __UpperCAmelCase						)
            and len(config.architectures						) > 0
        )
        if not self.args.only_pretrain_model and has_model_class_in_config:
            try:
                snake_case   :				Union[str, Any]				 =   "TF" + config.architectures[0]  # prepend 'TF' for tensorflow model
                snake_case   :				Tuple				 =   __import__("transformers"						,     fromlist=[model_class]						)
                snake_case   :				List[str]				 =   getattr(__UpperCAmelCase						,     __UpperCAmelCase						)
                snake_case   :				Dict				 =   model_cls(__UpperCAmelCase						)
            except ImportError:
                raise ImportError(
                    F'{model_class} does not exist. If you just want to test the pretrained model, you might want to'
                    " set `--only_pretrain_model` or `args.only_pretrain_model=True`."						)
        else:
            snake_case   :				Optional[int]				 =   TF_MODEL_WITH_LM_HEAD_MAPPING[config.__class__](__UpperCAmelCase						)

        # encoder-decoder has vocab size saved differently
        snake_case   :				Tuple				 =   config.vocab_size if hasattr(__UpperCAmelCase						,     "vocab_size"						) else config.encoder.vocab_size
        snake_case   :				Dict				 =   random_input_ids(__UpperCAmelCase						,     __UpperCAmelCase						,     __UpperCAmelCase						)

        @run_with_tf_optimizations(self.args.eager_mode						,     self.args.use_xla						)
        def encoder_decoder_train():
            snake_case   :				List[Any]				 =   model(__UpperCAmelCase						,     decoder_input_ids=__UpperCAmelCase						,     labels=__UpperCAmelCase						,     training=__UpperCAmelCase						)[0]
            snake_case   :				List[str]				 =   tf.gradients(__UpperCAmelCase						,     model.trainable_variables						)
            return gradients

        @run_with_tf_optimizations(self.args.eager_mode						,     self.args.use_xla						)
        def encoder_train():
            snake_case   :				Any				 =   model(__UpperCAmelCase						,     labels=__UpperCAmelCase						,     training=__UpperCAmelCase						)[0]
            snake_case   :				List[str]				 =   tf.gradients(__UpperCAmelCase						,     model.trainable_variables						)
            return gradients

        snake_case   :				Optional[Any]				 =   encoder_decoder_train if config.is_encoder_decoder else encoder_train

        return _train







    def 			lowerCamelCase		(     self						,     UpperCamelCase__						)     ->		float:

        '''simple docstring'''




        with self.args.strategy.scope():
            try:
                if self.args.is_tpu or self.args.use_xla:
                    # run additional 10 times to stabilize compilation for tpu
                    logger.info("Do inference on TPU. Running model 5 times to stabilize compilation"						)
                    timeit.repeat(__UpperCAmelCase						,     repeat=1						,     number=5						)

                # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
                snake_case   :				Union[str, Any]				 =   timeit.repeat(
                    __UpperCAmelCase						,     repeat=self.args.repeat						,     number=10						,     )

                return min(__UpperCAmelCase						) / 10.0
            except ResourceExhaustedError as e:
                self.print_fn(F'Doesn\'t fit on GPU. {e}'						)







    def 			lowerCamelCase		(     self						,     UpperCamelCase__						)     ->		[Memory, MemorySummary]:

        '''simple docstring'''




        logger.info(
            "Note that TensorFlow allocates more memory than "
            "it might need to speed up computation. "
            "The memory reported here corresponds to the memory "
            "reported by `nvidia-smi`, which can vary depending "
            "on total available memory on the GPU that is used."						)
        with self.args.strategy.scope():
            try:
                if self.args.trace_memory_line_by_line:
                    if not self.args.eager_mode:
                        raise ValueError(
                            "`args.eager_mode` is set to `False`. Make sure to run model in eager mode to measure memory"
                            " consumption line by line."						)
                    snake_case   :				Tuple				 =   start_memory_tracing("transformers"						)

                if self.args.is_tpu:
                    # tpu
                    raise NotImplementedError(
                        "Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking"
                        " with `args.memory=False`"						)
                elif self.args.is_gpu:
                    # gpu
                    if not is_pyanvml_available():
                        logger.warning(
                            "py3nvml not installed, we won\'t log GPU memory usage. "
                            "Install py3nvml (pip install py3nvml) to log information about GPU."						)
                        snake_case   :				Union[str, Any]				 =   "N/A"
                    else:
                        logger.info(
                            "Measuring total GPU usage on GPU device. Make sure to not have additional processes"
                            " running on the same GPU."						)
                        # init nvml
                        nvml.nvmlInit()
                        func()
                        snake_case   :				Optional[Any]				 =   nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx						)
                        snake_case   :				Optional[int]				 =   nvml.nvmlDeviceGetMemoryInfo(__UpperCAmelCase						)
                        snake_case   :				Tuple				 =   meminfo.used
                        snake_case   :				str				 =   Memory(__UpperCAmelCase						)
                        # shutdown nvml
                        nvml.nvmlShutdown()
                else:
                    # cpu
                    if self.args.trace_memory_line_by_line:
                        logger.info(
                            "When enabling line by line tracing, the max peak memory for CPU is inaccurate in"
                            " TensorFlow."						)
                        snake_case   :				Optional[int]				 =   None
                    else:
                        snake_case   :				Tuple				 =   measure_peak_memory_cpu(__UpperCAmelCase						)
                        snake_case   :				Union[str, Any]				 =   Memory(__UpperCAmelCase						) if isinstance(__UpperCAmelCase						,     __UpperCAmelCase						) else memory_bytes
                if self.args.trace_memory_line_by_line:
                    snake_case   :				Union[str, Any]				 =   stop_memory_tracing(__UpperCAmelCase						)
                    if memory is None:
                        snake_case   :				Optional[Any]				 =   summary.total
                else:
                    snake_case   :				List[Any]				 =   None

                return memory, summary
            except ResourceExhaustedError as e:
                self.print_fn(F'Doesn\'t fit on GPU. {e}'						)
                return "N/A", None









"""simple docstring"""



import importlib

import torch
import yaml
from omegaconf import OmegaConf
from taming.models.vqgan import VQModel







def   __lowerCAmelCase						(						lowercase  :  Union[str, Any]						,							lowercase  :  Optional[int]=False			)     ->						Union[str, Any]:


    """simple docstring"""





    snake_case   :				Dict				 =   OmegaConf.load(lowercase			)
    if display:
        print(yaml.dump(OmegaConf.to_container(lowercase			)			)			)
    return config







def   __lowerCAmelCase						(						lowercase  :  Dict						,							lowercase  :  Dict=None						,							lowercase  :  Dict=None			)     ->						Union[str, Any]:


    """simple docstring"""





    if conf_path is None:
        snake_case   :				Optional[Any]				 =   "./model_checkpoints/vqgan_only.yaml"
    snake_case   :				Union[str, Any]				 =   load_config(lowercase						,							display=lowercase			)
    snake_case   :				List[Any]				 =   VQModel(**config.model.params			)
    if ckpt_path is None:
        snake_case   :				Optional[int]				 =   "./model_checkpoints/vqgan_only.pt"
    snake_case   :				Union[str, Any]				 =   torch.load(lowercase						,							map_location=lowercase			)
    if ".ckpt" in ckpt_path:
        snake_case   :				Union[str, Any]				 =   sd["state_dict"]
    model.load_state_dict(lowercase						,							strict=lowercase			)
    model.to(lowercase			)
    del sd
    return model







def   __lowerCAmelCase						(						lowercase  :  str						,							lowercase  :  List[str]			)     ->						List[str]:


    """simple docstring"""





    snake_case ,snake_case ,snake_case   :				List[Any]				 =   model.encode(lowercase			)
    print(F'VQGAN --- {model.__class__.__name__}: latent shape: {z.shape[2:]}'			)
    snake_case   :				Union[str, Any]				 =   model.decode(lowercase			)
    return xrec







def   __lowerCAmelCase						(						lowercase  :  List[Any]						,							lowercase  :  str=False			)     ->						Optional[int]:


    """simple docstring"""





    snake_case ,snake_case   :				Any				 =   string.rsplit("."						,							1			)
    if reload:
        snake_case   :				List[Any]				 =   importlib.import_module(lowercase			)
        importlib.reload(lowercase			)
    return getattr(importlib.import_module(lowercase						,							package=lowercase			)						,							cls			)







def   __lowerCAmelCase						(						lowercase  :  List[str]			)     ->						Union[str, Any]:


    """simple docstring"""





    if "target" not in config:
        raise KeyError("Expected key `target` to instantiate."			)
    return get_obj_from_str(config["target"]			)(**config.get("params"						,							{}			)			)







def   __lowerCAmelCase						(						lowercase  :  Tuple						,							lowercase  :  List[str]						,							lowercase  :  Tuple=True						,							lowercase  :  Optional[Any]=True			)     ->						Optional[int]:


    """simple docstring"""





    snake_case   :				Optional[Any]				 =   instantiate_from_config(lowercase			)
    if sd is not None:
        model.load_state_dict(lowercase			)
    if gpu:
        model.cuda()
    if eval_mode:
        model.eval()
    return {"model": model}







def   __lowerCAmelCase						(						lowercase  :  List[str]						,							lowercase  :  int						,							lowercase  :  Dict						,							lowercase  :  Union[str, Any]			)     ->						Union[str, Any]:


    """simple docstring"""





    if ckpt:
        snake_case   :				Dict				 =   torch.load(lowercase						,							map_location="cpu"			)
        snake_case   :				Any				 =   pl_sd["global_step"]
        print(F'loaded model from global step {global_step}.'			)
    else:
        snake_case   :				Any				 =   {"state_dict": None}
        snake_case   :				List[str]				 =   None
    snake_case   :				Dict				 =   load_model_from_config(config.model						,							pl_sd["state_dict"]						,							gpu=lowercase						,							eval_mode=lowercase			)["model"]
    return model, global_step





