


"""simple docstring"""



import random
import timeit
from functools import wraps
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_tf_auto import TF_MODEL_MAPPING, TF_MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_tf_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_tf_available():
  import tensorflow as tf
  from tensorflow.python.framework.errors_impl import ResourceExhaustedError

  from .benchmark_args_tf import TensorFlowBenchmarkArguments

if is_pyanvml_available():
  import pyanvml.pyanvml as nvml

__snake_case             =   logging.get_logger(__name__)







def   __lowerCAmelCase						(						lowercase  :  bool						,							lowercase  :  bool			)     ->						List[str]:


    """simple docstring"""





    def run_func(lowercase  :  Optional[Any]			):
        @wraps(UpperCamelCase__			)
        def run_in_eager_mode(*lowercase  :  int						,							**lowercase  :  Dict			):
            return func(*UpperCamelCase__						,							**UpperCamelCase__			)

        @wraps(UpperCamelCase__			)
        @tf.function(experimental_compile=UpperCamelCase__			)
        def run_in_graph_mode(*lowercase  :  List[Any]						,							**lowercase  :  Tuple			):
            return func(*UpperCamelCase__						,							**UpperCamelCase__			)

        if do_eager_mode is True:
            if use_xla is not False:
                raise ValueError(
                    "Cannot run model in XLA, if `args.eager_mode` is set to `True`. Please set `args.eager_mode=False`."			)
            return run_in_eager_mode
        else:
            return run_in_graph_mode

    return run_func







def   __lowerCAmelCase						(						lowercase  :  int						,							lowercase  :  int						,							lowercase  :  int			)     ->						List[str]:


    """simple docstring"""





    snake_case   :				int				 =   random.Random()
    snake_case   :				str				 =   [rng.randint(0						,							vocab_size - 1			) for i in range(batch_size * sequence_length			)]
    return tf.constant(UpperCamelCase__						,							shape=(batch_size, sequence_length)						,							dtype=tf.intaa			)


class _lowerCAmelCase    (				snake_case_     ):
   __UpperCAmelCase			:	TensorFlowBenchmarkArguments
   __UpperCAmelCase			:	PretrainedConfig
   __UpperCAmelCase			:	str							  =  "TensorFlow"







   @property
   def 			lowerCamelCase		(     self						)     ->		str:

       '''simple docstring'''




       return tf.__version__







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						,     UpperCamelCase__						,     UpperCamelCase__						)     ->		Union[str, Any]:

       '''simple docstring'''




       snake_case   :				Dict				 =   self.args.strategy
       if strategy is None:
           raise ValueError("A device strategy has to be initialized before using TensorFlow."						)
       snake_case   :				Dict				 =   self._prepare_inference_func(_a						,     _a						,     _a						)
       return self._measure_speed(_inference						)







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						,     UpperCamelCase__						,     UpperCamelCase__						)     ->		Union[str, Any]:

       '''simple docstring'''




       snake_case   :				Tuple				 =   self.args.strategy
       if strategy is None:
           raise ValueError("A device strategy has to be initialized before using TensorFlow."						)
       snake_case   :				Optional[int]				 =   self._prepare_train_func(_a						,     _a						,     _a						)
       return self._measure_speed(_train						)







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						,     UpperCamelCase__						,     UpperCamelCase__						)     ->		Any:

       '''simple docstring'''




       if self.args.is_gpu:
           tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]						,     _a						)
       snake_case   :				Tuple				 =   self.args.strategy
       if strategy is None:
           raise ValueError("A device strategy has to be initialized before using TensorFlow."						)
       snake_case   :				List[Any]				 =   self._prepare_inference_func(_a						,     _a						,     _a						)
       return self._measure_memory(_inference						)







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						,     UpperCamelCase__						,     UpperCamelCase__						)     ->		int:

       '''simple docstring'''




       if self.args.is_gpu:
           tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]						,     _a						)
       snake_case   :				Optional[Any]				 =   self.args.strategy
       if strategy is None:
           raise ValueError("A device strategy has to be initialized before using TensorFlow."						)

       snake_case   :				int				 =   self._prepare_train_func(_a						,     _a						,     _a						)
       return self._measure_memory(_train						)







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						,     UpperCamelCase__						,     UpperCamelCase__						)     ->		List[Any]:

       '''simple docstring'''




       snake_case   :				str				 =   self.config_dict[model_name]

       if self.args.fpaa:
           raise NotImplementedError("Mixed precision is currently not supported."						)

       snake_case   :				Dict				 =   (
           hasattr(_a						,     "architectures"						)
           and isinstance(config.architectures						,     _a						)
           and len(config.architectures						) > 0
       )
       if not self.args.only_pretrain_model and has_model_class_in_config:
           try:
               snake_case   :				Optional[int]				 =   "TF" + config.architectures[0]  # prepend 'TF' for tensorflow model
               snake_case   :				Any				 =   __import__("transformers"						,     fromlist=[model_class]						)
               snake_case   :				Dict				 =   getattr(_a						,     _a						)
               snake_case   :				Union[str, Any]				 =   model_cls(_a						)
           except ImportError:
               raise ImportError(
                   F'{model_class} does not exist. If you just want to test the pretrained model, you might want to'
                   " set `--only_pretrain_model` or `args.only_pretrain_model=True`."						)
       else:
           snake_case   :				Any				 =   TF_MODEL_MAPPING[config.__class__](_a						)

       # encoder-decoder has vocab size saved differently
       snake_case   :				Union[str, Any]				 =   config.vocab_size if hasattr(_a						,     "vocab_size"						) else config.encoder.vocab_size
       snake_case   :				List[Any]				 =   random_input_ids(_a						,     _a						,     _a						)

       @run_with_tf_optimizations(self.args.eager_mode						,     self.args.use_xla						)
       def encoder_decoder_forward():
           return model(_a						,     decoder_input_ids=_a						,     training=_a						)

       @run_with_tf_optimizations(self.args.eager_mode						,     self.args.use_xla						)
       def encoder_forward():
           return model(_a						,     training=_a						)

       snake_case   :				List[str]				 =   encoder_decoder_forward if config.is_encoder_decoder else encoder_forward

       return _inference







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						,     UpperCamelCase__						,     UpperCamelCase__						)     ->		int:

       '''simple docstring'''




       snake_case   :				List[str]				 =   self.config_dict[model_name]

       if self.args.eager_mode is not False:
           raise ValueError("Training cannot be done in eager mode. Please make sure that `args.eager_mode = False`."						)

       if self.args.fpaa:
           raise NotImplementedError("Mixed precision is currently not supported."						)

       snake_case   :				Optional[Any]				 =   (
           hasattr(_a						,     "architectures"						)
           and isinstance(config.architectures						,     _a						)
           and len(config.architectures						) > 0
       )
       if not self.args.only_pretrain_model and has_model_class_in_config:
           try:
               snake_case   :				List[str]				 =   "TF" + config.architectures[0]  # prepend 'TF' for tensorflow model
               snake_case   :				List[Any]				 =   __import__("transformers"						,     fromlist=[model_class]						)
               snake_case   :				Optional[Any]				 =   getattr(_a						,     _a						)
               snake_case   :				Tuple				 =   model_cls(_a						)
           except ImportError:
               raise ImportError(
                   F'{model_class} does not exist. If you just want to test the pretrained model, you might want to'
                   " set `--only_pretrain_model` or `args.only_pretrain_model=True`."						)
       else:
           snake_case   :				Union[str, Any]				 =   TF_MODEL_WITH_LM_HEAD_MAPPING[config.__class__](_a						)

       # encoder-decoder has vocab size saved differently
       snake_case   :				str				 =   config.vocab_size if hasattr(_a						,     "vocab_size"						) else config.encoder.vocab_size
       snake_case   :				Optional[int]				 =   random_input_ids(_a						,     _a						,     _a						)

       @run_with_tf_optimizations(self.args.eager_mode						,     self.args.use_xla						)
       def encoder_decoder_train():
           snake_case   :				Union[str, Any]				 =   model(_a						,     decoder_input_ids=_a						,     labels=_a						,     training=_a						)[0]
           snake_case   :				Union[str, Any]				 =   tf.gradients(_a						,     model.trainable_variables						)
           return gradients

       @run_with_tf_optimizations(self.args.eager_mode						,     self.args.use_xla						)
       def encoder_train():
           snake_case   :				int				 =   model(_a						,     labels=_a						,     training=_a						)[0]
           snake_case   :				Any				 =   tf.gradients(_a						,     model.trainable_variables						)
           return gradients

       snake_case   :				str				 =   encoder_decoder_train if config.is_encoder_decoder else encoder_train

       return _train







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						)     ->		int:

       '''simple docstring'''




       with self.args.strategy.scope():
           try:
               if self.args.is_tpu or self.args.use_xla:
                   # run additional 10 times to stabilize compilation for tpu
                   logger.info("Do inference on TPU. Running model 5 times to stabilize compilation"						)
                   timeit.repeat(_a						,     repeat=1						,     number=5						)

               # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
               snake_case   :				Any				 =   timeit.repeat(
                   _a						,     repeat=self.args.repeat						,     number=10						,     )

               return min(_a						) / 10.0
           except ResourceExhaustedError as e:
               self.print_fn(F'Doesn\'t fit on GPU. {e}'						)







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						)     ->		Optional[Any]:

       '''simple docstring'''




       logger.info(
           "Note that TensorFlow allocates more memory than "
           "it might need to speed up computation. "
           "The memory reported here corresponds to the memory "
           "reported by `nvidia-smi`, which can vary depending "
           "on total available memory on the GPU that is used."						)
       with self.args.strategy.scope():
           try:
               if self.args.trace_memory_line_by_line:
                   if not self.args.eager_mode:
                       raise ValueError(
                           "`args.eager_mode` is set to `False`. Make sure to run model in eager mode to measure memory"
                           " consumption line by line."						)
                   snake_case   :				Dict				 =   start_memory_tracing("transformers"						)

               if self.args.is_tpu:
                   # tpu
                   raise NotImplementedError(
                       "Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking"
                       " with `args.memory=False`"						)
               elif self.args.is_gpu:
                   # gpu
                   if not is_pyanvml_available():
                       logger.warning(
                           "py3nvml not installed, we won\'t log GPU memory usage. "
                           "Install py3nvml (pip install py3nvml) to log information about GPU."						)
                       snake_case   :				Optional[int]				 =   "N/A"
                   else:
                       logger.info(
                           "Measuring total GPU usage on GPU device. Make sure to not have additional processes"
                           " running on the same GPU."						)
                       # init nvml
                       nvml.nvmlInit()
                       func()
                       snake_case   :				Any				 =   nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx						)
                       snake_case   :				str				 =   nvml.nvmlDeviceGetMemoryInfo(_a						)
                       snake_case   :				Union[str, Any]				 =   meminfo.used
                       snake_case   :				List[str]				 =   Memory(_a						)
                       # shutdown nvml
                       nvml.nvmlShutdown()
               else:
                   # cpu
                   if self.args.trace_memory_line_by_line:
                       logger.info(
                           "When enabling line by line tracing, the max peak memory for CPU is inaccurate in"
                           " TensorFlow."						)
                       snake_case   :				int				 =   None
                   else:
                       snake_case   :				List[str]				 =   measure_peak_memory_cpu(_a						)
                       snake_case   :				Any				 =   Memory(_a						) if isinstance(_a						,     _a						) else memory_bytes
               if self.args.trace_memory_line_by_line:
                   snake_case   :				List[Any]				 =   stop_memory_tracing(_a						)
                   if memory is None:
                       snake_case   :				Optional[int]				 =   summary.total
               else:
                   snake_case   :				Tuple				 =   None

               return memory, summary
           except ResourceExhaustedError as e:
               self.print_fn(F'Doesn\'t fit on GPU. {e}'						)
               return "N/A", None









"""simple docstring"""



import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
__snake_case             =   data_utils.TransfoXLTokenizer
__snake_case             =   data_utils.TransfoXLCorpus
__snake_case             =   data_utils
__snake_case             =   data_utils







def   __lowerCAmelCase						(						lowercase  :  Optional[int]						,							lowercase  :  int						,							lowercase  :  List[Any]						,							lowercase  :  Union[str, Any]			)     ->						List[Any]:


    """simple docstring"""





    if transfo_xl_dataset_file:
        # Convert a pre-processed corpus (see original TensorFlow repo)
        with open(lowercase						,							"rb"			) as fp:
            snake_case   :				int				 =   pickle.load(lowercase						,							encoding="latin1"			)
        # Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
        snake_case   :				int				 =   pytorch_dump_folder_path + "/" + VOCAB_FILES_NAMES["pretrained_vocab_file"]
        print(F'Save vocabulary to {pytorch_vocab_dump_path}'			)
        snake_case   :				str				 =   corpus.vocab.__dict__
        torch.save(lowercase						,							lowercase			)

        snake_case   :				str				 =   corpus.__dict__
        corpus_dict_no_vocab.pop("vocab"						,							lowercase			)
        snake_case   :				Dict				 =   pytorch_dump_folder_path + "/" + CORPUS_NAME
        print(F'Save dataset to {pytorch_dataset_dump_path}'			)
        torch.save(lowercase						,							lowercase			)

    if tf_checkpoint_path:
        # Convert a pre-trained TensorFlow model
        snake_case   :				Union[str, Any]				 =   os.path.abspath(lowercase			)
        snake_case   :				str				 =   os.path.abspath(lowercase			)

        print(F'Converting Transformer XL checkpoint from {tf_path} with config at {config_path}.'			)
        # Initialise PyTorch model
        if transfo_xl_config_file == "":
            snake_case   :				int				 =   TransfoXLConfig()
        else:
            snake_case   :				Optional[int]				 =   TransfoXLConfig.from_json_file(lowercase			)
        print(F'Building PyTorch model from configuration: {config}'			)
        snake_case   :				str				 =   TransfoXLLMHeadModel(lowercase			)

        snake_case   :				str				 =   load_tf_weights_in_transfo_xl(lowercase						,							lowercase						,							lowercase			)
        # Save pytorch-model
        snake_case   :				Union[str, Any]				 =   os.path.join(lowercase						,							lowercase			)
        snake_case   :				Optional[Any]				 =   os.path.join(lowercase						,							lowercase			)
        print(F'Save PyTorch model to {os.path.abspath(lowercase			)}'			)
        torch.save(model.state_dict()						,							lowercase			)
        print(F'Save configuration file to {os.path.abspath(lowercase			)}'			)
        with open(lowercase						,							"w"						,							encoding="utf-8"			) as f:
            f.write(config.to_json_string()			)


if __name__ == "__main__":
  __snake_case             =   argparse.ArgumentParser()
  parser.add_argument(
      """--pytorch_dump_folder_path""",
      default=None,
      type=str,
      required=True,
      help="""Path to the folder to store the PyTorch model or dataset/vocab.""",
  )
  parser.add_argument(
      """--tf_checkpoint_path""",
      default="""""",
      type=str,
      help="""An optional path to a TensorFlow checkpoint path to be converted.""",
  )
  parser.add_argument(
      """--transfo_xl_config_file""",
      default="""""",
      type=str,
      help=(
          """An optional config json file corresponding to the pre-trained BERT model. \n"""
          """This specifies the model architecture."""
      ),
  )
  parser.add_argument(
      """--transfo_xl_dataset_file""",
      default="""""",
      type=str,
      help="""An optional dataset file to be converted in a vocabulary.""",
  )
  __snake_case             =   parser.parse_args()
  convert_transfo_xl_checkpoint_to_pytorch(
      args.tf_checkpoint_path,
      args.transfo_xl_config_file,
      args.pytorch_dump_folder_path,
      args.transfo_xl_dataset_file,
  )





