


"""simple docstring"""



import argparse
import collections
import os
import re

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_table.py
__snake_case             =   """src/transformers"""
__snake_case             =   """docs/source/en"""
__snake_case             =   """."""







def   __lowerCAmelCase						(						lowercase  :  List[str]						,							lowercase  :  List[Any]						,							lowercase  :  int			)     ->						Optional[int]:


    """simple docstring"""





    with open(lowercase						,							"r"						,							encoding="utf-8"						,							newline="\n"			) as f:
        snake_case   :				List[str]				 =   f.readlines()
    # Find the start prompt.
    snake_case   :				str				 =   0
    while not lines[start_index].startswith(lowercase			):
        start_index += 1
    start_index += 1

    snake_case   :				Tuple				 =   start_index
    while not lines[end_index].startswith(lowercase			):
        end_index += 1
    end_index -= 1

    while len(lines[start_index]			) <= 1:
        start_index += 1
    while len(lines[end_index]			) <= 1:
        end_index -= 1
    end_index += 1
    return "".join(lines[start_index:end_index]			), start_index, end_index, lines


# Add here suffixes that are used to identify models, separated by |
__snake_case             =   """Model|Encoder|Decoder|ForConditionalGeneration"""
# Regexes that match TF/Flax/PT model names.
__snake_case             =   re.compile(R"""TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")
__snake_case             =   re.compile(R"""Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
__snake_case             =   re.compile(R"""(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")


# This is to make sure the transformers module imported is the one in the repo.
__snake_case             =   direct_transformers_import(TRANSFORMERS_PATH)







def   __lowerCAmelCase						(						lowercase  :  str			)     ->						Any:


    """simple docstring"""





    snake_case   :				Dict				 =   re.finditer(".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)"						,							lowercase			)
    return [m.group(0			) for m in matches]







def   __lowerCAmelCase						(						lowercase  :  List[str]						,							lowercase  :  Optional[Any]			)     ->						List[Any]:


    """simple docstring"""





    snake_case   :				Optional[Any]				 =   2 if text == "✅" or text == "❌" else len(lowercase			)
    snake_case   :				Optional[int]				 =   (width - text_length) // 2
    snake_case   :				Dict				 =   width - text_length - left_indent
    return " " * left_indent + text + " " * right_indent







def   __lowerCAmelCase						(						)     ->						Any:


    """simple docstring"""





    snake_case   :				Any				 =   transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
    snake_case   :				Dict				 =   {
        name: config_maping_names[code]
        for code, name in transformers_module.MODEL_NAMES_MAPPING.items()
        if code in config_maping_names
    }
    snake_case   :				int				 =   {name: config.replace("Config"						,							""			) for name, config in model_name_to_config.items()}

    # Dictionaries flagging if each model prefix has a slow/fast tokenizer, backend in PT/TF/Flax.
    snake_case   :				int				 =   collections.defaultdict(lowercase			)
    snake_case   :				Dict				 =   collections.defaultdict(lowercase			)
    snake_case   :				List[str]				 =   collections.defaultdict(lowercase			)
    snake_case   :				Union[str, Any]				 =   collections.defaultdict(lowercase			)
    snake_case   :				str				 =   collections.defaultdict(lowercase			)

    # Let's lookup through all transformers object (once).
    for attr_name in dir(lowercase			):
        snake_case   :				Optional[Any]				 =   None
        if attr_name.endswith("Tokenizer"			):
            snake_case   :				Tuple				 =   slow_tokenizers
            snake_case   :				str				 =   attr_name[:-9]
        elif attr_name.endswith("TokenizerFast"			):
            snake_case   :				Union[str, Any]				 =   fast_tokenizers
            snake_case   :				str				 =   attr_name[:-13]
        elif _re_tf_models.match(lowercase			) is not None:
            snake_case   :				Optional[int]				 =   tf_models
            snake_case   :				Tuple				 =   _re_tf_models.match(lowercase			).groups()[0]
        elif _re_flax_models.match(lowercase			) is not None:
            snake_case   :				Optional[Any]				 =   flax_models
            snake_case   :				List[str]				 =   _re_flax_models.match(lowercase			).groups()[0]
        elif _re_pt_models.match(lowercase			) is not None:
            snake_case   :				Dict				 =   pt_models
            snake_case   :				List[str]				 =   _re_pt_models.match(lowercase			).groups()[0]

        if lookup_dict is not None:
            while len(lowercase			) > 0:
                if attr_name in model_name_to_prefix.values():
                    snake_case   :				str				 =   True
                    break
                # Try again after removing the last word in the name
                snake_case   :				List[str]				 =   "".join(camel_case_split(lowercase			)[:-1]			)

    # Let's build that table!
    snake_case   :				Dict				 =   list(model_name_to_config.keys()			)
    model_names.sort(key=str.lower			)
    snake_case   :				Union[str, Any]				 =   ["Model", "Tokenizer slow", "Tokenizer fast", "PyTorch support", "TensorFlow support", "Flax Support"]
    # We'll need widths to properly display everything in the center (+2 is to leave one extra space on each side).
    snake_case   :				Union[str, Any]				 =   [len(lowercase			) + 2 for c in columns]
    snake_case   :				Dict				 =   max([len(lowercase			) for name in model_names]			) + 2

    # Build the table per se
    snake_case   :				Optional[int]				 =   "|" + "|".join([_center_text(lowercase						,							lowercase			) for c, w in zip(lowercase						,							lowercase			)]			) + "|\n"
    # Use ":-----:" format to center-aligned table cell texts
    table += "|" + "|".join([":" + "-" * (w - 2) + ":" for w in widths]			) + "|\n"

    snake_case   :				List[Any]				 =   {True: "✅", False: "❌"}
    for name in model_names:
        snake_case   :				List[str]				 =   model_name_to_prefix[name]
        snake_case   :				str				 =   [
            name,
            check[slow_tokenizers[prefix]],
            check[fast_tokenizers[prefix]],
            check[pt_models[prefix]],
            check[tf_models[prefix]],
            check[flax_models[prefix]],
        ]
        table += "|" + "|".join([_center_text(lowercase						,							lowercase			) for l, w in zip(lowercase						,							lowercase			)]			) + "|\n"
    return table







def   __lowerCAmelCase						(						lowercase  :  Union[str, Any]=False			)     ->						Any:


    """simple docstring"""





    snake_case ,snake_case ,snake_case ,snake_case   :				Optional[Any]				 =   _find_text_in_file(
        filename=os.path.join(lowercase						,							"index.md"			)						,							start_prompt="<!--This table is updated automatically from the auto modules"						,							end_prompt="<!-- End table-->"						,							)
    snake_case   :				Optional[int]				 =   get_model_table_from_auto_modules()

    if current_table != new_table:
        if overwrite:
            with open(os.path.join(lowercase						,							"index.md"			)						,							"w"						,							encoding="utf-8"						,							newline="\n"			) as f:
                f.writelines(lines[:start_index] + [new_table] + lines[end_index:]			)
        else:
            raise ValueError(
                "The model table in the `index.md` has not been updated. Run `make fix-copies` to fix this."			)


if __name__ == "__main__":
  __snake_case             =   argparse.ArgumentParser()
  parser.add_argument("""--fix_and_overwrite""", action="""store_true""", help="""Whether to fix inconsistencies.""")
  __snake_case             =   parser.parse_args()

  check_model_table(args.fix_and_overwrite)









"""simple docstring"""



import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
__snake_case             =   data_utils.TransfoXLTokenizer
__snake_case             =   data_utils.TransfoXLCorpus
__snake_case             =   data_utils
__snake_case             =   data_utils







def   __lowerCAmelCase						(						lowercase  :  Optional[int]						,							lowercase  :  int						,							lowercase  :  List[Any]						,							lowercase  :  Union[str, Any]			)     ->						List[Any]:


    """simple docstring"""





    if transfo_xl_dataset_file:
        # Convert a pre-processed corpus (see original TensorFlow repo)
        with open(lowercase						,							"rb"			) as fp:
            snake_case   :				int				 =   pickle.load(lowercase						,							encoding="latin1"			)
        # Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
        snake_case   :				int				 =   pytorch_dump_folder_path + "/" + VOCAB_FILES_NAMES["pretrained_vocab_file"]
        print(F'Save vocabulary to {pytorch_vocab_dump_path}'			)
        snake_case   :				str				 =   corpus.vocab.__dict__
        torch.save(lowercase						,							lowercase			)

        snake_case   :				str				 =   corpus.__dict__
        corpus_dict_no_vocab.pop("vocab"						,							lowercase			)
        snake_case   :				Dict				 =   pytorch_dump_folder_path + "/" + CORPUS_NAME
        print(F'Save dataset to {pytorch_dataset_dump_path}'			)
        torch.save(lowercase						,							lowercase			)

    if tf_checkpoint_path:
        # Convert a pre-trained TensorFlow model
        snake_case   :				Union[str, Any]				 =   os.path.abspath(lowercase			)
        snake_case   :				str				 =   os.path.abspath(lowercase			)

        print(F'Converting Transformer XL checkpoint from {tf_path} with config at {config_path}.'			)
        # Initialise PyTorch model
        if transfo_xl_config_file == "":
            snake_case   :				int				 =   TransfoXLConfig()
        else:
            snake_case   :				Optional[int]				 =   TransfoXLConfig.from_json_file(lowercase			)
        print(F'Building PyTorch model from configuration: {config}'			)
        snake_case   :				str				 =   TransfoXLLMHeadModel(lowercase			)

        snake_case   :				str				 =   load_tf_weights_in_transfo_xl(lowercase						,							lowercase						,							lowercase			)
        # Save pytorch-model
        snake_case   :				Union[str, Any]				 =   os.path.join(lowercase						,							lowercase			)
        snake_case   :				Optional[Any]				 =   os.path.join(lowercase						,							lowercase			)
        print(F'Save PyTorch model to {os.path.abspath(lowercase			)}'			)
        torch.save(model.state_dict()						,							lowercase			)
        print(F'Save configuration file to {os.path.abspath(lowercase			)}'			)
        with open(lowercase						,							"w"						,							encoding="utf-8"			) as f:
            f.write(config.to_json_string()			)


if __name__ == "__main__":
  __snake_case             =   argparse.ArgumentParser()
  parser.add_argument(
      """--pytorch_dump_folder_path""",
      default=None,
      type=str,
      required=True,
      help="""Path to the folder to store the PyTorch model or dataset/vocab.""",
  )
  parser.add_argument(
      """--tf_checkpoint_path""",
      default="""""",
      type=str,
      help="""An optional path to a TensorFlow checkpoint path to be converted.""",
  )
  parser.add_argument(
      """--transfo_xl_config_file""",
      default="""""",
      type=str,
      help=(
          """An optional config json file corresponding to the pre-trained BERT model. \n"""
          """This specifies the model architecture."""
      ),
  )
  parser.add_argument(
      """--transfo_xl_dataset_file""",
      default="""""",
      type=str,
      help="""An optional dataset file to be converted in a vocabulary.""",
  )
  __snake_case             =   parser.parse_args()
  convert_transfo_xl_checkpoint_to_pytorch(
      args.tf_checkpoint_path,
      args.transfo_xl_config_file,
      args.pytorch_dump_folder_path,
      args.transfo_xl_dataset_file,
  )





