





import json
import os
import shutil
import warnings
from argparse import ArgumentParser, Namespace
from pathlib import Path
from typing import List

from ..utils import logging
from . import BaseTransformersCLICommand


try:
     from cookiecutter.main import cookiecutter

     UpperCAmelCase     :     Dict							=	True
except ImportError:
     UpperCAmelCase     :     Optional[Any]							=	False

UpperCAmelCase     :     Optional[Any]							=	logging.get_logger(__name__)  # pylint: disable=invalid-name
def 					_A	(		SCREAMING_SNAKE_CASE					:       Namespace						):





  """simple docstring"""



  return AddNewModelCommand(args.testing	,	args.testing_file	,	path=args.path						)







class     __lowerCAmelCase	(   UpperCamelCase__):







      @staticmethod
      def        _lowercase  (   lowerCAmelCase__							)       ->						Optional[int]:



        '''simple docstring'''




        a__     : Union[str, Any]  =parser.add_parser("add-new-model"							)
        add_new_model_parser.add_argument("--testing"		,				action="store_true"		,				help="If in testing mode."							)
        add_new_model_parser.add_argument("--testing_file"		,				type=lowerCAmelCase__		,				help="Configuration file on which to run."							)
        add_new_model_parser.add_argument(
            "--path"		,				type=lowerCAmelCase__		,				help="Path to cookiecutter. Should only be used for testing purposes."							)
        add_new_model_parser.set_defaults(func=lowerCAmelCase__							)







      def __init__(   self		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__=None		,				*lowerCAmelCase__							)       ->						List[Any]:



        '''simple docstring'''




        a__     : List[str]  =testing
        a__     : str  =testing_file
        a__     : Union[str, Any]  =path







      def        _lowercase  (   self							)       ->						Optional[Any]:



        '''simple docstring'''




        warnings.warn(
            "The command `transformers-cli add-new-model` is deprecated and will be removed in v5 of Transformers. "
            "It is not actively maintained anymore, so might give a result that won't pass all tests and quality "
            "checks, you should use `transformers-cli add-new-model-like` instead."							)
        if not _has_cookiecutter:
          raise ImportError(
              "Model creation dependencies are required to use the `add_new_model` command. Install them by running "
              "the following at the root of your `transformers` clone:\n\n\t$ pip install -e .[modelcreation]\n"							)
        # Ensure that there is no other `cookiecutter-template-xxx` directory in the current working directory
        a__     : Union[str, Any]  =[directory for directory in os.listdir() if "cookiecutter-template-" == directory[:2_2]]
        if len(lowerCAmelCase__							) > 0:
          raise ValueError(
              "Several directories starting with `cookiecutter-template-` in current working directory. "
              "Please clean your directory by removing all folders starting with `cookiecutter-template-` or "
              "change your working directory."							)

        a__     : Tuple  =(
            Path(lowerCAmelCase__							).parent.parent.parent.parent if self._path is None else Path(self._path							).parent.parent
        )
        a__     : List[Any]  =path_to_transformer_root / "templates" / "adding_a_new_model"

        # Execute cookiecutter
        if not self._testing:
          cookiecutter(str(lowerCAmelCase__							)							)
        else:
          with open(self._testing_file		,				"r"							) as configuration_file:
            a__     : Optional[Any]  =json.load(lowerCAmelCase__							)

          cookiecutter(
              str(path_to_cookiecutter if self._path is None else self._path							)		,				no_input=lowerCAmelCase__		,				extra_context=lowerCAmelCase__		,				)

        a__     : str  =[directory for directory in os.listdir() if "cookiecutter-template-" in directory[:2_2]][0]

        # Retrieve configuration
        with open(directory + "/configuration.json"		,				"r"							) as configuration_file:
          a__     : Tuple  =json.load(lowerCAmelCase__							)

        a__     : Union[str, Any]  =configuration["lowercase_modelname"]
        a__     : Optional[Any]  =configuration["generate_tensorflow_pytorch_and_flax"]
        os.remove(F'''{directory}/configuration.json'''							)

        a__     : int  ="PyTorch" in generate_tensorflow_pytorch_and_flax
        a__     : Union[str, Any]  ="TensorFlow" in generate_tensorflow_pytorch_and_flax
        a__     : List[str]  ="Flax" in generate_tensorflow_pytorch_and_flax

        a__     : Any  =F'''{path_to_transformer_root}/src/transformers/models/{lowercase_model_name}'''
        os.makedirs(lowerCAmelCase__		,				exist_ok=lowerCAmelCase__							)
        os.makedirs(F'''{path_to_transformer_root}/tests/models/{lowercase_model_name}'''		,				exist_ok=lowerCAmelCase__							)

        # Tests require submodules as they have parent imports
        with open(F'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/__init__.py'''		,				"w"							):
          pass

        shutil.move(
            F'''{directory}/__init__.py'''		,				F'''{model_dir}/__init__.py'''		,				)
        shutil.move(
            F'''{directory}/configuration_{lowercase_model_name}.py'''		,				F'''{model_dir}/configuration_{lowercase_model_name}.py'''		,				)

        def remove_copy_lines(lowerCAmelCase__							):
          with open(lowerCAmelCase__		,				"r"							) as f:
            a__     : Tuple  =f.readlines()
          with open(lowerCAmelCase__		,				"w"							) as f:
            for line in lines:
              if "# Copied from transformers." not in line:
                f.write(lowerCAmelCase__							)

        if output_pytorch:
          if not self._testing:
            remove_copy_lines(F'''{directory}/modeling_{lowercase_model_name}.py'''							)

          shutil.move(
              F'''{directory}/modeling_{lowercase_model_name}.py'''		,				F'''{model_dir}/modeling_{lowercase_model_name}.py'''		,				)

          shutil.move(
              F'''{directory}/test_modeling_{lowercase_model_name}.py'''		,				F'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_{lowercase_model_name}.py'''		,				)
        else:
          os.remove(F'''{directory}/modeling_{lowercase_model_name}.py'''							)
          os.remove(F'''{directory}/test_modeling_{lowercase_model_name}.py'''							)

        if output_tensorflow:
          if not self._testing:
            remove_copy_lines(F'''{directory}/modeling_tf_{lowercase_model_name}.py'''							)

          shutil.move(
              F'''{directory}/modeling_tf_{lowercase_model_name}.py'''		,				F'''{model_dir}/modeling_tf_{lowercase_model_name}.py'''		,				)

          shutil.move(
              F'''{directory}/test_modeling_tf_{lowercase_model_name}.py'''		,				F'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_tf_{lowercase_model_name}.py'''		,				)
        else:
          os.remove(F'''{directory}/modeling_tf_{lowercase_model_name}.py'''							)
          os.remove(F'''{directory}/test_modeling_tf_{lowercase_model_name}.py'''							)

        if output_flax:
          if not self._testing:
            remove_copy_lines(F'''{directory}/modeling_flax_{lowercase_model_name}.py'''							)

          shutil.move(
              F'''{directory}/modeling_flax_{lowercase_model_name}.py'''		,				F'''{model_dir}/modeling_flax_{lowercase_model_name}.py'''		,				)

          shutil.move(
              F'''{directory}/test_modeling_flax_{lowercase_model_name}.py'''		,				F'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_flax_{lowercase_model_name}.py'''		,				)
        else:
          os.remove(F'''{directory}/modeling_flax_{lowercase_model_name}.py'''							)
          os.remove(F'''{directory}/test_modeling_flax_{lowercase_model_name}.py'''							)

        shutil.move(
            F'''{directory}/{lowercase_model_name}.md'''		,				F'''{path_to_transformer_root}/docs/source/en/model_doc/{lowercase_model_name}.md'''		,				)

        shutil.move(
            F'''{directory}/tokenization_{lowercase_model_name}.py'''		,				F'''{model_dir}/tokenization_{lowercase_model_name}.py'''		,				)

        shutil.move(
            F'''{directory}/tokenization_fast_{lowercase_model_name}.py'''		,				F'''{model_dir}/tokenization_{lowercase_model_name}_fast.py'''		,				)

        from os import fdopen, remove
        from shutil import copymode, move
        from tempfile import mkstemp

        def replace(lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__							):
          # Create temp file
          a__ ,							a__     : List[str]  =mkstemp()
          a__     : Union[str, Any]  =False
          with fdopen(lowerCAmelCase__		,				"w"							) as new_file:
            with open(lowerCAmelCase__							) as old_file:
              for line in old_file:
                new_file.write(lowerCAmelCase__							)
                if line_to_copy_below in line:
                  a__     : Optional[int]  =True
                  for line_to_copy in lines_to_copy:
                    new_file.write(lowerCAmelCase__							)

          if not line_found:
            raise ValueError(F'''Line {line_to_copy_below} was not found in file.'''							)

          # Copy the file permissions from the old file to the new file
          copymode(lowerCAmelCase__		,				lowerCAmelCase__							)
          # Remove original file
          remove(lowerCAmelCase__							)
          # Move new file
          move(lowerCAmelCase__		,				lowerCAmelCase__							)

        def skip_units(lowerCAmelCase__							):
          return (
              ("generating PyTorch" in line and not output_pytorch)
              or ("generating TensorFlow" in line and not output_tensorflow)
              or ("generating Flax" in line and not output_flax)
          )

        def replace_in_files(lowerCAmelCase__							):
          with open(lowerCAmelCase__							) as datafile:
            a__     : Union[str, Any]  =[]
            a__     : Tuple  =False
            a__     : Dict  =False
            for line in datafile:
              if "# To replace in: " in line and "##" not in line:
                a__     : List[str]  =line.split("\""							)[1]
                a__     : Any  =skip_units(lowerCAmelCase__							)
              elif "# Below: " in line and "##" not in line:
                a__     : Dict  =line.split("\""							)[1]
                a__     : List[str]  =skip_units(lowerCAmelCase__							)
              elif "# End." in line and "##" not in line:
                if not skip_file and not skip_snippet:
                  replace(lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__							)

                a__     : Optional[Any]  =[]
              elif "# Replace with" in line and "##" not in line:
                a__     : Tuple  =[]
              elif "##" not in line:
                lines_to_copy.append(lowerCAmelCase__							)

          remove(lowerCAmelCase__							)

        replace_in_files(F'''{directory}/to_replace_{lowercase_model_name}.py'''							)
        os.rmdir(lowerCAmelCase__							)













from typing import Optional

from .. import Features, NamedSplit
from ..packaged_modules.text.text import Text
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader







class     __lowerCAmelCase	(   UpperCamelCase__):







      def __init__(   self		,				lowerCAmelCase__		,				lowerCAmelCase__ = None		,				lowerCAmelCase__ = None		,				lowerCAmelCase__ = None		,				lowerCAmelCase__ = False		,				lowerCAmelCase__ = False		,				lowerCAmelCase__ = None		,				**lowerCAmelCase__		,				)       ->						int:



        '''simple docstring'''




        super().__init__(
            lowerCAmelCase__		,				split=lowerCAmelCase__		,				features=lowerCAmelCase__		,				cache_dir=lowerCAmelCase__		,				keep_in_memory=lowerCAmelCase__		,				streaming=lowerCAmelCase__		,				num_proc=lowerCAmelCase__		,				**lowerCAmelCase__		,				)
        a__     : Tuple  =path_or_paths if isinstance(lowerCAmelCase__		,				lowerCAmelCase__							) else {self.split: path_or_paths}
        a__     : List[str]  =Text(
            cache_dir=lowerCAmelCase__		,				data_files=lowerCAmelCase__		,				features=lowerCAmelCase__		,				**lowerCAmelCase__		,				)







      def        _lowercase  (   self							)       ->						List[Any]:



        '''simple docstring'''




        if self.streaming:
          a__     : str  =self.builder.as_streaming_dataset(split=self.split							)
        # Build regular (map-style) dataset
        else:
          a__     : Dict  =None
          a__     : Optional[Any]  =None
          a__     : Union[str, Any]  =None
          a__     : Tuple  =None

          self.builder.download_and_prepare(
              download_config=lowerCAmelCase__		,				download_mode=lowerCAmelCase__		,				verification_mode=lowerCAmelCase__		,				base_path=lowerCAmelCase__		,				num_proc=self.num_proc		,				)
          a__     : Tuple  =self.builder.as_dataset(
              split=self.split		,				verification_mode=lowerCAmelCase__		,				in_memory=self.keep_in_memory							)
        return dataset






