





import json
import os
import shutil
import warnings
from argparse import ArgumentParser, Namespace
from pathlib import Path
from typing import List

from ..utils import logging
from . import BaseTransformersCLICommand


try:
					from cookiecutter.main import cookiecutter

					UpperCAmelCase     :     Dict							=	True
except ImportError:
					UpperCAmelCase     :     Optional[Any]							=	False

UpperCAmelCase     :     Optional[Any]							=	logging.get_logger(__name__)  # pylint: disable=invalid-name
def 					_A	(		SCREAMING_SNAKE_CASE					:       Namespace						):





		"""simple docstring"""



		return AddNewModelCommand(args.testing	,	args.testing_file	,	path=args.path						)







class     __lowerCAmelCase	(   UpperCamelCase__):







						@staticmethod
						def        _lowercase  (   lowerCAmelCase__							)       ->						Optional[int]:



								'''simple docstring'''




								a__     : Union[str, Any]  =parser.add_parser("add-new-model"							)
								add_new_model_parser.add_argument("--testing"		,				action="store_true"		,				help="If in testing mode."							)
								add_new_model_parser.add_argument("--testing_file"		,				type=lowerCAmelCase__		,				help="Configuration file on which to run."							)
								add_new_model_parser.add_argument(
								    "--path"		,				type=lowerCAmelCase__		,				help="Path to cookiecutter. Should only be used for testing purposes."							)
								add_new_model_parser.set_defaults(func=lowerCAmelCase__							)







						def __init__(   self		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__=None		,				*lowerCAmelCase__							)       ->						List[Any]:



								'''simple docstring'''




								a__     : List[str]  =testing
								a__     : str  =testing_file
								a__     : Union[str, Any]  =path







						def        _lowercase  (   self							)       ->						Optional[Any]:



								'''simple docstring'''




								warnings.warn(
								    "The command `transformers-cli add-new-model` is deprecated and will be removed in v5 of Transformers. "
								    "It is not actively maintained anymore, so might give a result that won't pass all tests and quality "
								    "checks, you should use `transformers-cli add-new-model-like` instead."							)
								if not _has_cookiecutter:
										raise ImportError(
										    "Model creation dependencies are required to use the `add_new_model` command. Install them by running "
										    "the following at the root of your `transformers` clone:\n\n\t$ pip install -e .[modelcreation]\n"							)
								# Ensure that there is no other `cookiecutter-template-xxx` directory in the current working directory
								a__     : Union[str, Any]  =[directory for directory in os.listdir() if "cookiecutter-template-" == directory[:2_2]]
								if len(lowerCAmelCase__							) > 0:
										raise ValueError(
										    "Several directories starting with `cookiecutter-template-` in current working directory. "
										    "Please clean your directory by removing all folders starting with `cookiecutter-template-` or "
										    "change your working directory."							)

								a__     : Tuple  =(
								    Path(lowerCAmelCase__							).parent.parent.parent.parent if self._path is None else Path(self._path							).parent.parent
								)
								a__     : List[Any]  =path_to_transformer_root / "templates" / "adding_a_new_model"

								# Execute cookiecutter
								if not self._testing:
										cookiecutter(str(lowerCAmelCase__							)							)
								else:
										with open(self._testing_file		,				"r"							) as configuration_file:
												a__     : Optional[Any]  =json.load(lowerCAmelCase__							)

										cookiecutter(
										    str(path_to_cookiecutter if self._path is None else self._path							)		,				no_input=lowerCAmelCase__		,				extra_context=lowerCAmelCase__		,				)

								a__     : str  =[directory for directory in os.listdir() if "cookiecutter-template-" in directory[:2_2]][0]

								# Retrieve configuration
								with open(directory + "/configuration.json"		,				"r"							) as configuration_file:
										a__     : Tuple  =json.load(lowerCAmelCase__							)

								a__     : Union[str, Any]  =configuration["lowercase_modelname"]
								a__     : Optional[Any]  =configuration["generate_tensorflow_pytorch_and_flax"]
								os.remove(F'''{directory}/configuration.json'''							)

								a__     : int  ="PyTorch" in generate_tensorflow_pytorch_and_flax
								a__     : Union[str, Any]  ="TensorFlow" in generate_tensorflow_pytorch_and_flax
								a__     : List[str]  ="Flax" in generate_tensorflow_pytorch_and_flax

								a__     : Any  =F'''{path_to_transformer_root}/src/transformers/models/{lowercase_model_name}'''
								os.makedirs(lowerCAmelCase__		,				exist_ok=lowerCAmelCase__							)
								os.makedirs(F'''{path_to_transformer_root}/tests/models/{lowercase_model_name}'''		,				exist_ok=lowerCAmelCase__							)

								# Tests require submodules as they have parent imports
								with open(F'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/__init__.py'''		,				"w"							):
										pass

								shutil.move(
								    F'''{directory}/__init__.py'''		,				F'''{model_dir}/__init__.py'''		,				)
								shutil.move(
								    F'''{directory}/configuration_{lowercase_model_name}.py'''		,				F'''{model_dir}/configuration_{lowercase_model_name}.py'''		,				)

								def remove_copy_lines(lowerCAmelCase__							):
										with open(lowerCAmelCase__		,				"r"							) as f:
												a__     : Tuple  =f.readlines()
										with open(lowerCAmelCase__		,				"w"							) as f:
												for line in lines:
														if "# Copied from transformers." not in line:
																f.write(lowerCAmelCase__							)

								if output_pytorch:
										if not self._testing:
												remove_copy_lines(F'''{directory}/modeling_{lowercase_model_name}.py'''							)

										shutil.move(
										    F'''{directory}/modeling_{lowercase_model_name}.py'''		,				F'''{model_dir}/modeling_{lowercase_model_name}.py'''		,				)

										shutil.move(
										    F'''{directory}/test_modeling_{lowercase_model_name}.py'''		,				F'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_{lowercase_model_name}.py'''		,				)
								else:
										os.remove(F'''{directory}/modeling_{lowercase_model_name}.py'''							)
										os.remove(F'''{directory}/test_modeling_{lowercase_model_name}.py'''							)

								if output_tensorflow:
										if not self._testing:
												remove_copy_lines(F'''{directory}/modeling_tf_{lowercase_model_name}.py'''							)

										shutil.move(
										    F'''{directory}/modeling_tf_{lowercase_model_name}.py'''		,				F'''{model_dir}/modeling_tf_{lowercase_model_name}.py'''		,				)

										shutil.move(
										    F'''{directory}/test_modeling_tf_{lowercase_model_name}.py'''		,				F'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_tf_{lowercase_model_name}.py'''		,				)
								else:
										os.remove(F'''{directory}/modeling_tf_{lowercase_model_name}.py'''							)
										os.remove(F'''{directory}/test_modeling_tf_{lowercase_model_name}.py'''							)

								if output_flax:
										if not self._testing:
												remove_copy_lines(F'''{directory}/modeling_flax_{lowercase_model_name}.py'''							)

										shutil.move(
										    F'''{directory}/modeling_flax_{lowercase_model_name}.py'''		,				F'''{model_dir}/modeling_flax_{lowercase_model_name}.py'''		,				)

										shutil.move(
										    F'''{directory}/test_modeling_flax_{lowercase_model_name}.py'''		,				F'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_flax_{lowercase_model_name}.py'''		,				)
								else:
										os.remove(F'''{directory}/modeling_flax_{lowercase_model_name}.py'''							)
										os.remove(F'''{directory}/test_modeling_flax_{lowercase_model_name}.py'''							)

								shutil.move(
								    F'''{directory}/{lowercase_model_name}.md'''		,				F'''{path_to_transformer_root}/docs/source/en/model_doc/{lowercase_model_name}.md'''		,				)

								shutil.move(
								    F'''{directory}/tokenization_{lowercase_model_name}.py'''		,				F'''{model_dir}/tokenization_{lowercase_model_name}.py'''		,				)

								shutil.move(
								    F'''{directory}/tokenization_fast_{lowercase_model_name}.py'''		,				F'''{model_dir}/tokenization_{lowercase_model_name}_fast.py'''		,				)

								from os import fdopen, remove
								from shutil import copymode, move
								from tempfile import mkstemp

								def replace(lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__							):
										# Create temp file
										a__     : List[str]  =mkstemp()
										a__     : Union[str, Any]  =False
										with fdopen(lowerCAmelCase__		,				"w"							) as new_file:
												with open(lowerCAmelCase__							) as old_file:
														for line in old_file:
																new_file.write(lowerCAmelCase__							)
																if line_to_copy_below in line:
																		a__     : Optional[int]  =True
																		for line_to_copy in lines_to_copy:
																				new_file.write(lowerCAmelCase__							)

										if not line_found:
												raise ValueError(F'''Line {line_to_copy_below} was not found in file.'''							)

										# Copy the file permissions from the old file to the new file
										copymode(lowerCAmelCase__		,				lowerCAmelCase__							)
										# Remove original file
										remove(lowerCAmelCase__							)
										# Move new file
										move(lowerCAmelCase__		,				lowerCAmelCase__							)

								def skip_units(lowerCAmelCase__							):
										return (
										    ("generating PyTorch" in line and not output_pytorch)
										    or ("generating TensorFlow" in line and not output_tensorflow)
										    or ("generating Flax" in line and not output_flax)
										)

								def replace_in_files(lowerCAmelCase__							):
										with open(lowerCAmelCase__							) as datafile:
												a__     : Union[str, Any]  =[]
												a__     : Tuple  =False
												a__     : Dict  =False
												for line in datafile:
														if "# To replace in: " in line and "##" not in line:
																a__     : List[str]  =line.split("\""							)[1]
																a__     : Any  =skip_units(lowerCAmelCase__							)
														elif "# Below: " in line and "##" not in line:
																a__     : Dict  =line.split("\""							)[1]
																a__     : List[str]  =skip_units(lowerCAmelCase__							)
														elif "# End." in line and "##" not in line:
																if not skip_file and not skip_snippet:
																		replace(lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__							)

																a__     : Optional[Any]  =[]
														elif "# Replace with" in line and "##" not in line:
																a__     : Tuple  =[]
														elif "##" not in line:
																lines_to_copy.append(lowerCAmelCase__							)

										remove(lowerCAmelCase__							)

								replace_in_files(F'''{directory}/to_replace_{lowercase_model_name}.py'''							)
								os.rmdir(lowerCAmelCase__							)













import warnings

from transformers import AutoTokenizer
from transformers.utils import is_torch_available
from transformers.utils.generic import ExplicitEnum

from ...processing_utils import ProcessorMixin


if is_torch_available():
     import torch







class     __lowerCAmelCase	(   UpperCamelCase__):
      _lowercase    : Any            =      """char"""
      _lowercase    : str            =      """bpe"""
      _lowercase    : List[Any]            =      """wp"""
UpperCAmelCase     :     List[Any]							=	(DecodeType.CHARACTER, DecodeType.BPE, DecodeType.WORDPIECE)







class     __lowerCAmelCase	(   UpperCamelCase__):
      _lowercase    : Tuple            =      ["""image_processor""", """char_tokenizer"""]
      _lowercase    : Any            =      """ViTImageProcessor"""
      _lowercase    : Optional[Any]            =      """MgpstrTokenizer"""







      def __init__(   self		,				lowerCAmelCase__=None		,				lowerCAmelCase__=None		,				**lowerCAmelCase__							)       ->						Optional[Any]:



        '''simple docstring'''




        a__     : Optional[int]  =None
        if "feature_extractor" in kwargs:
          warnings.warn(
              "The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"
              " instead."		,				lowerCAmelCase__		,				)
          a__     : List[str]  =kwargs.pop("feature_extractor"							)

        a__     : List[str]  =image_processor if image_processor is not None else feature_extractor
        if image_processor is None:
          raise ValueError("You need to specify an `image_processor`."							)
        if tokenizer is None:
          raise ValueError("You need to specify a `tokenizer`."							)

        a__     : str  =tokenizer
        a__     : List[str]  =AutoTokenizer.from_pretrained("gpt2"							)
        a__     : Optional[int]  =AutoTokenizer.from_pretrained("bert-base-uncased"							)

        super().__init__(lowerCAmelCase__		,				lowerCAmelCase__							)







      def __call__(   self		,				lowerCAmelCase__=None		,				lowerCAmelCase__=None		,				lowerCAmelCase__=None		,				**lowerCAmelCase__							)       ->						Union[str, Any]:



        '''simple docstring'''




        if images is None and text is None:
          raise ValueError("You need to specify either an `images` or `text` input to process."							)

        if images is not None:
          a__     : Union[str, Any]  =self.image_processor(lowerCAmelCase__		,				return_tensors=lowerCAmelCase__		,				**lowerCAmelCase__							)
        if text is not None:
          a__     : int  =self.char_tokenizer(lowerCAmelCase__		,				return_tensors=lowerCAmelCase__		,				**lowerCAmelCase__							)

        if text is None:
          return inputs
        elif images is None:
          return encodings
        else:
          a__     : Tuple  =encodings["input_ids"]
          return inputs







      def        _lowercase  (   self		,				lowerCAmelCase__							)       ->						Optional[Any]:



        '''simple docstring'''




        a__ ,							a__ ,							a__     : Any  =sequences
        a__     : Union[str, Any]  =char_preds.size(0							)

        a__ ,							a__     : Dict  =self._decode_helper(lowerCAmelCase__		,				"char"							)
        a__ ,							a__     : List[Any]  =self._decode_helper(lowerCAmelCase__		,				"bpe"							)
        a__ ,							a__     : Optional[int]  =self._decode_helper(lowerCAmelCase__		,				"wp"							)

        a__     : List[Any]  =[]
        a__     : Dict  =[]
        for i in range(lowerCAmelCase__							):
          a__     : int  =[char_scores[i], bpe_scores[i], wp_scores[i]]
          a__     : Tuple  =[char_strs[i], bpe_strs[i], wp_strs[i]]
          a__     : Any  =scores.index(max(lowerCAmelCase__							)							)
          final_strs.append(strs[max_score_index]							)
          final_scores.append(scores[max_score_index]							)

        a__     : Dict  ={}
        a__     : str  =final_strs
        a__     : Optional[int]  =final_scores
        a__     : Union[str, Any]  =char_strs
        a__     : List[str]  =bpe_strs
        a__     : Union[str, Any]  =wp_strs
        return out







      def        _lowercase  (   self		,				lowerCAmelCase__		,				lowerCAmelCase__							)       ->						Tuple:



        '''simple docstring'''




        if format == DecodeType.CHARACTER:
          a__     : Optional[Any]  =self.char_decode
          a__     : Dict  =1
          a__     : Tuple  ="[s]"
        elif format == DecodeType.BPE:
          a__     : str  =self.bpe_decode
          a__     : Dict  =2
          a__     : Optional[int]  ="#"
        elif format == DecodeType.WORDPIECE:
          a__     : Union[str, Any]  =self.wp_decode
          a__     : List[Any]  =1_0_2
          a__     : Dict  ="[SEP]"
        else:
          raise ValueError(F'''Format {format} is not supported.'''							)

        a__ ,							a__     : Any  =[], []
        a__     : str  =pred_logits.size(0							)
        a__     : Optional[Any]  =pred_logits.size(1							)
        a__ ,							a__     : Optional[int]  =pred_logits.topk(1		,				dim=-1		,				largest=lowerCAmelCase__		,				sorted=lowerCAmelCase__							)
        a__     : Optional[Any]  =preds_index.view(-1		,				lowerCAmelCase__							)[:, 1:]
        a__     : Dict  =decoder(lowerCAmelCase__							)
        a__ ,							a__     : Any  =torch.nn.functional.softmax(lowerCAmelCase__		,				dim=2							).max(dim=2							)
        a__     : int  =preds_max_prob[:, 1:]

        for index in range(lowerCAmelCase__							):
          a__     : Optional[Any]  =preds_str[index].find(lowerCAmelCase__							)
          a__     : Optional[int]  =preds_str[index][:pred_eos]
          a__     : List[Any]  =preds_index[index].cpu().tolist()
          a__     : List[Any]  =pred_index.index(lowerCAmelCase__							) if eos_token in pred_index else -1
          a__     : Union[str, Any]  =preds_max_prob[index][: pred_eos_index + 1]
          a__     : List[Any]  =pred_max_prob.cumprod(dim=0							)[-1] if pred_max_prob.nelement() != 0 else 0.0
          dec_strs.append(lowerCAmelCase__							)
          conf_scores.append(lowerCAmelCase__							)

        return dec_strs, conf_scores







      def        _lowercase  (   self		,				lowerCAmelCase__							)       ->						str:



        '''simple docstring'''




        a__     : int  =[seq.replace(" "		,				""							) for seq in self.char_tokenizer.batch_decode(lowerCAmelCase__							)]
        return decode_strs







      def        _lowercase  (   self		,				lowerCAmelCase__							)       ->						Tuple:



        '''simple docstring'''




        return self.bpe_tokenizer.batch_decode(lowerCAmelCase__							)







      def        _lowercase  (   self		,				lowerCAmelCase__							)       ->						Optional[Any]:



        '''simple docstring'''




        a__     : List[Any]  =[seq.replace(" "		,				""							) for seq in self.wp_tokenizer.batch_decode(lowerCAmelCase__							)]
        return decode_strs






