

from argparse import ArgumentParser, Namespace

from ..utils import logging
from . import BaseTransformersCLICommand







def       A_				(   _UpperCAmelCase   ):
						return ConvertCommand(
						    args.model_type    ,     args.tf_checkpoint    ,     args.pytorch_dump_output    ,     args.config    ,     args.finetuning_task_name   )


lowerCAmelCase							:  Optional[Any]            =						"""
transformers can only be used from the commandline to convert TensorFlow models in PyTorch, In that case, it requires
TensorFlow to be installed. Please see https://www.tensorflow.org/install/ for installation instructions.
"""



class   __lowercase    (			UpperCAmelCase_     ):




			"""simple docstring"""







			@staticmethod
			def 	_SCREAMING_SNAKE_CASE      (			lowerCAmelCase__  :					ArgumentParser):
									SCREAMING_SNAKE_CASE_:			Tuple  		=	parser.add_parser(
									    "convert"       ,						help="CLI tool to run convert model from original author checkpoints to Transformers PyTorch checkpoints."       ,						)
									train_parser.add_argument("--model_type"       ,						type=lowerCAmelCase__       ,						required=lowerCAmelCase__       ,						help="Model's type.")
									train_parser.add_argument(
									    "--tf_checkpoint"       ,						type=lowerCAmelCase__       ,						required=lowerCAmelCase__       ,						help="TensorFlow checkpoint path or folder.")
									train_parser.add_argument(
									    "--pytorch_dump_output"       ,						type=lowerCAmelCase__       ,						required=lowerCAmelCase__       ,						help="Path to the PyTorch saved model output.")
									train_parser.add_argument("--config"       ,						type=lowerCAmelCase__       ,						default=""       ,						help="Configuration file path or folder.")
									train_parser.add_argument(
									    "--finetuning_task_name"       ,						type=lowerCAmelCase__       ,						default=lowerCAmelCase__       ,						help="Optional fine-tuning task name if the TF model was a finetuned model."       ,						)
									train_parser.set_defaults(func=lowerCAmelCase__)



			def __init__(			self  :					Optional[int]       ,						lowerCAmelCase__  :					str       ,						lowerCAmelCase__  :					str       ,						lowerCAmelCase__  :					str       ,						lowerCAmelCase__  :					str       ,						lowerCAmelCase__  :					str       ,						*lowerCAmelCase__  :					Optional[Any]       ,						):
									SCREAMING_SNAKE_CASE_:			Tuple  		=	logging.get_logger("transformers-cli/converting")

									self._logger.info(F"Loading model {model_type}")
									SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	model_type
									SCREAMING_SNAKE_CASE_:			Dict  		=	tf_checkpoint
									SCREAMING_SNAKE_CASE_:			str  		=	pytorch_dump_output
									SCREAMING_SNAKE_CASE_:			Any  		=	config
									SCREAMING_SNAKE_CASE_:			str  		=	finetuning_task_name




			def 	_SCREAMING_SNAKE_CASE      (			self  :					int):
									if self._model_type == "albert":
															try:
																					from ..models.albert.convert_albert_original_tf_checkpoint_to_pytorch import (
																					    convert_tf_checkpoint_to_pytorch,
																					)
															except ImportError:
																					raise ImportError(lowerCAmelCase__)

															convert_tf_checkpoint_to_pytorch(self._tf_checkpoint       ,						self._config       ,						self._pytorch_dump_output)
									elif self._model_type == "bert":
															try:
																					from ..models.bert.convert_bert_original_tf_checkpoint_to_pytorch import (
																					    convert_tf_checkpoint_to_pytorch,
																					)
															except ImportError:
																					raise ImportError(lowerCAmelCase__)

															convert_tf_checkpoint_to_pytorch(self._tf_checkpoint       ,						self._config       ,						self._pytorch_dump_output)
									elif self._model_type == "funnel":
															try:
																					from ..models.funnel.convert_funnel_original_tf_checkpoint_to_pytorch import (
																					    convert_tf_checkpoint_to_pytorch,
																					)
															except ImportError:
																					raise ImportError(lowerCAmelCase__)

															convert_tf_checkpoint_to_pytorch(self._tf_checkpoint       ,						self._config       ,						self._pytorch_dump_output)
									elif self._model_type == "t5":
															try:
																					from ..models.ta.convert_ta_original_tf_checkpoint_to_pytorch import convert_tf_checkpoint_to_pytorch
															except ImportError:
																					raise ImportError(lowerCAmelCase__)

															convert_tf_checkpoint_to_pytorch(self._tf_checkpoint       ,						self._config       ,						self._pytorch_dump_output)
									elif self._model_type == "gpt":
															from ..models.openai.convert_openai_original_tf_checkpoint_to_pytorch import (
															    convert_openai_checkpoint_to_pytorch,
															)

															convert_openai_checkpoint_to_pytorch(self._tf_checkpoint       ,						self._config       ,						self._pytorch_dump_output)
									elif self._model_type == "transfo_xl":
															try:
																					from ..models.transfo_xl.convert_transfo_xl_original_tf_checkpoint_to_pytorch import (
																					    convert_transfo_xl_checkpoint_to_pytorch,
																					)
															except ImportError:
																					raise ImportError(lowerCAmelCase__)

															if "ckpt" in self._tf_checkpoint.lower():
																					SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	self._tf_checkpoint
																					SCREAMING_SNAKE_CASE_:			List[Any]  		=	""
															else:
																					SCREAMING_SNAKE_CASE_:			Tuple  		=	self._tf_checkpoint
																					SCREAMING_SNAKE_CASE_:			Any  		=	""
															convert_transfo_xl_checkpoint_to_pytorch(
															    lowerCAmelCase__       ,						self._config       ,						self._pytorch_dump_output       ,						lowerCAmelCase__)
									elif self._model_type == "gpt2":
															try:
																					from ..models.gpta.convert_gpta_original_tf_checkpoint_to_pytorch import (
																					    convert_gpta_checkpoint_to_pytorch,
																					)
															except ImportError:
																					raise ImportError(lowerCAmelCase__)

															convert_gpta_checkpoint_to_pytorch(self._tf_checkpoint       ,						self._config       ,						self._pytorch_dump_output)
									elif self._model_type == "xlnet":
															try:
																					from ..models.xlnet.convert_xlnet_original_tf_checkpoint_to_pytorch import (
																					    convert_xlnet_checkpoint_to_pytorch,
																					)
															except ImportError:
																					raise ImportError(lowerCAmelCase__)

															convert_xlnet_checkpoint_to_pytorch(
															    self._tf_checkpoint       ,						self._config       ,						self._pytorch_dump_output       ,						self._finetuning_task_name)
									elif self._model_type == "xlm":
															from ..models.xlm.convert_xlm_original_pytorch_checkpoint_to_pytorch import (
															    convert_xlm_checkpoint_to_pytorch,
															)

															convert_xlm_checkpoint_to_pytorch(self._tf_checkpoint       ,						self._pytorch_dump_output)
									elif self._model_type == "lxmert":
															from ..models.lxmert.convert_lxmert_original_tf_checkpoint_to_pytorch import (
															    convert_lxmert_checkpoint_to_pytorch,
															)

															convert_lxmert_checkpoint_to_pytorch(self._tf_checkpoint       ,						self._pytorch_dump_output)
									elif self._model_type == "rembert":
															from ..models.rembert.convert_rembert_tf_checkpoint_to_pytorch import (
															    convert_rembert_tf_checkpoint_to_pytorch,
															)

															convert_rembert_tf_checkpoint_to_pytorch(self._tf_checkpoint       ,						self._config       ,						self._pytorch_dump_output)
									else:
															raise ValueError(
															    "--model_type should be selected in the list [bert, gpt, gpt2, t5, transfo_xl, xlnet, xlm, lxmert]")









import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


lowerCAmelCase							:  Any            =						logging.get_logger(__name__)

lowerCAmelCase							:  Dict            =						{"""vocab_file""": """vocab.txt"""}

lowerCAmelCase							:  List[str]            =						{
    """vocab_file""": {
        """YituTech/conv-bert-base""": """https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt""",
        """YituTech/conv-bert-medium-small""": (
            """https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt"""
        ),
        """YituTech/conv-bert-small""": """https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt""",
    }
}

lowerCAmelCase							:  List[Any]            =						{
    """YituTech/conv-bert-base""": 512,
    """YituTech/conv-bert-medium-small""": 512,
    """YituTech/conv-bert-small""": 512,
}


lowerCAmelCase							:  Tuple            =						{
    """YituTech/conv-bert-base""": {"""do_lower_case""": True},
    """YituTech/conv-bert-medium-small""": {"""do_lower_case""": True},
    """YituTech/conv-bert-small""": {"""do_lower_case""": True},
}



class   __lowercase    (			UpperCAmelCase_     ):




			"""simple docstring"""







			_UpperCAmelCase       :							str      							=					VOCAB_FILES_NAMES
			_UpperCAmelCase       :							List[str]      							=					PRETRAINED_VOCAB_FILES_MAP
			_UpperCAmelCase       :							List[Any]      							=					PRETRAINED_INIT_CONFIGURATION
			_UpperCAmelCase       :							Optional[Any]      							=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_UpperCAmelCase       :							Dict      							=					ConvBertTokenizer



			def __init__(			self  :					Optional[Any]       ,						lowerCAmelCase__  :					Optional[int]=None       ,						lowerCAmelCase__  :					List[str]=None       ,						lowerCAmelCase__  :					str=True       ,						lowerCAmelCase__  :					Any="[UNK]"       ,						lowerCAmelCase__  :					Optional[Any]="[SEP]"       ,						lowerCAmelCase__  :					Any="[PAD]"       ,						lowerCAmelCase__  :					Dict="[CLS]"       ,						lowerCAmelCase__  :					Dict="[MASK]"       ,						lowerCAmelCase__  :					Tuple=True       ,						lowerCAmelCase__  :					str=None       ,						**lowerCAmelCase__  :					Dict       ,						):
									super().__init__(
									    lowerCAmelCase__       ,						tokenizer_file=lowerCAmelCase__       ,						do_lower_case=lowerCAmelCase__       ,						unk_token=lowerCAmelCase__       ,						sep_token=lowerCAmelCase__       ,						pad_token=lowerCAmelCase__       ,						cls_token=lowerCAmelCase__       ,						mask_token=lowerCAmelCase__       ,						tokenize_chinese_chars=lowerCAmelCase__       ,						strip_accents=lowerCAmelCase__       ,						**lowerCAmelCase__       ,						)

									SCREAMING_SNAKE_CASE_:			List[str]  		=	json.loads(self.backend_tokenizer.normalizer.__getstate__())
									if (
									    normalizer_state.get("lowercase"       ,						lowerCAmelCase__) != do_lower_case
									    or normalizer_state.get("strip_accents"       ,						lowerCAmelCase__) != strip_accents
									    or normalizer_state.get("handle_chinese_chars"       ,						lowerCAmelCase__) != tokenize_chinese_chars
									):
															SCREAMING_SNAKE_CASE_:			Optional[int]  		=	getattr(lowerCAmelCase__       ,						normalizer_state.pop("type"))
															SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	do_lower_case
															SCREAMING_SNAKE_CASE_:			List[str]  		=	strip_accents
															SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	tokenize_chinese_chars
															SCREAMING_SNAKE_CASE_:			Optional[int]  		=	normalizer_class(**lowerCAmelCase__)

									SCREAMING_SNAKE_CASE_:			Optional[int]  		=	do_lower_case



			def 	_SCREAMING_SNAKE_CASE      (			self  :					str       ,						lowerCAmelCase__  :					Optional[int]       ,						lowerCAmelCase__  :					Union[str, Any]=None):
									SCREAMING_SNAKE_CASE_:			List[str]  		=	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

									if token_ids_a:
															output += token_ids_a + [self.sep_token_id]

									return output



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Optional[Any]       ,						lowerCAmelCase__  :					List[int]       ,						lowerCAmelCase__  :					Optional[List[int]] = None):
									SCREAMING_SNAKE_CASE_:			Optional[int]  		=	[self.sep_token_id]
									SCREAMING_SNAKE_CASE_:			int  		=	[self.cls_token_id]
									if token_ids_a is None:
															return len(cls + token_ids_a + sep) * [0]
									return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]




			def 	_SCREAMING_SNAKE_CASE      (			self  :					List[Any]       ,						lowerCAmelCase__  :					str       ,						lowerCAmelCase__  :					Optional[str] = None):
									SCREAMING_SNAKE_CASE_:			Any  		=	self._tokenizer.model.save(lowerCAmelCase__       ,						name=lowerCAmelCase__)
									return tuple(lowerCAmelCase__)






