

'''simple docstring'''



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


__snake_case			:     int										=      logging.get_logger(__name__)

__snake_case			:     int										=      {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__snake_case			:     str										=      {
    'vocab_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-german-cased': 'https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt',
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-german-cased': (
            'https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json'
        ),
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
    },
}

__snake_case			:     Optional[Any]										=      {
    'distilbert-base-uncased': 512,
    'distilbert-base-uncased-distilled-squad': 512,
    'distilbert-base-cased': 512,
    'distilbert-base-cased-distilled-squad': 512,
    'distilbert-base-german-cased': 512,
    'distilbert-base-multilingual-cased': 512,
}


__snake_case			:     List[str]										=      {
    'distilbert-base-uncased': {'do_lower_case': True},
    'distilbert-base-uncased-distilled-squad': {'do_lower_case': True},
    'distilbert-base-cased': {'do_lower_case': False},
    'distilbert-base-cased-distilled-squad': {'do_lower_case': False},
    'distilbert-base-german-cased': {'do_lower_case': False},
    'distilbert-base-multilingual-cased': {'do_lower_case': False},
}



class lowerCamelCase			(  lowercase_						):



			'''simple docstring'''






			__snake_case        =	VOCAB_FILES_NAMES
			__snake_case        =	PRETRAINED_VOCAB_FILES_MAP
			__snake_case        =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			__snake_case        =	PRETRAINED_INIT_CONFIGURATION
			__snake_case        =	['input_ids', 'attention_mask']
			__snake_case        =	DistilBertTokenizer


			def __init__(				self	:						Dict		,      lowerCAmelCase_	:						int=None		,      lowerCAmelCase_	:						List[str]=None		,      lowerCAmelCase_	:						Optional[int]=True		,      lowerCAmelCase_	:						Any="[UNK]"		,      lowerCAmelCase_	:						Dict="[SEP]"		,      lowerCAmelCase_	:						Union[str, Any]="[PAD]"		,      lowerCAmelCase_	:						Union[str, Any]="[CLS]"		,      lowerCAmelCase_	:						List[Any]="[MASK]"		,      lowerCAmelCase_	:						Union[str, Any]=True		,      lowerCAmelCase_	:						Optional[Any]=None		,      **lowerCAmelCase_	:						Any		,      )      ->		List[Any]:





									'''simple docstring'''
									super().__init__(
									    lowerCAmelCase_		,      tokenizer_file=lowerCAmelCase_		,      do_lower_case=lowerCAmelCase_		,      unk_token=lowerCAmelCase_		,      sep_token=lowerCAmelCase_		,      pad_token=lowerCAmelCase_		,      cls_token=lowerCAmelCase_		,      mask_token=lowerCAmelCase_		,      tokenize_chinese_chars=lowerCAmelCase_		,      strip_accents=lowerCAmelCase_		,      **lowerCAmelCase_		,      )

									A__					:       int     =json.loads(self.backend_tokenizer.normalizer.__getstate__()			)
									if (
									    normalizer_state.get("""lowercase"""		,      lowerCAmelCase_			) != do_lower_case
									    or normalizer_state.get("""strip_accents"""		,      lowerCAmelCase_			) != strip_accents
									    or normalizer_state.get("""handle_chinese_chars"""		,      lowerCAmelCase_			) != tokenize_chinese_chars
									):
															A__					:       int     =getattr(lowerCAmelCase_		,      normalizer_state.pop("""type"""			)			)
															A__					:       Tuple     =do_lower_case
															A__					:       Optional[Any]     =strip_accents
															A__					:       int     =tokenize_chinese_chars
															A__					:       Optional[int]     =normalizer_class(**lowerCAmelCase_			)

									A__					:       List[str]     =do_lower_case


			def lowercase__						(				self	:						Tuple		,      lowerCAmelCase_	:						Optional[Any]		,      lowerCAmelCase_	:						int=None			)      ->		Any:





									'''simple docstring'''
									A__					:       Optional[Any]     =[self.cls_token_id] + token_ids_a + [self.sep_token_id]

									if token_ids_a:
															output += token_ids_a + [self.sep_token_id]

									return output


			def lowercase__						(				self	:						List[Any]		,      lowerCAmelCase_	:						List[int]		,      lowerCAmelCase_	:						Optional[List[int]] = None			)      ->		List[int]:





									'''simple docstring'''
									A__					:       Optional[int]     =[self.sep_token_id]
									A__					:       Tuple     =[self.cls_token_id]
									if token_ids_a is None:
															return len(cls + token_ids_a + sep			) * [0]
									return len(cls + token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]



			def lowercase__						(				self	:						Union[str, Any]		,      lowerCAmelCase_	:						str		,      lowerCAmelCase_	:						Optional[str] = None			)      ->		Tuple[str]:





									'''simple docstring'''
									A__					:       List[str]     =self._tokenizer.model.save(lowerCAmelCase_		,      name=lowerCAmelCase_			)
									return tuple(lowerCAmelCase_			)




'''simple docstring'''



from argparse import ArgumentParser

from .add_new_model import AddNewModelCommand
from .add_new_model_like import AddNewModelLikeCommand
from .convert import ConvertCommand
from .download import DownloadCommand
from .env import EnvironmentCommand
from .lfs import LfsCommands
from .pt_to_tf import PTtoTFCommand
from .run import RunCommand
from .serving import ServeCommand
from .user import UserCommands
def 						__lowerCamelCase	(       )	->							List[Any]:






						"""simple docstring"""





						A__					:       List[Any]     =ArgumentParser("""Transformers CLI tool""",		usage="""transformers-cli <command> [<args>]"""	)
						A__					:       List[Any]     =parser.add_subparsers(help="""transformers-cli command helpers"""	)

						# Register commands
						ConvertCommand.register_subcommand(__snake_case	)
						DownloadCommand.register_subcommand(__snake_case	)
						EnvironmentCommand.register_subcommand(__snake_case	)
						RunCommand.register_subcommand(__snake_case	)
						ServeCommand.register_subcommand(__snake_case	)
						UserCommands.register_subcommand(__snake_case	)
						AddNewModelCommand.register_subcommand(__snake_case	)
						AddNewModelLikeCommand.register_subcommand(__snake_case	)
						LfsCommands.register_subcommand(__snake_case	)
						PTtoTFCommand.register_subcommand(__snake_case	)

						# Let's go
						A__					:       List[str]     =parser.parse_args()

						if not hasattr(__snake_case,		"""func"""	):
												parser.print_help()
												exit(1	)

						# Run
						A__					:       Optional[Any]     =args.func(__snake_case	)
						service.run()


if __name__ == "__main__":
		main()

