





import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

UpperCamelCase__     =	logging.get_logger(__name__)


UpperCamelCase__     =	{name: getattr(transformers, name + "Fast") for name in SLOW_TO_FAST_CONVERTERS}





def       _UpperCamelCase       (a__				:Optional[Any]				,     a__				:Union[str, Any]				,     a__				:str				,     a__				:List[str]    ):





							if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
														raise ValueError(f"""Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()    )}."""    )

							if tokenizer_name is None:
														UpperCamelCase__			       =     TOKENIZER_CLASSES
							else:
														UpperCamelCase__			       =     {tokenizer_name: getattr(a__				,     tokenizer_name + """Fast"""    )}

							logger.info(f"""Loading tokenizer classes: {tokenizer_names}"""    )

							for tokenizer_name in tokenizer_names:
														UpperCamelCase__			       =     TOKENIZER_CLASSES[tokenizer_name]

														UpperCamelCase__			       =     True
														if checkpoint_name is None:
																					UpperCamelCase__			       =     list(tokenizer_class.max_model_input_sizes.keys()    )
														else:
																					UpperCamelCase__			       =     [checkpoint_name]

														logger.info(f"""For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}"""    )

														for checkpoint in checkpoint_names:
																					logger.info(f"""Loading {tokenizer_class.__class__.__name__} {checkpoint}"""    )

																					# Load tokenizer
																					UpperCamelCase__			       =     tokenizer_class.from_pretrained(a__				,     force_download=a__    )

																					# Save fast tokenizer
																					logger.info(f"""Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}"""    )

																					# For organization names we create sub-directories
																					if "/" in checkpoint:
																												UpperCamelCase__       ,       UpperCamelCase__			       =     checkpoint.split("""/"""    )
																												UpperCamelCase__			       =     os.path.join(a__				,     a__    )
																					elif add_prefix:
																												UpperCamelCase__			       =     checkpoint
																												UpperCamelCase__			       =     dump_path
																					else:
																												UpperCamelCase__			       =     None
																												UpperCamelCase__			       =     dump_path

																					logger.info(f"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""    )

																					if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()    )[0]:
																												UpperCamelCase__			       =     list(tokenizer.pretrained_vocab_files_map.values()    )[0][checkpoint]
																												UpperCamelCase__			       =     file_path.split(a__    )[-1][0]
																												if next_char == "/":
																																			UpperCamelCase__			       =     os.path.join(a__				,     a__    )
																																			UpperCamelCase__			       =     None

																												logger.info(f"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""    )

																					UpperCamelCase__			       =     tokenizer.save_pretrained(
																					    a__				,     legacy_format=a__				,     filename_prefix=a__    )
																					logger.info(f"""=> File names {file_names}"""    )

																					for file_name in file_names:
																												if not file_name.endswith("""tokenizer.json"""    ):
																																			os.remove(a__    )
																																			logger.info(f"""=> removing {file_name}"""    )


if __name__ == "__main__":
					UpperCamelCase__     =	argparse.ArgumentParser()
					# Required parameters
					parser.add_argument(
					    "--dump_path", default=None, type=str, required=True, help="Path to output generated fast tokenizer files."
					)
					parser.add_argument(
					    "--tokenizer_name",
					    default=None,
					    type=str,
					    help=(
					        f"""Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will """
					        "download and convert all the checkpoints from AWS."
					    ),
					)
					parser.add_argument(
					    "--checkpoint_name",
					    default=None,
					    type=str,
					    help="Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.",
					)
					parser.add_argument(
					    "--force_download",
					    action="store_true",
					    help="Re-download checkpoints.",
					)
					UpperCamelCase__     =	parser.parse_args()

					convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)











from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


UpperCamelCase__     =	{
    "configuration_swiftformer": [
        "SWIFTFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "SwiftFormerConfig",
        "SwiftFormerOnnxConfig",
    ]
}

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					UpperCamelCase__     =	[
					    "SWIFTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST",
					    "SwiftFormerForImageClassification",
					    "SwiftFormerModel",
					    "SwiftFormerPreTrainedModel",
					]

if TYPE_CHECKING:
					from .configuration_swiftformer import (
					    SWIFTFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP,
					    SwiftFormerConfig,
					    SwiftFormerOnnxConfig,
					)

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_swiftformer import (
										    SWIFTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
										    SwiftFormerForImageClassification,
										    SwiftFormerModel,
										    SwiftFormerPreTrainedModel,
										)

else:
					import sys

					UpperCamelCase__     =	_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)




