


'''simple docstring'''



import argparse
import logging
import pickle
import random
import time

import numpy as np

from transformers import BertTokenizer, GPTaTokenizer, RobertaTokenizer


logging.basicConfig(
    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s""", datefmt="""%m/%d/%Y %H:%M:%S""", level=logging.INFO
)
_lowercase      :      Optional[Any]            =					logging.getLogger(__name__)


def 				lowerCamelCase__				(	):




						'''simple docstring'''






						UpperCAmelCase					    =     argparse.ArgumentParser(
						    description='''Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids).'''					)
						parser.add_argument('''--file_path'''				,       type=A				,       default='''data/dump.txt'''				,       help='''The path to the data.'''					)
						parser.add_argument('''--tokenizer_type'''				,       type=A				,       default='''bert'''				,       choices=['''bert''', '''roberta''', '''gpt2''']					)
						parser.add_argument('''--tokenizer_name'''				,       type=A				,       default='''bert-base-uncased'''				,       help='''The tokenizer to use.'''					)
						parser.add_argument('''--dump_file'''				,       type=A				,       default='''data/dump'''				,       help='''The dump file prefix.'''					)
						UpperCAmelCase					    =     parser.parse_args()

						logger.info(f"""Loading Tokenizer ({args.tokenizer_name})"""					)
						if args.tokenizer_type == "bert":
												UpperCAmelCase					    =     BertTokenizer.from_pretrained(args.tokenizer_name					)
												UpperCAmelCase					    =     tokenizer.special_tokens_map['''cls_token''']  # `[CLS]`
												UpperCAmelCase					    =     tokenizer.special_tokens_map['''sep_token''']  # `[SEP]`
						elif args.tokenizer_type == "roberta":
												UpperCAmelCase					    =     RobertaTokenizer.from_pretrained(args.tokenizer_name					)
												UpperCAmelCase					    =     tokenizer.special_tokens_map['''cls_token''']  # `<s>`
												UpperCAmelCase					    =     tokenizer.special_tokens_map['''sep_token''']  # `</s>`
						elif args.tokenizer_type == "gpt2":
												UpperCAmelCase					    =     GPTaTokenizer.from_pretrained(args.tokenizer_name					)
												UpperCAmelCase					    =     tokenizer.special_tokens_map['''bos_token''']  # `<|endoftext|>`
												UpperCAmelCase					    =     tokenizer.special_tokens_map['''eos_token''']  # `<|endoftext|>`

						logger.info(f"""Loading text from {args.file_path}"""					)
						with open(args.file_path				,       '''r'''				,       encoding='''utf8'''					) as fp:
												UpperCAmelCase					    =     fp.readlines()

						logger.info('''Start encoding'''					)
						logger.info(f"""{len(A					)} examples to process."""					)

						UpperCAmelCase					    =     []
						UpperCAmelCase					    =     0
						UpperCAmelCase					    =     1_00_00
						UpperCAmelCase					    =     time.time()
						for text in data:
												UpperCAmelCase					    =     f"""{bos} {text.strip()} {sep}"""
												UpperCAmelCase					    =     tokenizer.encode(A				,       add_special_tokens=A					)
												rslt.append(A					)

												iter += 1
												if iter % interval == 0:
																		UpperCAmelCase					    =     time.time()
																		logger.info(f"""{iter} examples processed. - {(end-start):.2f}s/{interval}expl"""					)
																		UpperCAmelCase					    =     time.time()
						logger.info('''Finished binarization'''					)
						logger.info(f"""{len(A					)} examples processed."""					)

						UpperCAmelCase					    =     f"""{args.dump_file}.{args.tokenizer_name}.pickle"""
						UpperCAmelCase					    =     tokenizer.vocab_size
						if vocab_size < (1 << 16):
												UpperCAmelCase					    =     [np.uintaa(A					) for d in rslt]
						else:
												UpperCAmelCase					    =     [np.intaa(A					) for d in rslt]
						random.shuffle(rslt_					)
						logger.info(f"""Dump to {dp_file}"""					)
						with open(A				,       '''wb'''					) as handle:
												pickle.dump(rslt_				,       A				,       protocol=pickle.HIGHEST_PROTOCOL					)


if __name__ == "__main__":
	main()









'''simple docstring'''



from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_lowercase      :      List[Any]            =					{
    """configuration_x_clip""": [
        """XCLIP_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """XCLIPConfig""",
        """XCLIPTextConfig""",
        """XCLIPVisionConfig""",
    ],
    """processing_x_clip""": ["""XCLIPProcessor"""],
}

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_lowercase      :      List[str]            =					[
	    """XCLIP_PRETRAINED_MODEL_ARCHIVE_LIST""",
	    """XCLIPModel""",
	    """XCLIPPreTrainedModel""",
	    """XCLIPTextModel""",
	    """XCLIPVisionModel""",
	]

if TYPE_CHECKING:
	from .configuration_x_clip import (
	    XCLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
	    XCLIPConfig,
	    XCLIPTextConfig,
	    XCLIPVisionConfig,
	)
	from .processing_x_clip import XCLIPProcessor

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_x_clip import (
		    XCLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
		    XCLIPModel,
		    XCLIPPreTrainedModel,
		    XCLIPTextModel,
		    XCLIPVisionModel,
		)

else:
	import sys

	_lowercase      :      List[Any]            =					_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)





