
import argparse
import logging
import pickle
import random
import time

import numpy as np

from transformers import BertTokenizer, GPTaTokenizer, RobertaTokenizer


logging.basicConfig(
    format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt='%m/%d/%Y %H:%M:%S', level=logging.INFO
)
lowercase__          =logging.getLogger(__name__)




def        __UpperCamelCase					(    ):
 __a			:  List[Any]		    =  argparse.ArgumentParser(
     description='''Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids).'''  )
 parser.add_argument('''--file_path'''  ,       type=lowerCAmelCase__  ,       default='''data/dump.txt'''  ,       help='''The path to the data.'''  )
 parser.add_argument('''--tokenizer_type'''  ,       type=lowerCAmelCase__  ,       default='''bert'''  ,       choices=['''bert''', '''roberta''', '''gpt2''']  )
 parser.add_argument('''--tokenizer_name'''  ,       type=lowerCAmelCase__  ,       default='''bert-base-uncased'''  ,       help='''The tokenizer to use.'''  )
 parser.add_argument('''--dump_file'''  ,       type=lowerCAmelCase__  ,       default='''data/dump'''  ,       help='''The dump file prefix.'''  )
 __a			:  Optional[int]		    =  parser.parse_args()

 logger.info(f"Loading Tokenizer ({args.tokenizer_name})"  )
 if args.tokenizer_type == "bert":
  __a			:  Dict		    =  BertTokenizer.from_pretrained(args.tokenizer_name  )
  __a			:  int		    =  tokenizer.special_tokens_map['''cls_token''']  # `[CLS]`
  __a			:  int		    =  tokenizer.special_tokens_map['''sep_token''']  # `[SEP]`
 elif args.tokenizer_type == "roberta":
  __a			:  int		    =  RobertaTokenizer.from_pretrained(args.tokenizer_name  )
  __a			:  int		    =  tokenizer.special_tokens_map['''cls_token''']  # `<s>`
  __a			:  Optional[Any]		    =  tokenizer.special_tokens_map['''sep_token''']  # `</s>`
 elif args.tokenizer_type == "gpt2":
  __a			:  Optional[int]		    =  GPTaTokenizer.from_pretrained(args.tokenizer_name  )
  __a			:  List[Any]		    =  tokenizer.special_tokens_map['''bos_token''']  # `<|endoftext|>`
  __a			:  Dict		    =  tokenizer.special_tokens_map['''eos_token''']  # `<|endoftext|>`

 logger.info(f"Loading text from {args.file_path}"  )
 with open(args.file_path  ,       '''r'''  ,       encoding='''utf8'''  ) as fp:
  __a			:  Union[str, Any]		    =  fp.readlines()

 logger.info('''Start encoding'''  )
 logger.info(f"{len(lowerCAmelCase__  )} examples to process."  )

 __a			:  Optional[int]		    =  []
 __a			:  Union[str, Any]		    =  0
 __a			:  List[Any]		    =  1_0_0_0_0
 __a			:  Tuple		    =  time.time()
 for text in data:
  __a			:  Optional[Any]		    =  f"{bos} {text.strip()} {sep}"
  __a			:  Dict		    =  tokenizer.encode(lowerCAmelCase__  ,       add_special_tokens=lowerCAmelCase__  )
  rslt.append(lowerCAmelCase__  )

  iter += 1
  if iter % interval == 0:
   __a			:  List[Any]		    =  time.time()
   logger.info(f"{iter} examples processed. - {(end-start):.2f}s/{interval}expl"  )
   __a			:  Any		    =  time.time()
 logger.info('''Finished binarization'''  )
 logger.info(f"{len(lowerCAmelCase__  )} examples processed."  )

 __a			:  Optional[int]		    =  f"{args.dump_file}.{args.tokenizer_name}.pickle"
 __a			:  Dict		    =  tokenizer.vocab_size
 if vocab_size < (1 << 1_6):
  __a			:  Optional[int]		    =  [np.uintaa(lowerCAmelCase__  ) for d in rslt]
 else:
  __a			:  List[str]		    =  [np.intaa(lowerCAmelCase__  ) for d in rslt]
 random.shuffle(rslt_  )
 logger.info(f"Dump to {dp_file}"  )
 with open(lowerCAmelCase__  ,       '''wb'''  ) as handle:
  pickle.dump(rslt_  ,       lowerCAmelCase__  ,       protocol=pickle.HIGHEST_PROTOCOL  )


if __name__ == "__main__":
      main()


import argparse
import os
import re

import packaging.version


lowercase__          ='examples/'
lowercase__          ={
    'examples': (re.compile(R'^check_min_version\("[^"]+"\)\s*$', re.MULTILINE), 'check_min_version("VERSION")\n'),
    'init': (re.compile(R'^__version__\s+=\s+"([^"]+)"\s*$', re.MULTILINE), '__version__ = "VERSION"\n'),
    'setup': (re.compile(R'^(\s*)version\s*=\s*"[^"]+",', re.MULTILINE), R'\1version="VERSION",'),
    'doc': (re.compile(R'^(\s*)release\s*=\s*"[^"]+"$', re.MULTILINE), 'release = "VERSION"\n'),
}
lowercase__          ={
    'init': 'src/transformers/__init__.py',
    'setup': 'setup.py',
}
lowercase__          ='README.md'




def        __UpperCamelCase					(    lowerCAmelCase__							:    List[Any]  ,       lowerCAmelCase__							:    Dict  ,       lowerCAmelCase__							:    Tuple  ):
 with open(lowerCAmelCase__  ,       '''r'''  ,       encoding='''utf-8'''  ,       newline='''\n'''  ) as f:
  __a			:  Tuple		    =  f.read()
 __a    ,	__a			:  Optional[int]		    =  REPLACE_PATTERNS[pattern]
 __a			:  List[Any]		    =  replace.replace('''VERSION'''  ,       lowerCAmelCase__  )
 __a			:  Any		    =  re_pattern.sub(lowerCAmelCase__  ,       lowerCAmelCase__  )
 with open(lowerCAmelCase__  ,       '''w'''  ,       encoding='''utf-8'''  ,       newline='''\n'''  ) as f:
  f.write(lowerCAmelCase__  )




def        __UpperCamelCase					(    lowerCAmelCase__							:    Tuple  ):
 for folder, directories, fnames in os.walk(lowerCAmelCase__  ):
  # Removing some of the folders with non-actively maintained examples from the walk
  if "research_projects" in directories:
   directories.remove('''research_projects'''  )
  if "legacy" in directories:
   directories.remove('''legacy'''  )
  for fname in fnames:
   if fname.endswith('''.py'''  ):
    update_version_in_file(os.path.join(lowerCAmelCase__  ,       lowerCAmelCase__  )  ,       lowerCAmelCase__  ,       pattern='''examples'''  )




def        __UpperCamelCase					(    lowerCAmelCase__							:    Optional[int]  ,       lowerCAmelCase__							:    Dict=False  ):
 for pattern, fname in REPLACE_FILES.items():
  update_version_in_file(lowerCAmelCase__  ,       lowerCAmelCase__  ,       lowerCAmelCase__  )
 if not patch:
  update_version_in_examples(lowerCAmelCase__  )




def        __UpperCamelCase					(    ):
 __a			:  Optional[int]		    =  '''🤗 Transformers currently provides the following architectures'''
 __a			:  int		    =  '''1. Want to contribute a new model?'''
 with open(lowerCAmelCase__  ,       '''r'''  ,       encoding='''utf-8'''  ,       newline='''\n'''  ) as f:
  __a			:  Tuple		    =  f.readlines()

 # Find the start of the list.
 __a			:  Optional[int]		    =  0
 while not lines[start_index].startswith(_start_prompt  ):
  start_index += 1
 start_index += 1

 __a			:  Any		    =  start_index
 # Update the lines in the model list.
 while not lines[index].startswith(_end_prompt  ):
  if lines[index].startswith('''1.'''  ):
   __a			:  str		    =  lines[index].replace(
       '''https://huggingface.co/docs/transformers/main/model_doc'''  ,       '''https://huggingface.co/docs/transformers/model_doc'''  ,       )
  index += 1

 with open(lowerCAmelCase__  ,       '''w'''  ,       encoding='''utf-8'''  ,       newline='''\n'''  ) as f:
  f.writelines(lowerCAmelCase__  )




def        __UpperCamelCase					(    ):
 with open(REPLACE_FILES['''init''']  ,       '''r'''  ) as f:
  __a			:  Optional[int]		    =  f.read()
 __a			:  str		    =  REPLACE_PATTERNS['''init'''][0].search(lowerCAmelCase__  ).groups()[0]
 return packaging.version.parse(lowerCAmelCase__  )




def        __UpperCamelCase					(    lowerCAmelCase__							:    Union[str, Any]=False  ):
 __a			:  str		    =  get_version()
 if patch and default_version.is_devrelease:
  raise ValueError('''Can\'t create a patch version from the dev branch, checkout a released version!'''  )
 if default_version.is_devrelease:
  __a			:  Union[str, Any]		    =  default_version.base_version
 elif patch:
  __a			:  Tuple		    =  f"{default_version.major}.{default_version.minor}.{default_version.micro + 1}"
 else:
  __a			:  List[str]		    =  f"{default_version.major}.{default_version.minor + 1}.0"

 # Now let's ask nicely if that's the right one.
 __a			:  List[str]		    =  input(f"Which version are you releasing? [{default_version}]"  )
 if len(lowerCAmelCase__  ) == 0:
  __a			:  Tuple		    =  default_version

 print(f"Updating version to {version}."  )
 global_version_update(lowerCAmelCase__  ,       patch=lowerCAmelCase__  )
 if not patch:
  print('''Cleaning main README, don\'t forget to run `make fix-copies`.'''  )
  clean_main_ref_in_model_list()




def        __UpperCamelCase					(    ):
 __a			:  Dict		    =  get_version()
 __a			:  str		    =  f"{current_version.major}.{current_version.minor + 1}.0.dev0"
 __a			:  Any		    =  current_version.base_version

 # Check with the user we got that right.
 __a			:  Any		    =  input(f"Which version are we developing now? [{dev_version}]"  )
 if len(lowerCAmelCase__  ) == 0:
  __a			:  Any		    =  dev_version

 print(f"Updating version to {version}."  )
 global_version_update(lowerCAmelCase__  )
 print('''Cleaning main README, don\'t forget to run `make fix-copies`.'''  )
 clean_main_ref_in_model_list()


if __name__ == "__main__":
      lowercase__          =argparse.ArgumentParser()
      parser.add_argument('--post_release', action='store_true', help='Whether this is pre or post release.')
      parser.add_argument('--patch', action='store_true', help='Whether or not this is a patch release.')
      lowercase__          =parser.parse_args()
      if not args.post_release:
            pre_release_work(patch=args.patch)
      elif args.patch:
            print('Nothing to do after a patch :-)')
      else:
            post_release_work()
