




'''simple docstring'''



import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class 		lowercase_   (			A					,  unittest.TestCase       ):


   """simple docstring"""

   lowerCamelCase_      =						GPTaTokenizer
   lowerCamelCase_      =						GPTaTokenizerFast
   lowerCamelCase_      =						True
   lowerCamelCase_      =						{'''add_prefix_space''': True}
   lowerCamelCase_      =						False



   def 							lowerCAmelCase_      (   self :     Dict   ):

      """simple docstring"""
      super().setUp()

      # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
      _SCREAMING_SNAKE_CASE	=				[
          "l",
          "o",
          "w",
          "e",
          "r",
          "s",
          "t",
          "i",
          "d",
          "n",
          "\u0120",
          "\u0120l",
          "\u0120n",
          "\u0120lo",
          "\u0120low",
          "er",
          "\u0120lowest",
          "\u0120newer",
          "\u0120wider",
          "<unk>",
          "<|endoftext|>",
      ]
      _SCREAMING_SNAKE_CASE	=				dict(zip(__lowerCamelCase  ,				range(len(__lowerCamelCase   )   )   )   )
      _SCREAMING_SNAKE_CASE	=				["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
      _SCREAMING_SNAKE_CASE	=				{"unk_token": "<unk>"}

      _SCREAMING_SNAKE_CASE	=				os.path.join(self.tmpdirname  ,				VOCAB_FILES_NAMES["vocab_file"]   )
      _SCREAMING_SNAKE_CASE	=				os.path.join(self.tmpdirname  ,				VOCAB_FILES_NAMES["merges_file"]   )
      with open(self.vocab_file  ,				"w"  ,				encoding="utf-8"   ) as fp:
         fp.write(json.dumps(__lowerCamelCase   ) + "\n"   )
      with open(self.merges_file  ,				"w"  ,				encoding="utf-8"   ) as fp:
         fp.write("\n".join(__lowerCamelCase   )   )



   def 							lowerCAmelCase_      (   self :     Any  ,				**__lowerCamelCase :     Any   ):

      """simple docstring"""
      kwargs.update(self.special_tokens_map   )
      return GPTaTokenizer.from_pretrained(self.tmpdirname  ,				**__lowerCamelCase   )



   def 							lowerCAmelCase_      (   self :     Optional[Any]  ,				**__lowerCamelCase :     List[str]   ):

      """simple docstring"""
      kwargs.update(self.special_tokens_map   )
      return GPTaTokenizerFast.from_pretrained(self.tmpdirname  ,				**__lowerCamelCase   )



   def 							lowerCAmelCase_      (   self :     Any  ,				__lowerCamelCase :     int   ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				"lower newer"
      _SCREAMING_SNAKE_CASE	=				"lower newer"
      return input_text, output_text



   def 							lowerCAmelCase_      (   self :     Dict   ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				GPTaTokenizer(self.vocab_file  ,				self.merges_file  ,				**self.special_tokens_map   )
      _SCREAMING_SNAKE_CASE	=				"lower newer"
      _SCREAMING_SNAKE_CASE	=				["\u0120low", "er", "\u0120", "n", "e", "w", "er"]
      _SCREAMING_SNAKE_CASE	=				tokenizer.tokenize(__lowerCamelCase  ,				add_prefix_space=__lowerCamelCase   )
      self.assertListEqual(__lowerCamelCase  ,				__lowerCamelCase   )

      _SCREAMING_SNAKE_CASE	=				tokens + [tokenizer.unk_token]
      _SCREAMING_SNAKE_CASE	=				[1_4, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
      self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase   )  ,				__lowerCamelCase   )



   def 							lowerCAmelCase_      (   self :     Optional[int]   ):

      """simple docstring"""
      if not self.test_rust_tokenizer:
         return

      _SCREAMING_SNAKE_CASE	=				self.get_tokenizer()
      _SCREAMING_SNAKE_CASE	=				self.get_rust_tokenizer(add_prefix_space=__lowerCamelCase   )

      _SCREAMING_SNAKE_CASE	=				"lower newer"

      # Testing tokenization
      _SCREAMING_SNAKE_CASE	=				tokenizer.tokenize(__lowerCamelCase  ,				add_prefix_space=__lowerCamelCase   )
      _SCREAMING_SNAKE_CASE	=				rust_tokenizer.tokenize(__lowerCamelCase   )
      self.assertListEqual(__lowerCamelCase  ,				__lowerCamelCase   )

      # Testing conversion to ids without special tokens
      _SCREAMING_SNAKE_CASE	=				tokenizer.encode(__lowerCamelCase  ,				add_special_tokens=__lowerCamelCase  ,				add_prefix_space=__lowerCamelCase   )
      _SCREAMING_SNAKE_CASE	=				rust_tokenizer.encode(__lowerCamelCase  ,				add_special_tokens=__lowerCamelCase   )
      self.assertListEqual(__lowerCamelCase  ,				__lowerCamelCase   )

      # Testing conversion to ids with special tokens
      _SCREAMING_SNAKE_CASE	=				self.get_rust_tokenizer(add_prefix_space=__lowerCamelCase   )
      _SCREAMING_SNAKE_CASE	=				tokenizer.encode(__lowerCamelCase  ,				add_prefix_space=__lowerCamelCase   )
      _SCREAMING_SNAKE_CASE	=				rust_tokenizer.encode(__lowerCamelCase   )
      self.assertListEqual(__lowerCamelCase  ,				__lowerCamelCase   )

      # Testing the unknown token
      _SCREAMING_SNAKE_CASE	=				tokens + [rust_tokenizer.unk_token]
      _SCREAMING_SNAKE_CASE	=				[1_4, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
      self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(__lowerCamelCase   )  ,				__lowerCamelCase   )



   def 							lowerCAmelCase_      (   self :     str  ,				*__lowerCamelCase :     Optional[int]  ,				**__lowerCamelCase :     List[Any]   ):

      """simple docstring"""
      # It's very difficult to mix/test pretokenization with byte-level
      # And get both GPT2 and Roberta to work at the same time (mostly an issue of adding a space before the string)
      pass



   def 							lowerCAmelCase_      (   self :     Dict  ,				__lowerCamelCase :     int=1_5   ):

      """simple docstring"""
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""   ):
            _SCREAMING_SNAKE_CASE	=				self.rust_tokenizer_class.from_pretrained(__lowerCamelCase  ,				**__lowerCamelCase   )

            # Simple input
            _SCREAMING_SNAKE_CASE	=				"This is a simple input"
            _SCREAMING_SNAKE_CASE	=				["This is a simple input 1", "This is a simple input 2"]
            _SCREAMING_SNAKE_CASE	=				("This is a simple input", "This is a pair")
            _SCREAMING_SNAKE_CASE	=				[
                ("This is a simple input 1", "This is a simple input 2"),
                ("This is a simple pair 1", "This is a simple pair 2"),
            ]

            # Simple input tests
            self.assertRaises(__lowerCamelCase  ,				tokenizer_r.encode  ,				__lowerCamelCase  ,				max_length=__lowerCamelCase  ,				padding="max_length"   )

            # Simple input
            self.assertRaises(__lowerCamelCase  ,				tokenizer_r.encode_plus  ,				__lowerCamelCase  ,				max_length=__lowerCamelCase  ,				padding="max_length"   )

            # Simple input
            self.assertRaises(
                __lowerCamelCase  ,				tokenizer_r.batch_encode_plus  ,				__lowerCamelCase  ,				max_length=__lowerCamelCase  ,				padding="max_length"  ,				)

            # Pair input
            self.assertRaises(__lowerCamelCase  ,				tokenizer_r.encode  ,				__lowerCamelCase  ,				max_length=__lowerCamelCase  ,				padding="max_length"   )

            # Pair input
            self.assertRaises(__lowerCamelCase  ,				tokenizer_r.encode_plus  ,				__lowerCamelCase  ,				max_length=__lowerCamelCase  ,				padding="max_length"   )

            # Pair input
            self.assertRaises(
                __lowerCamelCase  ,				tokenizer_r.batch_encode_plus  ,				__lowerCamelCase  ,				max_length=__lowerCamelCase  ,				padding="max_length"  ,				)



   def 							lowerCAmelCase_      (   self :     List[str]   ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				GPTaTokenizer.from_pretrained(self.tmpdirname  ,				pad_token="<pad>"   )

      # Simple input
      _SCREAMING_SNAKE_CASE	=				"This is a simple input"
      _SCREAMING_SNAKE_CASE	=				["This is a simple input looooooooong", "This is a simple input"]
      _SCREAMING_SNAKE_CASE	=				("This is a simple input", "This is a pair")
      _SCREAMING_SNAKE_CASE	=				[
          ("This is a simple input loooooong", "This is a simple input"),
          ("This is a simple pair loooooong", "This is a simple pair"),
      ]

      _SCREAMING_SNAKE_CASE	=				tokenizer.pad_token_id

      _SCREAMING_SNAKE_CASE	=				tokenizer(__lowerCamelCase  ,				padding="max_length"  ,				max_length=3_0  ,				return_tensors="np"   )
      _SCREAMING_SNAKE_CASE	=				tokenizer(__lowerCamelCase  ,				padding=__lowerCamelCase  ,				truncate=__lowerCamelCase  ,				return_tensors="np"   )
      _SCREAMING_SNAKE_CASE	=				tokenizer(*__lowerCamelCase  ,				padding="max_length"  ,				max_length=6_0  ,				return_tensors="np"   )
      _SCREAMING_SNAKE_CASE	=				tokenizer(__lowerCamelCase  ,				padding=__lowerCamelCase  ,				truncate=__lowerCamelCase  ,				return_tensors="np"   )

      # s
      # test single string max_length padding
      self.assertEqual(out_s["input_ids"].shape[-1]  ,				3_0   )
      self.assertTrue(pad_token_id in out_s["input_ids"]   )
      self.assertTrue(0 in out_s["attention_mask"]   )

      # s2
      # test automatic padding
      self.assertEqual(out_sa["input_ids"].shape[-1]  ,				3_3   )
      # long slice doesn't have padding
      self.assertFalse(pad_token_id in out_sa["input_ids"][0]   )
      self.assertFalse(0 in out_sa["attention_mask"][0]   )
      # short slice does have padding
      self.assertTrue(pad_token_id in out_sa["input_ids"][1]   )
      self.assertTrue(0 in out_sa["attention_mask"][1]   )

      # p
      # test single pair max_length padding
      self.assertEqual(out_p["input_ids"].shape[-1]  ,				6_0   )
      self.assertTrue(pad_token_id in out_p["input_ids"]   )
      self.assertTrue(0 in out_p["attention_mask"]   )

      # p2
      # test automatic padding pair
      self.assertEqual(out_pa["input_ids"].shape[-1]  ,				5_2   )
      # long slice pair doesn't have padding
      self.assertFalse(pad_token_id in out_pa["input_ids"][0]   )
      self.assertFalse(0 in out_pa["attention_mask"][0]   )
      # short slice pair does have padding
      self.assertTrue(pad_token_id in out_pa["input_ids"][1]   )
      self.assertTrue(0 in out_pa["attention_mask"][1]   )



   def 							lowerCAmelCase_      (   self :     Union[str, Any]   ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				"$$$"
      _SCREAMING_SNAKE_CASE	=				GPTaTokenizer.from_pretrained(self.tmpdirname  ,				bos_token=__lowerCamelCase  ,				add_bos_token=__lowerCamelCase   )

      _SCREAMING_SNAKE_CASE	=				"This is a simple input"
      _SCREAMING_SNAKE_CASE	=				["This is a simple input 1", "This is a simple input 2"]

      _SCREAMING_SNAKE_CASE	=				tokenizer.bos_token_id

      _SCREAMING_SNAKE_CASE	=				tokenizer(__lowerCamelCase   )
      _SCREAMING_SNAKE_CASE	=				tokenizer(__lowerCamelCase   )

      self.assertEqual(out_s.input_ids[0]  ,				__lowerCamelCase   )
      self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids   )   )

      _SCREAMING_SNAKE_CASE	=				tokenizer.decode(out_s.input_ids   )
      _SCREAMING_SNAKE_CASE	=				tokenizer.batch_decode(out_sa.input_ids   )

      self.assertEqual(decode_s.split()[0]  ,				__lowerCamelCase   )
      self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa   )   )



   def 							lowerCAmelCase_      (   self :     Tuple   ):

      """simple docstring"""
      pass



   def 							lowerCAmelCase_      (   self :     Optional[int]   ):

      """simple docstring"""
      # TODO: change to self.get_tokenizers() when the fast version is implemented
      _SCREAMING_SNAKE_CASE	=				[self.get_tokenizer(do_lower_case=__lowerCamelCase  ,				add_bos_token=__lowerCamelCase   )]
      for tokenizer in tokenizers:
         with self.subTest(F"""{tokenizer.__class__.__name__}"""   ):
            _SCREAMING_SNAKE_CASE	=				"Encode this."
            _SCREAMING_SNAKE_CASE	=				"This one too please."
            _SCREAMING_SNAKE_CASE	=				tokenizer.encode(__lowerCamelCase  ,				add_special_tokens=__lowerCamelCase   )
            encoded_sequence += tokenizer.encode(__lowerCamelCase  ,				add_special_tokens=__lowerCamelCase   )
            _SCREAMING_SNAKE_CASE	=				tokenizer.encode_plus(
                __lowerCamelCase  ,				__lowerCamelCase  ,				add_special_tokens=__lowerCamelCase  ,				return_special_tokens_mask=__lowerCamelCase  ,				)
            _SCREAMING_SNAKE_CASE	=				encoded_sequence_dict["input_ids"]
            _SCREAMING_SNAKE_CASE	=				encoded_sequence_dict["special_tokens_mask"]
            self.assertEqual(len(__lowerCamelCase   )  ,				len(__lowerCamelCase   )   )

            _SCREAMING_SNAKE_CASE	=				[
                (x if not special_tokens_mask[i] else None) for i, x in enumerate(__lowerCamelCase   )
            ]
            _SCREAMING_SNAKE_CASE	=				[x for x in filtered_sequence if x is not None]
            self.assertEqual(__lowerCamelCase  ,				__lowerCamelCase   )







@require_tokenizers
class 		lowercase_   (			unittest.TestCase       ):


   """simple docstring"""

   def 							lowerCAmelCase_      (   self :     Union[str, Any]   ):

      """simple docstring"""
      # More context:
      # https://huggingface.co/wjmcat/opt-350m-paddle/discussions/1
      # https://huggingface.slack.com/archives/C01N44FJDHT/p1653511495183519
      # https://github.com/huggingface/transformers/pull/17088#discussion_r871246439

      _SCREAMING_SNAKE_CASE	=				AutoTokenizer.from_pretrained("facebook/opt-350m"  ,				from_slow=__lowerCamelCase   )
      _SCREAMING_SNAKE_CASE	=				"A photo of a cat"

      _SCREAMING_SNAKE_CASE	=				tokenizer.encode(
          __lowerCamelCase  ,				)
      self.assertEqual(__lowerCamelCase  ,				[2, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8]   )
      tokenizer.save_pretrained("test_opt"   )

      _SCREAMING_SNAKE_CASE	=				AutoTokenizer.from_pretrained("./test_opt"   )
      _SCREAMING_SNAKE_CASE	=				tokenizer.encode(
          __lowerCamelCase  ,				)
      self.assertEqual(__lowerCamelCase  ,				[2, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8]   )



   def 							lowerCAmelCase_      (   self :     Tuple   ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				AutoTokenizer.from_pretrained("facebook/opt-350m"  ,				use_slow=__lowerCamelCase   )
      _SCREAMING_SNAKE_CASE	=				"A photo of a cat"

      _SCREAMING_SNAKE_CASE	=				tokenizer.encode(
          __lowerCamelCase  ,				)
      # Same as above
      self.assertEqual(__lowerCamelCase  ,				[2, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8]   )



   @unittest.skip("This test is failing because of a bug in the fast tokenizer"   )
   def 							lowerCAmelCase_      (   self :     Tuple   ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				AutoTokenizer.from_pretrained("facebook/opt-350m"  ,				from_slow=__lowerCamelCase   )

      _SCREAMING_SNAKE_CASE	=				"bos"
      _SCREAMING_SNAKE_CASE	=				tokenizer.get_vocab()["bos"]

      _SCREAMING_SNAKE_CASE	=				"A photo of a cat"
      _SCREAMING_SNAKE_CASE	=				tokenizer.encode(
          __lowerCamelCase  ,				)
      # We changed the bos token
      self.assertEqual(__lowerCamelCase  ,				[3_1_9_5_7, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8]   )
      tokenizer.save_pretrained("./tok"   )
      _SCREAMING_SNAKE_CASE	=				AutoTokenizer.from_pretrained("./tok"   )
      self.assertTrue(tokenizer.is_fast   )
      _SCREAMING_SNAKE_CASE	=				tokenizer.encode(
          __lowerCamelCase  ,				)
      self.assertEqual(__lowerCamelCase  ,				[3_1_9_5_7, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8]   )







'''simple docstring'''



import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

lowerCamelCase_			=						logging.get_logger(__name__)


lowerCamelCase_			=						{name: getattr(transformers, name + 'Fast') for name in SLOW_TO_FAST_CONVERTERS}
def 			SCREAMING_SNAKE_CASE_			(							__A    :			Optional[int]	,					__A    :			Tuple	,					__A    :			List[str]	,					__A    :			List[str]  )			->			List[Any]:
   if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
      raise ValueError(f"""Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()  )}."""  )

   if tokenizer_name is None:
      _SCREAMING_SNAKE_CASE	=				TOKENIZER_CLASSES
   else:
      _SCREAMING_SNAKE_CASE	=				{tokenizer_name: getattr(__A	,					tokenizer_name + "Fast"  )}

   logger.info(f"""Loading tokenizer classes: {tokenizer_names}"""  )

   for tokenizer_name in tokenizer_names:
      _SCREAMING_SNAKE_CASE	=				TOKENIZER_CLASSES[tokenizer_name]

      _SCREAMING_SNAKE_CASE	=				True
      if checkpoint_name is None:
         _SCREAMING_SNAKE_CASE	=				list(tokenizer_class.max_model_input_sizes.keys()  )
      else:
         _SCREAMING_SNAKE_CASE	=				[checkpoint_name]

      logger.info(f"""For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}"""  )

      for checkpoint in checkpoint_names:
         logger.info(f"""Loading {tokenizer_class.__class__.__name__} {checkpoint}"""  )

         # Load tokenizer
         _SCREAMING_SNAKE_CASE	=				tokenizer_class.from_pretrained(__A	,					force_download=__A  )

         # Save fast tokenizer
         logger.info(f"""Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}"""  )

         # For organization names we create sub-directories
         if "/" in checkpoint:
            _SCREAMING_SNAKE_CASE,	_SCREAMING_SNAKE_CASE	=				checkpoint.split("/"  )
            _SCREAMING_SNAKE_CASE	=				os.path.join(__A	,					__A  )
         elif add_prefix:
            _SCREAMING_SNAKE_CASE	=				checkpoint
            _SCREAMING_SNAKE_CASE	=				dump_path
         else:
            _SCREAMING_SNAKE_CASE	=				None
            _SCREAMING_SNAKE_CASE	=				dump_path

         logger.info(f"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""  )

         if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()  )[0]:
            _SCREAMING_SNAKE_CASE	=				list(tokenizer.pretrained_vocab_files_map.values()  )[0][checkpoint]
            _SCREAMING_SNAKE_CASE	=				file_path.split(__A  )[-1][0]
            if next_char == "/":
               _SCREAMING_SNAKE_CASE	=				os.path.join(__A	,					__A  )
               _SCREAMING_SNAKE_CASE	=				None

            logger.info(f"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""  )

         _SCREAMING_SNAKE_CASE	=				tokenizer.save_pretrained(
             __A	,					legacy_format=__A	,					filename_prefix=__A  )
         logger.info(f"""=> File names {file_names}"""  )

         for file_name in file_names:
            if not file_name.endswith("tokenizer.json"  ):
               os.remove(__A  )
               logger.info(f"""=> removing {file_name}"""  )


if __name__ == "__main__":
 lowerCamelCase_			=						argparse.ArgumentParser()
 # Required parameters
 parser.add_argument(
     '--dump_path', default=None, type=str, required=True, help='Path to output generated fast tokenizer files.'
 )
 parser.add_argument(
     '--tokenizer_name',
     default=None,
     type=str,
     help=(
         f'''Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will '''
         'download and convert all the checkpoints from AWS.'
     ),
 )
 parser.add_argument(
     '--checkpoint_name',
     default=None,
     type=str,
     help='Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.',
 )
 parser.add_argument(
     '--force_download',
     action='store_true',
     help='Re-download checkpoints.',
 )
 lowerCamelCase_			=						parser.parse_args()

 convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)

