


"""simple docstring"""



import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class _lowerCAmelCase    (				snake_case_   ,							unittest.TestCase     ):
   __UpperCAmelCase			:	List[str]							  =  CodeGenTokenizer
   __UpperCAmelCase			:	Union[str, Any]							  =  CodeGenTokenizerFast
   __UpperCAmelCase			:	Optional[Any]							  =  True
   __UpperCAmelCase			:	Any							  =  {'''add_prefix_space''': True}
   __UpperCAmelCase			:	str							  =  False







   def 			lowerCamelCase		(     self						)     ->		Optional[int]:

       '''simple docstring'''




       super().setUp()

       # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
       snake_case   :				Optional[int]				 =   [
           "l",
           "o",
           "w",
           "e",
           "r",
           "s",
           "t",
           "i",
           "d",
           "n",
           "\u0120",
           "\u0120l",
           "\u0120n",
           "\u0120lo",
           "\u0120low",
           "er",
           "\u0120lowest",
           "\u0120newer",
           "\u0120wider",
           "<unk>",
           "<|endoftext|>",
       ]
       snake_case   :				str				 =   dict(zip(UpperCamelCase__						,     range(len(UpperCamelCase__						)						)						)						)
       snake_case   :				List[str]				 =   ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
       snake_case   :				Optional[Any]				 =   {"unk_token": "<unk>"}

       snake_case   :				Tuple				 =   os.path.join(self.tmpdirname						,     VOCAB_FILES_NAMES["vocab_file"]						)
       snake_case   :				Tuple				 =   os.path.join(self.tmpdirname						,     VOCAB_FILES_NAMES["merges_file"]						)
       with open(self.vocab_file						,     "w"						,     encoding="utf-8"						) as fp:
           fp.write(json.dumps(UpperCamelCase__						) + "\n"						)
       with open(self.merges_file						,     "w"						,     encoding="utf-8"						) as fp:
           fp.write("\n".join(UpperCamelCase__						)						)







   def 			lowerCamelCase		(     self						,     **UpperCamelCase__						)     ->		str:

       '''simple docstring'''




       kwargs.update(self.special_tokens_map						)
       return CodeGenTokenizer.from_pretrained(self.tmpdirname						,     **UpperCamelCase__						)







   def 			lowerCamelCase		(     self						,     **UpperCamelCase__						)     ->		str:

       '''simple docstring'''




       kwargs.update(self.special_tokens_map						)
       return CodeGenTokenizerFast.from_pretrained(self.tmpdirname						,     **UpperCamelCase__						)







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						)     ->		str:

       '''simple docstring'''




       snake_case   :				Tuple				 =   "lower newer"
       snake_case   :				str				 =   "lower newer"
       return input_text, output_text







   def 			lowerCamelCase		(     self						)     ->		Any:

       '''simple docstring'''




       snake_case   :				Optional[int]				 =   CodeGenTokenizer(self.vocab_file						,     self.merges_file						,     **self.special_tokens_map						)
       snake_case   :				Tuple				 =   "lower newer"
       snake_case   :				Any				 =   ["\u0120low", "er", "\u0120", "n", "e", "w", "er"]
       snake_case   :				Optional[Any]				 =   tokenizer.tokenize(UpperCamelCase__						,     add_prefix_space=UpperCamelCase__						)
       self.assertListEqual(UpperCamelCase__						,     UpperCamelCase__						)

       snake_case   :				Union[str, Any]				 =   tokens + [tokenizer.unk_token]
       snake_case   :				str				 =   [14, 15, 10, 9, 3, 2, 15, 19]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__						)						,     UpperCamelCase__						)







   def 			lowerCamelCase		(     self						)     ->		Tuple:

       '''simple docstring'''




       if not self.test_rust_tokenizer:
           return

       snake_case   :				Union[str, Any]				 =   self.get_tokenizer()
       snake_case   :				Any				 =   self.get_rust_tokenizer(add_prefix_space=UpperCamelCase__						)

       snake_case   :				Optional[int]				 =   "lower newer"

       # Testing tokenization
       snake_case   :				Optional[int]				 =   tokenizer.tokenize(UpperCamelCase__						,     add_prefix_space=UpperCamelCase__						)
       snake_case   :				Optional[Any]				 =   rust_tokenizer.tokenize(UpperCamelCase__						)
       self.assertListEqual(UpperCamelCase__						,     UpperCamelCase__						)

       # Testing conversion to ids without special tokens
       snake_case   :				List[str]				 =   tokenizer.encode(UpperCamelCase__						,     add_special_tokens=UpperCamelCase__						,     add_prefix_space=UpperCamelCase__						)
       snake_case   :				Optional[int]				 =   rust_tokenizer.encode(UpperCamelCase__						,     add_special_tokens=UpperCamelCase__						)
       self.assertListEqual(UpperCamelCase__						,     UpperCamelCase__						)

       # Testing conversion to ids with special tokens
       snake_case   :				Dict				 =   self.get_rust_tokenizer(add_prefix_space=UpperCamelCase__						)
       snake_case   :				int				 =   tokenizer.encode(UpperCamelCase__						,     add_prefix_space=UpperCamelCase__						)
       snake_case   :				Any				 =   rust_tokenizer.encode(UpperCamelCase__						)
       self.assertListEqual(UpperCamelCase__						,     UpperCamelCase__						)

       # Testing the unknown token
       snake_case   :				Any				 =   tokens + [rust_tokenizer.unk_token]
       snake_case   :				List[Any]				 =   [14, 15, 10, 9, 3, 2, 15, 19]
       self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(UpperCamelCase__						)						,     UpperCamelCase__						)







   def 			lowerCamelCase		(     self						,     *UpperCamelCase__						,     **UpperCamelCase__						)     ->		Union[str, Any]:

       '''simple docstring'''




       pass







   def 			lowerCamelCase		(     self						,     UpperCamelCase__=15						)     ->		str:

       '''simple docstring'''




       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'						):
               snake_case   :				List[Any]				 =   self.rust_tokenizer_class.from_pretrained(UpperCamelCase__						,     **UpperCamelCase__						)

               # Simple input
               snake_case   :				Dict				 =   "This is a simple input"
               snake_case   :				str				 =   ["This is a simple input 1", "This is a simple input 2"]
               snake_case   :				Any				 =   ("This is a simple input", "This is a pair")
               snake_case   :				Optional[Any]				 =   [
                   ("This is a simple input 1", "This is a simple input 2"),
                   ("This is a simple pair 1", "This is a simple pair 2"),
               ]

               # Simple input tests
               self.assertRaises(UpperCamelCase__						,     tokenizer_r.encode						,     UpperCamelCase__						,     max_length=UpperCamelCase__						,     padding="max_length"						)

               # Simple input
               self.assertRaises(UpperCamelCase__						,     tokenizer_r.encode_plus						,     UpperCamelCase__						,     max_length=UpperCamelCase__						,     padding="max_length"						)

               # Simple input
               self.assertRaises(
                   UpperCamelCase__						,     tokenizer_r.batch_encode_plus						,     UpperCamelCase__						,     max_length=UpperCamelCase__						,     padding="max_length"						,     )

               # Pair input
               self.assertRaises(UpperCamelCase__						,     tokenizer_r.encode						,     UpperCamelCase__						,     max_length=UpperCamelCase__						,     padding="max_length"						)

               # Pair input
               self.assertRaises(UpperCamelCase__						,     tokenizer_r.encode_plus						,     UpperCamelCase__						,     max_length=UpperCamelCase__						,     padding="max_length"						)

               # Pair input
               self.assertRaises(
                   UpperCamelCase__						,     tokenizer_r.batch_encode_plus						,     UpperCamelCase__						,     max_length=UpperCamelCase__						,     padding="max_length"						,     )







   def 			lowerCamelCase		(     self						)     ->		Dict:

       '''simple docstring'''




       snake_case   :				List[str]				 =   CodeGenTokenizer.from_pretrained(self.tmpdirname						,     pad_token="<pad>"						)

       # Simple input
       snake_case   :				List[str]				 =   "This is a simple input"
       snake_case   :				Dict				 =   ["This is a simple input looooooooong", "This is a simple input"]
       snake_case   :				List[Any]				 =   ("This is a simple input", "This is a pair")
       snake_case   :				Any				 =   [
           ("This is a simple input loooooong", "This is a simple input"),
           ("This is a simple pair loooooong", "This is a simple pair"),
       ]

       snake_case   :				int				 =   tokenizer.pad_token_id

       snake_case   :				Optional[Any]				 =   tokenizer(UpperCamelCase__						,     padding="max_length"						,     max_length=30						,     return_tensors="np"						)
       snake_case   :				Dict				 =   tokenizer(UpperCamelCase__						,     padding=UpperCamelCase__						,     truncate=UpperCamelCase__						,     return_tensors="np"						)
       snake_case   :				Union[str, Any]				 =   tokenizer(*UpperCamelCase__						,     padding="max_length"						,     max_length=60						,     return_tensors="np"						)
       snake_case   :				Tuple				 =   tokenizer(UpperCamelCase__						,     padding=UpperCamelCase__						,     truncate=UpperCamelCase__						,     return_tensors="np"						)

       # s
       # test single string max_length padding
       self.assertEqual(out_s["input_ids"].shape[-1]						,     30						)
       self.assertTrue(pad_token_id in out_s["input_ids"]						)
       self.assertTrue(0 in out_s["attention_mask"]						)

       # s2
       # test automatic padding
       self.assertEqual(out_sa["input_ids"].shape[-1]						,     33						)
       # long slice doesn't have padding
       self.assertFalse(pad_token_id in out_sa["input_ids"][0]						)
       self.assertFalse(0 in out_sa["attention_mask"][0]						)
       # short slice does have padding
       self.assertTrue(pad_token_id in out_sa["input_ids"][1]						)
       self.assertTrue(0 in out_sa["attention_mask"][1]						)

       # p
       # test single pair max_length padding
       self.assertEqual(out_p["input_ids"].shape[-1]						,     60						)
       self.assertTrue(pad_token_id in out_p["input_ids"]						)
       self.assertTrue(0 in out_p["attention_mask"]						)

       # p2
       # test automatic padding pair
       self.assertEqual(out_pa["input_ids"].shape[-1]						,     52						)
       # long slice pair doesn't have padding
       self.assertFalse(pad_token_id in out_pa["input_ids"][0]						)
       self.assertFalse(0 in out_pa["attention_mask"][0]						)
       # short slice pair does have padding
       self.assertTrue(pad_token_id in out_pa["input_ids"][1]						)
       self.assertTrue(0 in out_pa["attention_mask"][1]						)







   def 			lowerCamelCase		(     self						)     ->		List[Any]:

       '''simple docstring'''




       snake_case   :				str				 =   "$$$"
       snake_case   :				List[str]				 =   CodeGenTokenizer.from_pretrained(self.tmpdirname						,     bos_token=UpperCamelCase__						,     add_bos_token=UpperCamelCase__						)

       snake_case   :				List[Any]				 =   "This is a simple input"
       snake_case   :				List[Any]				 =   ["This is a simple input 1", "This is a simple input 2"]

       snake_case   :				Any				 =   tokenizer.bos_token_id

       snake_case   :				Optional[int]				 =   tokenizer(UpperCamelCase__						)
       snake_case   :				Optional[int]				 =   tokenizer(UpperCamelCase__						)

       self.assertEqual(out_s.input_ids[0]						,     UpperCamelCase__						)
       self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids						)						)

       snake_case   :				Optional[Any]				 =   tokenizer.decode(out_s.input_ids						)
       snake_case   :				Any				 =   tokenizer.batch_decode(out_sa.input_ids						)

       self.assertEqual(decode_s.split()[0]						,     UpperCamelCase__						)
       self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa						)						)







   @slow
   def 			lowerCamelCase		(     self						)     ->		Optional[int]:

       '''simple docstring'''




       snake_case   :				int				 =   CodeGenTokenizer.from_pretrained("Salesforce/codegen-350M-mono"						)

       snake_case   :				Optional[Any]				 =   "\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"
       snake_case   :				Optional[int]				 =   "\nif len_a > len_b:      result = a\nelse:      result = b"

       snake_case   :				Tuple				 =   tokenizer.encode(UpperCamelCase__						)
       snake_case   :				Optional[int]				 =   ["^#", re.escape("<|endoftext|>"						), "^'''", "^\"\"\"", "\n\n\n"]
       snake_case   :				Tuple				 =   tokenizer.decode(UpperCamelCase__						,     truncate_before_pattern=UpperCamelCase__						)
       self.assertEqual(UpperCamelCase__						,     UpperCamelCase__						)







   def 			lowerCamelCase		(     self						)     ->		List[str]:

       '''simple docstring'''




       pass









"""simple docstring"""



import string







def   __lowerCAmelCase						(						lowercase  :  str			)     ->						str:


    """simple docstring"""





    snake_case   :				List[str]				 =   ""
    for i in sequence:
        snake_case   :				Optional[Any]				 =   ord(lowercase			)
        if 65 <= extract <= 90:
            output += chr(155 - extract			)
        elif 97 <= extract <= 122:
            output += chr(219 - extract			)
        else:
            output += i
    return output







def   __lowerCAmelCase						(						lowercase  :  str			)     ->						str:


    """simple docstring"""





    snake_case   :				Dict				 =   string.ascii_letters
    snake_case   :				List[Any]				 =   string.ascii_lowercase[::-1] + string.ascii_uppercase[::-1]
    return "".join(
        letters_reversed[letters.index(lowercase			)] if c in letters else c for c in sequence			)







def   __lowerCAmelCase						(						)     ->						None:


    """simple docstring"""





    from timeit import timeit

    print("Running performance benchmarks..."			)
    snake_case   :				Optional[int]				 =   "from string import printable ; from __main__ import atbash, atbash_slow"
    print(F'> atbash_slow(): {timeit("atbash_slow(printable)"						,							setup=lowercase			)} seconds'			)
    print(F'>      atbash(): {timeit("atbash(printable)"						,							setup=lowercase			)} seconds'			)


if __name__ == "__main__":
  for example in ("ABCDEFGH", "123GGjj", "testStringtest", "with space"):
    print(F'''{example} encrypted in atbash: {atbash(example)}''')
  benchmark()





