






import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class 				__UpperCAmelCase		(_UpperCAmelCase	,unittest.TestCase   ):
			__snake_case    :				Any									=							CodeGenTokenizer
			__snake_case    :				Optional[int]									=							CodeGenTokenizerFast
			__snake_case    :				Optional[int]									=							True
			__snake_case    :				int									=							{"add_prefix_space": True}
			__snake_case    :				List[str]									=							False







			def 							UpperCamelCase				(     self:  str					):







									'''simple docstring'''




									super().setUp()

									# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
									_SCREAMING_SNAKE_CASE     				=  [
									    """l""",
									    """o""",
									    """w""",
									    """e""",
									    """r""",
									    """s""",
									    """t""",
									    """i""",
									    """d""",
									    """n""",
									    """\u0120""",
									    """\u0120l""",
									    """\u0120n""",
									    """\u0120lo""",
									    """\u0120low""",
									    """er""",
									    """\u0120lowest""",
									    """\u0120newer""",
									    """\u0120wider""",
									    """<unk>""",
									    """<|endoftext|>""",
									]
									_SCREAMING_SNAKE_CASE     				=  dict(zip(UpperCAmelCase_				,     range(len(UpperCAmelCase_					)					)					)					)
									_SCREAMING_SNAKE_CASE     				=  ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
									_SCREAMING_SNAKE_CASE     				=  {"""unk_token""": """<unk>"""}

									_SCREAMING_SNAKE_CASE     				=  os.path.join(self.tmpdirname				,     VOCAB_FILES_NAMES["""vocab_file"""]					)
									_SCREAMING_SNAKE_CASE     				=  os.path.join(self.tmpdirname				,     VOCAB_FILES_NAMES["""merges_file"""]					)
									with open(self.vocab_file				,     """w"""				,     encoding="""utf-8"""					) as fp:
															fp.write(json.dumps(UpperCAmelCase_					) + """\n"""					)
									with open(self.merges_file				,     """w"""				,     encoding="""utf-8"""					) as fp:
															fp.write("""\n""".join(UpperCAmelCase_					)					)







			def 							UpperCamelCase				(     self:  Tuple				,     **UpperCAmelCase_:  Any					):







									'''simple docstring'''




									kwargs.update(self.special_tokens_map					)
									return CodeGenTokenizer.from_pretrained(self.tmpdirname				,     **UpperCAmelCase_					)







			def 							UpperCamelCase				(     self:  Any				,     **UpperCAmelCase_:  List[Any]					):







									'''simple docstring'''




									kwargs.update(self.special_tokens_map					)
									return CodeGenTokenizerFast.from_pretrained(self.tmpdirname				,     **UpperCAmelCase_					)







			def 							UpperCamelCase				(     self:  int				,     UpperCAmelCase_:  List[Any]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  """lower newer"""
									_SCREAMING_SNAKE_CASE     				=  """lower newer"""
									return input_text, output_text







			def 							UpperCamelCase				(     self:  Any					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  CodeGenTokenizer(self.vocab_file				,     self.merges_file				,     **self.special_tokens_map					)
									_SCREAMING_SNAKE_CASE     				=  """lower newer"""
									_SCREAMING_SNAKE_CASE     				=  ["""\u0120low""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
									_SCREAMING_SNAKE_CASE     				=  tokenizer.tokenize(UpperCAmelCase_				,     add_prefix_space=UpperCAmelCase_					)
									self.assertListEqual(UpperCAmelCase_				,     UpperCAmelCase_					)

									_SCREAMING_SNAKE_CASE     				=  tokens + [tokenizer.unk_token]
									_SCREAMING_SNAKE_CASE     				=  [14, 15, 10, 9, 3, 2, 15, 19]
									self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_					)				,     UpperCAmelCase_					)







			def 							UpperCamelCase				(     self:  Union[str, Any]					):







									'''simple docstring'''




									if not self.test_rust_tokenizer:
															return

									_SCREAMING_SNAKE_CASE     				=  self.get_tokenizer()
									_SCREAMING_SNAKE_CASE     				=  self.get_rust_tokenizer(add_prefix_space=UpperCAmelCase_					)

									_SCREAMING_SNAKE_CASE     				=  """lower newer"""

									# Testing tokenization
									_SCREAMING_SNAKE_CASE     				=  tokenizer.tokenize(UpperCAmelCase_				,     add_prefix_space=UpperCAmelCase_					)
									_SCREAMING_SNAKE_CASE     				=  rust_tokenizer.tokenize(UpperCAmelCase_					)
									self.assertListEqual(UpperCAmelCase_				,     UpperCAmelCase_					)

									# Testing conversion to ids without special tokens
									_SCREAMING_SNAKE_CASE     				=  tokenizer.encode(UpperCAmelCase_				,     add_special_tokens=UpperCAmelCase_				,     add_prefix_space=UpperCAmelCase_					)
									_SCREAMING_SNAKE_CASE     				=  rust_tokenizer.encode(UpperCAmelCase_				,     add_special_tokens=UpperCAmelCase_					)
									self.assertListEqual(UpperCAmelCase_				,     UpperCAmelCase_					)

									# Testing conversion to ids with special tokens
									_SCREAMING_SNAKE_CASE     				=  self.get_rust_tokenizer(add_prefix_space=UpperCAmelCase_					)
									_SCREAMING_SNAKE_CASE     				=  tokenizer.encode(UpperCAmelCase_				,     add_prefix_space=UpperCAmelCase_					)
									_SCREAMING_SNAKE_CASE     				=  rust_tokenizer.encode(UpperCAmelCase_					)
									self.assertListEqual(UpperCAmelCase_				,     UpperCAmelCase_					)

									# Testing the unknown token
									_SCREAMING_SNAKE_CASE     				=  tokens + [rust_tokenizer.unk_token]
									_SCREAMING_SNAKE_CASE     				=  [14, 15, 10, 9, 3, 2, 15, 19]
									self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(UpperCAmelCase_					)				,     UpperCAmelCase_					)







			def 							UpperCamelCase				(     self:  str				,     *UpperCAmelCase_:  List[str]				,     **UpperCAmelCase_:  Optional[Any]					):







									'''simple docstring'''




									pass







			def 							UpperCamelCase				(     self:  Dict				,     UpperCAmelCase_:  Union[str, Any]=15					):







									'''simple docstring'''




									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'					):
																					_SCREAMING_SNAKE_CASE     				=  self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_				,     **UpperCAmelCase_					)

																					# Simple input
																					_SCREAMING_SNAKE_CASE     				=  """This is a simple input"""
																					_SCREAMING_SNAKE_CASE     				=  ["""This is a simple input 1""", """This is a simple input 2"""]
																					_SCREAMING_SNAKE_CASE     				=  ("""This is a simple input""", """This is a pair""")
																					_SCREAMING_SNAKE_CASE     				=  [
																					    ("""This is a simple input 1""", """This is a simple input 2"""),
																					    ("""This is a simple pair 1""", """This is a simple pair 2"""),
																					]

																					# Simple input tests
																					self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding="""max_length"""					)

																					# Simple input
																					self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding="""max_length"""					)

																					# Simple input
																					self.assertRaises(
																					    UpperCAmelCase_				,     tokenizer_r.batch_encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding="""max_length"""				,     )

																					# Pair input
																					self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding="""max_length"""					)

																					# Pair input
																					self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding="""max_length"""					)

																					# Pair input
																					self.assertRaises(
																					    UpperCAmelCase_				,     tokenizer_r.batch_encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding="""max_length"""				,     )







			def 							UpperCamelCase				(     self:  List[str]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  CodeGenTokenizer.from_pretrained(self.tmpdirname				,     pad_token="""<pad>"""					)

									# Simple input
									_SCREAMING_SNAKE_CASE     				=  """This is a simple input"""
									_SCREAMING_SNAKE_CASE     				=  ["""This is a simple input looooooooong""", """This is a simple input"""]
									_SCREAMING_SNAKE_CASE     				=  ("""This is a simple input""", """This is a pair""")
									_SCREAMING_SNAKE_CASE     				=  [
									    ("""This is a simple input loooooong""", """This is a simple input"""),
									    ("""This is a simple pair loooooong""", """This is a simple pair"""),
									]

									_SCREAMING_SNAKE_CASE     				=  tokenizer.pad_token_id

									_SCREAMING_SNAKE_CASE     				=  tokenizer(UpperCAmelCase_				,     padding="""max_length"""				,     max_length=30				,     return_tensors="""np"""					)
									_SCREAMING_SNAKE_CASE     				=  tokenizer(UpperCAmelCase_				,     padding=UpperCAmelCase_				,     truncate=UpperCAmelCase_				,     return_tensors="""np"""					)
									_SCREAMING_SNAKE_CASE     				=  tokenizer(*UpperCAmelCase_				,     padding="""max_length"""				,     max_length=60				,     return_tensors="""np"""					)
									_SCREAMING_SNAKE_CASE     				=  tokenizer(UpperCAmelCase_				,     padding=UpperCAmelCase_				,     truncate=UpperCAmelCase_				,     return_tensors="""np"""					)

									# s
									# test single string max_length padding
									self.assertEqual(out_s["""input_ids"""].shape[-1]				,     30					)
									self.assertTrue(pad_token_id in out_s["""input_ids"""]					)
									self.assertTrue(0 in out_s["""attention_mask"""]					)

									# s2
									# test automatic padding
									self.assertEqual(out_sa["""input_ids"""].shape[-1]				,     33					)
									# long slice doesn't have padding
									self.assertFalse(pad_token_id in out_sa["""input_ids"""][0]					)
									self.assertFalse(0 in out_sa["""attention_mask"""][0]					)
									# short slice does have padding
									self.assertTrue(pad_token_id in out_sa["""input_ids"""][1]					)
									self.assertTrue(0 in out_sa["""attention_mask"""][1]					)

									# p
									# test single pair max_length padding
									self.assertEqual(out_p["""input_ids"""].shape[-1]				,     60					)
									self.assertTrue(pad_token_id in out_p["""input_ids"""]					)
									self.assertTrue(0 in out_p["""attention_mask"""]					)

									# p2
									# test automatic padding pair
									self.assertEqual(out_pa["""input_ids"""].shape[-1]				,     52					)
									# long slice pair doesn't have padding
									self.assertFalse(pad_token_id in out_pa["""input_ids"""][0]					)
									self.assertFalse(0 in out_pa["""attention_mask"""][0]					)
									# short slice pair does have padding
									self.assertTrue(pad_token_id in out_pa["""input_ids"""][1]					)
									self.assertTrue(0 in out_pa["""attention_mask"""][1]					)







			def 							UpperCamelCase				(     self:  Optional[Any]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  """$$$"""
									_SCREAMING_SNAKE_CASE     				=  CodeGenTokenizer.from_pretrained(self.tmpdirname				,     bos_token=UpperCAmelCase_				,     add_bos_token=UpperCAmelCase_					)

									_SCREAMING_SNAKE_CASE     				=  """This is a simple input"""
									_SCREAMING_SNAKE_CASE     				=  ["""This is a simple input 1""", """This is a simple input 2"""]

									_SCREAMING_SNAKE_CASE     				=  tokenizer.bos_token_id

									_SCREAMING_SNAKE_CASE     				=  tokenizer(UpperCAmelCase_					)
									_SCREAMING_SNAKE_CASE     				=  tokenizer(UpperCAmelCase_					)

									self.assertEqual(out_s.input_ids[0]				,     UpperCAmelCase_					)
									self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids					)					)

									_SCREAMING_SNAKE_CASE     				=  tokenizer.decode(out_s.input_ids					)
									_SCREAMING_SNAKE_CASE     				=  tokenizer.batch_decode(out_sa.input_ids					)

									self.assertEqual(decode_s.split()[0]				,     UpperCAmelCase_					)
									self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa					)					)







			@slow
			def 							UpperCamelCase				(     self:  Any					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  CodeGenTokenizer.from_pretrained("""Salesforce/codegen-350M-mono"""					)

									_SCREAMING_SNAKE_CASE     				=  """\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"""
									_SCREAMING_SNAKE_CASE     				=  """\nif len_a > len_b:      result = a\nelse:      result = b"""

									_SCREAMING_SNAKE_CASE     				=  tokenizer.encode(UpperCAmelCase_					)
									_SCREAMING_SNAKE_CASE     				=  ["""^#""", re.escape("""<|endoftext|>"""					), """^'''""", """^\"\"\"""", """\n\n\n"""]
									_SCREAMING_SNAKE_CASE     				=  tokenizer.decode(UpperCAmelCase_				,     truncate_before_pattern=UpperCAmelCase_					)
									self.assertEqual(UpperCAmelCase_				,     UpperCAmelCase_					)







			def 							UpperCamelCase				(     self:  Union[str, Any]					):







									'''simple docstring'''




									pass









import copy
import re



class 				__UpperCAmelCase		:
			__snake_case    :				Any									=							"hp"
			__snake_case    :				str									=							{}
			__snake_case    :				List[Any]									=							None







			@classmethod
			def 							UpperCamelCase				(     cls:  Optional[Any]				,     UpperCAmelCase_:  int				,     UpperCAmelCase_:  Optional[int]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  prefix
									_SCREAMING_SNAKE_CASE     				=  defaults
									cls.build_naming_info()







			@staticmethod
			def 							UpperCamelCase				(     UpperCAmelCase_:  Any				,     UpperCAmelCase_:  str					):







									'''simple docstring'''




									if len(UpperCAmelCase_					) == 0:
															return ""
									_SCREAMING_SNAKE_CASE     				=  None
									if any(char.isdigit() for char in word					):
															raise Exception(F'Parameters should not contain numbers: \'{word}\' contains a number'					)
									if word in info["short_word"]:
															return info["short_word"][word]
									for prefix_len in range(1				,     len(UpperCAmelCase_					) + 1					):
															_SCREAMING_SNAKE_CASE     				=  word[:prefix_len]
															if prefix in info["reverse_short_word"]:
																					continue
															else:
																					_SCREAMING_SNAKE_CASE     				=  prefix
																					break

									if short_word is None:
															# Paranoid fallback
															def int_to_alphabetic(UpperCAmelCase_:  List[Any]					):
																					_SCREAMING_SNAKE_CASE     				=  """"""
																					while integer != 0:
																											_SCREAMING_SNAKE_CASE     				=  chr(ord("""A"""					) + integer % 10					) + s
																											integer //= 10
																					return s

															_SCREAMING_SNAKE_CASE     				=  0
															while True:
																					_SCREAMING_SNAKE_CASE     				=  word + """#""" + int_to_alphabetic(UpperCAmelCase_					)
																					if sword in info["reverse_short_word"]:
																											continue
																					else:
																											_SCREAMING_SNAKE_CASE     				=  sword
																											break

									_SCREAMING_SNAKE_CASE     				=  short_word
									_SCREAMING_SNAKE_CASE     				=  word
									return short_word







			@staticmethod
			def 							UpperCamelCase				(     UpperCAmelCase_:  List[str]				,     UpperCAmelCase_:  List[str]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  param_name.split("""_"""					)

									_SCREAMING_SNAKE_CASE     				=  [TrialShortNamer.shortname_for_word(UpperCAmelCase_				,     UpperCAmelCase_					) for word in words]

									# We try to create a separatorless short name, but if there is a collision we have to fallback
									# to a separated short name
									_SCREAMING_SNAKE_CASE     				=  ["""""", """_"""]

									for separator in separators:
															_SCREAMING_SNAKE_CASE     				=  separator.join(UpperCAmelCase_					)
															if shortname not in info["reverse_short_param"]:
																					_SCREAMING_SNAKE_CASE     				=  shortname
																					_SCREAMING_SNAKE_CASE     				=  param_name
																					return shortname

									return param_name







			@staticmethod
			def 							UpperCamelCase				(     UpperCAmelCase_:  Optional[int]				,     UpperCAmelCase_:  Optional[int]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  TrialShortNamer.shortname_for_key(UpperCAmelCase_				,     UpperCAmelCase_					)
									_SCREAMING_SNAKE_CASE     				=  short_name
									_SCREAMING_SNAKE_CASE     				=  param_name







			@classmethod
			def 							UpperCamelCase				(     cls:  str					):







									'''simple docstring'''




									if cls.NAMING_INFO is not None:
															return

									_SCREAMING_SNAKE_CASE     				=  {
									    """short_word""": {},
									    """reverse_short_word""": {},
									    """short_param""": {},
									    """reverse_short_param""": {},
									}

									_SCREAMING_SNAKE_CASE     				=  list(cls.DEFAULTS.keys()					)

									for k in field_keys:
															cls.add_new_param_name(UpperCAmelCase_				,     UpperCAmelCase_					)

									_SCREAMING_SNAKE_CASE     				=  info







			@classmethod
			def 							UpperCamelCase				(     cls:  Any				,     UpperCAmelCase_:  Optional[Any]					):







									'''simple docstring'''




									cls.build_naming_info()
									assert cls.PREFIX is not None
									_SCREAMING_SNAKE_CASE     				=  [copy.copy(cls.PREFIX					)]

									for k, v in params.items():
															if k not in cls.DEFAULTS:
																					raise Exception(F'You should provide a default value for the param name {k} with value {v}'					)
															if v == cls.DEFAULTS[k]:
																					# The default value is not added to the name
																					continue

															_SCREAMING_SNAKE_CASE     				=  cls.NAMING_INFO["""short_param"""][k]

															if isinstance(UpperCAmelCase_				,     UpperCAmelCase_					):
																					_SCREAMING_SNAKE_CASE     				=  1 if v else 0

															_SCREAMING_SNAKE_CASE     				=  """""" if isinstance(UpperCAmelCase_				,     (int, float)					) else """-"""
															_SCREAMING_SNAKE_CASE     				=  F'{key}{sep}{v}'
															name.append(UpperCAmelCase_					)

									return "_".join(UpperCAmelCase_					)







			@classmethod
			def 							UpperCamelCase				(     cls:  int				,     UpperCAmelCase_:  int					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  repr[len(cls.PREFIX					) + 1 :]
									if repr == "":
															_SCREAMING_SNAKE_CASE     				=  []
									else:
															_SCREAMING_SNAKE_CASE     				=  repr.split("""_"""					)

									_SCREAMING_SNAKE_CASE     				=  {}

									for value in values:
															if "-" in value:
																					_SCREAMING_SNAKE_CASE				,					_SCREAMING_SNAKE_CASE     				=  value.split("""-"""					)
															else:
																					_SCREAMING_SNAKE_CASE     				=  re.sub("""[0-9.]"""				,     """"""				,     UpperCAmelCase_					)
																					_SCREAMING_SNAKE_CASE     				=  float(re.sub("""[^0-9.]"""				,     """"""				,     UpperCAmelCase_					)					)

															_SCREAMING_SNAKE_CASE     				=  cls.NAMING_INFO["""reverse_short_param"""][p_k]

															_SCREAMING_SNAKE_CASE     				=  p_v

									for k in cls.DEFAULTS:
															if k not in parameters:
																					_SCREAMING_SNAKE_CASE     				=  cls.DEFAULTS[k]

									return parameters

