






import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin



class 				__UpperCAmelCase		(_UpperCAmelCase	,unittest.TestCase   ):
			__snake_case    :				int									=							TransfoXLTokenizer
			__snake_case    :				Tuple									=							False
			__snake_case    :				List[Any]									=							False







			def 							UpperCamelCase				(     self:  int					):







									'''simple docstring'''




									super().setUp()

									_SCREAMING_SNAKE_CASE     				=  [
									    """<unk>""",
									    """[CLS]""",
									    """[SEP]""",
									    """want""",
									    """unwanted""",
									    """wa""",
									    """un""",
									    """running""",
									    """,""",
									    """low""",
									    """l""",
									]
									_SCREAMING_SNAKE_CASE     				=  os.path.join(self.tmpdirname				,     VOCAB_FILES_NAMES["""vocab_file"""]					)
									with open(self.vocab_file				,     """w"""				,     encoding="""utf-8"""					) as vocab_writer:
															vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]					)					)







			def 							UpperCamelCase				(     self:  Any				,     **UpperCAmelCase_:  Optional[Any]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  True
									return TransfoXLTokenizer.from_pretrained(self.tmpdirname				,     **UpperCAmelCase_					)







			def 							UpperCamelCase				(     self:  Optional[int]				,     UpperCAmelCase_:  List[str]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  """<unk> UNwanted , running"""
									_SCREAMING_SNAKE_CASE     				=  """<unk> unwanted, running"""
									return input_text, output_text







			def 							UpperCamelCase				(     self:  Union[str, Any]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  TransfoXLTokenizer(vocab_file=self.vocab_file				,     lower_case=UpperCAmelCase_					)

									_SCREAMING_SNAKE_CASE     				=  tokenizer.tokenize("""<unk> UNwanted , running"""					)
									self.assertListEqual(UpperCAmelCase_				,     ["""<unk>""", """unwanted""", """,""", """running"""]					)

									self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_					)				,     [0, 4, 8, 7]					)







			def 							UpperCamelCase				(     self:  int					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  TransfoXLTokenizer(lower_case=UpperCAmelCase_					)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """					)				,     ["""hello""", """!""", """how""", """are""", """you""", """?"""]					)







			def 							UpperCamelCase				(     self:  Optional[Any]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  TransfoXLTokenizer(lower_case=UpperCAmelCase_					)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """					)				,     ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]					)







			def 							UpperCamelCase				(     self:  List[str]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  TransfoXLTokenizer(lower_case=UpperCAmelCase_					)
									_SCREAMING_SNAKE_CASE     				=  """Hello (bracket) and side-scrolled [and] Henry's $5,000 with 3.34 m. What's up!?"""
									_SCREAMING_SNAKE_CASE     				=  [
									    """Hello""",
									    """(""",
									    """bracket""",
									    """)""",
									    """and""",
									    """side""",
									    """@-@""",
									    """scrolled""",
									    """[""",
									    """and""",
									    """]""",
									    """Henry""",
									    """'s""",
									    """$""",
									    """5""",
									    """@,@""",
									    """000""",
									    """with""",
									    """3""",
									    """@.@""",
									    """34""",
									    """m""",
									    """.""",
									    """What""",
									    """'s""",
									    """up""",
									    """!""",
									    """?""",
									]

									self.assertListEqual(tokenizer.tokenize(UpperCAmelCase_					)				,     UpperCAmelCase_					)

									self.assertEqual(tokenizer.convert_tokens_to_string(UpperCAmelCase_					)				,     UpperCAmelCase_					)







			def 							UpperCamelCase				(     self:  List[str]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  self.get_tokenizer()
									_SCREAMING_SNAKE_CASE     				=  len(UpperCAmelCase_					)

									tokenizer.add_tokens(["""new1""", """new2"""]					)
									tokenizer.move_added_token("""new1"""				,     1					)

									# Check that moved token is not copied (duplicate)
									self.assertEqual(len(UpperCAmelCase_					)				,     original_len + 2					)
									# Check that token is moved to specified id
									self.assertEqual(tokenizer.encode("""new1"""					)				,     [1]					)
									self.assertEqual(tokenizer.decode([1]					)				,     """new1"""					)









import json
import os
import unittest

from transformers import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
from transformers.models.openai.tokenization_openai import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_spacy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class 				__UpperCAmelCase		(_UpperCAmelCase	,unittest.TestCase   ):
			__snake_case    :				Optional[Any]									=							OpenAIGPTTokenizer
			__snake_case    :				Dict									=							OpenAIGPTTokenizerFast
			__snake_case    :				Optional[Any]									=							True
			__snake_case    :				Dict									=							False







			def 							UpperCamelCase				(     self:  Union[str, Any]					):







									'''simple docstring'''




									super().setUp()

									# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
									_SCREAMING_SNAKE_CASE     				=  [
									    """l""",
									    """o""",
									    """w""",
									    """e""",
									    """r""",
									    """s""",
									    """t""",
									    """i""",
									    """d""",
									    """n""",
									    """w</w>""",
									    """r</w>""",
									    """t</w>""",
									    """lo""",
									    """low""",
									    """er</w>""",
									    """low</w>""",
									    """lowest</w>""",
									    """newer</w>""",
									    """wider</w>""",
									    """<unk>""",
									]
									_SCREAMING_SNAKE_CASE     				=  dict(zip(UpperCAmelCase_				,     range(len(UpperCAmelCase_					)					)					)					)
									_SCREAMING_SNAKE_CASE     				=  ["""#version: 0.2""", """l o""", """lo w""", """e r</w>""", """"""]

									_SCREAMING_SNAKE_CASE     				=  os.path.join(self.tmpdirname				,     VOCAB_FILES_NAMES["""vocab_file"""]					)
									_SCREAMING_SNAKE_CASE     				=  os.path.join(self.tmpdirname				,     VOCAB_FILES_NAMES["""merges_file"""]					)
									with open(self.vocab_file				,     """w"""					) as fp:
															fp.write(json.dumps(UpperCAmelCase_					)					)
									with open(self.merges_file				,     """w"""					) as fp:
															fp.write("""\n""".join(UpperCAmelCase_					)					)







			def 							UpperCamelCase				(     self:  List[Any]				,     UpperCAmelCase_:  Dict					):







									'''simple docstring'''




									return "lower newer", "lower newer"







			def 							UpperCamelCase				(     self:  Any					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  OpenAIGPTTokenizer(self.vocab_file				,     self.merges_file					)

									_SCREAMING_SNAKE_CASE     				=  """lower"""
									_SCREAMING_SNAKE_CASE     				=  ["""low""", """er</w>"""]
									_SCREAMING_SNAKE_CASE     				=  tokenizer.tokenize(UpperCAmelCase_					)
									self.assertListEqual(UpperCAmelCase_				,     UpperCAmelCase_					)

									_SCREAMING_SNAKE_CASE     				=  tokens + ["""<unk>"""]
									_SCREAMING_SNAKE_CASE     				=  [14, 15, 20]
									self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_					)				,     UpperCAmelCase_					)







			def 							UpperCamelCase				(     self:  Optional[Any]				,     UpperCAmelCase_:  Union[str, Any]=15					):







									'''simple docstring'''




									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'					):
																					_SCREAMING_SNAKE_CASE     				=  self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_				,     **UpperCAmelCase_					)

																					# Simple input
																					_SCREAMING_SNAKE_CASE     				=  """This is a simple input"""
																					_SCREAMING_SNAKE_CASE     				=  ["""This is a simple input 1""", """This is a simple input 2"""]
																					_SCREAMING_SNAKE_CASE     				=  ("""This is a simple input""", """This is a pair""")
																					_SCREAMING_SNAKE_CASE     				=  [
																					    ("""This is a simple input 1""", """This is a simple input 2"""),
																					    ("""This is a simple pair 1""", """This is a simple pair 2"""),
																					]

																					# Simple input tests
																					self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding="""max_length"""					)

																					# Simple input
																					self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding="""max_length"""					)

																					# Simple input
																					self.assertRaises(
																					    UpperCAmelCase_				,     tokenizer_r.batch_encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding="""max_length"""				,     )

																					# Pair input
																					self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding="""max_length"""					)

																					# Pair input
																					self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding="""max_length"""					)

																					# Pair input
																					self.assertRaises(
																					    UpperCAmelCase_				,     tokenizer_r.batch_encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding="""max_length"""				,     )







			def 							UpperCamelCase				(     self:  List[Any]					):







									'''simple docstring'''




									pass



@require_ftfy
@require_spacy
@require_tokenizers
class 				__UpperCAmelCase		(_UpperCAmelCase   ):
			pass

