


"""simple docstring"""



import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin







class _lowerCAmelCase    (				snake_case_   ,							unittest.TestCase     ):
   __UpperCAmelCase			:	str							  =  TransfoXLTokenizer
   __UpperCAmelCase			:	Union[str, Any]							  =  False
   __UpperCAmelCase			:	Union[str, Any]							  =  False







   def 			lowerCamelCase		(     self						)     ->		Tuple:

       '''simple docstring'''




       super().setUp()

       snake_case   :				Any				 =   [
           "<unk>",
           "[CLS]",
           "[SEP]",
           "want",
           "unwanted",
           "wa",
           "un",
           "running",
           ",",
           "low",
           "l",
       ]
       snake_case   :				str				 =   os.path.join(self.tmpdirname						,     VOCAB_FILES_NAMES["vocab_file"]						)
       with open(self.vocab_file						,     "w"						,     encoding="utf-8"						) as vocab_writer:
           vocab_writer.write("".join([x + "\n" for x in vocab_tokens]						)						)







   def 			lowerCamelCase		(     self						,     **UpperCamelCase__						)     ->		List[str]:

       '''simple docstring'''




       snake_case   :				Tuple				 =   True
       return TransfoXLTokenizer.from_pretrained(self.tmpdirname						,     **UpperCamelCase__						)







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						)     ->		List[Any]:

       '''simple docstring'''




       snake_case   :				int				 =   "<unk> UNwanted , running"
       snake_case   :				List[Any]				 =   "<unk> unwanted, running"
       return input_text, output_text







   def 			lowerCamelCase		(     self						)     ->		Dict:

       '''simple docstring'''




       snake_case   :				str				 =   TransfoXLTokenizer(vocab_file=self.vocab_file						,     lower_case=UpperCamelCase__						)

       snake_case   :				str				 =   tokenizer.tokenize("<unk> UNwanted , running"						)
       self.assertListEqual(UpperCamelCase__						,     ["<unk>", "unwanted", ",", "running"]						)

       self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__						)						,     [0, 4, 8, 7]						)







   def 			lowerCamelCase		(     self						)     ->		Any:

       '''simple docstring'''




       snake_case   :				List[str]				 =   TransfoXLTokenizer(lower_case=UpperCamelCase__						)

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo ! how  \n Are yoU ?  "						)						,     ["hello", "!", "how", "are", "you", "?"]						)







   def 			lowerCamelCase		(     self						)     ->		int:

       '''simple docstring'''




       snake_case   :				Tuple				 =   TransfoXLTokenizer(lower_case=UpperCamelCase__						)

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo ! how  \n Are yoU ?  "						)						,     ["HeLLo", "!", "how", "Are", "yoU", "?"]						)







   def 			lowerCamelCase		(     self						)     ->		Tuple:

       '''simple docstring'''




       snake_case   :				Optional[Any]				 =   TransfoXLTokenizer(lower_case=UpperCamelCase__						)
       snake_case   :				List[str]				 =   "Hello (bracket) and side-scrolled [and] Henry's $5,000 with 3.34 m. What's up!?"
       snake_case   :				Any				 =   [
           "Hello",
           "(",
           "bracket",
           ")",
           "and",
           "side",
           "@-@",
           "scrolled",
           "[",
           "and",
           "]",
           "Henry",
           "'s",
           "$",
           "5",
           "@,@",
           "000",
           "with",
           "3",
           "@.@",
           "34",
           "m",
           ".",
           "What",
           "'s",
           "up",
           "!",
           "?",
       ]

       self.assertListEqual(tokenizer.tokenize(UpperCamelCase__						)						,     UpperCamelCase__						)

       self.assertEqual(tokenizer.convert_tokens_to_string(UpperCamelCase__						)						,     UpperCamelCase__						)







   def 			lowerCamelCase		(     self						)     ->		Tuple:

       '''simple docstring'''




       snake_case   :				List[str]				 =   self.get_tokenizer()
       snake_case   :				int				 =   len(UpperCamelCase__						)

       tokenizer.add_tokens(["new1", "new2"]						)
       tokenizer.move_added_token("new1"						,     1						)

       # Check that moved token is not copied (duplicate)
       self.assertEqual(len(UpperCamelCase__						)						,     original_len + 2						)
       # Check that token is moved to specified id
       self.assertEqual(tokenizer.encode("new1"						)						,     [1]						)
       self.assertEqual(tokenizer.decode([1]						)						,     "new1"						)









"""simple docstring"""



import doctest
import logging
import os
import unittest
from pathlib import Path
from typing import List, Union

import transformers
from transformers.testing_utils import require_tf, require_torch, slow


__snake_case             =   logging.getLogger()







@unittest.skip('''Temporarily disable the doc tests.'''     )
@require_torch
@require_tf
@slow
class _lowerCAmelCase    (				unittest.TestCase     ):







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						,     UpperCamelCase__ = None						,     UpperCamelCase__ = None						,     UpperCamelCase__ = None						,     UpperCamelCase__ = True						,     )     ->		Tuple:

       '''simple docstring'''




       snake_case   :				List[Any]				 =   [file for file in os.listdir(UpperCamelCase__						) if os.path.isfile(os.path.join(UpperCamelCase__						,     UpperCamelCase__						)						)]

       if identifier is not None:
           snake_case   :				Optional[Any]				 =   [file for file in files if identifier in file]

       if n_identifier is not None:
           if isinstance(UpperCamelCase__						,     UpperCamelCase__						):
               for n_ in n_identifier:
                   snake_case   :				str				 =   [file for file in files if n_ not in file]
           else:
               snake_case   :				str				 =   [file for file in files if n_identifier not in file]

       snake_case   :				Tuple				 =   ignore_files or []
       ignore_files.append("__init__.py"						)
       snake_case   :				Optional[Any]				 =   [file for file in files if file not in ignore_files]

       for file in files:
           # Open all files
           print("Testing"						,     UpperCamelCase__						)

           if only_modules:
               snake_case   :				str				 =   file.split("."						)[0]
               try:
                   snake_case   :				Optional[int]				 =   getattr(UpperCamelCase__						,     UpperCamelCase__						)
                   snake_case   :				str				 =   doctest.DocTestSuite(UpperCamelCase__						)
                   snake_case   :				Optional[Any]				 =   unittest.TextTestRunner().run(UpperCamelCase__						)
                   self.assertIs(len(result.failures						)						,     0						)
               except AttributeError:
                   logger.info(F'{module_identifier} is not a module.'						)
           else:
               snake_case   :				Tuple				 =   doctest.testfile(str(".." / directory / file						)						,     optionflags=doctest.ELLIPSIS						)
               self.assertIs(result.failed						,     0						)







   def 			lowerCamelCase		(     self						)     ->		str:

       '''simple docstring'''




       snake_case   :				Tuple				 =   Path("src/transformers"						)
       snake_case   :				List[Any]				 =   "modeling"
       snake_case   :				Dict				 =   [
           "modeling_ctrl.py",
           "modeling_tf_ctrl.py",
       ]
       self.analyze_directory(UpperCamelCase__						,     identifier=UpperCamelCase__						,     ignore_files=UpperCamelCase__						)







   def 			lowerCamelCase		(     self						)     ->		Dict:

       '''simple docstring'''




       snake_case   :				Optional[Any]				 =   Path("src/transformers"						)
       snake_case   :				Optional[Any]				 =   "tokenization"
       self.analyze_directory(UpperCamelCase__						,     identifier=UpperCamelCase__						)







   def 			lowerCamelCase		(     self						)     ->		List[str]:

       '''simple docstring'''




       snake_case   :				Union[str, Any]				 =   Path("src/transformers"						)
       snake_case   :				Optional[Any]				 =   "configuration"
       self.analyze_directory(UpperCamelCase__						,     identifier=UpperCamelCase__						)







   def 			lowerCamelCase		(     self						)     ->		Optional[Any]:

       '''simple docstring'''




       snake_case   :				List[str]				 =   Path("src/transformers"						)
       snake_case   :				List[str]				 =   ["configuration", "modeling", "tokenization"]
       self.analyze_directory(UpperCamelCase__						,     n_identifier=UpperCamelCase__						)







   def 			lowerCamelCase		(     self						)     ->		List[Any]:

       '''simple docstring'''




       snake_case   :				Any				 =   Path("docs/source"						)
       snake_case   :				Tuple				 =   ["favicon.ico"]
       self.analyze_directory(UpperCamelCase__						,     ignore_files=UpperCamelCase__						,     only_modules=UpperCamelCase__						)





