import ast
import os
import re
import shutil
import tempfile
import unittest
from unittest import mock

import torch

from accelerate.test_utils.examples import compare_against_test
from accelerate.test_utils.testing import TempDirTestCase, require_trackers, run_command, slow
from accelerate.utils import write_basic_config


# DataLoaders built from `test_samples/MRPC` for quick testing
# Should mock `{script_name}.get_dataloaders` via:
# @mock.patch("{script_name}.get_dataloaders", mocked_dataloaders)

UpperCamelCase			    =     [
    '''cross_validation.py''',
    '''gradient_accumulation.py''',
    '''local_sgd.py''',
    '''multi_process_metrics.py''',
    '''memory.py''',
    '''automatic_gradient_accumulation.py''',
    '''fsdp_with_peak_mem_tracking.py''',
    '''deepspeed_with_config_support.py''',
    '''megatron_lm_gpt_pretraining.py''',
]



class 				__UpperCAmelCase		(unittest.TestCase   ):







			def 							UpperCamelCase				(     self:  Tuple				,     UpperCAmelCase_:  str				,     UpperCAmelCase_:  bool				,     UpperCAmelCase_:  str = None				,     UpperCAmelCase_:  list = None					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  None
									_SCREAMING_SNAKE_CASE     				=  os.path.abspath(os.path.join("""examples"""				,     """by_feature"""					)					)
									_SCREAMING_SNAKE_CASE     				=  os.path.abspath("""examples"""					)
									for item in os.listdir(UpperCAmelCase_					):
															if item not in EXCLUDE_EXAMPLES:
																					_SCREAMING_SNAKE_CASE     				=  os.path.join(UpperCAmelCase_				,     UpperCAmelCase_					)
																					if os.path.isfile(UpperCAmelCase_					) and ".py" in item_path:
																											with self.subTest(
																											    tested_script=UpperCAmelCase_				,     feature_script=UpperCAmelCase_				,     tested_section="""main()""" if parser_only else """training_function()"""				,     ):
																																	_SCREAMING_SNAKE_CASE     				=  compare_against_test(
																																	    os.path.join(UpperCAmelCase_				,     UpperCAmelCase_					)				,     UpperCAmelCase_				,     UpperCAmelCase_				,     UpperCAmelCase_					)
																																	_SCREAMING_SNAKE_CASE     				=  """\n""".join(UpperCAmelCase_					)
																																	if special_strings is not None:
																																							for string in special_strings:
																																													_SCREAMING_SNAKE_CASE     				=  diff.replace(UpperCAmelCase_				,     """"""					)
																																	self.assertEqual(UpperCAmelCase_				,     """"""					)







			def 							UpperCamelCase				(     self:  List[str]					):







									'''simple docstring'''




									self.one_complete_example("""complete_nlp_example.py"""				,     UpperCAmelCase_					)
									self.one_complete_example("""complete_nlp_example.py"""				,     UpperCAmelCase_					)







			def 							UpperCamelCase				(     self:  Dict					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  os.path.abspath(os.path.join("""examples"""				,     """cv_example.py"""					)					)
									_SCREAMING_SNAKE_CASE     				=  [
									    """ """ * 16 + """{\n\n""",
									    """ """ * 20 + """\"accuracy\": eval_metric[\"accuracy\"],\n\n""",
									    """ """ * 20 + """\"f1\": eval_metric[\"f1\"],\n\n""",
									    """ """ * 20 + """\"train_loss\": total_loss.item() / len(train_dataloader),\n\n""",
									    """ """ * 20 + """\"epoch\": epoch,\n\n""",
									    """ """ * 16 + """},\n\n""",
									    """ """ * 16 + """step=epoch,\n""",
									    """ """ * 12,
									    """ """ * 8 + """for step, batch in enumerate(active_dataloader):\n""",
									]
									self.one_complete_example("""complete_cv_example.py"""				,     UpperCAmelCase_				,     UpperCAmelCase_				,     UpperCAmelCase_					)
									self.one_complete_example("""complete_cv_example.py"""				,     UpperCAmelCase_				,     UpperCAmelCase_				,     UpperCAmelCase_					)



@mock.patch.dict(os.environ	,{"TESTING_MOCKED_DATALOADERS": "1"}   )
class 				__UpperCAmelCase		(_UpperCAmelCase   ):
			__snake_case    :				Any									=							False







			@classmethod
			def 							UpperCamelCase				(     cls:  List[Any]					):







									'''simple docstring'''




									super().setUpClass()
									_SCREAMING_SNAKE_CASE     				=  tempfile.mkdtemp()
									_SCREAMING_SNAKE_CASE     				=  os.path.join(cls._tmpdir				,     """default_config.yml"""					)

									write_basic_config(save_location=cls.configPath					)
									_SCREAMING_SNAKE_CASE     				=  ["""accelerate""", """launch""", """--config_file""", cls.configPath]







			@classmethod
			def 							UpperCamelCase				(     cls:  Optional[Any]					):







									'''simple docstring'''




									super().tearDownClass()
									shutil.rmtree(cls._tmpdir					)







			def 							UpperCamelCase				(     self:  int					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  F'\n        examples/by_feature/checkpointing.py\n        --checkpointing_steps epoch\n        --output_dir {self.tmpdir}\n        '.split()
									run_command(self._launch_args + testargs					)
									self.assertTrue(os.path.exists(os.path.join(self.tmpdir				,     """epoch_0"""					)					)					)







			def 							UpperCamelCase				(     self:  int					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  F'\n        examples/by_feature/checkpointing.py\n        --checkpointing_steps 1\n        --output_dir {self.tmpdir}\n        '.split()
									_SCREAMING_SNAKE_CASE     				=  run_command(self._launch_args + testargs					)
									self.assertTrue(os.path.exists(os.path.join(self.tmpdir				,     """step_2"""					)					)					)







			def 							UpperCamelCase				(     self:  Optional[int]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  F'\n        examples/by_feature/checkpointing.py\n        --resume_from_checkpoint {os.path.join(self.tmpdir				,     "epoch_0"					)}\n        '.split()
									_SCREAMING_SNAKE_CASE     				=  run_command(self._launch_args + testargs				,     return_stdout=UpperCAmelCase_					)
									self.assertNotIn("""epoch 0:"""				,     UpperCAmelCase_					)
									self.assertIn("""epoch 1:"""				,     UpperCAmelCase_					)







			def 							UpperCamelCase				(     self:  Union[str, Any]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  F'\n        examples/by_feature/checkpointing.py\n        --resume_from_checkpoint {os.path.join(self.tmpdir				,     "step_2"					)}\n        '.split()
									_SCREAMING_SNAKE_CASE     				=  run_command(self._launch_args + testargs				,     return_stdout=UpperCAmelCase_					)
									if torch.cuda.is_available():
															_SCREAMING_SNAKE_CASE     				=  torch.cuda.device_count()
									else:
															_SCREAMING_SNAKE_CASE     				=  1
									if num_processes > 1:
															self.assertNotIn("""epoch 0:"""				,     UpperCAmelCase_					)
															self.assertIn("""epoch 1:"""				,     UpperCAmelCase_					)
									else:
															self.assertIn("""epoch 0:"""				,     UpperCAmelCase_					)
															self.assertIn("""epoch 1:"""				,     UpperCAmelCase_					)







			@slow
			def 							UpperCamelCase				(     self:  Optional[int]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  """
        examples/by_feature/cross_validation.py
        --num_folds 2
        """.split()
									with mock.patch.dict(os.environ				,     {"""TESTING_MOCKED_DATALOADERS""": """0"""}					):
															_SCREAMING_SNAKE_CASE     				=  run_command(self._launch_args + testargs				,     return_stdout=UpperCAmelCase_					)
															_SCREAMING_SNAKE_CASE     				=  re.findall("""({.+})"""				,     UpperCAmelCase_					)
															_SCREAMING_SNAKE_CASE     				=  [r for r in results if """accuracy""" in r][-1]
															_SCREAMING_SNAKE_CASE     				=  ast.literal_eval(UpperCAmelCase_					)
															self.assertGreaterEqual(results["""accuracy"""]				,     0.75					)







			def 							UpperCamelCase				(     self:  List[Any]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  ["""examples/by_feature/multi_process_metrics.py"""]
									run_command(self._launch_args + testargs					)







			@require_trackers
			@mock.patch.dict(os.environ				,     {"""WANDB_MODE""": """offline"""}					)
			def 							UpperCamelCase				(     self:  str					):







									'''simple docstring'''




									with tempfile.TemporaryDirectory() as tmpdir:
															_SCREAMING_SNAKE_CASE     				=  F'\n            examples/by_feature/tracking.py\n            --with_tracking\n            --project_dir {tmpdir}\n            '.split()
															run_command(self._launch_args + testargs					)
															self.assertTrue(os.path.exists(os.path.join(UpperCAmelCase_				,     """tracking"""					)					)					)







			def 							UpperCamelCase				(     self:  Any					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  ["""examples/by_feature/gradient_accumulation.py"""]
									run_command(self._launch_args + testargs					)







			def 							UpperCamelCase				(     self:  Union[str, Any]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  ["""examples/by_feature/local_sgd.py"""]
									run_command(self._launch_args + testargs					)









import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin



class 				__UpperCAmelCase		(_UpperCAmelCase	,unittest.TestCase   ):
			__snake_case    :				int									=							TransfoXLTokenizer
			__snake_case    :				Tuple									=							False
			__snake_case    :				List[Any]									=							False







			def 							UpperCamelCase				(     self:  int					):







									'''simple docstring'''




									super().setUp()

									_SCREAMING_SNAKE_CASE     				=  [
									    """<unk>""",
									    """[CLS]""",
									    """[SEP]""",
									    """want""",
									    """unwanted""",
									    """wa""",
									    """un""",
									    """running""",
									    """,""",
									    """low""",
									    """l""",
									]
									_SCREAMING_SNAKE_CASE     				=  os.path.join(self.tmpdirname				,     VOCAB_FILES_NAMES["""vocab_file"""]					)
									with open(self.vocab_file				,     """w"""				,     encoding="""utf-8"""					) as vocab_writer:
															vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]					)					)







			def 							UpperCamelCase				(     self:  Any				,     **UpperCAmelCase_:  Optional[Any]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  True
									return TransfoXLTokenizer.from_pretrained(self.tmpdirname				,     **UpperCAmelCase_					)







			def 							UpperCamelCase				(     self:  Optional[int]				,     UpperCAmelCase_:  List[str]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  """<unk> UNwanted , running"""
									_SCREAMING_SNAKE_CASE     				=  """<unk> unwanted, running"""
									return input_text, output_text







			def 							UpperCamelCase				(     self:  Union[str, Any]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  TransfoXLTokenizer(vocab_file=self.vocab_file				,     lower_case=UpperCAmelCase_					)

									_SCREAMING_SNAKE_CASE     				=  tokenizer.tokenize("""<unk> UNwanted , running"""					)
									self.assertListEqual(UpperCAmelCase_				,     ["""<unk>""", """unwanted""", """,""", """running"""]					)

									self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_					)				,     [0, 4, 8, 7]					)







			def 							UpperCamelCase				(     self:  int					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  TransfoXLTokenizer(lower_case=UpperCAmelCase_					)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """					)				,     ["""hello""", """!""", """how""", """are""", """you""", """?"""]					)







			def 							UpperCamelCase				(     self:  Optional[Any]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  TransfoXLTokenizer(lower_case=UpperCAmelCase_					)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """					)				,     ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]					)







			def 							UpperCamelCase				(     self:  List[str]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  TransfoXLTokenizer(lower_case=UpperCAmelCase_					)
									_SCREAMING_SNAKE_CASE     				=  """Hello (bracket) and side-scrolled [and] Henry's $5,000 with 3.34 m. What's up!?"""
									_SCREAMING_SNAKE_CASE     				=  [
									    """Hello""",
									    """(""",
									    """bracket""",
									    """)""",
									    """and""",
									    """side""",
									    """@-@""",
									    """scrolled""",
									    """[""",
									    """and""",
									    """]""",
									    """Henry""",
									    """'s""",
									    """$""",
									    """5""",
									    """@,@""",
									    """000""",
									    """with""",
									    """3""",
									    """@.@""",
									    """34""",
									    """m""",
									    """.""",
									    """What""",
									    """'s""",
									    """up""",
									    """!""",
									    """?""",
									]

									self.assertListEqual(tokenizer.tokenize(UpperCAmelCase_					)				,     UpperCAmelCase_					)

									self.assertEqual(tokenizer.convert_tokens_to_string(UpperCAmelCase_					)				,     UpperCAmelCase_					)







			def 							UpperCamelCase				(     self:  List[str]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  self.get_tokenizer()
									_SCREAMING_SNAKE_CASE     				=  len(UpperCAmelCase_					)

									tokenizer.add_tokens(["""new1""", """new2"""]					)
									tokenizer.move_added_token("""new1"""				,     1					)

									# Check that moved token is not copied (duplicate)
									self.assertEqual(len(UpperCAmelCase_					)				,     original_len + 2					)
									# Check that token is moved to specified id
									self.assertEqual(tokenizer.encode("""new1"""					)				,     [1]					)
									self.assertEqual(tokenizer.decode([1]					)				,     """new1"""					)

