


"""simple docstring"""







import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import MaMaaaTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)
from transformers.utils import is_sentencepiece_available


if is_sentencepiece_available():
					from transformers.models.mam_aaa.tokenization_mam_aaa import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


if is_sentencepiece_available():
					lowerCAmelCase__				    =       get_tests_dir('''fixtures/test_sentencepiece.model''')


if is_torch_available():
					from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

lowerCAmelCase__				    =       128_022
lowerCAmelCase__				    =       128_028


@require_sentencepiece
class 				SCREAMING_SNAKE_CASE__    (		lowercase       ,   unittest.TestCase    ):
							"""simple docstring"""


							a  :    Optional[int]       =MaMaaaTokenizer
							a  :    int       =False
							a  :    str       =False
							a  :    int       =True

							def 	lowercase__							(						self						):






									"""simple docstring"""





									super().setUp()

									lowerCAmelCase					:					str								=							["</s>", "<unk>", "▁This", "▁is", "▁a", "▁t", "est", "\u0120", "<pad>"]
									lowerCAmelCase					:					int								=							dict(zip(snake_case__		,					range(len(snake_case__						)						)						)						)
									lowerCAmelCase					:					List[str]								=							Path(self.tmpdirname						)
									save_json(snake_case__		,					save_dir / VOCAB_FILES_NAMES["vocab_file"]						)
									if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
											copyfile(snake_case__		,					save_dir / VOCAB_FILES_NAMES["spm_file"]						)

									lowerCAmelCase					:					int								=							MaMaaaTokenizer.from_pretrained(self.tmpdirname						)
									tokenizer.save_pretrained(self.tmpdirname						)

							def 	lowercase__							(						self		,					**snake_case__						):






									"""simple docstring"""





									return MaMaaaTokenizer.from_pretrained(self.tmpdirname		,					**snake_case__						)

							def 	lowercase__							(						self		,					snake_case__						):






									"""simple docstring"""





									return (
									    "This is a test",
									    "This is a test",
									)

							def 	lowercase__							(						self						):






									"""simple docstring"""





									lowerCAmelCase					:					Optional[int]								=							"</s>"
									lowerCAmelCase					:					List[Any]								=							0

									self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case__						)		,					snake_case__						)
									self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case__						)		,					snake_case__						)

							def 	lowercase__							(						self						):






									"""simple docstring"""





									lowerCAmelCase					:					int								=							self.get_tokenizer()
									lowerCAmelCase					:					Optional[int]								=							list(tokenizer.get_vocab().keys()						)

									self.assertEqual(vocab_keys[0]		,					"</s>"						)
									self.assertEqual(vocab_keys[1]		,					"<unk>"						)
									self.assertEqual(vocab_keys[-1]		,					"<s>"						)
									self.assertEqual(len(snake_case__						)		,					tokenizer.vocab_size + len(tokenizer.get_added_vocab()						)						)

							@unittest.skip("Skip this test while all models are still to be uploaded."						)
							def 	lowercase__							(						self						):






									"""simple docstring"""





									pass

							def 	lowercase__							(						self						):






									"""simple docstring"""





									lowerCAmelCase					:					Union[str, Any]								=							self.get_tokenizer()

									lowerCAmelCase					:					Union[str, Any]								=							tokenizer.tokenize("This is a test"						)
									self.assertListEqual(snake_case__		,					["▁This", "▁is", "▁a", "▁t", "est"]						)

									self.assertListEqual(
									    tokenizer.convert_tokens_to_ids(snake_case__						)		,					[2, 3, 4, 5, 6]		,					)

									lowerCAmelCase					:					int								=							tokenizer.convert_ids_to_tokens([2, 3, 4, 5, 6]						)
									self.assertListEqual(snake_case__		,					["▁This", "▁is", "▁a", "▁t", "est"]						)

									lowerCAmelCase					:					List[Any]								=							tokenizer.convert_tokens_to_string(snake_case__						)
									self.assertEqual(snake_case__		,					"This is a test"						)







							@slow
							def 	lowercase__							(						self						):






									"""simple docstring"""





									lowerCAmelCase					:					int								=							{"input_ids": [[128_022, 110_108, 397, 11, 38_272, 2_247, 124_811, 285, 18_105, 1_586, 207, 7, 39_534, 4_428, 397, 1_019, 18_105, 1_586, 207, 7, 41_337, 16_786, 241, 7, 20_214, 17, 125_690, 10_398, 7, 44_378, 58_069, 68_342, 7_798, 7_343, 11, 299, 33_310, 4, 158, 37_350, 94_077, 4_569, 299, 33_310, 90, 4, 52_840, 290, 4, 31_270, 112, 299, 682, 4, 52_840, 39_953, 14_079, 193, 52_519, 90_894, 17_894, 120_697, 11, 40_445, 551, 17, 1_019, 52_519, 90_894, 17_756, 963, 11, 40_445, 480, 17, 9_792, 1_120, 5_173, 1_393, 6_240, 16_786, 241, 120_996, 28, 1_245, 1_393, 118_240, 11_123, 1_019, 93_612, 2_691, 10_618, 98_058, 120_409, 1_928, 279, 4, 40_683, 367, 178, 207, 1_019, 103, 103_121, 506, 65_296, 5, 2], [128_022, 21_217, 367, 117, 125_450, 128, 719, 7, 7_308, 40, 93_612, 12_669, 1_116, 16_704, 71, 17_785, 3_699, 15_592, 35, 144, 9_584, 241, 11_943, 713, 950, 799, 2_247, 88_427, 150, 149, 118_813, 120_706, 1_019, 106_906, 81_518, 28, 1_224, 22_799, 397, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [128_022, 1_658, 123_311, 5_155, 5_578, 4_722, 279, 14_947, 2_366, 1_120, 1_197, 14, 1_348, 9_232, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
									# fmt: on

									self.tokenizer_integration_test_util(
									    expected_encoding=snake_case__		,					model_name="facebook/m2m100_418M"		,					revision="c168bae485c864188cf9aa0e4108b0b6934dc91e"		,					)


@require_torch
@require_sentencepiece
@require_tokenizers
class 				SCREAMING_SNAKE_CASE__    (		unittest.TestCase    ):
							"""simple docstring"""


							a  :    Any       ="facebook/m2m100_418M"
							a  :    List[Any]       =[
							    "In my opinion, there are two levels of response from the French government.",
							    "NSA Affair Emphasizes Complete Lack of Debate on Intelligence",
							]
							a  :    str       =[
							    "Selon moi, il y a deux niveaux de réponse de la part du gouvernement français.",
							    "L'affaire NSA souligne l'absence totale de débat sur le renseignement",
							]

							# fmt: off
							a  :    Optional[Any]       =[EN_CODE, 5_93, 19_49, 11_57_81, 4, 7_15_86, 42_34, 6_06_33, 12_62_33, 4_32, 12_38_08, 1_55_92, 11_97, 11_71_32, 12_06_18, 5, 2]

							@classmethod
							def 	lowercase__							(						cls						):






									"""simple docstring"""





									lowerCAmelCase					:					MaMaaaTokenizer								=							MaMaaaTokenizer.from_pretrained(
									    cls.checkpoint_name		,					src_lang="en"		,					tgt_lang="fr"						)
									lowerCAmelCase					:					List[Any]								=							1
									return cls

							def 	lowercase__							(						self						):






									"""simple docstring"""





									self.assertEqual(self.tokenizer.get_lang_id("ar"						)		,					128_006						)
									self.assertEqual(self.tokenizer.get_lang_id("en"						)		,					128_022						)
									self.assertEqual(self.tokenizer.get_lang_id("ro"						)		,					128_076						)
									self.assertEqual(self.tokenizer.get_lang_id("mr"						)		,					128_063						)

							def 	lowercase__							(						self						):






									"""simple docstring"""





									lowerCAmelCase					:					Union[str, Any]								=							self.tokenizer.get_vocab()
									self.assertEqual(len(snake_case__						)		,					self.tokenizer.vocab_size						)
									self.assertEqual(vocab["<unk>"]		,					3						)
									self.assertIn(self.tokenizer.get_lang_token("en"						)		,					snake_case__						)

							def 	lowercase__							(						self						):






									"""simple docstring"""





									lowerCAmelCase					:					Optional[int]								=							"en"
									lowerCAmelCase					:					int								=							self.tokenizer.batch_encode_plus(self.src_text						).input_ids[0]
									self.assertListEqual(self.expected_src_tokens		,					snake_case__						)

							def 	lowercase__							(						self						):






									"""simple docstring"""





									self.assertIn(snake_case__		,					self.tokenizer.all_special_ids						)
									# fmt: off
									lowerCAmelCase					:					Any								=							[FR_CODE, 5_364, 82, 8_642, 4, 294, 47, 8, 14_028, 136, 3_286, 9_706, 6, 90_797, 6, 144_012, 162, 88_128, 30_061, 5, 2]
									# fmt: on
									lowerCAmelCase					:					str								=							self.tokenizer.decode(snake_case__		,					skip_special_tokens=snake_case__						)
									lowerCAmelCase					:					Tuple								=							self.tokenizer.decode(generated_ids[1:]		,					skip_special_tokens=snake_case__						)
									self.assertEqual(snake_case__		,					snake_case__						)
									self.assertNotIn(self.tokenizer.eos_token		,					snake_case__						)

							def 	lowercase__							(						self						):






									"""simple docstring"""





									lowerCAmelCase					:					Any								=							tempfile.mkdtemp()
									lowerCAmelCase					:					List[str]								=							self.tokenizer.lang_token_to_id
									self.tokenizer.save_pretrained(snake_case__						)
									lowerCAmelCase					:					str								=							MaMaaaTokenizer.from_pretrained(snake_case__						)
									self.assertDictEqual(new_tok.lang_token_to_id		,					snake_case__						)

							@require_torch
							def 	lowercase__							(						self						):






									"""simple docstring"""





									lowerCAmelCase					:					Tuple								=							"en"
									lowerCAmelCase					:					Optional[int]								=							"fr"

									lowerCAmelCase					:					int								=							self.tokenizer(self.src_text		,					text_target=self.tgt_text		,					padding=snake_case__		,					return_tensors="pt"						)

									lowerCAmelCase					:					Any								=							shift_tokens_right(
									    batch["labels"]		,					self.tokenizer.pad_token_id		,					self.tokenizer.eos_token_id						)

									for k in batch:
											lowerCAmelCase					:					Any								=							batch[k].tolist()
									# batch = {k: v.tolist() for k,v in batch.items()}
									# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
									# batch.decoder_inputs_ids[0][0] ==
									assert batch.input_ids[1][0] == EN_CODE
									assert batch.input_ids[1][-1] == 2
									assert batch.labels[1][0] == FR_CODE
									assert batch.labels[1][-1] == 2
									assert batch.decoder_input_ids[1][:2] == [2, FR_CODE]

							@require_torch
							def 	lowercase__							(						self						):






									"""simple docstring"""





									lowerCAmelCase					:					Dict								=							"mr"
									self.assertListEqual(self.tokenizer.prefix_tokens		,					[self.tokenizer.get_lang_id("mr"						)]						)
									self.assertListEqual(self.tokenizer.suffix_tokens		,					[self.tokenizer.eos_token_id]						)

									lowerCAmelCase					:					int								=							"zh"
									self.assertListEqual(self.tokenizer.prefix_tokens		,					[self.tokenizer.get_lang_id("zh"						)]						)
									self.assertListEqual(self.tokenizer.suffix_tokens		,					[self.tokenizer.eos_token_id]						)

							@require_torch
							def 	lowercase__							(						self						):






									"""simple docstring"""





									lowerCAmelCase					:					str								=							"mr"
									self.tokenizer._switch_to_target_mode()
									self.assertListEqual(self.tokenizer.prefix_tokens		,					[self.tokenizer.get_lang_id("mr"						)]						)
									self.assertListEqual(self.tokenizer.suffix_tokens		,					[self.tokenizer.eos_token_id]						)
									self.tokenizer._switch_to_input_mode()
									self.assertListEqual(self.tokenizer.prefix_tokens		,					[self.tokenizer.get_lang_id(self.tokenizer.src_lang						)]						)

									lowerCAmelCase					:					Dict								=							"zh"
									self.tokenizer._switch_to_target_mode()
									self.assertListEqual(self.tokenizer.prefix_tokens		,					[self.tokenizer.get_lang_id("zh"						)]						)
									self.assertListEqual(self.tokenizer.suffix_tokens		,					[self.tokenizer.eos_token_id]						)
									self.tokenizer._switch_to_input_mode()
									self.assertListEqual(self.tokenizer.prefix_tokens		,					[self.tokenizer.get_lang_id(self.tokenizer.src_lang						)]						)







							@require_torch
							def 	lowercase__							(						self						):






									"""simple docstring"""





									lowerCAmelCase					:					Dict								=							self.tokenizer._build_translation_inputs("A test"		,					return_tensors="pt"		,					src_lang="en"		,					tgt_lang="ar"						)

									self.assertEqual(
									    nested_simplify(snake_case__						)		,					{
									        # en_XX, A, test, EOS
									        "input_ids": [[128_022, 58, 4_183, 2]],
									        "attention_mask": [[1, 1, 1, 1]],
									        # ar_AR
									        "forced_bos_token_id": 128_006,
									    }		,					)









"""simple docstring"""







import argparse
import json
import logging
import os
import shutil
import sys
import tempfile
import unittest
from unittest import mock

import torch
from accelerate.utils import write_basic_config

from transformers.testing_utils import TestCasePlus, get_gpu_count, run_command, slow, torch_device
from transformers.utils import is_apex_available


logging.basicConfig(level=logging.DEBUG)

lowerCAmelCase__				    =       logging.getLogger()
def  a__      (    ):


  '''simple docstring'''







  lowerCAmelCase					:					Tuple								=							argparse.ArgumentParser()
  parser.add_argument("-f"    )
  lowerCAmelCase					:					Tuple								=							parser.parse_args()
  return args.f
def  a__      (    SCREAMING_SNAKE_CASE   :   int    ):


  '''simple docstring'''







  lowerCAmelCase					:					Union[str, Any]								=							{}
  lowerCAmelCase					:					List[str]								=							os.path.join(SCREAMING_SNAKE_CASE    ,				"all_results.json"    )
  if os.path.exists(SCREAMING_SNAKE_CASE    ):
    with open(SCREAMING_SNAKE_CASE    ,				"r"    ) as f:
      lowerCAmelCase					:					str								=							json.load(SCREAMING_SNAKE_CASE    )
  else:
    raise ValueError(f"""can't find {path}"""    )
  return results
def  a__      (    ):


  '''simple docstring'''







  lowerCAmelCase					:					Tuple								=							torch.cuda.is_available() and torch_device == "cuda"
  return is_using_cuda and is_apex_available()


lowerCAmelCase__				    =       logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)





class 				SCREAMING_SNAKE_CASE__    (		lowercase    ):
       """simple docstring"""


       @classmethod
       def 	lowercase__							(						cls						):






         """simple docstring"""





         lowerCAmelCase					:					Any								=							tempfile.mkdtemp()
         lowerCAmelCase					:					List[Any]								=							os.path.join(cls.tmpdir		,					"default_config.yml"						)
         write_basic_config(save_location=cls.configPath						)
         lowerCAmelCase					:					Optional[int]								=							["accelerate", "launch", "--config_file", cls.configPath]

       @classmethod
       def 	lowercase__							(						cls						):






         """simple docstring"""





         shutil.rmtree(cls.tmpdir						)

       @mock.patch.dict(os.environ		,					{"WANDB_MODE": "offline"}						)
       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					Optional[int]								=							self.get_auto_remove_tmp_dir()
         lowerCAmelCase					:					int								=							f"""
            {self.examples_dir}/pytorch/text-classification/run_glue_no_trainer.py
            --model_name_or_path distilbert-base-uncased
            --output_dir {tmp_dir}
            --train_file ./tests/fixtures/tests_samples/MRPC/train.csv
            --validation_file ./tests/fixtures/tests_samples/MRPC/dev.csv
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --learning_rate=1e-4
            --seed=42
            --checkpointing_steps epoch
            --with_tracking
        """.split()

         if is_cuda_and_apex_available():
           testargs.append("--fp16"						)

         run_command(self._launch_args + testargs						)
         lowerCAmelCase					:					Dict								=							get_results(snake_case__						)
         self.assertGreaterEqual(result["eval_accuracy"]		,					0.75						)
         self.assertTrue(os.path.exists(os.path.join(snake_case__		,					"epoch_0"						)						)						)
         self.assertTrue(os.path.exists(os.path.join(snake_case__		,					"glue_no_trainer"						)						)						)

       @mock.patch.dict(os.environ		,					{"WANDB_MODE": "offline"}						)
       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					List[Any]								=							self.get_auto_remove_tmp_dir()
         lowerCAmelCase					:					Tuple								=							f"""
            {self.examples_dir}/pytorch/language-modeling/run_clm_no_trainer.py
            --model_name_or_path distilgpt2
            --train_file ./tests/fixtures/sample_text.txt
            --validation_file ./tests/fixtures/sample_text.txt
            --block_size 128
            --per_device_train_batch_size 5
            --per_device_eval_batch_size 5
            --num_train_epochs 2
            --output_dir {tmp_dir}
            --checkpointing_steps epoch
            --with_tracking
        """.split()

         if torch.cuda.device_count() > 1:
           # Skipping because there are not enough batches to train the model + would need a drop_last to work.
           return

         run_command(self._launch_args + testargs						)
         lowerCAmelCase					:					List[str]								=							get_results(snake_case__						)
         self.assertLess(result["perplexity"]		,					100						)
         self.assertTrue(os.path.exists(os.path.join(snake_case__		,					"epoch_0"						)						)						)
         self.assertTrue(os.path.exists(os.path.join(snake_case__		,					"clm_no_trainer"						)						)						)

       @mock.patch.dict(os.environ		,					{"WANDB_MODE": "offline"}						)
       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					Union[str, Any]								=							self.get_auto_remove_tmp_dir()
         lowerCAmelCase					:					Tuple								=							f"""
            {self.examples_dir}/pytorch/language-modeling/run_mlm_no_trainer.py
            --model_name_or_path distilroberta-base
            --train_file ./tests/fixtures/sample_text.txt
            --validation_file ./tests/fixtures/sample_text.txt
            --output_dir {tmp_dir}
            --num_train_epochs=1
            --checkpointing_steps epoch
            --with_tracking
        """.split()

         run_command(self._launch_args + testargs						)
         lowerCAmelCase					:					Any								=							get_results(snake_case__						)
         self.assertLess(result["perplexity"]		,					42						)
         self.assertTrue(os.path.exists(os.path.join(snake_case__		,					"epoch_0"						)						)						)
         self.assertTrue(os.path.exists(os.path.join(snake_case__		,					"mlm_no_trainer"						)						)						)

       @mock.patch.dict(os.environ		,					{"WANDB_MODE": "offline"}						)
       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					Optional[int]								=							7 if get_gpu_count() > 1 else 2

         lowerCAmelCase					:					Optional[Any]								=							self.get_auto_remove_tmp_dir()
         lowerCAmelCase					:					int								=							f"""
            {self.examples_dir}/pytorch/token-classification/run_ner_no_trainer.py
            --model_name_or_path bert-base-uncased
            --train_file tests/fixtures/tests_samples/conll/sample.json
            --validation_file tests/fixtures/tests_samples/conll/sample.json
            --output_dir {tmp_dir}
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=2
            --num_train_epochs={epochs}
            --seed 7
            --checkpointing_steps epoch
            --with_tracking
        """.split()

         run_command(self._launch_args + testargs						)
         lowerCAmelCase					:					str								=							get_results(snake_case__						)
         self.assertGreaterEqual(result["eval_accuracy"]		,					0.75						)
         self.assertLess(result["train_loss"]		,					0.5						)
         self.assertTrue(os.path.exists(os.path.join(snake_case__		,					"epoch_0"						)						)						)
         self.assertTrue(os.path.exists(os.path.join(snake_case__		,					"ner_no_trainer"						)						)						)

       @unittest.skip(reason="Fix me @muellerzr"						)
       @mock.patch.dict(os.environ		,					{"WANDB_MODE": "offline"}						)
       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					Optional[int]								=							self.get_auto_remove_tmp_dir()
         lowerCAmelCase					:					List[str]								=							f"""
            {self.examples_dir}/pytorch/question-answering/run_qa_no_trainer.py
            --model_name_or_path bert-base-uncased
            --version_2_with_negative
            --train_file tests/fixtures/tests_samples/SQUAD/sample.json
            --validation_file tests/fixtures/tests_samples/SQUAD/sample.json
            --output_dir {tmp_dir}
            --seed=42
            --max_train_steps=10
            --num_warmup_steps=2
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --checkpointing_steps epoch
            --with_tracking
        """.split()

         run_command(self._launch_args + testargs						)
         lowerCAmelCase					:					Optional[Any]								=							get_results(snake_case__						)
         # Because we use --version_2_with_negative the testing script uses SQuAD v2 metrics.
         self.assertGreaterEqual(result["eval_f1"]		,					28						)
         self.assertGreaterEqual(result["eval_exact"]		,					28						)
         self.assertTrue(os.path.exists(os.path.join(snake_case__		,					"epoch_0"						)						)						)
         self.assertTrue(os.path.exists(os.path.join(snake_case__		,					"qa_no_trainer"						)						)						)

       @mock.patch.dict(os.environ		,					{"WANDB_MODE": "offline"}						)
       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					List[Any]								=							self.get_auto_remove_tmp_dir()
         lowerCAmelCase					:					List[Any]								=							f"""
            {self.examples_dir}/pytorch/multiple-choice/run_swag_no_trainer.py
            --model_name_or_path bert-base-uncased
            --train_file tests/fixtures/tests_samples/swag/sample.json
            --validation_file tests/fixtures/tests_samples/swag/sample.json
            --output_dir {tmp_dir}
            --max_train_steps=20
            --num_warmup_steps=2
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --with_tracking
        """.split()

         run_command(self._launch_args + testargs						)
         lowerCAmelCase					:					Tuple								=							get_results(snake_case__						)
         self.assertGreaterEqual(result["eval_accuracy"]		,					0.8						)
         self.assertTrue(os.path.exists(os.path.join(snake_case__		,					"swag_no_trainer"						)						)						)

       @slow
       @mock.patch.dict(os.environ		,					{"WANDB_MODE": "offline"}						)
       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					str								=							self.get_auto_remove_tmp_dir()
         lowerCAmelCase					:					Tuple								=							f"""
            {self.examples_dir}/pytorch/summarization/run_summarization_no_trainer.py
            --model_name_or_path t5-small
            --train_file tests/fixtures/tests_samples/xsum/sample.json
            --validation_file tests/fixtures/tests_samples/xsum/sample.json
            --output_dir {tmp_dir}
            --max_train_steps=50
            --num_warmup_steps=8
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --checkpointing_steps epoch
            --with_tracking
        """.split()

         run_command(self._launch_args + testargs						)
         lowerCAmelCase					:					str								=							get_results(snake_case__						)
         self.assertGreaterEqual(result["eval_rouge1"]		,					10						)
         self.assertGreaterEqual(result["eval_rouge2"]		,					2						)
         self.assertGreaterEqual(result["eval_rougeL"]		,					7						)
         self.assertGreaterEqual(result["eval_rougeLsum"]		,					7						)
         self.assertTrue(os.path.exists(os.path.join(snake_case__		,					"epoch_0"						)						)						)
         self.assertTrue(os.path.exists(os.path.join(snake_case__		,					"summarization_no_trainer"						)						)						)

       @slow
       @mock.patch.dict(os.environ		,					{"WANDB_MODE": "offline"}						)
       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					Tuple								=							self.get_auto_remove_tmp_dir()
         lowerCAmelCase					:					Tuple								=							f"""
            {self.examples_dir}/pytorch/translation/run_translation_no_trainer.py
            --model_name_or_path sshleifer/student_marian_en_ro_6_1
            --source_lang en
            --target_lang ro
            --train_file tests/fixtures/tests_samples/wmt16/sample.json
            --validation_file tests/fixtures/tests_samples/wmt16/sample.json
            --output_dir {tmp_dir}
            --max_train_steps=50
            --num_warmup_steps=8
            --num_beams=6
            --learning_rate=3e-3
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --source_lang en_XX
            --target_lang ro_RO
            --checkpointing_steps epoch
            --with_tracking
        """.split()

         run_command(self._launch_args + testargs						)
         lowerCAmelCase					:					int								=							get_results(snake_case__						)
         self.assertGreaterEqual(result["eval_bleu"]		,					30						)
         self.assertTrue(os.path.exists(os.path.join(snake_case__		,					"epoch_0"						)						)						)
         self.assertTrue(os.path.exists(os.path.join(snake_case__		,					"translation_no_trainer"						)						)						)

       @slow
       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					Optional[int]								=							logging.StreamHandler(sys.stdout						)
         logger.addHandler(snake_case__						)

         lowerCAmelCase					:					Optional[Any]								=							self.get_auto_remove_tmp_dir()
         lowerCAmelCase					:					Any								=							f"""
            {self.examples_dir}/pytorch/semantic-segmentation/run_semantic_segmentation_no_trainer.py
            --dataset_name huggingface/semantic-segmentation-test-sample
            --output_dir {tmp_dir}
            --max_train_steps=10
            --num_warmup_steps=2
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --checkpointing_steps epoch
        """.split()

         run_command(self._launch_args + testargs						)
         lowerCAmelCase					:					int								=							get_results(snake_case__						)
         self.assertGreaterEqual(result["eval_overall_accuracy"]		,					0.10						)







       @mock.patch.dict(os.environ		,					{"WANDB_MODE": "offline"}						)
       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					str								=							self.get_auto_remove_tmp_dir()
         lowerCAmelCase					:					Union[str, Any]								=							f"""
            {self.examples_dir}/pytorch/image-classification/run_image_classification_no_trainer.py
            --model_name_or_path google/vit-base-patch16-224-in21k
            --dataset_name hf-internal-testing/cats_vs_dogs_sample
            --learning_rate 1e-4
            --per_device_train_batch_size 2
            --per_device_eval_batch_size 1
            --max_train_steps 2
            --train_val_split 0.1
            --seed 42
            --output_dir {tmp_dir}
            --with_tracking
            --checkpointing_steps 1
        """.split()

         if is_cuda_and_apex_available():
           testargs.append("--fp16"						)

         run_command(self._launch_args + testargs						)
         lowerCAmelCase					:					Dict								=							get_results(snake_case__						)
         # The base model scores a 25%
         self.assertGreaterEqual(result["eval_accuracy"]		,					0.6						)
         self.assertTrue(os.path.exists(os.path.join(snake_case__		,					"step_1"						)						)						)
         self.assertTrue(os.path.exists(os.path.join(snake_case__		,					"image_classification_no_trainer"						)						)						)





