"""simple docstring"""
import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized, parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
					from sagemaker import Session, TrainingJobAnalytics
					from sagemaker.huggingface import HuggingFace



@pytest.mark.skipif(
    literal_eval(os.getenv("""TEST_SAGEMAKER"""       ,      """False"""   )   ) is not True       ,      reason="""Skipping test because should only be run when releasing minor transformers version"""       ,      )
@pytest.mark.usefixtures("""sm_env"""   )
@parameterized_class(
    [
        {
            """framework""": """pytorch""",
            """script""": """run_glue.py""",
            """model_name_or_path""": """distilbert-base-cased""",
            """instance_type""": """ml.p3.16xlarge""",
            """results""": {"""train_runtime""": 650, """eval_accuracy""": 0.7, """eval_loss""": 0.6},
        },
        {
            """framework""": """pytorch""",
            """script""": """run_ddp.py""",
            """model_name_or_path""": """distilbert-base-cased""",
            """instance_type""": """ml.p3.16xlarge""",
            """results""": {"""train_runtime""": 600, """eval_accuracy""": 0.7, """eval_loss""": 0.6},
        },
        {
            """framework""": """tensorflow""",
            """script""": """run_tf_dist.py""",
            """model_name_or_path""": """distilbert-base-cased""",
            """instance_type""": """ml.p3.16xlarge""",
            """results""": {"""train_runtime""": 600, """eval_accuracy""": 0.6, """eval_loss""": 0.7},
        },
    ]   )
class 					lowerCAmelCase_      (     unittest.TestCase   ):



				"""simple docstring"""

				def 						snake_case	(			self      ):






										"""simple docstring"""

										if self.framework == "pytorch":
																subprocess.run(
																    F"""cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py""".split()						,      encoding='utf-8'						,      check=lowerCAmelCase						,      )
										assert hasattr(self						,      'env'      )

				def 						snake_case	(			self						,      lowerCAmelCase      ):






										"""simple docstring"""

										snake_case        =     F"""{self.env.base_job_name}-{instance_count}-{"ddp" if "ddp" in self.script else "smd"}"""
										# distributed data settings
										snake_case        =     {'smdistributed': {'dataparallel': {'enabled': True}}} if self.script != 'run_ddp.py' else None

										# creates estimator
										return HuggingFace(
										    entry_point=self.script						,      source_dir=self.env.test_path						,      role=self.env.role						,      image_uri=self.env.image_uri						,      base_job_name=lowerCAmelCase						,      instance_count=lowerCAmelCase						,      instance_type=self.instance_type						,      debugger_hook_config=lowerCAmelCase						,      hyperparameters={**self.env.distributed_hyperparameters, 'model_name_or_path': self.model_name_or_path}						,      metric_definitions=self.env.metric_definitions						,      distribution=lowerCAmelCase						,      py_version='py36'						,      )

				def 						snake_case	(			self						,      lowerCAmelCase      ):






										"""simple docstring"""

										TrainingJobAnalytics(lowerCAmelCase      ).export_csv(F"""{self.env.test_path}/{job_name}_metrics.csv"""      )

				@parameterized.expand([(2,)]      )
				def 						snake_case	(			self						,      lowerCAmelCase      ):






										"""simple docstring"""

										snake_case        =     self.create_estimator(lowerCAmelCase      )

										# run training
										estimator.fit()

										# result dataframe
										snake_case        =     TrainingJobAnalytics(estimator.latest_training_job.name      ).dataframe()

										# extract kpis
										snake_case        =     list(result_metrics_df[result_metrics_df.metric_name == 'eval_accuracy']['value']      )
										snake_case        =     list(result_metrics_df[result_metrics_df.metric_name == 'eval_loss']['value']      )
										# get train time from SageMaker job, this includes starting, preprocessing, stopping
										snake_case        =     (
										    Session().describe_training_job(estimator.latest_training_job.name      ).get('TrainingTimeInSeconds'						,      99_99_99      )
										)

										# assert kpis
										assert train_runtime <= self.results["train_runtime"]
										assert all(t >= self.results['eval_accuracy'] for t in eval_accuracy      )
										assert all(t <= self.results['eval_loss'] for t in eval_loss      )

										# dump tests result into json file to share in PR
										with open(F"""{estimator.latest_training_job.name}.json"""						,      'w'      ) as outfile:
																json.dump({'train_time': train_runtime, 'eval_accuracy': eval_accuracy, 'eval_loss': eval_loss}						,      lowerCAmelCase      )



"""simple docstring"""
import json
import os
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from requests.exceptions import HTTPError

from transformers.utils import (
    CONFIG_NAME,
    FLAX_WEIGHTS_NAME,
    TF2_WEIGHTS_NAME,
    TRANSFORMERS_CACHE,
    WEIGHTS_NAME,
    cached_file,
    get_file_from_repo,
    has_file,
)


SCREAMING_SNAKE_CASE__   =					"hf-internal-testing/tiny-random-bert"
SCREAMING_SNAKE_CASE__   =					os.path.join(TRANSFORMERS_CACHE, "models--hf-internal-testing--tiny-random-bert")
SCREAMING_SNAKE_CASE__   =					"9b8c223d42b2188cb49d29af482996f9d0f3e5a6"



class 					lowerCAmelCase_      (     unittest.TestCase   ):



				"""simple docstring"""

				def 						snake_case	(			self      ):






										"""simple docstring"""

										snake_case        =     cached_file(lowerCAmelCase						,      lowerCAmelCase      )
										# Should have downloaded the file in here
										self.assertTrue(os.path.isdir(lowerCAmelCase      )      )
										# Cache should contain at least those three subfolders:
										for subfolder in ["blobs", "refs", "snapshots"]:
																self.assertTrue(os.path.isdir(os.path.join(lowerCAmelCase						,      lowerCAmelCase      )      )      )
										with open(os.path.join(lowerCAmelCase						,      'refs'						,      'main'      )      ) as f:
																snake_case        =     f.read()
										self.assertEqual(lowerCAmelCase						,      os.path.join(lowerCAmelCase						,      'snapshots'						,      lowerCAmelCase						,      lowerCAmelCase      )      )
										self.assertTrue(os.path.isfile(lowerCAmelCase      )      )

										# File is cached at the same place the second time.
										snake_case        =     cached_file(lowerCAmelCase						,      lowerCAmelCase      )
										self.assertEqual(lowerCAmelCase						,      lowerCAmelCase      )

										# Using a specific revision to test the full commit hash.
										snake_case        =     cached_file(lowerCAmelCase						,      lowerCAmelCase						,      revision='9b8c223'      )
										self.assertEqual(lowerCAmelCase						,      os.path.join(lowerCAmelCase						,      'snapshots'						,      lowerCAmelCase						,      lowerCAmelCase      )      )

				def 						snake_case	(			self      ):






										"""simple docstring"""

										with self.assertRaisesRegex(lowerCAmelCase						,      'is not a valid model identifier'      ):
																snake_case        =     cached_file('tiny-random-bert'						,      lowerCAmelCase      )

										with self.assertRaisesRegex(lowerCAmelCase						,      'is not a valid git identifier'      ):
																snake_case        =     cached_file(lowerCAmelCase						,      lowerCAmelCase						,      revision='aaaa'      )

										with self.assertRaisesRegex(lowerCAmelCase						,      'does not appear to have a file named'      ):
																snake_case        =     cached_file(lowerCAmelCase						,      'conf'      )

				def 						snake_case	(			self      ):






										"""simple docstring"""

										with self.assertRaisesRegex(lowerCAmelCase						,      'does not appear to have a file named'      ):
																snake_case        =     cached_file(lowerCAmelCase						,      'conf'      )

										with open(os.path.join(lowerCAmelCase						,      'refs'						,      'main'      )      ) as f:
																snake_case        =     f.read()
										self.assertTrue(os.path.isfile(os.path.join(lowerCAmelCase						,      '.no_exist'						,      lowerCAmelCase						,      'conf'      )      )      )

										snake_case        =     cached_file(lowerCAmelCase						,      'conf'						,      _raise_exceptions_for_missing_entries=lowerCAmelCase      )
										self.assertIsNone(lowerCAmelCase      )

										snake_case        =     cached_file(lowerCAmelCase						,      'conf'						,      local_files_only=lowerCAmelCase						,      _raise_exceptions_for_missing_entries=lowerCAmelCase      )
										self.assertIsNone(lowerCAmelCase      )

										snake_case        =     mock.Mock()
										snake_case        =     5_00
										snake_case        =     {}
										snake_case        =     HTTPError
										snake_case        =     {}

										# Under the mock environment we get a 500 error when trying to reach the tokenizer.
										with mock.patch('requests.Session.request'						,      return_value=lowerCAmelCase      ) as mock_head:
																snake_case        =     cached_file(lowerCAmelCase						,      'conf'						,      _raise_exceptions_for_connection_errors=lowerCAmelCase      )
																self.assertIsNone(lowerCAmelCase      )
																# This check we did call the fake head request
																mock_head.assert_called()

				def 						snake_case	(			self      ):






										"""simple docstring"""

										self.assertTrue(has_file('hf-internal-testing/tiny-bert-pt-only'						,      lowerCAmelCase      )      )
										self.assertFalse(has_file('hf-internal-testing/tiny-bert-pt-only'						,      lowerCAmelCase      )      )
										self.assertFalse(has_file('hf-internal-testing/tiny-bert-pt-only'						,      lowerCAmelCase      )      )

				def 						snake_case	(			self      ):






										"""simple docstring"""

										self.assertIsNone(get_file_from_repo('bert-base-cased'						,      'ahah.txt'      )      )

										# The function raises if the repository does not exist.
										with self.assertRaisesRegex(lowerCAmelCase						,      'is not a valid model identifier'      ):
																get_file_from_repo('bert-base-case'						,      lowerCAmelCase      )

										# The function raises if the revision does not exist.
										with self.assertRaisesRegex(lowerCAmelCase						,      'is not a valid git identifier'      ):
																get_file_from_repo('bert-base-cased'						,      lowerCAmelCase						,      revision='ahaha'      )

										snake_case        =     get_file_from_repo('bert-base-cased'						,      lowerCAmelCase      )
										# The name is the cached name which is not very easy to test, so instead we load the content.
										snake_case        =     json.loads(open(lowerCAmelCase						,      'r'      ).read()      )
										self.assertEqual(config['hidden_size']						,      7_68      )

				def 						snake_case	(			self      ):






										"""simple docstring"""

										with tempfile.TemporaryDirectory() as tmp_dir:
																snake_case        =     Path(lowerCAmelCase      ) / 'a.txt'
																filename.touch()
																self.assertEqual(get_file_from_repo(lowerCAmelCase						,      'a.txt'      )						,      str(lowerCAmelCase      )      )

																self.assertIsNone(get_file_from_repo(lowerCAmelCase						,      'b.txt'      )      )


