



from typing import Dict

from transformers import EvalPrediction, HfArgumentParser, TrainingArguments, is_torch_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    get_torch_dist_unique_port,
    require_torch_multi_gpu,
    require_torch_neuroncore,
)
from transformers.training_args import ParallelMode
from transformers.utils import logging


SCREAMING_SNAKE_CASE_:Tuple					=						logging.get_logger(__name__)


if is_torch_available():
      import torch
      from torch import nn
      from torch.utils.data import Dataset

      from transformers import Trainer






      class SCREAMING_SNAKE_CASE__					(		_lowerCamelCase   ):




          '''simple docstring'''






          def __init__(	self,		lowerCamelCase__ = 101		):
              A			:	Tuple										=       length






          def __len__(	self		):
              return self.length






          def __getitem__(	self,		lowerCamelCase__		):
              return i






      class SCREAMING_SNAKE_CASE__					:




          '''simple docstring'''






          def __call__(	self,		lowerCamelCase__		):
              return {"input_ids": torch.tensor(lowercase_		), "labels": torch.tensor(lowercase_		)}






      class SCREAMING_SNAKE_CASE__					(		nn.Module   ):




          '''simple docstring'''






          def __init__(	self		):
              super().__init__()
              # Add some (unused) params otherwise DDP will complain.
              A			:	str										=       nn.Linear(120,		80		)






          def       _lowerCAmelCase (	self,		lowerCamelCase__,		lowerCamelCase__=None		):
              if labels is not None:
                  return torch.tensor(0.0,		device=input_ids.device		), input_ids
              else:
                  return input_ids






class SCREAMING_SNAKE_CASE__					(		_lowerCamelCase   ):




    '''simple docstring'''






    @require_torch_neuroncore
    def       _lowerCAmelCase (	self		):
        A			:	int										=       f'''--nproc_per_node=2\n            --master_port={get_torch_dist_unique_port()}\n            {self.test_file_dir}/test_trainer_distributed.py\n        '''.split()
        A			:	Optional[Any]										=       self.get_auto_remove_tmp_dir()
        A			:	Dict										=       f'''--output_dir {output_dir}'''.split()
        A			:	Optional[int]										=       ["""torchrun"""] + distributed_args + args
        execute_subprocess_async(lowercase_,		env=self.get_env()		)
        # successful return here == success - any errors would have caused an error in the sub-call






class SCREAMING_SNAKE_CASE__					(		_lowerCamelCase   ):




    '''simple docstring'''






    @require_torch_multi_gpu
    def       _lowerCAmelCase (	self		):
        A			:	int										=       f'''--nproc_per_node={torch.cuda.device_count()}\n            --master_port={get_torch_dist_unique_port()}\n            {self.test_file_dir}/test_trainer_distributed.py\n        '''.split()
        A			:	int										=       self.get_auto_remove_tmp_dir()
        A			:	Tuple										=       f'''--output_dir {output_dir}'''.split()
        A			:	Union[str, Any]										=       ["""torchrun"""] + distributed_args + args
        execute_subprocess_async(lowercase_,		env=self.get_env()		)
        # successful return here == success - any errors would have caused an error in the sub-call


if __name__ == "__main__":
      # The script below is meant to be run under torch.distributed, on a machine with multiple GPUs:
      #
      # PYTHONPATH="src" python -m torch.distributed.run --nproc_per_node 2 --output_dir output_dir ./tests/test_trainer_distributed.py

      SCREAMING_SNAKE_CASE_:List[str]					=						HfArgumentParser((TrainingArguments,))
      SCREAMING_SNAKE_CASE_:List[Any]					=						parser.parse_args_into_dataclasses()[0]

      logger.warning(
          F"""Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}, """
          F"""distributed training: {training_args.parallel_mode != ParallelMode.NOT_DISTRIBUTED}"""
      )

      # Essentially, what we want to verify in the distributed case is that we get all samples back,
      # in the right order. (this is crucial for prediction for instance)
      for dataset_length in [101, 40, 7]:
            SCREAMING_SNAKE_CASE_:str					=						DummyDataset(dataset_length)




            def      __UpperCamelCase     (  _lowerCAmelCase		)	->  Dict:



                """simple docstring"""






                A			:	str										=       list(range(len(__UpperCAmelCase		)		)		)
                A			:	Optional[Any]										=       p.predictions.tolist() == sequential and p.label_ids.tolist() == sequential
                if not success and training_args.local_rank == 0:
                    logger.warning(
                        """Predictions and/or labels do not match expected results:\n  - predictions: """
                        f'''{p.predictions.tolist()}\n  - labels: {p.label_ids.tolist()}\n  - expected: {sequential}'''		)
                return {"success": success}

            SCREAMING_SNAKE_CASE_:List[Any]					=						Trainer(
                model=DummyModel(),
                args=training_args,
                data_collator=DummyDataCollator(),
                eval_dataset=dataset,
                compute_metrics=compute_metrics,
            )
            SCREAMING_SNAKE_CASE_:int					=						trainer.evaluate()
            logger.info(metrics)
            if metrics["eval_success"] is not True:
                  logger.error(metrics)
                  exit(1)

            SCREAMING_SNAKE_CASE_:Dict					=						trainer.predict(dataset)
            logger.info(p.metrics)
            if p.metrics["test_success"] is not True:
                  logger.error(p.metrics)
                  exit(1)

            SCREAMING_SNAKE_CASE_:Union[str, Any]					=						2

            SCREAMING_SNAKE_CASE_:Optional[Any]					=						trainer.evaluate()
            logger.info(metrics)
            if metrics["eval_success"] is not True:
                  logger.error(metrics)
                  exit(1)

            SCREAMING_SNAKE_CASE_:Tuple					=						trainer.predict(dataset)
            logger.info(p.metrics)
            if p.metrics["test_success"] is not True:
                  logger.error(p.metrics)
                  exit(1)




            SCREAMING_SNAKE_CASE_:Union[str, Any]					=						None







import inspect
import re
from hashlib import shaaaa
from typing import Dict, List

from .arrow import arrow
from .audiofolder import audiofolder
from .csv import csv
from .imagefolder import imagefolder
from .json import json
from .pandas import pandas
from .parquet import parquet
from .sql import sql  # noqa F401
from .text import text




def      __UpperCamelCase     (  _lowerCAmelCase		)	->  str:



    """simple docstring"""






    A			:	int										=       []
    for line in lines:
        A			:	int										=       re.sub(R"""#.*"""				,					""""""				,					_lowerCAmelCase		)  # remove comments
        if line:
            filtered_lines.append(_lowerCAmelCase		)
    A			:	Tuple										=       """\n""".join(_lowerCAmelCase		)

    # Make a hash from all this code
    A			:	Union[str, Any]										=       full_str.encode("""utf-8"""		)
    return shaaaa(_lowerCAmelCase		).hexdigest()


# get importable module names and hash for caching
SCREAMING_SNAKE_CASE_:List[Any]					=						{
    """csv""": (csv.__name__, _hash_python_lines(inspect.getsource(csv).splitlines())),
    """json""": (json.__name__, _hash_python_lines(inspect.getsource(json).splitlines())),
    """pandas""": (pandas.__name__, _hash_python_lines(inspect.getsource(pandas).splitlines())),
    """parquet""": (parquet.__name__, _hash_python_lines(inspect.getsource(parquet).splitlines())),
    """arrow""": (arrow.__name__, _hash_python_lines(inspect.getsource(arrow).splitlines())),
    """text""": (text.__name__, _hash_python_lines(inspect.getsource(text).splitlines())),
    """imagefolder""": (imagefolder.__name__, _hash_python_lines(inspect.getsource(imagefolder).splitlines())),
    """audiofolder""": (audiofolder.__name__, _hash_python_lines(inspect.getsource(audiofolder).splitlines())),
}

# Used to infer the module to use based on the data files extensions
SCREAMING_SNAKE_CASE_:Optional[Any]					=						{
    """.csv""": ("""csv""", {}),
    """.tsv""": ("""csv""", {"""sep""": """\t"""}),
    """.json""": ("""json""", {}),
    """.jsonl""": ("""json""", {}),
    """.parquet""": ("""parquet""", {}),
    """.arrow""": ("""arrow""", {}),
    """.txt""": ("""text""", {}),
}
_EXTENSION_TO_MODULE.update({ext: ("""imagefolder""", {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext.upper(): ("""imagefolder""", {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext: ("""audiofolder""", {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext.upper(): ("""audiofolder""", {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
SCREAMING_SNAKE_CASE_:Optional[int]					=						{"""imagefolder""", """audiofolder"""}

# Used to filter data files based on extensions given a module name
SCREAMING_SNAKE_CASE_:Dict[str, List[str]]					=						{}
for _ext, (_module, _) in _EXTENSION_TO_MODULE.items():
     _MODULE_TO_EXTENSIONS.setdefault(_module, []).append(_ext)

_MODULE_TO_EXTENSIONS["imagefolder"].append(""".zip""")




_MODULE_TO_EXTENSIONS["audiofolder"].append(""".zip""")


