


import os

import pytest
import yaml

from datasets.features.features import Features, Value
from datasets.info import DatasetInfo, DatasetInfosDict
@pytest.mark.parametrize(
    '''files'''						,    [
        ['''full:README.md''', '''dataset_infos.json'''],
        ['''empty:README.md''', '''dataset_infos.json'''],
        ['''dataset_infos.json'''],
        ['''full:README.md'''],
    ]						,    )
def       _a			(				SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE					):
		lowercase__						=				tmp_path_factory.mktemp('''dset_infos_dir'''					)
		if "full:README.md" in files:
				with open(dataset_infos_dir / '''README.md'''						,    '''w'''					) as f:
						f.write('''---\ndataset_info:\n  dataset_size: 42\n---'''					)
		if "empty:README.md" in files:
				with open(dataset_infos_dir / '''README.md'''						,    '''w'''					) as f:
						f.write(''''''					)
						 # we want to support dataset_infos.json for backward compatibility
		if "dataset_infos.json" in files:
				with open(dataset_infos_dir / '''dataset_infos.json'''						,    '''w'''					) as f:
						f.write('''{"default": {"dataset_size": 42}}'''					)
		lowercase__						=				DatasetInfosDict.from_directory(SCREAMING_SNAKE_CASE					)
		assert dataset_infos
		assert dataset_infos["default"].dataset_size == 42
@pytest.mark.parametrize(
    '''dataset_info'''						,    [
        DatasetInfo(),
        DatasetInfo(
            description='''foo'''						,    features=Features({'''a''': Value('''int32'''					)}					)						,    builder_name='''builder'''						,    config_name='''config'''						,    version='''1.0.0'''						,    splits=[{'''name''': '''train'''}]						,    download_size=42						,    ),
    ]						,    )
def       _a			(				SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE					):
		lowercase__						=				str(SCREAMING_SNAKE_CASE					)
		dataset_info.write_to_directory(SCREAMING_SNAKE_CASE					)
		lowercase__						=				DatasetInfo.from_directory(SCREAMING_SNAKE_CASE					)
		assert dataset_info == reloaded
		assert os.path.exists(os.path.join(SCREAMING_SNAKE_CASE						,    '''dataset_info.json'''					)					)
def       _a			(				):
		lowercase__						=				DatasetInfo(
		    description='''foo'''						,    citation='''bar'''						,    homepage='''https://foo.bar'''						,    license='''CC0'''						,    features=Features({'''a''': Value('''int32'''					)}					)						,    post_processed={}						,    supervised_keys=()						,    task_templates=[]						,    builder_name='''builder'''						,    config_name='''config'''						,    version='''1.0.0'''						,    splits=[{'''name''': '''train''', '''num_examples''': 42}]						,    download_checksums={}						,    download_size=13_37						,    post_processing_size=4_42						,    dataset_size=12_34						,    size_in_bytes=13_37 + 4_42 + 12_34						,    )
		lowercase__						=				dataset_info._to_yaml_dict()
		assert sorted(SCREAMING_SNAKE_CASE					) == sorted(DatasetInfo._INCLUDED_INFO_IN_YAML					)
		for key in DatasetInfo._INCLUDED_INFO_IN_YAML:
				assert key in dataset_info_yaml_dict
				assert isinstance(dataset_info_yaml_dict[key]						,    (list, dict, int, str)					)
		lowercase__						=				yaml.safe_dump(SCREAMING_SNAKE_CASE					)
		lowercase__						=				yaml.safe_load(SCREAMING_SNAKE_CASE					)
		assert dataset_info_yaml_dict == reloaded
def       _a			(				):
		lowercase__						=				DatasetInfo()
		lowercase__						=				dataset_info._to_yaml_dict()
		assert dataset_info_yaml_dict == {}
@pytest.mark.parametrize(
    '''dataset_infos_dict'''						,    [
        DatasetInfosDict(),
        DatasetInfosDict({'''default''': DatasetInfo()}					),
        DatasetInfosDict({'''my_config_name''': DatasetInfo()}					),
        DatasetInfosDict(
            {
                '''default''': DatasetInfo(
                    description='''foo'''						,    features=Features({'''a''': Value('''int32'''					)}					)						,    builder_name='''builder'''						,    config_name='''config'''						,    version='''1.0.0'''						,    splits=[{'''name''': '''train'''}]						,    download_size=42						,    )
            }					),
        DatasetInfosDict(
            {
                '''v1''': DatasetInfo(dataset_size=42					),
                '''v2''': DatasetInfo(dataset_size=13_37					),
            }					),
    ]						,    )
def       _a			(				SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE					):
		lowercase__						=				str(SCREAMING_SNAKE_CASE					)
		dataset_infos_dict.write_to_directory(SCREAMING_SNAKE_CASE					)
		lowercase__						=				DatasetInfosDict.from_directory(SCREAMING_SNAKE_CASE					)

		# the config_name of the dataset_infos_dict take over the attribute
		for config_name, dataset_info in dataset_infos_dict.items():
				lowercase__						=				config_name
				# the yaml representation doesn't include fields like description or citation
				# so we just test that we can recover what we can from the yaml
				lowercase__						=				DatasetInfo._from_yaml_dict(dataset_info._to_yaml_dict()					)
		assert dataset_infos_dict == reloaded

		if dataset_infos_dict:
				assert os.path.exists(os.path.join(SCREAMING_SNAKE_CASE						,    '''README.md'''					)					)






import json
import logging
import os
import socket

import git
import numpy as np
import torch


logging.basicConfig(
    format='%(asctime)s - %(levelname)s - %(name)s - PID: %(process)d -  %(message)s',
    datefmt='%m/%d/%Y %H:%M:%S',
    level=logging.INFO,
)
lowerCAmelCase	=							logging.getLogger(__name__)
def       _a			(				SCREAMING_SNAKE_CASE					):


		"""simple docstring"""




		lowercase__						=				git.Repo(search_parent_directories=SCREAMING_SNAKE_CASE					)
		lowercase__						=				{
		    '''repo_id''': str(SCREAMING_SNAKE_CASE					),
		    '''repo_sha''': str(repo.head.object.hexsha					),
		    '''repo_branch''': str(repo.active_branch					),
		}

		with open(os.path.join(SCREAMING_SNAKE_CASE						,    '''git_log.json'''					)						,    '''w'''					) as f:
				json.dump(SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE						,    indent=4					)
def       _a			(				SCREAMING_SNAKE_CASE					):


		"""simple docstring"""




		if params.n_gpu <= 0:
				lowercase__						=				0
				lowercase__						=				-1
				lowercase__						=				True
				lowercase__						=				False
				return

		assert torch.cuda.is_available()

		logger.info('''Initializing GPUs'''					)
		if params.n_gpu > 1:
				assert params.local_rank != -1

				lowercase__						=				int(os.environ['''WORLD_SIZE''']					)
				lowercase__						=				int(os.environ['''N_GPU_NODE''']					)
				lowercase__						=				int(os.environ['''RANK''']					)

				# number of nodes / node ID
				lowercase__						=				params.world_size // params.n_gpu_per_node
				lowercase__						=				params.global_rank // params.n_gpu_per_node
				lowercase__						=				True

				assert params.n_nodes == int(os.environ['''N_NODES''']					)
				assert params.node_id == int(os.environ['''NODE_RANK''']					)

		# local job (single GPU)
		else:
				assert params.local_rank == -1

				lowercase__						=				1
				lowercase__						=				0
				lowercase__						=				0
				lowercase__						=				0
				lowercase__						=				1
				lowercase__						=				1
				lowercase__						=				False

		# sanity checks
		assert params.n_nodes >= 1
		assert 0 <= params.node_id < params.n_nodes
		assert 0 <= params.local_rank <= params.global_rank < params.world_size
		assert params.world_size == params.n_nodes * params.n_gpu_per_node

		# define whether this is the master process / if we are in multi-node distributed mode
		lowercase__						=				params.node_id == 0 and params.local_rank == 0
		lowercase__						=				params.n_nodes > 1

		# summary
		lowercase__						=				f'--- Global rank: {params.global_rank} - '
		logger.info(PREFIX + '''Number of nodes: %i''' % params.n_nodes					)
		logger.info(PREFIX + '''Node ID        : %i''' % params.node_id					)
		logger.info(PREFIX + '''Local rank     : %i''' % params.local_rank					)
		logger.info(PREFIX + '''World size     : %i''' % params.world_size					)
		logger.info(PREFIX + '''GPUs per node  : %i''' % params.n_gpu_per_node					)
		logger.info(PREFIX + '''Master         : %s''' % str(params.is_master					)					)
		logger.info(PREFIX + '''Multi-node     : %s''' % str(params.multi_node					)					)
		logger.info(PREFIX + '''Multi-GPU      : %s''' % str(params.multi_gpu					)					)
		logger.info(PREFIX + '''Hostname       : %s''' % socket.gethostname()					)

		# set GPU device
		torch.cuda.set_device(params.local_rank					)

		# initialize multi-GPU
		if params.multi_gpu:
				logger.info('''Initializing PyTorch distributed'''					)
				torch.distributed.init_process_group(
				    init_method='''env://'''						,    backend='''nccl'''						,    )
def       _a			(				SCREAMING_SNAKE_CASE					):


		"""simple docstring"""




		np.random.seed(args.seed					)
		torch.manual_seed(args.seed					)
		if args.n_gpu > 0:
				torch.cuda.manual_seed_all(args.seed					)


