


import importlib.util
import os
import platform
from argparse import ArgumentParser

import huggingface_hub

from .. import __version__ as version
from ..utils import (
    is_accelerate_available,
    is_flax_available,
    is_safetensors_available,
    is_tf_available,
    is_torch_available,
)
from . import BaseTransformersCLICommand

def 						_UpperCAmelCase    (      SCREAMING_SNAKE_CASE__      : Tuple    ):
     return EnvironmentCommand()

def 						_UpperCAmelCase    (      SCREAMING_SNAKE_CASE__      : List[str]    ):
     return EnvironmentCommand(args.accelerate_config_file    )






class    UpperCAmelCase__	( A_		):



    """simple docstring"""






    @staticmethod
    def _a (      A_   )  ->	Dict:
         __UpperCamelCase   							=parser.add_parser('env'   )
         download_parser.set_defaults(func=A_   )
         download_parser.add_argument(
             '--accelerate-config_file' ,       default=A_ ,       help='The accelerate config file to use for the default values in the launching script.' ,       )
         download_parser.set_defaults(func=A_   )






    def __init__(      self ,       A_ ,       *A_   )  ->	None:
         __UpperCamelCase   							=accelerate_config_file






    def _a (      self   )  ->	Optional[Any]:
         __UpperCamelCase   							='not installed'
         if is_safetensors_available():
              import safetensors

              __UpperCamelCase   							=safetensors.__version__
         elif importlib.util.find_spec('safetensors'   ) is not None:
              import safetensors

              __UpperCamelCase   							=f'{safetensors.__version__} but is ignored because of PyTorch version too old.'

         __UpperCamelCase   							='not installed'
         __UpperCamelCase   							=__UpperCamelCase   							='not found'
         if is_accelerate_available():
              import accelerate
              from accelerate.commands.config import default_config_file, load_config_from_file

              __UpperCamelCase   							=accelerate.__version__
              # Get the default from the config file.
              if self._accelerate_config_file is not None or os.path.isfile(A_   ):
                   __UpperCamelCase   							=load_config_from_file(self._accelerate_config_file   ).to_dict()

              __UpperCamelCase   							=(
                  '\n'.join([f'\t- {prop}: {val}' for prop, val in accelerate_config.items()]   )
                  if isinstance(A_ ,       A_   )
                  else f'\t{accelerate_config}'
              )

         __UpperCamelCase   							='not installed'
         __UpperCamelCase   							='NA'
         if is_torch_available():
              import torch

              __UpperCamelCase   							=torch.__version__
              __UpperCamelCase   							=torch.cuda.is_available()

         __UpperCamelCase   							='not installed'
         __UpperCamelCase   							='NA'
         if is_tf_available():
              import tensorflow as tf

              __UpperCamelCase   							=tf.__version__
              try:
                   # deprecated in v2.1
                   __UpperCamelCase   							=tf.test.is_gpu_available()
              except AttributeError:
                   # returns list of devices, convert to bool
                   __UpperCamelCase   							=bool(tf.config.list_physical_devices('GPU'   )   )

         __UpperCamelCase   							='not installed'
         __UpperCamelCase   							='not installed'
         __UpperCamelCase   							='not installed'
         __UpperCamelCase   							='NA'
         if is_flax_available():
              import flax
              import jax
              import jaxlib

              __UpperCamelCase   							=flax.__version__
              __UpperCamelCase   							=jax.__version__
              __UpperCamelCase   							=jaxlib.__version__
              __UpperCamelCase   							=jax.lib.xla_bridge.get_backend().platform

         __UpperCamelCase   							={
             '`transformers` version': version,
             'Platform': platform.platform(),
             'Python version': platform.python_version(),
             'Huggingface_hub version': huggingface_hub.__version__,
             'Safetensors version': f'{safetensors_version}',
             'Accelerate version': f'{accelerate_version}',
             'Accelerate config': f'{accelerate_config_str}',
             'PyTorch version (GPU?)': f'{pt_version} ({pt_cuda_available})',
             'Tensorflow version (GPU?)': f'{tf_version} ({tf_cuda_available})',
             'Flax version (CPU?/GPU?/TPU?)': f'{flax_version} ({jax_backend})',
             'Jax version': f'{jax_version}',
             'JaxLib version': f'{jaxlib_version}',
             'Using GPU in script?': '<fill in>',
             'Using distributed or parallel set-up in script?': '<fill in>',
         }

         print('\nCopy-and-paste the text below in your GitHub issue and FILL OUT the two last points.\n'   )
         print(self.format_dict(A_   )   )

         return info






    @staticmethod
    def _a (      A_   )  ->	Union[str, Any]:
         return "\n".join([f'- {prop}: {val}' for prop, val in d.items()]   ) + "\n"









import os
from typing import BinaryIO, Optional, Union

import numpy as np
import pyarrow.parquet as pq

from .. import Audio, Dataset, Features, Image, NamedSplit, Value, config
from ..features.features import FeatureType, _visit
from ..formatting import query_table
from ..packaged_modules import _PACKAGED_DATASETS_MODULES
from ..packaged_modules.parquet.parquet import Parquet
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader

def 						_UpperCAmelCase    (      SCREAMING_SNAKE_CASE__      : Features    ):
     __UpperCamelCase   							=np.inf

     def set_batch_size(SCREAMING_SNAKE_CASE__      : FeatureType    ) -> None:
          nonlocal batch_size
          if isinstance(SCREAMING_SNAKE_CASE__      ,					SCREAMING_SNAKE_CASE__    ):
               __UpperCamelCase   							=min(SCREAMING_SNAKE_CASE__      ,					config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS    )
          elif isinstance(SCREAMING_SNAKE_CASE__      ,					SCREAMING_SNAKE_CASE__    ):
               __UpperCamelCase   							=min(SCREAMING_SNAKE_CASE__      ,					config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS    )
          elif isinstance(SCREAMING_SNAKE_CASE__      ,					SCREAMING_SNAKE_CASE__    ) and feature.dtype == "binary":
               __UpperCamelCase   							=min(SCREAMING_SNAKE_CASE__      ,					config.PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS    )

     _visit(SCREAMING_SNAKE_CASE__      ,					SCREAMING_SNAKE_CASE__    )

     return None if batch_size is np.inf else batch_size






class    UpperCAmelCase__	( A_		):



    """simple docstring"""






    def __init__(      self ,       A_ ,       A_ = None ,       A_ = None ,       A_ = None ,       A_ = False ,       A_ = False ,       A_ = None ,       **A_ ,       )  ->	Dict:
         super().__init__(
             A_ ,       split=A_ ,       features=A_ ,       cache_dir=A_ ,       keep_in_memory=A_ ,       streaming=A_ ,       num_proc=A_ ,       **A_ ,       )
         __UpperCamelCase   							=path_or_paths if isinstance(A_ ,       A_   ) else {self.split: path_or_paths}
         __UpperCamelCase   							=_PACKAGED_DATASETS_MODULES['parquet'][1]
         __UpperCamelCase   							=Parquet(
             cache_dir=A_ ,       data_files=A_ ,       features=A_ ,       hash=A_ ,       **A_ ,       )






    def _a (      self   )  ->	List[Any]:
         # Build iterable dataset
         if self.streaming:
              __UpperCamelCase   							=self.builder.as_streaming_dataset(split=self.split   )
         # Build regular (map-style) dataset
         else:
              __UpperCamelCase   							=None
              __UpperCamelCase   							=None
              __UpperCamelCase   							=None
              __UpperCamelCase   							=None

              self.builder.download_and_prepare(
                  download_config=A_ ,       download_mode=A_ ,       verification_mode=A_ ,       base_path=A_ ,       num_proc=self.num_proc ,       )
              __UpperCamelCase   							=self.builder.as_dataset(
                  split=self.split ,       verification_mode=A_ ,       in_memory=self.keep_in_memory   )
         return dataset






class    UpperCAmelCase__	:



    """simple docstring"""






    def __init__(      self ,       A_ ,       A_ ,       A_ = None ,       **A_ ,       )  ->	List[Any]:
         __UpperCamelCase   							=dataset
         __UpperCamelCase   							=path_or_buf
         __UpperCamelCase   							=batch_size or get_writer_batch_size(dataset.features   )
         __UpperCamelCase   							=parquet_writer_kwargs






    def _a (      self   )  ->	int:
         __UpperCamelCase   							=self.batch_size if self.batch_size else config.DEFAULT_MAX_BATCH_SIZE

         if isinstance(self.path_or_buf ,       (str, bytes, os.PathLike)   ):
              with open(self.path_or_buf ,       'wb+'   ) as buffer:
                   __UpperCamelCase   							=self._write(file_obj=A_ ,       batch_size=A_ ,       **self.parquet_writer_kwargs   )
         else:
              __UpperCamelCase   							=self._write(file_obj=self.path_or_buf ,       batch_size=A_ ,       **self.parquet_writer_kwargs   )
         return written






    def _a (      self ,       A_ ,       A_ ,       **A_   )  ->	int:
         __UpperCamelCase   							=0
         __UpperCamelCase   							=parquet_writer_kwargs.pop('path_or_buf' ,       A_   )
         __UpperCamelCase   							=self.dataset.features.arrow_schema

         __UpperCamelCase   							=pq.ParquetWriter(A_ ,       schema=A_ ,       **A_   )

         for offset in logging.tqdm(
             range(0 ,       len(self.dataset   ) ,       A_   ) ,       unit='ba' ,       disable=not logging.is_progress_bar_enabled() ,       desc='Creating parquet from Arrow format' ,       ):
              __UpperCamelCase   							=query_table(
                  table=self.dataset._data ,       key=slice(A_ ,       offset + batch_size   ) ,       indices=self.dataset._indices if self.dataset._indices is not None else None ,       )
              writer.write_table(A_   )
              written += batch.nbytes
         writer.close()
         return written





