
import functools
import logging
import os
import sys
import threading
from logging import (
    CRITICAL,  # NOQA
    DEBUG,  # NOQA
    ERROR,  # NOQA
    FATAL,  # NOQA
    INFO,  # NOQA
    NOTSET,  # NOQA
    WARN,  # NOQA
    WARNING,  # NOQA
)
from typing import Optional

import huggingface_hub.utils as hf_hub_utils
from tqdm import auto as tqdm_lib


lowercase__          =threading.Lock()
lowercase__          =None

lowercase__          ={
    'debug': logging.DEBUG,
    'info': logging.INFO,
    'warning': logging.WARNING,
    'error': logging.ERROR,
    'critical': logging.CRITICAL,
}

lowercase__          =logging.WARNING

lowercase__          =True




def        __UpperCamelCase					(    ):
 __a			:  Dict		    =  os.getenv('''TRANSFORMERS_VERBOSITY'''  ,       lowerCAmelCase__  )
 if env_level_str:
  if env_level_str in log_levels:
   return log_levels[env_level_str]
  else:
   logging.getLogger().warning(
       f"Unknown option TRANSFORMERS_VERBOSITY={env_level_str}, "
       f"has to be one of: { ', '.join(log_levels.keys()  ) }"  )
 return _default_log_level




def        __UpperCamelCase					(    ):
 return __name__.split('''.'''  )[0]




def        __UpperCamelCase					(    ):
 return logging.getLogger(_get_library_name()  )




def        __UpperCamelCase					(    ):
 global _default_handler

 with _lock:
  if _default_handler:
   # This library has already configured the library root logger.
   return
  __a			:  str		    =  logging.StreamHandler()  # Set sys.stderr as stream.
  __a			:  Union[str, Any]		    =  sys.stderr.flush

  # Apply our default configuration to the library root logger.
  __a			:  Union[str, Any]		    =  _get_library_root_logger()
  library_root_logger.addHandler(_default_handler  )
  library_root_logger.setLevel(_get_default_logging_level()  )
  __a			:  Union[str, Any]		    =  False




def        __UpperCamelCase					(    ):
 global _default_handler

 with _lock:
  if not _default_handler:
   return

  __a			:  Optional[Any]		    =  _get_library_root_logger()
  library_root_logger.removeHandler(_default_handler  )
  library_root_logger.setLevel(logging.NOTSET  )
  __a			:  str		    =  None




def        __UpperCamelCase					(    ):
 return log_levels




def        __UpperCamelCase					(    lowerCAmelCase__							:    Optional[str] = None  ):

 if name is None:
  __a			:  str		    =  _get_library_name()

 _configure_library_root_logger()
 return logging.getLogger(lowerCAmelCase__  )




def        __UpperCamelCase					(    ):
 _configure_library_root_logger()
 return _get_library_root_logger().getEffectiveLevel()




def        __UpperCamelCase					(    lowerCAmelCase__							:    int  ):
 _configure_library_root_logger()
 _get_library_root_logger().setLevel(lowerCAmelCase__  )




def        __UpperCamelCase					(    ):
 return set_verbosity(lowerCAmelCase__  )




def        __UpperCamelCase					(    ):
 return set_verbosity(lowerCAmelCase__  )




def        __UpperCamelCase					(    ):
 return set_verbosity(lowerCAmelCase__  )




def        __UpperCamelCase					(    ):
 return set_verbosity(lowerCAmelCase__  )




def        __UpperCamelCase					(    ):
 _configure_library_root_logger()

 assert _default_handler is not None
 _get_library_root_logger().removeHandler(_default_handler  )




def        __UpperCamelCase					(    ):
 _configure_library_root_logger()

 assert _default_handler is not None
 _get_library_root_logger().addHandler(_default_handler  )




def        __UpperCamelCase					(    lowerCAmelCase__							:    logging.Handler  ):
 _configure_library_root_logger()

 assert handler is not None
 _get_library_root_logger().addHandler(lowerCAmelCase__  )




def        __UpperCamelCase					(    lowerCAmelCase__							:    logging.Handler  ):
 _configure_library_root_logger()

 assert handler is not None and handler not in _get_library_root_logger().handlers
 _get_library_root_logger().removeHandler(lowerCAmelCase__  )




def        __UpperCamelCase					(    ):
 _configure_library_root_logger()
 __a			:  Tuple		    =  False




def        __UpperCamelCase					(    ):
 _configure_library_root_logger()
 __a			:  Dict		    =  True




def        __UpperCamelCase					(    ):
 __a			:  Optional[Any]		    =  _get_library_root_logger().handlers

 for handler in handlers:
  __a			:  Optional[int]		    =  logging.Formatter('''[%(levelname)s|%(filename)s:%(lineno)s] %(asctime)s >> %(message)s'''  )
  handler.setFormatter(lowerCAmelCase__  )




def        __UpperCamelCase					(    ):
 __a			:  Tuple		    =  _get_library_root_logger().handlers

 for handler in handlers:
  handler.setFormatter(lowerCAmelCase__  )




def        __UpperCamelCase					(    self							:    List[Any]  ,       *lowerCAmelCase__							:    List[str]  ,       **lowerCAmelCase__							:    Optional[int]  ):
 __a			:  Union[str, Any]		    =  os.getenv('''TRANSFORMERS_NO_ADVISORY_WARNINGS'''  ,       lowerCAmelCase__  )
 if no_advisory_warnings:
  return
 self.warning(*lowerCAmelCase__  ,       **lowerCAmelCase__  )


lowercase__          =warning_advice




@functools.lru_cache(lowerCAmelCase__  )
def        __UpperCamelCase					(    self							:    Any  ,       *lowerCAmelCase__							:    Optional[Any]  ,       **lowerCAmelCase__							:    Any  ):
 self.warning(*lowerCAmelCase__  ,       **lowerCAmelCase__  )


lowercase__          =warning_once



class 		UpperCamelCase__  :
   def __init__(self			: List[Any] ,    *snake_case_			: Any ,    **snake_case_			: List[Any]       ):  # pylint: disable=unused-argument
    __a			:  Optional[int]		    =  args[0] if args else None
   def __iter__(self			: Dict       ):
    return iter(self._iterator       )
   def __getattr__(self			: List[str] ,    snake_case_			: List[Any]       ):

    def empty_fn(*snake_case_			: Tuple ,    **snake_case_			: Union[str, Any]       ):  # pylint: disable=unused-argument
     return

    return empty_fn
   def __enter__(self			: Union[str, Any]       ):
    return self







   def __exit__(self			: Tuple ,    snake_case_			: List[Any] ,    snake_case_			: List[str] ,    snake_case_			: List[str]       ):
    return





class 		UpperCamelCase__  :
   def __call__(self			: List[Any] ,    *snake_case_			: List[str] ,    **snake_case_			: List[Any]       ):
    if _tqdm_active:
     return tqdm_lib.tqdm(*snake_case_ ,    **snake_case_       )
    else:
     return EmptyTqdm(*snake_case_ ,    **snake_case_       )
   def 				lowerCAmelCase      (self			: str ,    *snake_case_			: Any ,    **snake_case_			: List[Any]       ):
    __a			:  str		    =  None
    if _tqdm_active:
     return tqdm_lib.tqdm.set_lock(*snake_case_ ,    **snake_case_       )







   def 				lowerCAmelCase      (self			: str       ):
    if _tqdm_active:
     return tqdm_lib.tqdm.get_lock()





lowercase__          =_tqdm_cls()




def        __UpperCamelCase					(    ):
 global _tqdm_active
 return bool(_tqdm_active  )




def        __UpperCamelCase					(    ):
 global _tqdm_active
 __a			:  str		    =  True
 hf_hub_utils.enable_progress_bars()




def        __UpperCamelCase					(    ):
 global _tqdm_active
 __a			:  Optional[int]		    =  False
 hf_hub_utils.disable_progress_bars()


import json
import os
import tempfile

import datasets
from utils import generate_example_dataset, get_duration


lowercase__          =50000
lowercase__          =5000

lowercase__       ,      lowercase__          =os.path.split(__file__)
lowercase__          =os.path.join(RESULTS_BASEPATH, 'results', RESULTS_FILENAME.replace('.py', '.json'))




@get_duration
def        __UpperCamelCase					(    lowerCAmelCase__							:    datasets.Dataset  ,       lowerCAmelCase__							:    List[str]  ):
 for i in range(lowerCAmelCase__  ):
  __a			:  str		    =  dataset[i]




@get_duration
def        __UpperCamelCase					(    lowerCAmelCase__							:    datasets.Dataset  ,       lowerCAmelCase__							:    Optional[Any]  ,       lowerCAmelCase__							:    Tuple  ):
 for i in range(0  ,       len(lowerCAmelCase__  )  ,       lowerCAmelCase__  ):
  __a			:  Optional[int]		    =  dataset[i : i + batch_size]




@get_duration
def        __UpperCamelCase					(    lowerCAmelCase__							:    datasets.Dataset  ,       lowerCAmelCase__							:    Union[str, Any]  ,       lowerCAmelCase__							:    str  ):
 with dataset.formatted_as(type=lowerCAmelCase__  ):
  for i in range(lowerCAmelCase__  ):
   __a			:  Dict		    =  dataset[i]




@get_duration
def        __UpperCamelCase					(    lowerCAmelCase__							:    datasets.Dataset  ,       lowerCAmelCase__							:    List[str]  ,       lowerCAmelCase__							:    Optional[Any]  ,       lowerCAmelCase__							:    int  ):
 with dataset.formatted_as(type=lowerCAmelCase__  ):
  for i in range(0  ,       lowerCAmelCase__  ,       lowerCAmelCase__  ):
   __a			:  int		    =  dataset[i : i + batch_size]




def        __UpperCamelCase					(    ):
 __a			:  Any		    =  {'''num examples''': SPEED_TEST_N_EXAMPLES}
 __a			:  List[Any]		    =  [
     (read, {'''length''': SMALL_TEST}),
     (read, {'''length''': SPEED_TEST_N_EXAMPLES}),
     (read_batch, {'''length''': SPEED_TEST_N_EXAMPLES, '''batch_size''': 1_0}),
     (read_batch, {'''length''': SPEED_TEST_N_EXAMPLES, '''batch_size''': 1_0_0}),
     (read_batch, {'''length''': SPEED_TEST_N_EXAMPLES, '''batch_size''': 1_0_0_0}),
     (read_formatted, {'''type''': '''numpy''', '''length''': SMALL_TEST}),
     (read_formatted, {'''type''': '''pandas''', '''length''': SMALL_TEST}),
     (read_formatted, {'''type''': '''torch''', '''length''': SMALL_TEST}),
     (read_formatted, {'''type''': '''tensorflow''', '''length''': SMALL_TEST}),
     (read_formatted_batch, {'''type''': '''numpy''', '''length''': SMALL_TEST, '''batch_size''': 1_0}),
     (read_formatted_batch, {'''type''': '''numpy''', '''length''': SMALL_TEST, '''batch_size''': 1_0_0_0}),
 ]

 __a			:  Union[str, Any]		    =  [
     (read, {'''length''': SMALL_TEST}),
     (read, {'''length''': SPEED_TEST_N_EXAMPLES}),
     (read_batch, {'''length''': SPEED_TEST_N_EXAMPLES, '''batch_size''': 1_0}),
     (read_batch, {'''length''': SPEED_TEST_N_EXAMPLES, '''batch_size''': 1_0_0}),
     (read_batch, {'''length''': SPEED_TEST_N_EXAMPLES, '''batch_size''': 1_0_0_0}),
     (read_formatted, {'''type''': '''numpy''', '''length''': SMALL_TEST}),
     (read_formatted_batch, {'''type''': '''numpy''', '''length''': SMALL_TEST, '''batch_size''': 1_0}),
     (read_formatted_batch, {'''type''': '''numpy''', '''length''': SMALL_TEST, '''batch_size''': 1_0_0_0}),
 ]
 with tempfile.TemporaryDirectory() as tmp_dir:
  print('''generating dataset'''  )
  __a			:  Optional[Any]		    =  datasets.Features(
      {'''list''': datasets.Sequence(datasets.Value('''float32'''  )  ), '''numbers''': datasets.Value('''float32'''  )}  )
  __a			:  Optional[int]		    =  generate_example_dataset(
      os.path.join(lowerCAmelCase__  ,       '''dataset.arrow'''  )  ,       lowerCAmelCase__  ,       num_examples=lowerCAmelCase__  ,       seq_shapes={'''list''': (1_0_0,)}  ,       )
  print('''first set of iterations'''  )
  for func, kwargs in functions:
   print(func.__name__  ,       str(lowerCAmelCase__  )  )
   __a			:  str		    =  func(lowerCAmelCase__  ,       **lowerCAmelCase__  )

  print('''shuffling dataset'''  )
  __a			:  int		    =  dataset.shuffle()
  print('''Second set of iterations (after shuffling'''  )
  for func, kwargs in functions_shuffled:
   print('''shuffled '''  ,       func.__name__  ,       str(lowerCAmelCase__  )  )
   __a			:  List[Any]		    =  func(
       lowerCAmelCase__  ,       **lowerCAmelCase__  )

 with open(lowerCAmelCase__  ,       '''wb'''  ) as f:
  f.write(json.dumps(lowerCAmelCase__  ).encode('''utf-8'''  )  )


if __name__ == "__main__":  # useful to run the profiler
      benchmark_iterating()
