


"""simple docstring"""







from collections import Counter
from pathlib import Path
from typing import Optional, Tuple

import yaml


class    SCREAMING_SNAKE_CASE__    (		yaml.SafeLoader    ):
       """simple docstring"""


       def 	lowercase__							(						self		,					snake_case__						):






         """simple docstring"""





         lowerCAmelCase					:					Union[str, Any]								=							[self.constructed_objects[key_node] for key_node, _ in node.value]
         lowerCAmelCase					:					Optional[Any]								=							[tuple(snake_case__						) if isinstance(snake_case__		,					snake_case__						) else key for key in keys]
         lowerCAmelCase					:					Optional[Any]								=							Counter(snake_case__						)
         lowerCAmelCase					:					List[str]								=							[key for key in counter if counter[key] > 1]
         if duplicate_keys:
           raise TypeError(f"""Got duplicate yaml keys: {duplicate_keys}"""						)







       def 	lowercase__							(						self		,					snake_case__		,					snake_case__=False						):






         """simple docstring"""





         lowerCAmelCase					:					Optional[Any]								=							super().construct_mapping(snake_case__		,					deep=snake_case__						)
         self._check_no_duplicates_on_constructed_node(snake_case__						)
         return mapping

def  a__      (    SCREAMING_SNAKE_CASE   :   str    ):


  '''simple docstring'''







  lowerCAmelCase					:					Tuple								=							list(readme_content.splitlines()    )
  if full_content and full_content[0] == "---" and "---" in full_content[1:]:
    lowerCAmelCase					:					str								=							full_content[1:].index("---"    ) + 1
    lowerCAmelCase					:					Union[str, Any]								=							"\n".join(full_content[1:sep_idx]    )
    return yamlblock, "\n".join(full_content[sep_idx + 1 :]    )

  return None, "\n".join(SCREAMING_SNAKE_CASE    )


class    SCREAMING_SNAKE_CASE__    (		lowercase    ):
       """simple docstring"""


       a  :    int       ={"train_eval_index"}  # train-eval-index in the YAML metadata

       @classmethod
       def 	lowercase__							(						cls		,					snake_case__						):






         """simple docstring"""





         with open(snake_case__		,					encoding="utf-8"						) as readme_file:
           lowerCAmelCase					:					Dict								=							_split_yaml_from_readme(readme_file.read()						)
         if yaml_string is not None:
           return cls.from_yaml_string(snake_case__						)
         else:
           return cls()

       def 	lowercase__							(						self		,					snake_case__						):






         """simple docstring"""





         if path.exists():
           with open(snake_case__		,					encoding="utf-8"						) as readme_file:
             lowerCAmelCase					:					Tuple								=							readme_file.read()
         else:
           lowerCAmelCase					:					List[Any]								=							None
         lowerCAmelCase					:					List[str]								=							self._to_readme(snake_case__						)
         with open(snake_case__		,					"w"		,					encoding="utf-8"						) as readme_file:
           readme_file.write(snake_case__						)

       def 	lowercase__							(						self		,					snake_case__ = None						):






         """simple docstring"""





         if readme_content is not None:
           lowerCAmelCase					:					List[Any]								=							_split_yaml_from_readme(snake_case__						)
           lowerCAmelCase					:					int								=							"---\n" + self.to_yaml_string() + "---\n" + content
         else:
           lowerCAmelCase					:					List[Any]								=							"---\n" + self.to_yaml_string() + "---\n"
         return full_content

       @classmethod
       def 	lowercase__							(						cls		,					snake_case__						):






         """simple docstring"""





         lowerCAmelCase					:					Optional[Any]								=							yaml.load(snake_case__		,					Loader=_NoDuplicateSafeLoader						) or {}

         # Convert the YAML keys to DatasetMetadata fields
         lowerCAmelCase					:					int								=							{
             (key.replace("-"		,					"_"						) if key.replace("-"		,					"_"						) in cls._FIELDS_WITH_DASHES else key): value
             for key, value in metadata_dict.items()
         }
         return cls(**snake_case__						)







       def 	lowercase__							(						self						):






         """simple docstring"""





         return yaml.safe_dump(
             {
                 (key.replace("_"		,					"-"						) if key in self._FIELDS_WITH_DASHES else key): value
                 for key, value in self.items()
             }		,					sort_keys=snake_case__		,					allow_unicode=snake_case__		,					encoding="utf-8"		,					).decode("utf-8"						)

lowerCAmelCase__				    =       {
    '''image-classification''': [],
    '''translation''': [],
    '''image-segmentation''': [],
    '''fill-mask''': [],
    '''automatic-speech-recognition''': [],
    '''token-classification''': [],
    '''sentence-similarity''': [],
    '''audio-classification''': [],
    '''question-answering''': [],
    '''summarization''': [],
    '''zero-shot-classification''': [],
    '''table-to-text''': [],
    '''feature-extraction''': [],
    '''other''': [],
    '''multiple-choice''': [],
    '''text-classification''': [],
    '''text-to-image''': [],
    '''text2text-generation''': [],
    '''zero-shot-image-classification''': [],
    '''tabular-classification''': [],
    '''tabular-regression''': [],
    '''image-to-image''': [],
    '''tabular-to-text''': [],
    '''unconditional-image-generation''': [],
    '''text-retrieval''': [],
    '''text-to-speech''': [],
    '''object-detection''': [],
    '''audio-to-audio''': [],
    '''text-generation''': [],
    '''conversational''': [],
    '''table-question-answering''': [],
    '''visual-question-answering''': [],
    '''image-to-text''': [],
    '''reinforcement-learning''': [],
    '''voice-activity-detection''': [],
    '''time-series-forecasting''': [],
    '''document-question-answering''': [],
}


if __name__ == "__main__":
     from argparse import ArgumentParser

     lowerCAmelCase__				    =       ArgumentParser(usage='''Validate the yaml metadata block of a README.md file.''')
     ap.add_argument('''readme_filepath''')
     lowerCAmelCase__				    =       ap.parse_args()

     lowerCAmelCase__				    =       Path(args.readme_filepath)
     lowerCAmelCase__				    =       DatasetMetadata.from_readme(readme_filepath)
     print(dataset_metadata)
     dataset_metadata.to_readme(readme_filepath)









"""simple docstring"""







import multiprocessing
from typing import TYPE_CHECKING, Optional, Union

from .. import Dataset, Features, config
from ..formatting import query_table
from ..packaged_modules.sql.sql import Sql
from ..utils import logging
from .abc import AbstractDatasetInputStream


if TYPE_CHECKING:
     import sqlitea

     import sqlalchemy


class 				SCREAMING_SNAKE_CASE__    (		lowercase    ):
       """simple docstring"""


       def __init__(						self		,					snake_case__		,					snake_case__		,					snake_case__ = None		,					snake_case__ = None		,					snake_case__ = False		,					**snake_case__		,					):






         """simple docstring"""





         super().__init__(features=snake_case__		,					cache_dir=snake_case__		,					keep_in_memory=snake_case__		,					**snake_case__						)
         lowerCAmelCase					:					Dict								=							Sql(
             cache_dir=snake_case__		,					features=snake_case__		,					sql=snake_case__		,					con=snake_case__		,					**snake_case__		,					)







       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					Tuple								=							None
         lowerCAmelCase					:					Optional[int]								=							None
         lowerCAmelCase					:					str								=							None
         lowerCAmelCase					:					int								=							None

         self.builder.download_and_prepare(
             download_config=snake_case__		,					download_mode=snake_case__		,					verification_mode=snake_case__		,					base_path=snake_case__		,					)

         # Build dataset for splits
         lowerCAmelCase					:					str								=							self.builder.as_dataset(
             split="train"		,					verification_mode=snake_case__		,					in_memory=self.keep_in_memory						)
         return dataset


class 				SCREAMING_SNAKE_CASE__    :
       """simple docstring"""


       def __init__(						self		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__ = None		,					snake_case__ = None		,					**snake_case__		,					):






         """simple docstring"""





         if num_proc is not None and num_proc <= 0:
           raise ValueError(f"""num_proc {num_proc} must be an integer > 0."""						)

         lowerCAmelCase					:					Tuple								=							dataset
         lowerCAmelCase					:					Tuple								=							name
         lowerCAmelCase					:					List[str]								=							con
         lowerCAmelCase					:					List[Any]								=							batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
         lowerCAmelCase					:					Optional[int]								=							num_proc
         lowerCAmelCase					:					Optional[int]								=							to_sql_kwargs

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					List[str]								=							self.to_sql_kwargs.pop("sql"		,					snake_case__						)
         lowerCAmelCase					:					List[Any]								=							self.to_sql_kwargs.pop("con"		,					snake_case__						)
         lowerCAmelCase					:					Dict								=							self.to_sql_kwargs.pop("index"		,					snake_case__						)

         lowerCAmelCase					:					Dict								=							self._write(index=snake_case__		,					**self.to_sql_kwargs						)
         return written

       def 	lowercase__							(						self		,					snake_case__						):






         """simple docstring"""





         lowerCAmelCase						,  lowerCAmelCase						,  lowerCAmelCase					:					Optional[int]								=							args
         lowerCAmelCase					:					Optional[int]								=							{**to_sql_kwargs, "if_exists": "append"} if offset > 0 else to_sql_kwargs
         lowerCAmelCase					:					List[Any]								=							query_table(
             table=self.dataset.data		,					key=slice(snake_case__		,					offset + self.batch_size						)		,					indices=self.dataset._indices		,					)
         lowerCAmelCase					:					Tuple								=							batch.to_pandas()
         lowerCAmelCase					:					str								=							df.to_sql(self.name		,					self.con		,					index=snake_case__		,					**snake_case__						)
         return num_rows or len(snake_case__						)







       def 	lowercase__							(						self		,					snake_case__		,					**snake_case__						):






         """simple docstring"""





         lowerCAmelCase					:					Tuple								=							0

         if self.num_proc is None or self.num_proc == 1:
           for offset in logging.tqdm(
               range(0		,					len(self.dataset						)		,					self.batch_size						)		,					unit="ba"		,					disable=not logging.is_progress_bar_enabled()		,					desc="Creating SQL from Arrow format"		,					):
             written += self._batch_sql((offset, index, to_sql_kwargs)						)
         else:
           lowerCAmelCase						,  lowerCAmelCase					:					Dict								=							len(self.dataset						), self.batch_size
           with multiprocessing.Pool(self.num_proc						) as pool:
             for num_rows in logging.tqdm(
                 pool.imap(
                     self._batch_sql		,					[(offset, index, to_sql_kwargs) for offset in range(0		,					snake_case__		,					snake_case__						)]		,					)		,					total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size		,					unit="ba"		,					disable=not logging.is_progress_bar_enabled()		,					desc="Creating SQL from Arrow format"		,					):
               written += num_rows

         return written





