




"""simple docstring"""




import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader
class       a__   ( lowerCamelCase__				):






						def __init__(       self		,      _A		,      _A = None		,      _A = None		,      _A = None		,      _A = False		,      _A = False		,      _A = None		,      _A = None		,      **_A		,      ):




											"""simple docstring"""



											super().__init__(
											    __A		,      split=__A		,      features=__A		,      cache_dir=__A		,      keep_in_memory=__A		,      streaming=__A		,      num_proc=__A		,      **__A		,      )
											__lowerCAmelCase           =	field
											__lowerCAmelCase           =	path_or_paths if isinstance(__A		,      __A      ) else {self.split: path_or_paths}
											__lowerCAmelCase           =	Json(
											    cache_dir=__A		,      data_files=__A		,      features=__A		,      field=__A		,      **__A		,      )






						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											if self.streaming:
																__lowerCAmelCase           =	self.builder.as_streaming_dataset(split=self.split      )
											# Build regular (map-style) dataset
											else:
																__lowerCAmelCase           =	None
																__lowerCAmelCase           =	None
																__lowerCAmelCase           =	None
																__lowerCAmelCase           =	None

																self.builder.download_and_prepare(
																    download_config=__A		,      download_mode=__A		,      verification_mode=__A		,      base_path=__A		,      num_proc=self.num_proc		,      )
																__lowerCAmelCase           =	self.builder.as_dataset(
																    split=self.split		,      verification_mode=__A		,      in_memory=self.keep_in_memory      )
											return dataset






class       a__   :






						def __init__(       self		,      _A		,      _A		,      _A = None		,      _A = None		,      **_A		,      ):




											"""simple docstring"""



											if num_proc is not None and num_proc <= 0:
																raise ValueError(f"""num_proc {num_proc} must be an integer > 0."""      )

											__lowerCAmelCase           =	dataset
											__lowerCAmelCase           =	path_or_buf
											__lowerCAmelCase           =	batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
											__lowerCAmelCase           =	num_proc
											__lowerCAmelCase           =	'''utf-8'''
											__lowerCAmelCase           =	to_json_kwargs






						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											__lowerCAmelCase           =	self.to_json_kwargs.pop("path_or_buf"		,      __A      )
											__lowerCAmelCase           =	self.to_json_kwargs.pop("orient"		,      "records"      )
											__lowerCAmelCase           =	self.to_json_kwargs.pop("lines"		,      True if orient == "records" else False      )
											__lowerCAmelCase           =	self.to_json_kwargs.pop("index"		,      False if orient in ["split", "table"] else True      )
											__lowerCAmelCase           =	self.to_json_kwargs.pop("compression"		,      __A      )

											if compression not in [None, "infer", "gzip", "bz2", "xz"]:
																raise NotImplementedError(f"""`datasets` currently does not support {compression} compression"""      )

											if isinstance(self.path_or_buf		,      (str, bytes, os.PathLike)      ):
																with fsspec.open(self.path_or_buf		,      "wb"		,      compression=__A      ) as buffer:
																					__lowerCAmelCase           =	self._write(file_obj=__A		,      orient=__A		,      lines=__A		,      index=__A		,      **self.to_json_kwargs      )
											else:
																if compression:
																					raise NotImplementedError(
																					    f"""The compression parameter is not supported when writing to a buffer, but compression={compression}"""
																					    " was passed. Please provide a local path instead."      )
																__lowerCAmelCase           =	self._write(
																    file_obj=self.path_or_buf		,      orient=__A		,      lines=__A		,      index=__A		,      **self.to_json_kwargs      )
											return written






						def     __SCREAMING_SNAKE_CASE(       self		,      _A      ):




											"""simple docstring"""



											__lowerCAmelCase           =	args

											__lowerCAmelCase           =	query_table(
											    table=self.dataset.data		,      key=slice(__A		,      offset + self.batch_size      )		,      indices=self.dataset._indices		,      )
											__lowerCAmelCase           =	batch.to_pandas().to_json(
											    path_or_buf=__A		,      orient=__A		,      lines=__A		,      index=__A		,      **__A      )
											if not json_str.endswith("\n"      ):
																json_str += "\n"
											return json_str.encode(self.encoding      )






						def     __SCREAMING_SNAKE_CASE(       self		,      _A		,      _A		,      _A		,      _A		,      **_A		,      ):




											"""simple docstring"""



											__lowerCAmelCase           =	0

											if self.num_proc is None or self.num_proc == 1:
																for offset in logging.tqdm(
																    range(0		,      len(self.dataset      )		,      self.batch_size      )		,      unit="ba"		,      disable=not logging.is_progress_bar_enabled()		,      desc="Creating json from Arrow format"		,      ):
																					__lowerCAmelCase           =	self._batch_json((offset, orient, lines, index, to_json_kwargs)      )
																					written += file_obj.write(__A      )
											else:
																__lowerCAmelCase           =	len(self.dataset      ), self.batch_size
																with multiprocessing.Pool(self.num_proc      ) as pool:
																					for json_str in logging.tqdm(
																					    pool.imap(
																					        self._batch_json		,      [(offset, orient, lines, index, to_json_kwargs) for offset in range(0		,      __A		,      __A      )]		,      )		,      total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size		,      unit="ba"		,      disable=not logging.is_progress_bar_enabled()		,      desc="Creating json from Arrow format"		,      ):
																										written += file_obj.write(__A      )

											return written










from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase__         =      logging.get_logger(__name__)

UpperCamelCase__         =      {
    """alibaba-damo/mgp-str-base""": """https://huggingface.co/alibaba-damo/mgp-str-base/resolve/main/config.json""",
}
class       a__   ( snake_case__				):
						_a							:   Dict =   """mgp-str"""






						def __init__(       self		,      _A=[3_2, 1_2_8]		,      _A=4		,      _A=3		,      _A=2_7		,      _A=3_8		,      _A=5_0_2_5_7		,      _A=3_0_5_2_2		,      _A=7_6_8		,      _A=1_2		,      _A=1_2		,      _A=4.0		,      _A=True		,      _A=False		,      _A=1E-5		,      _A=0.0		,      _A=0.0		,      _A=0.0		,      _A=False		,      _A=0.02		,      **_A		,      ):




											"""simple docstring"""



											super().__init__(**_A      )

											__lowerCAmelCase           =	image_size
											__lowerCAmelCase           =	patch_size
											__lowerCAmelCase           =	num_channels
											__lowerCAmelCase           =	max_token_length
											__lowerCAmelCase           =	num_character_labels
											__lowerCAmelCase           =	num_bpe_labels
											__lowerCAmelCase           =	num_wordpiece_labels
											__lowerCAmelCase           =	hidden_size
											__lowerCAmelCase           =	num_hidden_layers
											__lowerCAmelCase           =	num_attention_heads
											__lowerCAmelCase           =	mlp_ratio
											__lowerCAmelCase           =	distilled
											__lowerCAmelCase           =	layer_norm_eps
											__lowerCAmelCase           =	drop_rate
											__lowerCAmelCase           =	qkv_bias
											__lowerCAmelCase           =	attn_drop_rate
											__lowerCAmelCase           =	drop_path_rate
											__lowerCAmelCase           =	output_aa_attentions
											__lowerCAmelCase           =	initializer_range




