


"""simple docstring"""







import time
import warnings
from abc import ABC
from copy import deepcopy
from typing import Optional

import torch

from ..utils import add_start_docstrings, logging


lowerCAmelCase__				    =       logging.get_logger(__name__)


lowerCAmelCase__				    =       r'''
    Args:
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        scores (`torch.FloatTensor` of shape `(batch_size, config.vocab_size)`):
            Prediction scores of a language modeling head. These can be scores for each vocabulary token before SoftMax
            or scores for each vocabulary token after SoftMax.
        kwargs (`Dict[str, Any]`, *optional*):
            Additional stopping criteria specific kwargs.

    Return:
        `bool`. `False` indicates we should continue, `True` indicates we should stop.

'''


class 				SCREAMING_SNAKE_CASE__    (		lowercase    ):
       """simple docstring"""


       @add_start_docstrings(snake_case__						)
       def __call__(						self		,					snake_case__		,					snake_case__		,					**snake_case__						):






         """simple docstring"""





         raise NotImplementedError("StoppingCriteria needs to be subclassed"						)


class 				SCREAMING_SNAKE_CASE__    (		lowercase    ):
       """simple docstring"""


       def __init__(						self		,					snake_case__		,					snake_case__ = None						):






         """simple docstring"""





         lowerCAmelCase					:					List[str]								=							max_length
         lowerCAmelCase					:					Union[str, Any]								=							max_position_embeddings







       @add_start_docstrings(snake_case__						)
       def __call__(						self		,					snake_case__		,					snake_case__		,					**snake_case__						):






         """simple docstring"""





         lowerCAmelCase					:					Dict								=							input_ids.shape[-1]
         lowerCAmelCase					:					List[str]								=							cur_len >= self.max_length
         if self.max_position_embeddings is not None and not is_done and cur_len >= self.max_position_embeddings:
           logger.warning_once(
               "This is a friendly reminder - the current text generation call will exceed the model's predefined "
               f"""maximum length ({self.max_position_embeddings}). Depending on the model, you may observe """
               "exceptions, performance degradation, or nothing at all."						)
         return is_done


class 				SCREAMING_SNAKE_CASE__    (		lowercase    ):
       """simple docstring"""


       def __init__(						self		,					snake_case__		,					snake_case__						):






         """simple docstring"""





         warnings.warn(
             "The class `MaxNewTokensCriteria` is deprecated. "
             f"""Please use `MaxLengthCriteria(max_length={start_length + max_new_tokens})` """
             "with `max_length = start_length + max_new_tokens` instead."		,					snake_case__		,					)
         lowerCAmelCase					:					Any								=							start_length
         lowerCAmelCase					:					Union[str, Any]								=							max_new_tokens
         lowerCAmelCase					:					Union[str, Any]								=							start_length + max_new_tokens







       @add_start_docstrings(snake_case__						)
       def __call__(						self		,					snake_case__		,					snake_case__		,					**snake_case__						):






         """simple docstring"""





         return input_ids.shape[-1] >= self.max_length


class 				SCREAMING_SNAKE_CASE__    (		lowercase    ):
       """simple docstring"""


       def __init__(						self		,					snake_case__		,					snake_case__ = None						):






         """simple docstring"""





         lowerCAmelCase					:					Dict								=							max_time
         lowerCAmelCase					:					List[Any]								=							time.time() if initial_timestamp is None else initial_timestamp







       @add_start_docstrings(snake_case__						)
       def __call__(						self		,					snake_case__		,					snake_case__		,					**snake_case__						):






         """simple docstring"""





         return time.time() - self.initial_timestamp > self.max_time


class 				SCREAMING_SNAKE_CASE__    (		lowercase    ):
       """simple docstring"""


       @add_start_docstrings(snake_case__						)
       def __call__(						self		,					snake_case__		,					snake_case__		,					**snake_case__						):






         """simple docstring"""





         return any(criteria(snake_case__		,					snake_case__						) for criteria in self						)







       @property
       def 	lowercase__							(						self						):






         """simple docstring"""





         for stopping_criterium in self:
           if isinstance(snake_case__		,					snake_case__						):
             return stopping_criterium.max_length
           elif isinstance(snake_case__		,					snake_case__						):
             return stopping_criterium.max_length
         return None

def  a__      (    SCREAMING_SNAKE_CASE   :   StoppingCriteriaList    ,				SCREAMING_SNAKE_CASE   :   int    ):


  '''simple docstring'''







  lowerCAmelCase					:					Tuple								=							stopping_criteria.max_length
  lowerCAmelCase					:					Optional[int]								=							deepcopy(SCREAMING_SNAKE_CASE    )
  if stopping_max_length is not None and stopping_max_length != max_length:
    warnings.warn("You set different `max_length` for stopping criteria and `max_length` parameter"    ,				SCREAMING_SNAKE_CASE    )
  elif stopping_max_length is None:
    new_stopping_criteria.append(MaxLengthCriteria(max_length=SCREAMING_SNAKE_CASE    )    )
  return new_stopping_criteria









"""simple docstring"""







import pyarrow.parquet as pq
import pytest

from datasets import Audio, Dataset, DatasetDict, Features, NamedSplit, Sequence, Value, config
from datasets.features.image import Image
from datasets.io.parquet import ParquetDatasetReader, ParquetDatasetWriter, get_writer_batch_size

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases
def  a__      (    SCREAMING_SNAKE_CASE   :   str    ,				SCREAMING_SNAKE_CASE   :   Any    ):


  '''simple docstring'''







  assert isinstance(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    )
  assert dataset.num_rows == 4
  assert dataset.num_columns == 3
  assert dataset.column_names == ["col_1", "col_2", "col_3"]
  for feature, expected_dtype in expected_features.items():
    assert dataset.features[feature].dtype == expected_dtype
@pytest.mark.parametrize("keep_in_memory"    ,				[False, True]    )
def  a__      (    SCREAMING_SNAKE_CASE   :   Any    ,				SCREAMING_SNAKE_CASE   :   List[str]    ,				SCREAMING_SNAKE_CASE   :   int    ):


  '''simple docstring'''







  lowerCAmelCase					:					List[Any]								=							tmp_path / "cache"
  lowerCAmelCase					:					Optional[int]								=							{"col_1": "string", "col_2": "int64", "col_3": "float64"}
  with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
    lowerCAmelCase					:					str								=							ParquetDatasetReader(SCREAMING_SNAKE_CASE    ,				cache_dir=SCREAMING_SNAKE_CASE    ,				keep_in_memory=SCREAMING_SNAKE_CASE    ).read()
  _check_parquet_dataset(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    )
@pytest.mark.parametrize(
    "features"    ,				[
        None,
        {"col_1": "string", "col_2": "int64", "col_3": "float64"},
        {"col_1": "string", "col_2": "string", "col_3": "string"},
        {"col_1": "int32", "col_2": "int32", "col_3": "int32"},
        {"col_1": "float32", "col_2": "float32", "col_3": "float32"},
    ]    ,				)
def  a__      (    SCREAMING_SNAKE_CASE   :   Tuple    ,				SCREAMING_SNAKE_CASE   :   Optional[Any]    ,				SCREAMING_SNAKE_CASE   :   Tuple    ):


  '''simple docstring'''







  lowerCAmelCase					:					Optional[int]								=							tmp_path / "cache"
  lowerCAmelCase					:					Optional[Any]								=							{"col_1": "string", "col_2": "int64", "col_3": "float64"}
  lowerCAmelCase					:					Tuple								=							features.copy() if features else default_expected_features
  lowerCAmelCase					:					List[str]								=							(
      Features({feature: Value(SCREAMING_SNAKE_CASE    ) for feature, dtype in features.items()}    ) if features is not None else None
  )
  lowerCAmelCase					:					List[str]								=							ParquetDatasetReader(SCREAMING_SNAKE_CASE    ,				features=SCREAMING_SNAKE_CASE    ,				cache_dir=SCREAMING_SNAKE_CASE    ).read()
  _check_parquet_dataset(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    )
@pytest.mark.parametrize("split"    ,				[None, NamedSplit("train"    ), "train", "test"]    )
def  a__      (    SCREAMING_SNAKE_CASE   :   Union[str, Any]    ,				SCREAMING_SNAKE_CASE   :   Union[str, Any]    ,				SCREAMING_SNAKE_CASE   :   Optional[int]    ):


  '''simple docstring'''







  lowerCAmelCase					:					List[Any]								=							tmp_path / "cache"
  lowerCAmelCase					:					int								=							{"col_1": "string", "col_2": "int64", "col_3": "float64"}
  lowerCAmelCase					:					Tuple								=							ParquetDatasetReader(SCREAMING_SNAKE_CASE    ,				cache_dir=SCREAMING_SNAKE_CASE    ,				split=SCREAMING_SNAKE_CASE    ).read()
  _check_parquet_dataset(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    )
  assert dataset.split == split if split else "train"
@pytest.mark.parametrize("path_type"    ,				[str, list]    )
def  a__      (    SCREAMING_SNAKE_CASE   :   List[Any]    ,				SCREAMING_SNAKE_CASE   :   Dict    ,				SCREAMING_SNAKE_CASE   :   List[str]    ):


  '''simple docstring'''







  if issubclass(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    ):
    lowerCAmelCase					:					Optional[Any]								=							parquet_path
  elif issubclass(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    ):
    lowerCAmelCase					:					Dict								=							[parquet_path]
  lowerCAmelCase					:					Optional[Any]								=							tmp_path / "cache"
  lowerCAmelCase					:					Any								=							{"col_1": "string", "col_2": "int64", "col_3": "float64"}
  lowerCAmelCase					:					Optional[Any]								=							ParquetDatasetReader(SCREAMING_SNAKE_CASE    ,				cache_dir=SCREAMING_SNAKE_CASE    ).read()
  _check_parquet_dataset(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    )
def  a__      (    SCREAMING_SNAKE_CASE   :   Dict    ,				SCREAMING_SNAKE_CASE   :   List[str]    ,				SCREAMING_SNAKE_CASE   :   Tuple=("train",)    ):


  '''simple docstring'''







  assert isinstance(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    )
  for split in splits:
    lowerCAmelCase					:					List[Any]								=							dataset_dict[split]
    assert dataset.num_rows == 4
    assert dataset.num_columns == 3
    assert dataset.column_names == ["col_1", "col_2", "col_3"]
    for feature, expected_dtype in expected_features.items():
      assert dataset.features[feature].dtype == expected_dtype
@pytest.mark.parametrize("keep_in_memory"    ,				[False, True]    )
def  a__      (    SCREAMING_SNAKE_CASE   :   Optional[int]    ,				SCREAMING_SNAKE_CASE   :   Tuple    ,				SCREAMING_SNAKE_CASE   :   Tuple    ):


  '''simple docstring'''







  lowerCAmelCase					:					Tuple								=							tmp_path / "cache"
  lowerCAmelCase					:					List[Any]								=							{"col_1": "string", "col_2": "int64", "col_3": "float64"}
  with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
    lowerCAmelCase					:					Optional[Any]								=							ParquetDatasetReader(
        {"train": parquet_path}    ,				cache_dir=SCREAMING_SNAKE_CASE    ,				keep_in_memory=SCREAMING_SNAKE_CASE    ).read()
  _check_parquet_datasetdict(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    )
@pytest.mark.parametrize(
    "features"    ,				[
        None,
        {"col_1": "string", "col_2": "int64", "col_3": "float64"},
        {"col_1": "string", "col_2": "string", "col_3": "string"},
        {"col_1": "int32", "col_2": "int32", "col_3": "int32"},
        {"col_1": "float32", "col_2": "float32", "col_3": "float32"},
    ]    ,				)
def  a__      (    SCREAMING_SNAKE_CASE   :   Optional[int]    ,				SCREAMING_SNAKE_CASE   :   Dict    ,				SCREAMING_SNAKE_CASE   :   Optional[int]    ):


  '''simple docstring'''







  lowerCAmelCase					:					Any								=							tmp_path / "cache"
  lowerCAmelCase					:					Union[str, Any]								=							{"col_1": "string", "col_2": "int64", "col_3": "float64"}
  lowerCAmelCase					:					List[Any]								=							features.copy() if features else default_expected_features
  lowerCAmelCase					:					List[Any]								=							(
      Features({feature: Value(SCREAMING_SNAKE_CASE    ) for feature, dtype in features.items()}    ) if features is not None else None
  )
  lowerCAmelCase					:					Optional[int]								=							ParquetDatasetReader({"train": parquet_path}    ,				features=SCREAMING_SNAKE_CASE    ,				cache_dir=SCREAMING_SNAKE_CASE    ).read()
  _check_parquet_datasetdict(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    )
@pytest.mark.parametrize("split"    ,				[None, NamedSplit("train"    ), "train", "test"]    )
def  a__      (    SCREAMING_SNAKE_CASE   :   str    ,				SCREAMING_SNAKE_CASE   :   Dict    ,				SCREAMING_SNAKE_CASE   :   List[str]    ):


  '''simple docstring'''







  if split:
    lowerCAmelCase					:					List[str]								=							{split: parquet_path}
  else:
    lowerCAmelCase					:					List[str]								=							"train"
    lowerCAmelCase					:					str								=							{"train": parquet_path, "test": parquet_path}
  lowerCAmelCase					:					Optional[int]								=							tmp_path / "cache"
  lowerCAmelCase					:					List[Any]								=							{"col_1": "string", "col_2": "int64", "col_3": "float64"}
  lowerCAmelCase					:					List[str]								=							ParquetDatasetReader(SCREAMING_SNAKE_CASE    ,				cache_dir=SCREAMING_SNAKE_CASE    ).read()
  _check_parquet_datasetdict(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    ,				splits=list(path.keys()    )    )
  assert all(dataset[split].split == split for split in path.keys()    )
def  a__      (    SCREAMING_SNAKE_CASE   :   Optional[Any]    ,				SCREAMING_SNAKE_CASE   :   Any    ):


  '''simple docstring'''







  lowerCAmelCase					:					Optional[int]								=							ParquetDatasetWriter(SCREAMING_SNAKE_CASE    ,				tmp_path / "foo.parquet"    )
  assert writer.write() > 0
  lowerCAmelCase					:					List[str]								=							pq.ParquetFile(tmp_path / "foo.parquet"    )
  lowerCAmelCase					:					Union[str, Any]								=							pf.read()
  assert dataset.data.table == output_table
def  a__      (    SCREAMING_SNAKE_CASE   :   List[Any]    ,				SCREAMING_SNAKE_CASE   :   List[str]    ):


  '''simple docstring'''







  lowerCAmelCase					:					str								=							str(shared_datadir / "test_image_rgb.jpg"    )
  lowerCAmelCase					:					List[str]								=							{"image": [image_path]}
  lowerCAmelCase					:					str								=							Features({"image": Image()}    )
  lowerCAmelCase					:					Optional[Any]								=							Dataset.from_dict(SCREAMING_SNAKE_CASE    ,				features=SCREAMING_SNAKE_CASE    )
  lowerCAmelCase					:					List[Any]								=							ParquetDatasetWriter(SCREAMING_SNAKE_CASE    ,				tmp_path / "foo.parquet"    )
  assert writer.write() > 0

  lowerCAmelCase					:					Any								=							Dataset.from_parquet(str(tmp_path / "foo.parquet"    )    )
  assert dataset.features == reloaded_dataset.features

  lowerCAmelCase					:					Optional[int]								=							ParquetDatasetReader(str(tmp_path / "foo.parquet"    )    ,				streaming=SCREAMING_SNAKE_CASE    ).read()
  assert dataset.features == reloaded_iterable_dataset.features





@pytest.mark.parametrize(
    "feature, expected"    ,				[
        (Features({"foo": Value("int32"    )}    ), None),
        (Features({"image": Image(), "foo": Value("int32"    )}    ), config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS),
        (Features({"nested": Sequence(Audio()    )}    ), config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS),
    ]    ,				)
def  a__      (    SCREAMING_SNAKE_CASE   :   str    ,				SCREAMING_SNAKE_CASE   :   List[str]    ):


  '''simple docstring'''







  assert get_writer_batch_size(SCREAMING_SNAKE_CASE    ) == expected





