

import pyarrow.parquet as pq
import pytest

from datasets import Audio, Dataset, DatasetDict, Features, NamedSplit, Sequence, Value, config
from datasets.features.image import Image
from datasets.io.parquet import ParquetDatasetReader, ParquetDatasetWriter, get_writer_batch_size

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases

def 						UpperCamelCase (lowercase_:			Any   ,				lowercase_:			Optional[int] )					->				Union[str, Any]:
	assert isinstance(lowercase_   ,				lowercase_ )
	assert dataset.num_rows == 4
	assert dataset.num_columns == 3
	assert dataset.column_names == ["col_1", "col_2", "col_3"]
	for feature, expected_dtype in expected_features.items():
		assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize("""keep_in_memory"""   ,				[False, True] )
def 						UpperCamelCase (lowercase_:			List[str]   ,				lowercase_:			Union[str, Any]   ,				lowercase_:			Optional[int] )					->				Union[str, Any]:
	A__							:   Union[str, Any]					       =		tmp_path / """cache"""
	A__							:   Union[str, Any]					       =		{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
	with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
		A__							:   Optional[Any]					       =		ParquetDatasetReader(lowercase_   ,				cache_dir=lowercase_   ,				keep_in_memory=lowercase_ ).read()
	_check_parquet_dataset(lowercase_   ,				lowercase_ )

@pytest.mark.parametrize(
    """features"""   ,				[
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]   ,				)
def 						UpperCamelCase (lowercase_:			int   ,				lowercase_:			Tuple   ,				lowercase_:			Any )					->				int:
	A__							:   List[str]					       =		tmp_path / """cache"""
	A__							:   Tuple					       =		{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
	A__							:   Dict					       =		features.copy() if features else default_expected_features
	A__							:   Optional[int]					       =		(
	    Features({feature: Value(lowercase_ ) for feature, dtype in features.items()} ) if features is not None else None
	)
	A__							:   str					       =		ParquetDatasetReader(lowercase_   ,				features=lowercase_   ,				cache_dir=lowercase_ ).read()
	_check_parquet_dataset(lowercase_   ,				lowercase_ )

@pytest.mark.parametrize("""split"""   ,				[None, NamedSplit("""train""" ), """train""", """test"""] )
def 						UpperCamelCase (lowercase_:			Optional[int]   ,				lowercase_:			str   ,				lowercase_:			int )					->				Optional[Any]:
	A__							:   Any					       =		tmp_path / """cache"""
	A__							:   List[Any]					       =		{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
	A__							:   Union[str, Any]					       =		ParquetDatasetReader(lowercase_   ,				cache_dir=lowercase_   ,				split=lowercase_ ).read()
	_check_parquet_dataset(lowercase_   ,				lowercase_ )
	assert dataset.split == split if split else "train"

@pytest.mark.parametrize("""path_type"""   ,				[str, list] )
def 						UpperCamelCase (lowercase_:			List[Any]   ,				lowercase_:			List[str]   ,				lowercase_:			int )					->				List[Any]:
	if issubclass(lowercase_   ,				lowercase_ ):
		A__							:   List[str]					       =		parquet_path
	elif issubclass(lowercase_   ,				lowercase_ ):
		A__							:   Tuple					       =		[parquet_path]
	A__							:   List[Any]					       =		tmp_path / """cache"""
	A__							:   List[Any]					       =		{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
	A__							:   List[str]					       =		ParquetDatasetReader(lowercase_   ,				cache_dir=lowercase_ ).read()
	_check_parquet_dataset(lowercase_   ,				lowercase_ )

def 						UpperCamelCase (lowercase_:			Dict   ,				lowercase_:			Union[str, Any]   ,				lowercase_:			Dict=("train",) )					->				Optional[Any]:
	assert isinstance(lowercase_   ,				lowercase_ )
	for split in splits:
		A__							:   Tuple					       =		dataset_dict[split]
		assert dataset.num_rows == 4
		assert dataset.num_columns == 3
		assert dataset.column_names == ["col_1", "col_2", "col_3"]
		for feature, expected_dtype in expected_features.items():
			assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize("""keep_in_memory"""   ,				[False, True] )
def 						UpperCamelCase (lowercase_:			Union[str, Any]   ,				lowercase_:			Dict   ,				lowercase_:			Optional[Any] )					->				Dict:
	A__							:   List[Any]					       =		tmp_path / """cache"""
	A__							:   Union[str, Any]					       =		{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
	with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
		A__							:   List[Any]					       =		ParquetDatasetReader(
		    {"""train""": parquet_path}   ,				cache_dir=lowercase_   ,				keep_in_memory=lowercase_ ).read()
	_check_parquet_datasetdict(lowercase_   ,				lowercase_ )

@pytest.mark.parametrize(
    """features"""   ,				[
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]   ,				)
def 						UpperCamelCase (lowercase_:			int   ,				lowercase_:			List[str]   ,				lowercase_:			Any )					->				List[Any]:
	A__							:   Any					       =		tmp_path / """cache"""
	A__							:   Dict					       =		{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
	A__							:   Tuple					       =		features.copy() if features else default_expected_features
	A__							:   Optional[int]					       =		(
	    Features({feature: Value(lowercase_ ) for feature, dtype in features.items()} ) if features is not None else None
	)
	A__							:   Any					       =		ParquetDatasetReader({"""train""": parquet_path}   ,				features=lowercase_   ,				cache_dir=lowercase_ ).read()
	_check_parquet_datasetdict(lowercase_   ,				lowercase_ )

@pytest.mark.parametrize("""split"""   ,				[None, NamedSplit("""train""" ), """train""", """test"""] )
def 						UpperCamelCase (lowercase_:			List[Any]   ,				lowercase_:			Tuple   ,				lowercase_:			int )					->				List[str]:
	if split:
		A__							:   Any					       =		{split: parquet_path}
	else:
		A__							:   Any					       =		"""train"""
		A__							:   Dict					       =		{"""train""": parquet_path, """test""": parquet_path}
	A__							:   Union[str, Any]					       =		tmp_path / """cache"""
	A__							:   int					       =		{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
	A__							:   Tuple					       =		ParquetDatasetReader(lowercase_   ,				cache_dir=lowercase_ ).read()
	_check_parquet_datasetdict(lowercase_   ,				lowercase_   ,				splits=list(path.keys() ) )
	assert all(dataset[split].split == split for split in path.keys() )

def 						UpperCamelCase (lowercase_:			Optional[int]   ,				lowercase_:			Optional[int] )					->				Optional[int]:
	A__							:   List[str]					       =		ParquetDatasetWriter(lowercase_   ,				tmp_path / """foo.parquet""" )
	assert writer.write() > 0
	A__							:   List[str]					       =		pq.ParquetFile(tmp_path / """foo.parquet""" )
	A__							:   Any					       =		pf.read()
	assert dataset.data.table == output_table

def 						UpperCamelCase (lowercase_:			Any   ,				lowercase_:			Tuple )					->				Union[str, Any]:
	A__							:   Optional[int]					       =		str(shared_datadir / """test_image_rgb.jpg""" )
	A__							:   List[str]					       =		{"""image""": [image_path]}
	A__							:   List[str]					       =		Features({"""image""": Image()} )
	A__							:   Tuple					       =		Dataset.from_dict(lowercase_   ,				features=lowercase_ )
	A__							:   Tuple					       =		ParquetDatasetWriter(lowercase_   ,				tmp_path / """foo.parquet""" )
	assert writer.write() > 0

	A__							:   Optional[int]					       =		Dataset.from_parquet(str(tmp_path / """foo.parquet""" ) )
	assert dataset.features == reloaded_dataset.features

	A__							:   Optional[int]					       =		ParquetDatasetReader(str(tmp_path / """foo.parquet""" )   ,				streaming=lowercase_ ).read()
	assert dataset.features == reloaded_iterable_dataset.features


@pytest.mark.parametrize(
    """feature, expected"""   ,				[
        (Features({"""foo""": Value("""int32""" )} ), None),
        (Features({"""image""": Image(), """foo""": Value("""int32""" )} ), config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS),
        (Features({"""nested""": Sequence(Audio() )} ), config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS),
    ]   ,				)
def 						UpperCamelCase (lowercase_:			str   ,				lowercase_:			str )					->				List[Any]:
	assert get_writer_batch_size(lowercase_ ) == expected






from ...configuration_utils import PretrainedConfig
from ...utils import logging


A_	:			str  	=      logging.get_logger(__name__)

A_	:			Any  	=      {
    'tiiuae/falcon-40b': 'https://huggingface.co/tiiuae/falcon-40b/resolve/main/config.json',
    'tiiuae/falcon-7b': 'https://huggingface.co/tiiuae/falcon-7b/resolve/main/config.json',
}



class    _a						(__magic_name__    ):


					'''simple docstring'''






					UpperCAmelCase__:       Dict  							=	'''falcon'''
					UpperCAmelCase__:       Any  							=	['''past_key_values''']






					def __init__(			self   ,	A__=6_5024   ,	A__=4544   ,	A__=32   ,	A__=71   ,	A__=1e-5   ,	A__=0.0_2   ,	A__=True   ,	A__=0.0   ,	A__=0.0   ,	A__=None   ,	A__=False   ,	A__=False   ,	A__=True   ,	A__=True   ,	A__=False   ,	A__=11   ,	A__=11   ,	**A__   ,	):
						A__							:   Dict					       =		vocab_size
						# Backward compatibility with n_embed kwarg
						A__							:   Union[str, Any]					       =		kwargs.pop("""n_embed"""   ,	A__      )
						A__							:   Optional[Any]					       =		hidden_size if n_embed is None else n_embed
						A__							:   Any					       =		num_hidden_layers
						A__							:   Any					       =		num_attention_heads
						A__							:   Optional[Any]					       =		layer_norm_epsilon
						A__							:   Tuple					       =		initializer_range
						A__							:   Tuple					       =		use_cache
						A__							:   str					       =		hidden_dropout
						A__							:   List[str]					       =		attention_dropout

						A__							:   List[Any]					       =		bos_token_id
						A__							:   Optional[Any]					       =		eos_token_id
						A__							:   Optional[Any]					       =		num_attention_heads if num_kv_heads is None else num_kv_heads
						A__							:   List[str]					       =		alibi
						A__							:   Tuple					       =		new_decoder_architecture
						A__							:   List[str]					       =		multi_query  # Ignored when new_decoder_architecture is True
						A__							:   List[Any]					       =		parallel_attn
						A__							:   int					       =		bias

						super().__init__(bos_token_id=A__   ,	eos_token_id=A__   ,	**A__      )






					@property
					def   __A      (			self      ):
						return self.hidden_size // self.num_attention_heads






					@property
					def   __A      (			self      ):
						return not self.alibi



