



import copy
import os
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import pytest

from datasets.arrow_writer import ArrowWriter, OptimizedTypedSequence, ParquetWriter, TypedSequence
from datasets.features import ArrayaD, ClassLabel, Features, Image, Value
from datasets.features.features import ArrayaDExtensionType, cast_to_python_objects
from datasets.keyhash import DuplicatedKeysError, InvalidKeyError

from .utils import require_pil



class 	lowerCamelCase__      (							lowerCamelCase__):




		'''simple docstring'''



		def 		lowerCAmelCase__				(self			)					->      Optional[Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    List[str]												=					pa.array(TypedSequence([1, 2, 3]			)			)
						self.assertEqual(arr.type		,pa.intaa()			)



		def 		lowerCAmelCase__				(self			)					->      int:

						"""simple docstring"""






						with self.assertRaises(__lowerCamelCase			):
										lowerCAmelCase__					:    Optional[int]												=					pa.array(TypedSequence([1, 2, 3]			)		,type=pa.intaa()			)



		def 		lowerCAmelCase__				(self			)					->      Dict:

						"""simple docstring"""






						with self.assertRaises(__lowerCamelCase			):
										lowerCAmelCase__					:    str												=					pa.array(TypedSequence([1, 2, 3]		,try_type=Value('''bool'''			)		,type=Value('''int64'''			)			)			)



		def 		lowerCAmelCase__				(self			)					->      Union[str, Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    Union[str, Any]												=					pa.array(TypedSequence([1, 2, 3]		,type=Value('''int32'''			)			)			)
						self.assertEqual(arr.type		,pa.intaa()			)



		def 		lowerCAmelCase__				(self			)					->      Optional[int]:

						"""simple docstring"""






						with self.assertRaises((TypeError, pa.lib.ArrowInvalid)			):
										lowerCAmelCase__					:    Tuple												=					pa.array(TypedSequence(['''foo''', '''bar''']		,type=Value('''int64'''			)			)			)



		def 		lowerCAmelCase__				(self			)					->      Union[str, Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    List[Any]												=					pa.array(TypedSequence([1, 2, 3]		,try_type=Value('''int32'''			)			)			)
						self.assertEqual(arr.type		,pa.intaa()			)



		def 		lowerCAmelCase__				(self			)					->      int:

						"""simple docstring"""






						lowerCAmelCase__					:    int												=					pa.array(TypedSequence(['''foo''', '''bar''']		,try_type=Value('''int64'''			)			)			)
						self.assertEqual(arr.type		,pa.string()			)



		def 		lowerCAmelCase__				(self			)					->      Tuple:

						"""simple docstring"""






						lowerCAmelCase__					:    List[Any]												=					pa.array(TypedSequence([[[1, 2, 3]]]		,type=ArrayaD((1, 3)		,'''int64'''			)			)			)
						self.assertEqual(arr.type		,ArrayaDExtensionType((1, 3)		,'''int64'''			)			)



		def 		lowerCAmelCase__				(self			)					->      Union[str, Any]:

						"""simple docstring"""






						with self.assertRaises((TypeError, pa.lib.ArrowInvalid)			):
										lowerCAmelCase__					:    List[Any]												=					pa.array(TypedSequence(['''foo''', '''bar''']		,type=ArrayaD((1, 3)		,'''int64'''			)			)			)



		def 		lowerCAmelCase__				(self			)					->      Dict:

						"""simple docstring"""






						lowerCAmelCase__					:    List[Any]												=					pa.array(TypedSequence([[[1, 2, 3]]]		,try_type=ArrayaD((1, 3)		,'''int64'''			)			)			)
						self.assertEqual(arr.type		,ArrayaDExtensionType((1, 3)		,'''int64'''			)			)



		def 		lowerCAmelCase__				(self			)					->      Union[str, Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    Dict												=					pa.array(TypedSequence(['''foo''', '''bar''']		,try_type=ArrayaD((1, 3)		,'''int64'''			)			)			)
						self.assertEqual(arr.type		,pa.string()			)



		@require_pil
		def 		lowerCAmelCase__				(self			)					->      int:

						"""simple docstring"""






						import PIL.Image

						lowerCAmelCase__					:    Any												=					PIL.Image.fromarray(np.arange(10		,dtype=np.uinta			).reshape(2		,5			)			)
						with patch(
						    '''datasets.arrow_writer.cast_to_python_objects'''		,side_effect=__lowerCamelCase			) as mock_cast_to_python_objects:
										lowerCAmelCase__					:    int												=					pa.array(TypedSequence([{'''path''': None, '''bytes''': B'''image_bytes'''}, pil_image]		,type=Image()			)			)
										lowerCAmelCase__					:    List[str]												=					mock_cast_to_python_objects.call_args_list[-1]
										self.assertIn('''optimize_list_casting'''		,__lowerCamelCase			)
										self.assertFalse(kwargs['''optimize_list_casting''']			)






def      lowerCAmelCase__		(   lowerCamelCase_		:		Any	,lowerCamelCase_		:		int):





				'''simple docstring'''

				lowerCAmelCase__					:    Optional[int]												=					pa.BufferReader(lowerCamelCase_) if isinstance(lowerCamelCase_	,pa.Buffer) else pa.memory_map(lowerCamelCase_)
				lowerCAmelCase__					:    int												=					pa.ipc.open_stream(lowerCamelCase_)
				lowerCAmelCase__					:    pa.Table												=					f.read_all()
				assert len(pa_table.to_batches()) == expected_num_chunks
				assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
				del pa_table






@pytest.mark.parametrize('''writer_batch_size'''	,[None, 1, 10])
@pytest.mark.parametrize(
    '''fields'''	,[None, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}])
def      lowerCAmelCase__		(   lowerCamelCase_		:		Any	,lowerCamelCase_		:		List[Any]):





				'''simple docstring'''

				lowerCAmelCase__					:    Union[str, Any]												=					pa.BufferOutputStream()
				lowerCAmelCase__					:    List[Any]												=					pa.schema(lowerCamelCase_) if fields else None
				with ArrowWriter(stream=lowerCamelCase_	,schema=lowerCamelCase_	,writer_batch_size=lowerCamelCase_) as writer:
								writer.write({'''col_1''': '''foo''', '''col_2''': 1})
								writer.write({'''col_1''': '''bar''', '''col_2''': 2})
								lowerCAmelCase__					:    List[Any]												=					writer.finalize()
				assert num_examples == 2
				assert num_bytes > 0
				if not fields:
								lowerCAmelCase__					:    List[Any]												=					{'''col_1''': pa.string(), '''col_2''': pa.intaa()}
				assert writer._schema == pa.schema(lowerCamelCase_	,metadata=writer._schema.metadata)
				_check_output(output.getvalue()	,expected_num_chunks=num_examples if writer_batch_size == 1 else 1)






def      lowerCAmelCase__		(   ):





				'''simple docstring'''

				lowerCAmelCase__					:    str												=					pa.BufferOutputStream()
				lowerCAmelCase__					:    Union[str, Any]												=					Features({'''labels''': ClassLabel(names=['''neg''', '''pos'''])})
				with ArrowWriter(stream=lowerCamelCase_	,features=lowerCamelCase_) as writer:
								writer.write({'''labels''': 0})
								writer.write({'''labels''': 1})
								lowerCAmelCase__					:    List[Any]												=					writer.finalize()
				assert num_examples == 2
				assert num_bytes > 0
				assert writer._schema == features.arrow_schema
				assert writer._schema.metadata == features.arrow_schema.metadata
				lowerCAmelCase__					:    int												=					pa.BufferReader(output.getvalue())
				lowerCAmelCase__					:    Dict												=					pa.ipc.open_stream(lowerCamelCase_)
				lowerCAmelCase__					:    pa.Table												=					f.read_all()
				lowerCAmelCase__					:    int												=					pa_table.schema
				assert pa_table.num_rows == 2
				assert schema == features.arrow_schema
				assert schema.metadata == features.arrow_schema.metadata
				assert features == Features.from_arrow_schema(lowerCamelCase_)






@pytest.mark.parametrize('''writer_batch_size'''	,[None, 1, 10])
def      lowerCAmelCase__		(   lowerCamelCase_		:		str):





				'''simple docstring'''

				lowerCAmelCase__					:    Dict												=					pa.BufferOutputStream()
				with ArrowWriter(
				    stream=lowerCamelCase_	,writer_batch_size=lowerCamelCase_	,hash_salt='''split_name'''	,check_duplicates=lowerCamelCase_	,) as writer:
								with pytest.raises(lowerCamelCase_):
												writer.write({'''col_1''': '''foo''', '''col_2''': 1}	,key=[1, 2])
												lowerCAmelCase__					:    int												=					writer.finalize()






@pytest.mark.parametrize('''writer_batch_size'''	,[None, 2, 10])
def      lowerCAmelCase__		(   lowerCamelCase_		:		List[Any]):





				'''simple docstring'''

				lowerCAmelCase__					:    Dict												=					pa.BufferOutputStream()
				with ArrowWriter(
				    stream=lowerCamelCase_	,writer_batch_size=lowerCamelCase_	,hash_salt='''split_name'''	,check_duplicates=lowerCamelCase_	,) as writer:
								with pytest.raises(lowerCamelCase_):
												writer.write({'''col_1''': '''foo''', '''col_2''': 1}	,key=10)
												writer.write({'''col_1''': '''bar''', '''col_2''': 2}	,key=10)
												lowerCAmelCase__					:    List[str]												=					writer.finalize()






@pytest.mark.parametrize('''writer_batch_size'''	,[None, 2, 10])
def      lowerCAmelCase__		(   lowerCamelCase_		:		Optional[Any]):





				'''simple docstring'''

				lowerCAmelCase__					:    Dict												=					pa.BufferOutputStream()
				with ArrowWriter(
				    stream=lowerCamelCase_	,writer_batch_size=lowerCamelCase_	,hash_salt='''split_name'''	,check_duplicates=lowerCamelCase_	,) as writer:
								writer.write({'''col_1''': '''foo''', '''col_2''': 1}	,key=1)
								writer.write({'''col_1''': '''bar''', '''col_2''': 2}	,key=2)
								lowerCAmelCase__					:    Optional[int]												=					writer.finalize()
				assert num_examples == 2
				assert num_bytes > 0
				_check_output(output.getvalue()	,expected_num_chunks=num_examples if writer_batch_size == 1 else 1)






@pytest.mark.parametrize('''writer_batch_size'''	,[None, 1, 10])
@pytest.mark.parametrize(
    '''fields'''	,[None, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}])
def      lowerCAmelCase__		(   lowerCamelCase_		:		Union[str, Any]	,lowerCamelCase_		:		Optional[int]):





				'''simple docstring'''

				lowerCAmelCase__					:    str												=					pa.BufferOutputStream()
				lowerCAmelCase__					:    Dict												=					pa.schema(lowerCamelCase_) if fields else None
				with ArrowWriter(stream=lowerCamelCase_	,schema=lowerCamelCase_	,writer_batch_size=lowerCamelCase_) as writer:
								writer.write_batch({'''col_1''': ['''foo''', '''bar'''], '''col_2''': [1, 2]})
								writer.write_batch({'''col_1''': [], '''col_2''': []})
								lowerCAmelCase__					:    Any												=					writer.finalize()
				assert num_examples == 2
				assert num_bytes > 0
				if not fields:
								lowerCAmelCase__					:    Any												=					{'''col_1''': pa.string(), '''col_2''': pa.intaa()}
				assert writer._schema == pa.schema(lowerCamelCase_	,metadata=writer._schema.metadata)
				_check_output(output.getvalue()	,expected_num_chunks=num_examples if writer_batch_size == 1 else 1)






@pytest.mark.parametrize('''writer_batch_size'''	,[None, 1, 10])
@pytest.mark.parametrize(
    '''fields'''	,[None, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}])
def      lowerCAmelCase__		(   lowerCamelCase_		:		Any	,lowerCamelCase_		:		List[str]):





				'''simple docstring'''

				lowerCAmelCase__					:    int												=					pa.BufferOutputStream()
				lowerCAmelCase__					:    List[str]												=					pa.schema(lowerCamelCase_) if fields else None
				with ArrowWriter(stream=lowerCamelCase_	,schema=lowerCamelCase_	,writer_batch_size=lowerCamelCase_) as writer:
								writer.write_table(pa.Table.from_pydict({'''col_1''': ['''foo''', '''bar'''], '''col_2''': [1, 2]}))
								lowerCAmelCase__					:    Any												=					writer.finalize()
				assert num_examples == 2
				assert num_bytes > 0
				if not fields:
								lowerCAmelCase__					:    Union[str, Any]												=					{'''col_1''': pa.string(), '''col_2''': pa.intaa()}
				assert writer._schema == pa.schema(lowerCamelCase_	,metadata=writer._schema.metadata)
				_check_output(output.getvalue()	,expected_num_chunks=num_examples if writer_batch_size == 1 else 1)






@pytest.mark.parametrize('''writer_batch_size'''	,[None, 1, 10])
@pytest.mark.parametrize(
    '''fields'''	,[None, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}])
def      lowerCAmelCase__		(   lowerCamelCase_		:		Tuple	,lowerCamelCase_		:		int):





				'''simple docstring'''

				lowerCAmelCase__					:    List[Any]												=					pa.BufferOutputStream()
				lowerCAmelCase__					:    Union[str, Any]												=					pa.schema(lowerCamelCase_) if fields else None
				with ArrowWriter(stream=lowerCamelCase_	,schema=lowerCamelCase_	,writer_batch_size=lowerCamelCase_) as writer:
								writer.write_row(pa.Table.from_pydict({'''col_1''': ['''foo'''], '''col_2''': [1]}))
								writer.write_row(pa.Table.from_pydict({'''col_1''': ['''bar'''], '''col_2''': [2]}))
								lowerCAmelCase__					:    int												=					writer.finalize()
				assert num_examples == 2
				assert num_bytes > 0
				if not fields:
								lowerCAmelCase__					:    List[Any]												=					{'''col_1''': pa.string(), '''col_2''': pa.intaa()}
				assert writer._schema == pa.schema(lowerCamelCase_	,metadata=writer._schema.metadata)
				_check_output(output.getvalue()	,expected_num_chunks=num_examples if writer_batch_size == 1 else 1)






def      lowerCAmelCase__		(   ):





				'''simple docstring'''

				with tempfile.TemporaryDirectory() as tmp_dir:
								lowerCAmelCase__					:    int												=					{'''col_1''': pa.string(), '''col_2''': pa.intaa()}
								lowerCAmelCase__					:    Dict												=					os.path.join(lowerCamelCase_	,'''test.arrow''')
								with ArrowWriter(path=lowerCamelCase_	,schema=pa.schema(lowerCamelCase_)) as writer:
												writer.write_batch({'''col_1''': ['''foo''', '''bar'''], '''col_2''': [1, 2]})
												lowerCAmelCase__					:    str												=					writer.finalize()
								assert num_examples == 2
								assert num_bytes > 0
								assert writer._schema == pa.schema(lowerCamelCase_	,metadata=writer._schema.metadata)
								_check_output(lowerCamelCase_	,1)






def      lowerCAmelCase__		(   lowerCamelCase_		:		List[str]):





				'''simple docstring'''

				if pa.types.is_list(lowerCamelCase_):
								return get_base_dtype(arr_type.value_type)
				else:
								return arr_type






def      lowerCAmelCase__		(   lowerCamelCase_		:		int	,lowerCamelCase_		:		Any):





				'''simple docstring'''

				if isinstance(lst[0]	,lowerCamelCase_):
								change_first_primitive_element_in_list(lst[0]	,lowerCamelCase_)
				else:
								lowerCAmelCase__					:    int												=					value






@pytest.mark.parametrize('''optimized_int_type, expected_dtype'''	,[(None, pa.intaa()), (Value('''int32'''), pa.intaa())])
@pytest.mark.parametrize('''sequence'''	,[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]])
def      lowerCAmelCase__		(   lowerCamelCase_		:		List[str]	,lowerCamelCase_		:		Tuple	,lowerCamelCase_		:		int):





				'''simple docstring'''

				lowerCAmelCase__					:    Union[str, Any]												=					pa.array(TypedSequence(lowerCamelCase_	,optimized_int_type=lowerCamelCase_))
				assert get_base_dtype(arr.type) == expected_dtype






@pytest.mark.parametrize(
    '''col, expected_dtype'''	,[
        ('''attention_mask''', pa.inta()),
        ('''special_tokens_mask''', pa.inta()),
        ('''token_type_ids''', pa.inta()),
        ('''input_ids''', pa.intaa()),
        ('''other''', pa.intaa()),
    ]	,)
@pytest.mark.parametrize('''sequence'''	,[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]])
def      lowerCAmelCase__		(   lowerCamelCase_		:		List[str]	,lowerCamelCase_		:		Optional[Any]	,lowerCamelCase_		:		Any):





				'''simple docstring'''

				lowerCAmelCase__					:    str												=					pa.array(OptimizedTypedSequence(lowerCamelCase_	,col=lowerCamelCase_))
				assert get_base_dtype(arr.type) == expected_dtype

				# not in range
				if col != "other":
								# avoids errors due to in-place modifications
								lowerCAmelCase__					:    Tuple												=					copy.deepcopy(lowerCamelCase_)
								lowerCAmelCase__					:    str												=					np.iinfo(expected_dtype.to_pandas_dtype()).max + 1
								change_first_primitive_element_in_list(lowerCamelCase_	,lowerCamelCase_)
								lowerCAmelCase__					:    Tuple												=					pa.array(OptimizedTypedSequence(lowerCamelCase_	,col=lowerCamelCase_))
								assert get_base_dtype(arr.type) == pa.intaa()






@pytest.mark.parametrize('''raise_exception'''	,[False, True])
def      lowerCAmelCase__		(   lowerCamelCase_		:		List[str]	,lowerCamelCase_		:		int):





				'''simple docstring'''

				lowerCAmelCase__					:    Any												=					str(tmp_path / '''dataset-train.arrow''')
				try:
								with ArrowWriter(path=lowerCamelCase_) as writer:
												if raise_exception:
																raise pa.lib.ArrowInvalid()
												else:
																writer.stream.close()
				except pa.lib.ArrowInvalid:
								pass
				finally:
								assert writer.stream.closed






def      lowerCAmelCase__		(   lowerCamelCase_		:		Optional[Any]):





				'''simple docstring'''

				lowerCAmelCase__					:    int												=					'''mock://dataset-train.arrow'''
				with ArrowWriter(path=lowerCamelCase_	,storage_options=mockfs.storage_options) as writer:
								assert isinstance(writer._fs	,type(lowerCamelCase_))
								assert writer._fs.storage_options == mockfs.storage_options
								writer.write({'''col_1''': '''foo''', '''col_2''': 1})
								writer.write({'''col_1''': '''bar''', '''col_2''': 2})
								lowerCAmelCase__					:    int												=					writer.finalize()
				assert num_examples == 2
				assert num_bytes > 0
				assert mockfs.exists(lowerCamelCase_)






def      lowerCAmelCase__		(   ):





				'''simple docstring'''

				lowerCAmelCase__					:    int												=					pa.BufferOutputStream()
				with ParquetWriter(stream=lowerCamelCase_) as writer:
								writer.write({'''col_1''': '''foo''', '''col_2''': 1})
								writer.write({'''col_1''': '''bar''', '''col_2''': 2})
								lowerCAmelCase__					:    Dict												=					writer.finalize()
				assert num_examples == 2
				assert num_bytes > 0
				lowerCAmelCase__					:    Optional[int]												=					pa.BufferReader(output.getvalue())
				lowerCAmelCase__					:    pa.Table												=					pq.read_table(lowerCamelCase_)
				assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}






@require_pil
@pytest.mark.parametrize('''embed_local_files'''	,[False, True])
def      lowerCAmelCase__		(   lowerCamelCase_		:		List[Any]	,lowerCamelCase_		:		List[Any]):





				'''simple docstring'''

				import PIL.Image

				lowerCAmelCase__					:    Optional[int]												=					str(tmp_path / '''test_image_rgb.jpg''')
				PIL.Image.fromarray(np.zeros((5, 5)	,dtype=np.uinta)).save(lowerCamelCase_	,format='''png''')
				lowerCAmelCase__					:    Tuple												=					pa.BufferOutputStream()
				with ParquetWriter(
				    stream=lowerCamelCase_	,features=Features({'''image''': Image()})	,embed_local_files=lowerCamelCase_) as writer:
								writer.write({'''image''': image_path})
								writer.finalize()
				lowerCAmelCase__					:    int												=					pa.BufferReader(output.getvalue())
				lowerCAmelCase__					:    pa.Table												=					pq.read_table(lowerCamelCase_)
				lowerCAmelCase__					:    Tuple												=					pa_table.to_pydict()
				if embed_local_files:
								assert isinstance(out['''image'''][0]['''path''']	,lowerCamelCase_)
								with open(lowerCamelCase_	,'''rb''') as f:
												assert out["image"][0]["bytes"] == f.read()
				else:
								assert out["image"][0]["path"] == image_path
								assert out["image"][0]["bytes"] is None






def      lowerCAmelCase__		(   ):





				'''simple docstring'''

				lowerCAmelCase__					:    Tuple												=					pa.schema([pa.field('''col_1'''	,pa.string()	,nullable=lowerCamelCase_)])
				lowerCAmelCase__					:    Optional[Any]												=					pa.BufferOutputStream()
				with ArrowWriter(stream=lowerCamelCase_) as writer:
								writer._build_writer(inferred_schema=lowerCamelCase_)
								assert writer._schema == pa.schema([pa.field('''col_1'''	,pa.string())])






import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


__snake_case    :		int  =get_tests_dir('fixtures/test_sentencepiece.model')



@require_sentencepiece
class 	lowerCamelCase__      (							lowerCamelCase__				,    unittest.TestCase):




		'''simple docstring'''



		snake_case_     =XLMProphetNetTokenizer
		snake_case_     =False
		snake_case_     =True



		def 		lowerCAmelCase__				(self			)					->      Tuple:

						"""simple docstring"""






						super().setUp()

						# We have a SentencePiece fixture for testing
						lowerCAmelCase__					:    int												=					XLMProphetNetTokenizer(__lowerCamelCase		,keep_accents=__lowerCamelCase			)
						tokenizer.save_pretrained(self.tmpdirname			)



		def 		lowerCAmelCase__				(self			)					->      Union[str, Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    str												=					'''[PAD]'''
						lowerCAmelCase__					:    Tuple												=					0

						self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCamelCase			)		,__lowerCamelCase			)
						self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCamelCase			)		,__lowerCamelCase			)



		def 		lowerCAmelCase__				(self			)					->      Optional[Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    Any												=					list(self.get_tokenizer().get_vocab().keys()			)

						self.assertEqual(vocab_keys[0]		,'''[PAD]'''			)
						self.assertEqual(vocab_keys[1]		,'''[CLS]'''			)
						self.assertEqual(vocab_keys[-1]		,'''j'''			)
						self.assertEqual(len(__lowerCamelCase			)		,10_12			)



		def 		lowerCAmelCase__				(self			)					->      Union[str, Any]:

						"""simple docstring"""






						self.assertEqual(self.get_tokenizer().vocab_size		,10_12			)



		def 		lowerCAmelCase__				(self			)					->      Tuple:

						"""simple docstring"""






						lowerCAmelCase__					:    Dict												=					XLMProphetNetTokenizer(__lowerCamelCase		,keep_accents=__lowerCamelCase			)

						lowerCAmelCase__					:    Tuple												=					tokenizer.tokenize('''This is a test'''			)
						self.assertListEqual(__lowerCamelCase		,['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']			)

						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(__lowerCamelCase			)		,[value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]		,)

						lowerCAmelCase__					:    Dict												=					tokenizer.tokenize('''I was born in 92000, and this is falsé.'''			)
						self.assertListEqual(
						    __lowerCamelCase		,[
						        SPIECE_UNDERLINE + '''I''',
						        SPIECE_UNDERLINE + '''was''',
						        SPIECE_UNDERLINE + '''b''',
						        '''or''',
						        '''n''',
						        SPIECE_UNDERLINE + '''in''',
						        SPIECE_UNDERLINE + '''''',
						        '''9''',
						        '''2''',
						        '''0''',
						        '''0''',
						        '''0''',
						        ''',''',
						        SPIECE_UNDERLINE + '''and''',
						        SPIECE_UNDERLINE + '''this''',
						        SPIECE_UNDERLINE + '''is''',
						        SPIECE_UNDERLINE + '''f''',
						        '''al''',
						        '''s''',
						        '''é''',
						        '''.''',
						    ]		,)
						lowerCAmelCase__					:    Optional[Any]												=					tokenizer.convert_tokens_to_ids(__lowerCamelCase			)
						self.assertListEqual(
						    __lowerCamelCase		,[
						        value + tokenizer.fairseq_offset
						        for value in [8, 21, 84, 55, 24, 19, 7, -9, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, -9, 4]
						    ]		,)

						lowerCAmelCase__					:    Optional[Any]												=					tokenizer.convert_ids_to_tokens(__lowerCamelCase			)
						self.assertListEqual(
						    __lowerCamelCase		,[
						        SPIECE_UNDERLINE + '''I''',
						        SPIECE_UNDERLINE + '''was''',
						        SPIECE_UNDERLINE + '''b''',
						        '''or''',
						        '''n''',
						        SPIECE_UNDERLINE + '''in''',
						        SPIECE_UNDERLINE + '''''',
						        '''[UNK]''',
						        '''2''',
						        '''0''',
						        '''0''',
						        '''0''',
						        ''',''',
						        SPIECE_UNDERLINE + '''and''',
						        SPIECE_UNDERLINE + '''this''',
						        SPIECE_UNDERLINE + '''is''',
						        SPIECE_UNDERLINE + '''f''',
						        '''al''',
						        '''s''',
						        '''[UNK]''',
						        '''.''',
						    ]		,)



		@cached_property
		def 		lowerCAmelCase__				(self			)					->      List[str]:

						"""simple docstring"""






						return XLMProphetNetTokenizer.from_pretrained('''microsoft/xprophetnet-large-wiki100-cased'''			)



		@slow
		def 		lowerCAmelCase__				(self			)					->      Tuple:

						"""simple docstring"""






						lowerCAmelCase__					:    Optional[int]												=					'''Hello World!'''
						lowerCAmelCase__					:    str												=					[3_53_89, 66_72, 49, 2]
						self.assertListEqual(__lowerCamelCase		,self.big_tokenizer.encode(__lowerCamelCase			)			)



		@slow
		def 		lowerCAmelCase__				(self			)					->      List[str]:

						"""simple docstring"""






						lowerCAmelCase__					:    Any												=					{'''input_ids''': [[1_10_73, 8_27_83, 18, 26, 8_27_83, 5_49, 5_15_40, 2_48, 1_72_09, 13_01, 2_17, 20, 21_51_86, 13_25, 1_47, 1_72_09, 13_01, 2_17, 20, 5_63_70, 53, 12_20_20, 20, 1_64_77, 27, 8_73_55, 45_48, 20, 47_28, 7_83_92, 17, 15_99_69, 18, 26, 2_44_91, 6_29, 15, 5_38, 2_27_04, 54_39, 15, 27_88, 2_44_91, 98_85, 15, 4_35_34, 6_05, 15, 8_14, 1_84_03, 3_32_00, 29, 15, 4_35_34, 2_44_58, 1_24_10, 1_11, 2_49_66, 8_36_69, 96_37, 14_40_68, 26, 8_50, 2_23_46, 27, 1_47, 2_49_66, 8_36_69, 8_34_90, 26, 3_91_13, 7_35, 27, 6_89, 6_56, 28_00, 13_39, 46_00, 53, 12_20_20, 11_57_85, 34, 8_16, 13_39, 4_68_87, 18, 1_47, 5_39_05, 19_51, 4_22_38, 4_11_70, 1_77_32, 8_34, 4_36, 15, 2_75_23, 9_87_33, 2_17, 1_47, 55_42, 49_81, 9_30, 1_73_47, 16, 2], [2_00_91, 6_29, 94, 8_27_86, 58, 4_90, 20, 15_28, 84, 5_39_05, 3_44, 8_05_92, 11_01_28, 1_88_22, 52_67, 13_06, 62, 15_25_37, 3_08, 79_97, 4_01, 12_44_27, 5_49, 3_54_42, 2_25, 1_09, 1_50_55, 2_57_48, 1_47, 71_19, 4_37_12, 34, 7_67, 13_53_66, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_92, 6_37_84, 11_94_66, 17, 14_78_08, 8_82_14, 18, 6_56, 81, 32, 32_96, 1_02_80, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
						# fmt: on

						self.tokenizer_integration_test_util(
						    expected_encoding=__lowerCamelCase		,model_name='''microsoft/xprophetnet-large-wiki100-cased'''		,revision='''1acad1643ddd54a44df6a1b797ada8373685d90e'''		,)

