






"""simple docstring"""







import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config





@pytest.fixture(scope="session"						)
def 						lowercase__()    ->List[str]:




						"""simple docstring"""



						lowercase__    :				Optional[int]=	10
						lowercase__    :				List[str]=	datasets.Features(
						    {
						        "tokens": datasets.Sequence(datasets.Value("string"						)						),
						        "labels": datasets.Sequence(datasets.ClassLabel(names=["negative", "positive"]						)						),
						        "answers": datasets.Sequence(
						            {
						                "text": datasets.Value("string"						),
						                "answer_start": datasets.Value("int32"						),
						            }						),
						        "id": datasets.Value("int64"						),
						    }						)
						lowercase__    :				Any=	datasets.Dataset.from_dict(
						    {
						        "tokens": [["foo"] * 5] * n,
						        "labels": [[1] * 5] * n,
						        "answers": [{"answer_start": [97], "text": ["1976"]}] * 10,
						        "id": list(range(A						)						),
						    } ,		features=A ,		)
						return dataset





@pytest.fixture(scope="session"						)
def 						lowercase__(A ,		A						)    ->Any:




						"""simple docstring"""



						lowercase__    :				List[Any]=	str(tmp_path_factory.mktemp("data"						) / "file.arrow"						)
						dataset.map(cache_file_name=A						)
						return filename


# FILE_CONTENT + files


a			:    Optional[int]				 =		"""\
    Text data.
    Second line of data."""





@pytest.fixture(scope="session"						)
def 						lowercase__(A						)    ->Optional[int]:




						"""simple docstring"""



						lowercase__    :				Dict=	tmp_path_factory.mktemp("data"						) / "file.txt"
						lowercase__    :				Optional[Any]=	FILE_CONTENT
						with open(A ,		"w"						) as f:
												f.write(A						)
						return filename





@pytest.fixture(scope="session"						)
def 						lowercase__(A						)    ->Optional[int]:




						"""simple docstring"""



						import bza

						lowercase__    :				Dict=	tmp_path_factory.mktemp("data"						) / "file.txt.bz2"
						lowercase__    :				Tuple=	bytes(A ,		"utf-8"						)
						with bza.open(A ,		"wb"						) as f:
												f.write(A						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A						)    ->Optional[Any]:




						"""simple docstring"""



						import gzip

						lowercase__    :				Union[str, Any]=	str(tmp_path_factory.mktemp("data"						) / "file.txt.gz"						)
						lowercase__    :				Tuple=	bytes(A ,		"utf-8"						)
						with gzip.open(A ,		"wb"						) as f:
												f.write(A						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A						)    ->Union[str, Any]:




						"""simple docstring"""



						if datasets.config.LZ4_AVAILABLE:
												import lza.frame

												lowercase__    :				Any=	tmp_path_factory.mktemp("data"						) / "file.txt.lz4"
												lowercase__    :				List[str]=	bytes(A ,		"utf-8"						)
												with lza.frame.open(A ,		"wb"						) as f:
																		f.write(A						)
												return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A ,		A						)    ->str:




						"""simple docstring"""



						if datasets.config.PY7ZR_AVAILABLE:
												import pyazr

												lowercase__    :				List[Any]=	tmp_path_factory.mktemp("data"						) / "file.txt.7z"
												with pyazr.SevenZipFile(A ,		"w"						) as archive:
																		archive.write(A ,		arcname=os.path.basename(A						)						)
												return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A ,		A						)    ->List[Any]:




						"""simple docstring"""



						import tarfile

						lowercase__    :				Dict=	tmp_path_factory.mktemp("data"						) / "file.txt.tar"
						with tarfile.TarFile(A ,		"w"						) as f:
												f.add(A ,		arcname=os.path.basename(A						)						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A						)    ->Dict:




						"""simple docstring"""



						import lzma

						lowercase__    :				Optional[Any]=	tmp_path_factory.mktemp("data"						) / "file.txt.xz"
						lowercase__    :				List[str]=	bytes(A ,		"utf-8"						)
						with lzma.open(A ,		"wb"						) as f:
												f.write(A						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A ,		A						)    ->Optional[int]:




						"""simple docstring"""



						import zipfile

						lowercase__    :				str=	tmp_path_factory.mktemp("data"						) / "file.txt.zip"
						with zipfile.ZipFile(A ,		"w"						) as f:
												f.write(A ,		arcname=os.path.basename(A						)						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A						)    ->List[Any]:




						"""simple docstring"""



						if datasets.config.ZSTANDARD_AVAILABLE:
												import zstandard as zstd

												lowercase__    :				List[Any]=	tmp_path_factory.mktemp("data"						) / "file.txt.zst"
												lowercase__    :				str=	bytes(A ,		"utf-8"						)
												with zstd.open(A ,		"wb"						) as f:
																		f.write(A						)
												return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A						)    ->Union[str, Any]:




						"""simple docstring"""



						lowercase__    :				List[str]=	tmp_path_factory.mktemp("data"						) / "file.xml"
						lowercase__    :				List[Any]=	textwrap.dedent(
						    "\\n    <?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n    <tmx version=\"1.4\">\n      <header segtype=\"sentence\" srclang=\"ca\" />\n      <body>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 1</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 1</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 2</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 2</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 3</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 3</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 4</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 4</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 5</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 5</seg></tuv>\n        </tu>\n      </body>\n    </tmx>"						)
						with open(A ,		"w"						) as f:
												f.write(A						)
						return filename


a			:    int				 =		[
    {"""col_1""": """0""", """col_2""": 0, """col_3""": 0.0},
    {"""col_1""": """1""", """col_2""": 1, """col_3""": 1.0},
    {"""col_1""": """2""", """col_2""": 2, """col_3""": 2.0},
    {"""col_1""": """3""", """col_2""": 3, """col_3""": 3.0},
]
a			:    List[str]				 =		[
    {"""col_1""": """4""", """col_2""": 4, """col_3""": 4.0},
    {"""col_1""": """5""", """col_2""": 5, """col_3""": 5.0},
]
a			:    str				 =		{
    """col_1""": ["""0""", """1""", """2""", """3"""],
    """col_2""": [0, 1, 2, 3],
    """col_3""": [0.0, 1.0, 2.0, 3.0],
}

a			:    str				 =		[
    {"""col_3""": 0.0, """col_1""": """0""", """col_2""": 0},
    {"""col_3""": 1.0, """col_1""": """1""", """col_2""": 1},
]

a			:    str				 =		[
    {"""col_1""": """s0""", """col_2""": 0, """col_3""": 0.0},
    {"""col_1""": """s1""", """col_2""": 1, """col_3""": 1.0},
    {"""col_1""": """s2""", """col_2""": 2, """col_3""": 2.0},
    {"""col_1""": """s3""", """col_2""": 3, """col_3""": 3.0},
]





@pytest.fixture(scope="session"						)
def 						lowercase__()    ->List[str]:




						"""simple docstring"""



						return DATA_DICT_OF_LISTS





@pytest.fixture(scope="session"						)
def 						lowercase__(A						)    ->Optional[Any]:




						"""simple docstring"""



						lowercase__    :				Dict=	datasets.Dataset.from_dict(A						)
						lowercase__    :				List[Any]=	str(tmp_path_factory.mktemp("data"						) / "dataset.arrow"						)
						dataset.map(cache_file_name=A						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A						)    ->str:




						"""simple docstring"""



						lowercase__    :				List[Any]=	str(tmp_path_factory.mktemp("data"						) / "dataset.sqlite"						)
						with contextlib.closing(sqlitea.connect(A						)						) as con:
												lowercase__    :				List[Any]=	con.cursor()
												cur.execute("CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)"						)
												for item in DATA:
																		cur.execute("INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)" ,		tuple(item.values()						)						)
												con.commit()
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A						)    ->Dict:




						"""simple docstring"""



						lowercase__    :				Any=	str(tmp_path_factory.mktemp("data"						) / "dataset.csv"						)
						with open(A ,		"w" ,		newline=""						) as f:
												lowercase__    :				Optional[Any]=	csv.DictWriter(A ,		fieldnames=["col_1", "col_2", "col_3"]						)
												writer.writeheader()
												for item in DATA:
																		writer.writerow(A						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A						)    ->Optional[int]:




						"""simple docstring"""



						lowercase__    :				int=	str(tmp_path_factory.mktemp("data"						) / "dataset2.csv"						)
						with open(A ,		"w" ,		newline=""						) as f:
												lowercase__    :				Union[str, Any]=	csv.DictWriter(A ,		fieldnames=["col_1", "col_2", "col_3"]						)
												writer.writeheader()
												for item in DATA:
																		writer.writerow(A						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A ,		A						)    ->List[str]:




						"""simple docstring"""



						import bza

						lowercase__    :				Optional[int]=	tmp_path_factory.mktemp("data"						) / "dataset.csv.bz2"
						with open(A ,		"rb"						) as f:
												lowercase__    :				str=	f.read()
						# data = bytes(FILE_CONTENT, "utf-8")
						with bza.open(A ,		"wb"						) as f:
												f.write(A						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A ,		A ,		A						)    ->str:




						"""simple docstring"""



						lowercase__    :				int=	tmp_path_factory.mktemp("data"						) / "dataset.csv.zip"
						with zipfile.ZipFile(A ,		"w"						) as f:
												f.write(A ,		arcname=os.path.basename(A						)						)
												f.write(A ,		arcname=os.path.basename(A						)						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A ,		A ,		A						)    ->List[Any]:




						"""simple docstring"""



						lowercase__    :				List[Any]=	tmp_path_factory.mktemp("data"						) / "dataset.csv.zip"
						with zipfile.ZipFile(A ,		"w"						) as f:
												f.write(A ,		arcname=os.path.basename(csv_path.replace(".csv" ,		".CSV"						)						)						)
												f.write(A ,		arcname=os.path.basename(csva_path.replace(".csv" ,		".CSV"						)						)						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A ,		A ,		A						)    ->Any:




						"""simple docstring"""



						lowercase__    :				Optional[int]=	tmp_path_factory.mktemp("data"						) / "dataset_with_dir.csv.zip"
						with zipfile.ZipFile(A ,		"w"						) as f:
												f.write(A ,		arcname=os.path.join("main_dir" ,		os.path.basename(A						)						)						)
												f.write(A ,		arcname=os.path.join("main_dir" ,		os.path.basename(A						)						)						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A						)    ->List[str]:




						"""simple docstring"""



						lowercase__    :				Union[str, Any]=	str(tmp_path_factory.mktemp("data"						) / "dataset.parquet"						)
						lowercase__    :				Optional[int]=	pa.schema(
						    {
						        "col_1": pa.string(),
						        "col_2": pa.intaa(),
						        "col_3": pa.floataa(),
						    }						)
						with open(A ,		"wb"						) as f:
												lowercase__    :				List[Any]=	pq.ParquetWriter(A ,		schema=A						)
												lowercase__    :				Tuple=	pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(A						)						)] for k in DATA[0]} ,		schema=A						)
												writer.write_table(A						)
												writer.close()
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A						)    ->str:




						"""simple docstring"""



						lowercase__    :				List[str]=	str(tmp_path_factory.mktemp("data"						) / "dataset.json"						)
						lowercase__    :				List[str]=	{"data": DATA}
						with open(A ,		"w"						) as f:
												json.dump(A ,		A						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A						)    ->Dict:




						"""simple docstring"""



						lowercase__    :				Union[str, Any]=	str(tmp_path_factory.mktemp("data"						) / "dataset.json"						)
						lowercase__    :				Union[str, Any]=	{"data": DATA_DICT_OF_LISTS}
						with open(A ,		"w"						) as f:
												json.dump(A ,		A						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A						)    ->Optional[Any]:




						"""simple docstring"""



						lowercase__    :				int=	str(tmp_path_factory.mktemp("data"						) / "dataset.jsonl"						)
						with open(A ,		"w"						) as f:
												for item in DATA:
																		f.write(json.dumps(A						) + "\n"						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A						)    ->Optional[int]:




						"""simple docstring"""



						lowercase__    :				Union[str, Any]=	str(tmp_path_factory.mktemp("data"						) / "dataset2.jsonl"						)
						with open(A ,		"w"						) as f:
												for item in DATA:
																		f.write(json.dumps(A						) + "\n"						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A						)    ->List[str]:




						"""simple docstring"""



						lowercase__    :				Optional[Any]=	str(tmp_path_factory.mktemp("data"						) / "dataset_312.jsonl"						)
						with open(A ,		"w"						) as f:
												for item in DATA_312:
																		f.write(json.dumps(A						) + "\n"						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A						)    ->Dict:




						"""simple docstring"""



						lowercase__    :				List[Any]=	str(tmp_path_factory.mktemp("data"						) / "dataset-str.jsonl"						)
						with open(A ,		"w"						) as f:
												for item in DATA_STR:
																		f.write(json.dumps(A						) + "\n"						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A ,		A						)    ->Union[str, Any]:




						"""simple docstring"""



						import gzip

						lowercase__    :				Dict=	str(tmp_path_factory.mktemp("data"						) / "dataset.txt.gz"						)
						with open(A ,		"rb"						) as orig_file:
												with gzip.open(A ,		"wb"						) as zipped_file:
																		zipped_file.writelines(A						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A ,		A						)    ->int:




						"""simple docstring"""



						import gzip

						lowercase__    :				Union[str, Any]=	str(tmp_path_factory.mktemp("data"						) / "dataset.jsonl.gz"						)
						with open(A ,		"rb"						) as orig_file:
												with gzip.open(A ,		"wb"						) as zipped_file:
																		zipped_file.writelines(A						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A ,		A ,		A						)    ->Optional[int]:




						"""simple docstring"""



						lowercase__    :				Tuple=	tmp_path_factory.mktemp("data"						) / "dataset.jsonl.zip"
						with zipfile.ZipFile(A ,		"w"						) as f:
												f.write(A ,		arcname=os.path.basename(A						)						)
												f.write(A ,		arcname=os.path.basename(A						)						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A ,		A ,		A ,		A						)    ->Union[str, Any]:




						"""simple docstring"""



						lowercase__    :				Any=	tmp_path_factory.mktemp("data"						) / "dataset_nested.jsonl.zip"
						with zipfile.ZipFile(A ,		"w"						) as f:
												f.write(A ,		arcname=os.path.join("nested" ,		os.path.basename(A						)						)						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A ,		A ,		A						)    ->Union[str, Any]:




						"""simple docstring"""



						lowercase__    :				Tuple=	tmp_path_factory.mktemp("data"						) / "dataset_with_dir.jsonl.zip"
						with zipfile.ZipFile(A ,		"w"						) as f:
												f.write(A ,		arcname=os.path.join("main_dir" ,		os.path.basename(A						)						)						)
												f.write(A ,		arcname=os.path.join("main_dir" ,		os.path.basename(A						)						)						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A ,		A ,		A						)    ->Optional[Any]:




						"""simple docstring"""



						lowercase__    :				Union[str, Any]=	tmp_path_factory.mktemp("data"						) / "dataset.jsonl.tar"
						with tarfile.TarFile(A ,		"w"						) as f:
												f.add(A ,		arcname=os.path.basename(A						)						)
												f.add(A ,		arcname=os.path.basename(A						)						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A ,		A ,		A ,		A						)    ->Union[str, Any]:




						"""simple docstring"""



						lowercase__    :				str=	tmp_path_factory.mktemp("data"						) / "dataset_nested.jsonl.tar"
						with tarfile.TarFile(A ,		"w"						) as f:
												f.add(A ,		arcname=os.path.join("nested" ,		os.path.basename(A						)						)						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A						)    ->Any:




						"""simple docstring"""



						lowercase__    :				Tuple=	["0", "1", "2", "3"]
						lowercase__    :				Any=	str(tmp_path_factory.mktemp("data"						) / "dataset.txt"						)
						with open(A ,		"w"						) as f:
												for item in data:
																		f.write(item + "\n"						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A						)    ->Optional[int]:




						"""simple docstring"""



						lowercase__    :				int=	["0", "1", "2", "3"]
						lowercase__    :				Tuple=	str(tmp_path_factory.mktemp("data"						) / "dataset2.txt"						)
						with open(A ,		"w"						) as f:
												for item in data:
																		f.write(item + "\n"						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A						)    ->Optional[Any]:




						"""simple docstring"""



						lowercase__    :				Any=	["0", "1", "2", "3"]
						lowercase__    :				Union[str, Any]=	tmp_path_factory.mktemp("data"						) / "dataset.abc"
						with open(A ,		"w"						) as f:
												for item in data:
																		f.write(item + "\n"						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A ,		A ,		A						)    ->Union[str, Any]:




						"""simple docstring"""



						lowercase__    :				Any=	tmp_path_factory.mktemp("data"						) / "dataset.text.zip"
						with zipfile.ZipFile(A ,		"w"						) as f:
												f.write(A ,		arcname=os.path.basename(A						)						)
												f.write(A ,		arcname=os.path.basename(A						)						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A ,		A ,		A						)    ->Optional[Any]:




						"""simple docstring"""



						lowercase__    :				Union[str, Any]=	tmp_path_factory.mktemp("data"						) / "dataset_with_dir.text.zip"
						with zipfile.ZipFile(A ,		"w"						) as f:
												f.write(A ,		arcname=os.path.join("main_dir" ,		os.path.basename(A						)						)						)
												f.write(A ,		arcname=os.path.join("main_dir" ,		os.path.basename(A						)						)						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A ,		A ,		A						)    ->Any:




						"""simple docstring"""



						lowercase__    :				int=	tmp_path_factory.mktemp("data"						) / "dataset.ext.zip"
						with zipfile.ZipFile(A ,		"w"						) as f:
												f.write(A ,		arcname=os.path.basename("unsupported.ext"						)						)
												f.write(A ,		arcname=os.path.basename("unsupported_2.ext"						)						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__(A						)    ->int:




						"""simple docstring"""



						lowercase__    :				Optional[int]=	"\n".join(["First", "Second\u2029with Unicode new line", "Third"]						)
						lowercase__    :				Optional[Any]=	str(tmp_path_factory.mktemp("data"						) / "dataset_with_unicode_new_lines.txt"						)
						with open(A ,		"w" ,		encoding="utf-8"						) as f:
												f.write(A						)
						return path





@pytest.fixture(scope="session"						)
def 						lowercase__()    ->List[Any]:




						"""simple docstring"""



						return os.path.join("tests" ,		"features" ,		"data" ,		"test_image_rgb.jpg"						)





@pytest.fixture(scope="session"						)
def 						lowercase__()    ->List[Any]:




						"""simple docstring"""



						return os.path.join("tests" ,		"features" ,		"data" ,		"test_audio_44100.wav"						)





@pytest.fixture(scope="session"						)
def 						lowercase__(A ,		A						)    ->Any:




						"""simple docstring"""



						lowercase__    :				Optional[Any]=	tmp_path_factory.mktemp("data"						) / "dataset.img.zip"
						with zipfile.ZipFile(A ,		"w"						) as f:
												f.write(A ,		arcname=os.path.basename(A						)						)
												f.write(A ,		arcname=os.path.basename(A						).replace(".jpg" ,		"2.jpg"						)						)
						return path







@pytest.fixture(scope="session"						)
def 						lowercase__(A						)    ->Union[str, Any]:




						"""simple docstring"""



						lowercase__    :				str=	tmp_path_factory.mktemp("data_dir"						)

						(data_dir / "subdir").mkdir()
						with open(data_dir / "subdir" / "train.txt" ,		"w"						) as f:
												f.write("foo\n" * 10						)
						with open(data_dir / "subdir" / "test.txt" ,		"w"						) as f:
												f.write("bar\n" * 10						)
						# hidden file
						with open(data_dir / "subdir" / ".test.txt" ,		"w"						) as f:
												f.write("bar\n" * 10						)

						# hidden directory
						(data_dir / ".subdir").mkdir()
						with open(data_dir / ".subdir" / "train.txt" ,		"w"						) as f:
												f.write("foo\n" * 10						)
						with open(data_dir / ".subdir" / "test.txt" ,		"w"						) as f:
												f.write("bar\n" * 10						)

						return data_dir














"""simple docstring"""







import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, PerceiverTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
			a			:    Optional[Any]				 =		"""pt"""
elif is_tf_available():
			a			:    Union[str, Any]				 =		"""tf"""
else:
			a			:    Any				 =		"""jax"""


class    __UpperCAmelCase(       SCREAMING_SNAKE_CASE__  ,  unittest.TestCase							):




						"""simple docstring"""




						__lowerCamelCase    	= PerceiverTokenizer
						__lowerCamelCase    	= False




						def 		UpperCAmelCase_	(       self      ):
												'''simple docstring'''
												super().setUp()
												lowercase__    :				str=	PerceiverTokenizer()
												tokenizer.save_pretrained(self.tmpdirname      )




						@cached_property
						def 		UpperCAmelCase_	(       self      ):
												'''simple docstring'''
												return PerceiverTokenizer.from_pretrained("deepmind/language-perceiver"      )




						def 		UpperCAmelCase_	(       self	,  **snake_case__      ):
												'''simple docstring'''
												return self.tokenizer_class.from_pretrained(self.tmpdirname	,  **snake_case__      )




						def 		UpperCAmelCase_	(       self	,  snake_case__	,  snake_case__=False	,  snake_case__=20	,  snake_case__=5      ):
												'''simple docstring'''
												# XXX The default common tokenizer tests assume that every ID is decodable on its own.
												# This assumption is invalid for Perceiver because single bytes might not be
												# valid utf-8 (byte 128 for instance).
												# Here we're overriding the smallest possible method to provide
												# a clean sequence without making the same assumption.

												lowercase__    :				Union[str, Any]=	[]
												for i in range(len(snake_case__      )      ):
																		try:
																								lowercase__    :				Any=	tokenizer.decode([i]	,  clean_up_tokenization_spaces=snake_case__      )
																		except UnicodeDecodeError:
																								pass
																		toks.append((i, tok)      )

												lowercase__    :				int=	list(filter(lambda snake_case__      : re.match(r"^[ a-zA-Z]+$"	,  t[1]      )	,  snake_case__      )      )
												lowercase__    :				Union[str, Any]=	list(filter(lambda snake_case__      : [t[0]] == tokenizer.encode(t[1]	,  add_special_tokens=snake_case__      )	,  snake_case__      )      )
												if max_length is not None and len(snake_case__      ) > max_length:
																		lowercase__    :				int=	toks[:max_length]
												if min_length is not None and len(snake_case__      ) < min_length and len(snake_case__      ) > 0:
																		while len(snake_case__      ) < min_length:
																								lowercase__    :				List[str]=	toks + toks
        # toks_str = [t[1] for t in toks]
												lowercase__    :				str=	[t[0] for t in toks]

												# Ensure consistency
												lowercase__    :				Optional[Any]=	tokenizer.decode(snake_case__	,  clean_up_tokenization_spaces=snake_case__      )
												if " " not in output_txt and len(snake_case__      ) > 1:
																		lowercase__    :				Dict=	(
																		    tokenizer.decode([toks_ids[0]]	,  clean_up_tokenization_spaces=snake_case__      )
																		    + " "
																		    + tokenizer.decode(toks_ids[1:]	,  clean_up_tokenization_spaces=snake_case__      )
																		)
												if with_prefix_space:
																		lowercase__    :				List[Any]=	" " + output_txt
												lowercase__    :				Any=	tokenizer.encode(snake_case__	,  add_special_tokens=snake_case__      )
												return output_txt, output_ids




						def 		UpperCAmelCase_	(       self      ):
												'''simple docstring'''
												lowercase__    :				Optional[Any]=	self.perceiver_tokenizer
												lowercase__    :				Union[str, Any]=	"Unicode €."
												lowercase__    :				Tuple=	tokenizer(snake_case__      )
												lowercase__    :				Dict=	[4, 91, 116, 111, 105, 117, 106, 107, 38, 232, 136, 178, 52, 5]
												self.assertEqual(encoded["input_ids"]	,  snake_case__      )

												# decoding
												lowercase__    :				List[str]=	tokenizer.decode(snake_case__      )
												self.assertEqual(snake_case__	,  "[CLS]Unicode €.[SEP]"      )

												lowercase__    :				List[str]=	tokenizer("e è é ê ë"      )
												lowercase__    :				int=	[4, 107, 38, 201, 174, 38, 201, 175, 38, 201, 176, 38, 201, 177, 5]
												self.assertEqual(encoded["input_ids"]	,  snake_case__      )
												# decoding
												lowercase__    :				Any=	tokenizer.decode(snake_case__      )
												self.assertEqual(snake_case__	,  "[CLS]e è é ê ë[SEP]"      )

												# encode/decode, but with `encode` instead of `__call__`
												self.assertEqual(tokenizer.decode(tokenizer.encode("e è é ê ë"      )      )	,  "[CLS]e è é ê ë[SEP]"      )




						def 		UpperCAmelCase_	(       self      ):
												'''simple docstring'''
												lowercase__    :				str=	self.perceiver_tokenizer
												lowercase__    :				Tuple=	["A long paragraph for summarization.", "Another paragraph for summarization."]
												# fmt: off
												lowercase__    :				List[Any]=	[4, 71, 38, 114, 117, 116, 109, 38, 118, 103, 120, 103, 109, 120, 103, 118, 110, 38, 108, 117, 120, 38, 121, 123, 115, 115, 103, 120, 111, 128, 103, 122, 111, 117, 116, 52, 5, 0]
												# fmt: on
												lowercase__    :				Dict=	tokenizer(snake_case__	,  padding=snake_case__	,  return_tensors=snake_case__      )
												self.assertIsInstance(snake_case__	,  snake_case__      )

												if FRAMEWORK != "jax":
																		lowercase__    :				List[Any]=	list(batch.input_ids.numpy()[0]      )
												else:
																		lowercase__    :				str=	list(batch.input_ids.tolist()[0]      )

												self.assertListEqual(snake_case__	,  snake_case__      )

												self.assertEqual((2, 38)	,  batch.input_ids.shape      )
												self.assertEqual((2, 38)	,  batch.attention_mask.shape      )




						def 		UpperCAmelCase_	(       self      ):
												'''simple docstring'''
												lowercase__    :				Optional[Any]=	self.perceiver_tokenizer
												lowercase__    :				int=	["A long paragraph for summarization.", "Another paragraph for summarization."]
												lowercase__    :				Union[str, Any]=	tokenizer(snake_case__	,  padding=snake_case__	,  return_tensors=snake_case__      )
												# check if input_ids are returned and no decoder_input_ids
												self.assertIn("input_ids"	,  snake_case__      )
												self.assertIn("attention_mask"	,  snake_case__      )
												self.assertNotIn("decoder_input_ids"	,  snake_case__      )
												self.assertNotIn("decoder_attention_mask"	,  snake_case__      )




						def 		UpperCAmelCase_	(       self      ):
												'''simple docstring'''
												lowercase__    :				Tuple=	self.perceiver_tokenizer
												lowercase__    :				int=	[
												    "Summary of the text.",
												    "Another summary.",
												]
												lowercase__    :				List[Any]=	tokenizer(
												    text_target=snake_case__	,  max_length=32	,  padding="max_length"	,  truncation=snake_case__	,  return_tensors=snake_case__      )
												self.assertEqual(32	,  targets["input_ids"].shape[1]      )




						def 		UpperCAmelCase_	(       self      ):
												'''simple docstring'''
												# safety check on max_len default value so we are sure the test works
												lowercase__    :				Union[str, Any]=	self.get_tokenizers()
												for tokenizer in tokenizers:
																		with self.subTest(F'''{tokenizer.__class__.__name__}'''      ):
																								self.assertNotEqual(tokenizer.model_max_length	,  42      )

        # Now let's start the test
												lowercase__    :				List[str]=	self.get_tokenizers()
												for tokenizer in tokenizers:
																		with self.subTest(F'''{tokenizer.__class__.__name__}'''      ):
																								# Isolate this from the other tests because we save additional tokens/etc
																								lowercase__    :				str=	tempfile.mkdtemp()

																								lowercase__    :				str=	" He is very happy, UNwant\u00E9d,running"
																								lowercase__    :				int=	tokenizer.encode(snake_case__	,  add_special_tokens=snake_case__      )
																								tokenizer.save_pretrained(snake_case__      )

																								lowercase__    :				List[str]=	tokenizer.__class__.from_pretrained(snake_case__      )
																								lowercase__    :				Tuple=	after_tokenizer.encode(snake_case__	,  add_special_tokens=snake_case__      )
																								self.assertListEqual(snake_case__	,  snake_case__      )

																								shutil.rmtree(snake_case__      )

												lowercase__    :				List[str]=	self.get_tokenizers(model_max_length=42      )
												for tokenizer in tokenizers:
																		with self.subTest(F'''{tokenizer.__class__.__name__}'''      ):
																								# Isolate this from the other tests because we save additional tokens/etc
																								lowercase__    :				Tuple=	tempfile.mkdtemp()

																								lowercase__    :				List[Any]=	" He is very happy, UNwant\u00E9d,running"
																								tokenizer.add_tokens(["bim", "bambam"]      )
																								lowercase__    :				Tuple=	tokenizer.additional_special_tokens
																								additional_special_tokens.append("new_additional_special_token"      )
																								tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens}      )
																								lowercase__    :				List[str]=	tokenizer.encode(snake_case__	,  add_special_tokens=snake_case__      )
																								tokenizer.save_pretrained(snake_case__      )

																								lowercase__    :				Tuple=	tokenizer.__class__.from_pretrained(snake_case__      )
																								lowercase__    :				Tuple=	after_tokenizer.encode(snake_case__	,  add_special_tokens=snake_case__      )
																								self.assertListEqual(snake_case__	,  snake_case__      )
																								self.assertIn("new_additional_special_token"	,  after_tokenizer.additional_special_tokens      )
																								self.assertEqual(after_tokenizer.model_max_length	,  42      )

																								lowercase__    :				Tuple=	tokenizer.__class__.from_pretrained(snake_case__	,  model_max_length=43      )
																								self.assertEqual(tokenizer.model_max_length	,  43      )

																								shutil.rmtree(snake_case__      )




						def 		UpperCAmelCase_	(       self      ):
												'''simple docstring'''
												lowercase__    :				Union[str, Any]=	[]
												if self.test_slow_tokenizer:
																		tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())      )

												if self.test_rust_tokenizer:
																		tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())      )

												for tokenizer_class, tokenizer_utils in tokenizer_list:
																		with tempfile.TemporaryDirectory() as tmp_dir:
																								tokenizer_utils.save_pretrained(snake_case__      )

																								with open(os.path.join(snake_case__	,  "special_tokens_map.json"      )	,  encoding="utf-8"      ) as json_file:
																														lowercase__    :				Optional[Any]=	json.load(snake_case__      )

																								with open(os.path.join(snake_case__	,  "tokenizer_config.json"      )	,  encoding="utf-8"      ) as json_file:
																														lowercase__    :				Optional[Any]=	json.load(snake_case__      )

																								lowercase__    :				List[str]=	[F'''<extra_id_{i}>''' for i in range(125      )]

																								lowercase__    :				Optional[int]=	added_tokens_extra_ids + [
																								    "an_additional_special_token"
																								]
																								lowercase__    :				Union[str, Any]=	added_tokens_extra_ids + [
																								    "an_additional_special_token"
																								]

																								with open(os.path.join(snake_case__	,  "special_tokens_map.json"      )	,  "w"	,  encoding="utf-8"      ) as outfile:
																														json.dump(snake_case__	,  snake_case__      )
																								with open(os.path.join(snake_case__	,  "tokenizer_config.json"      )	,  "w"	,  encoding="utf-8"      ) as outfile:
																														json.dump(snake_case__	,  snake_case__      )

																								# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
																								# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
																								# "special_tokens_map.json" files
																								lowercase__    :				Optional[int]=	tokenizer_class.from_pretrained(
																								    snake_case__	,  )
																								self.assertIn(
																								    "an_additional_special_token"	,  tokenizer_without_change_in_init.additional_special_tokens      )
																								self.assertEqual(
																								    ["an_additional_special_token"]	,  tokenizer_without_change_in_init.convert_ids_to_tokens(
																								        tokenizer_without_change_in_init.convert_tokens_to_ids(["an_additional_special_token"]      )      )	,  )

																								# Now we test that we can change the value of additional_special_tokens in the from_pretrained
																								lowercase__    :				int=	added_tokens_extra_ids + [AddedToken("a_new_additional_special_token"	,  lstrip=snake_case__      )]
																								lowercase__    :				int=	tokenizer_class.from_pretrained(
																								    snake_case__	,  additional_special_tokens=snake_case__	,  )

																								self.assertIn("a_new_additional_special_token"	,  tokenizer.additional_special_tokens      )
																								self.assertEqual(
																								    ["a_new_additional_special_token"]	,  tokenizer.convert_ids_to_tokens(
																								        tokenizer.convert_tokens_to_ids(["a_new_additional_special_token"]      )      )	,  )




						def 		UpperCAmelCase_	(       self      ):
												'''simple docstring'''
												lowercase__    :				Union[str, Any]=	self.perceiver_tokenizer
												self.assertEqual(tokenizer.decode([178]      )	,  "�"      )




						def 		UpperCAmelCase_	(       self      ):
												'''simple docstring'''
												pass




						def 		UpperCAmelCase_	(       self      ):
												'''simple docstring'''
												pass




						def 		UpperCAmelCase_	(       self      ):
												'''simple docstring'''
												pass




						def 		UpperCAmelCase_	(       self      ):
												'''simple docstring'''
												pass




						def 		UpperCAmelCase_	(       self      ):
												'''simple docstring'''
												# The default common tokenizer tests uses invalid tokens for Perceiver that can only accept one-character
												# strings and special added tokens as tokens
												lowercase__    :				Optional[int]=	self.get_tokenizers(fast=snake_case__	,  do_lower_case=snake_case__      )
												for tokenizer in tokenizers:
																		with self.subTest(F'''{tokenizer.__class__.__name__}'''      ):
																								lowercase__    :				Optional[int]=	["[CLS]", "t", "h", "i", "s", " ", "i", "s", " ", "a", " ", "t", "e", "s", "t", "[SEP]"]
																								lowercase__    :				Optional[int]=	tokenizer.convert_tokens_to_string(snake_case__      )

																								self.assertIsInstance(snake_case__	,  snake_case__      )






