

import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config





@pytest.fixture(scope="""session"""					)
def      snake_case_				()      ->     str:
   __lowerCAmelCase  :      Tuple    =						1_0
   __lowerCAmelCase  :      Optional[int]    =						datasets.Features(
       {
           """tokens""": datasets.Sequence(datasets.Value("""string"""					)					),
           """labels""": datasets.Sequence(datasets.ClassLabel(names=["""negative""", """positive"""]					)					),
           """answers""": datasets.Sequence(
               {
                   """text""": datasets.Value("""string"""					),
                   """answer_start""": datasets.Value("""int32"""					),
               }					),
           """id""": datasets.Value("""int64"""					),
       }					)
   __lowerCAmelCase  :      List[Any]    =						datasets.Dataset.from_dict(
       {
           """tokens""": [["""foo"""] * 5] * n,
           """labels""": [[1] * 5] * n,
           """answers""": [{"""answer_start""": [9_7], """text""": ["""1976"""]}] * 1_0,
           """id""": list(range(_a					)					),
       } ,       features=_a ,       )
   return dataset





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						Optional[Any] ,       __A					:						List[str]					)      ->     List[Any]:
   __lowerCAmelCase  :      Union[str, Any]    =						str(tmp_path_factory.mktemp("""data"""					) / """file.arrow"""					)
   dataset.map(cache_file_name=_a					)
   return filename


# FILE_CONTENT + files


__UpperCAmelCase								=			"""\
    Text data.
    Second line of data."""





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						Union[str, Any]					)      ->     Any:
   __lowerCAmelCase  :      List[Any]    =						tmp_path_factory.mktemp("""data"""					) / """file.txt"""
   __lowerCAmelCase  :      List[str]    =						FILE_CONTENT
   with open(_a ,       """w"""					) as f:
      f.write(_a					)
   return filename





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						List[Any]					)      ->     List[str]:
   import bza

   __lowerCAmelCase  :      Union[str, Any]    =						tmp_path_factory.mktemp("""data"""					) / """file.txt.bz2"""
   __lowerCAmelCase  :      List[str]    =						bytes(_a ,       """utf-8"""					)
   with bza.open(_a ,       """wb"""					) as f:
      f.write(_a					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						str					)      ->     Optional[Any]:
   import gzip

   __lowerCAmelCase  :      Any    =						str(tmp_path_factory.mktemp("""data"""					) / """file.txt.gz"""					)
   __lowerCAmelCase  :      Any    =						bytes(_a ,       """utf-8"""					)
   with gzip.open(_a ,       """wb"""					) as f:
      f.write(_a					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						Any					)      ->     Union[str, Any]:




   if datasets.config.LZ4_AVAILABLE:
      import lza.frame

      __lowerCAmelCase  :      Optional[Any]    =						tmp_path_factory.mktemp("""data"""					) / """file.txt.lz4"""
      __lowerCAmelCase  :      Optional[int]    =						bytes(_a ,       """utf-8"""					)
      with lza.frame.open(_a ,       """wb"""					) as f:
         f.write(_a					)
      return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						int ,       __A					:						Union[str, Any]					)      ->     Optional[Any]:




   if datasets.config.PY7ZR_AVAILABLE:
      import pyazr

      __lowerCAmelCase  :      int    =						tmp_path_factory.mktemp("""data"""					) / """file.txt.7z"""
      with pyazr.SevenZipFile(_a ,       """w"""					) as archive:
         archive.write(_a ,       arcname=os.path.basename(_a					)					)
      return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						List[str] ,       __A					:						List[str]					)      ->     Optional[int]:
   import tarfile

   __lowerCAmelCase  :      Any    =						tmp_path_factory.mktemp("""data"""					) / """file.txt.tar"""
   with tarfile.TarFile(_a ,       """w"""					) as f:
      f.add(_a ,       arcname=os.path.basename(_a					)					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						Union[str, Any]					)      ->     Any:
   import lzma

   __lowerCAmelCase  :      Dict    =						tmp_path_factory.mktemp("""data"""					) / """file.txt.xz"""
   __lowerCAmelCase  :      Any    =						bytes(_a ,       """utf-8"""					)
   with lzma.open(_a ,       """wb"""					) as f:
      f.write(_a					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						Dict ,       __A					:						Tuple					)      ->     int:
   import zipfile

   __lowerCAmelCase  :      Tuple    =						tmp_path_factory.mktemp("""data"""					) / """file.txt.zip"""
   with zipfile.ZipFile(_a ,       """w"""					) as f:
      f.write(_a ,       arcname=os.path.basename(_a					)					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						Tuple					)      ->     Optional[Any]:




   if datasets.config.ZSTANDARD_AVAILABLE:
      import zstandard as zstd

      __lowerCAmelCase  :      int    =						tmp_path_factory.mktemp("""data"""					) / """file.txt.zst"""
      __lowerCAmelCase  :      Tuple    =						bytes(_a ,       """utf-8"""					)
      with zstd.open(_a ,       """wb"""					) as f:
         f.write(_a					)
      return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						Tuple					)      ->     Dict:
   __lowerCAmelCase  :      Optional[Any]    =						tmp_path_factory.mktemp("""data"""					) / """file.xml"""
   __lowerCAmelCase  :      List[str]    =						textwrap.dedent(
       """\
    <?xml version=\"1.0\" encoding=\"UTF-8\" ?>
    <tmx version=\"1.4\">
      <header segtype=\"sentence\" srclang=\"ca\" />
      <body>
        <tu>
          <tuv xml:lang=\"ca\"><seg>Contingut 1</seg></tuv>
          <tuv xml:lang=\"en\"><seg>Content 1</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang=\"ca\"><seg>Contingut 2</seg></tuv>
          <tuv xml:lang=\"en\"><seg>Content 2</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang=\"ca\"><seg>Contingut 3</seg></tuv>
          <tuv xml:lang=\"en\"><seg>Content 3</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang=\"ca\"><seg>Contingut 4</seg></tuv>
          <tuv xml:lang=\"en\"><seg>Content 4</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang=\"ca\"><seg>Contingut 5</seg></tuv>
          <tuv xml:lang=\"en\"><seg>Content 5</seg></tuv>
        </tu>
      </body>
    </tmx>"""					)
   with open(_a ,       """w"""					) as f:
      f.write(_a					)
   return filename


__UpperCAmelCase								=			[
    {"""col_1""": """0""", """col_2""": 0, """col_3""": 0.0},
    {"""col_1""": """1""", """col_2""": 1, """col_3""": 1.0},
    {"""col_1""": """2""", """col_2""": 2, """col_3""": 2.0},
    {"""col_1""": """3""", """col_2""": 3, """col_3""": 3.0},
]
__UpperCAmelCase								=			[
    {"""col_1""": """4""", """col_2""": 4, """col_3""": 4.0},
    {"""col_1""": """5""", """col_2""": 5, """col_3""": 5.0},
]
__UpperCAmelCase								=			{
    """col_1""": ["""0""", """1""", """2""", """3"""],
    """col_2""": [0, 1, 2, 3],
    """col_3""": [0.0, 1.0, 2.0, 3.0],
}

__UpperCAmelCase								=			[
    {"""col_3""": 0.0, """col_1""": """0""", """col_2""": 0},
    {"""col_3""": 1.0, """col_1""": """1""", """col_2""": 1},
]

__UpperCAmelCase								=			[
    {"""col_1""": """s0""", """col_2""": 0, """col_3""": 0.0},
    {"""col_1""": """s1""", """col_2""": 1, """col_3""": 1.0},
    {"""col_1""": """s2""", """col_2""": 2, """col_3""": 2.0},
    {"""col_1""": """s3""", """col_2""": 3, """col_3""": 3.0},
]





@pytest.fixture(scope="""session"""					)
def      snake_case_				()      ->     Optional[int]:
   return DATA_DICT_OF_LISTS





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						Tuple					)      ->     List[str]:
   __lowerCAmelCase  :      Dict    =						datasets.Dataset.from_dict(_a					)
   __lowerCAmelCase  :      Any    =						str(tmp_path_factory.mktemp("""data"""					) / """dataset.arrow"""					)
   dataset.map(cache_file_name=_a					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						Tuple					)      ->     Optional[int]:
   __lowerCAmelCase  :      str    =						str(tmp_path_factory.mktemp("""data"""					) / """dataset.sqlite"""					)
   with contextlib.closing(sqlitea.connect(_a					)					) as con:
      __lowerCAmelCase  :      Union[str, Any]    =						con.cursor()
      cur.execute("""CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)"""					)
      for item in DATA:
         cur.execute("""INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)""" ,       tuple(item.values()					)					)
      con.commit()
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						Dict					)      ->     List[Any]:
   __lowerCAmelCase  :      List[str]    =						str(tmp_path_factory.mktemp("""data"""					) / """dataset.csv"""					)
   with open(_a ,       """w""" ,       newline=""""""					) as f:
      __lowerCAmelCase  :      List[str]    =						csv.DictWriter(_a ,       fieldnames=["""col_1""", """col_2""", """col_3"""]					)
      writer.writeheader()
      for item in DATA:
         writer.writerow(_a					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						List[str]					)      ->     Union[str, Any]:
   __lowerCAmelCase  :      int    =						str(tmp_path_factory.mktemp("""data"""					) / """dataset2.csv"""					)
   with open(_a ,       """w""" ,       newline=""""""					) as f:
      __lowerCAmelCase  :      str    =						csv.DictWriter(_a ,       fieldnames=["""col_1""", """col_2""", """col_3"""]					)
      writer.writeheader()
      for item in DATA:
         writer.writerow(_a					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						Optional[int] ,       __A					:						List[str]					)      ->     int:
   import bza

   __lowerCAmelCase  :      Dict    =						tmp_path_factory.mktemp("""data"""					) / """dataset.csv.bz2"""
   with open(_a ,       """rb"""					) as f:
      __lowerCAmelCase  :      Tuple    =						f.read()
   # data = bytes(FILE_CONTENT, "utf-8")
   with bza.open(_a ,       """wb"""					) as f:
      f.write(_a					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						Tuple ,       __A					:						Dict ,       __A					:						int					)      ->     Dict:
   __lowerCAmelCase  :      Optional[int]    =						tmp_path_factory.mktemp("""data"""					) / """dataset.csv.zip"""
   with zipfile.ZipFile(_a ,       """w"""					) as f:
      f.write(_a ,       arcname=os.path.basename(_a					)					)
      f.write(_a ,       arcname=os.path.basename(_a					)					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						Tuple ,       __A					:						Any ,       __A					:						Tuple					)      ->     Dict:
   __lowerCAmelCase  :      Optional[int]    =						tmp_path_factory.mktemp("""data"""					) / """dataset.csv.zip"""
   with zipfile.ZipFile(_a ,       """w"""					) as f:
      f.write(_a ,       arcname=os.path.basename(csv_path.replace(""".csv""" ,       """.CSV"""					)					)					)
      f.write(_a ,       arcname=os.path.basename(csva_path.replace(""".csv""" ,       """.CSV"""					)					)					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						str ,       __A					:						Dict ,       __A					:						str					)      ->     List[Any]:
   __lowerCAmelCase  :      List[Any]    =						tmp_path_factory.mktemp("""data"""					) / """dataset_with_dir.csv.zip"""
   with zipfile.ZipFile(_a ,       """w"""					) as f:
      f.write(_a ,       arcname=os.path.join("""main_dir""" ,       os.path.basename(_a					)					)					)
      f.write(_a ,       arcname=os.path.join("""main_dir""" ,       os.path.basename(_a					)					)					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						str					)      ->     Union[str, Any]:
   __lowerCAmelCase  :      int    =						str(tmp_path_factory.mktemp("""data"""					) / """dataset.parquet"""					)
   __lowerCAmelCase  :      int    =						pa.schema(
       {
           """col_1""": pa.string(),
           """col_2""": pa.intaa(),
           """col_3""": pa.floataa(),
       }					)
   with open(_a ,       """wb"""					) as f:
      __lowerCAmelCase  :      Union[str, Any]    =						pq.ParquetWriter(_a ,       schema=_a					)
      __lowerCAmelCase  :      Any    =						pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(_a					)					)] for k in DATA[0]} ,       schema=_a					)
      writer.write_table(_a					)
      writer.close()
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						Tuple					)      ->     Dict:
   __lowerCAmelCase  :      Optional[int]    =						str(tmp_path_factory.mktemp("""data"""					) / """dataset.json"""					)
   __lowerCAmelCase  :      Optional[Any]    =						{"""data""": DATA}
   with open(_a ,       """w"""					) as f:
      json.dump(_a ,       _a					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						Optional[int]					)      ->     Union[str, Any]:
   __lowerCAmelCase  :      List[str]    =						str(tmp_path_factory.mktemp("""data"""					) / """dataset.json"""					)
   __lowerCAmelCase  :      Dict    =						{"""data""": DATA_DICT_OF_LISTS}
   with open(_a ,       """w"""					) as f:
      json.dump(_a ,       _a					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						Union[str, Any]					)      ->     Optional[int]:
   __lowerCAmelCase  :      Dict    =						str(tmp_path_factory.mktemp("""data"""					) / """dataset.jsonl"""					)
   with open(_a ,       """w"""					) as f:
      for item in DATA:
         f.write(json.dumps(_a					) + """\n"""					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						int					)      ->     Tuple:
   __lowerCAmelCase  :      Tuple    =						str(tmp_path_factory.mktemp("""data"""					) / """dataset2.jsonl"""					)
   with open(_a ,       """w"""					) as f:
      for item in DATA:
         f.write(json.dumps(_a					) + """\n"""					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						Any					)      ->     Optional[Any]:
   __lowerCAmelCase  :      Union[str, Any]    =						str(tmp_path_factory.mktemp("""data"""					) / """dataset_312.jsonl"""					)
   with open(_a ,       """w"""					) as f:
      for item in DATA_312:
         f.write(json.dumps(_a					) + """\n"""					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						List[str]					)      ->     int:
   __lowerCAmelCase  :      List[Any]    =						str(tmp_path_factory.mktemp("""data"""					) / """dataset-str.jsonl"""					)
   with open(_a ,       """w"""					) as f:
      for item in DATA_STR:
         f.write(json.dumps(_a					) + """\n"""					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						Optional[int] ,       __A					:						List[str]					)      ->     Dict:
   import gzip

   __lowerCAmelCase  :      Tuple    =						str(tmp_path_factory.mktemp("""data"""					) / """dataset.txt.gz"""					)
   with open(_a ,       """rb"""					) as orig_file:
      with gzip.open(_a ,       """wb"""					) as zipped_file:
         zipped_file.writelines(_a					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						Dict ,       __A					:						List[Any]					)      ->     int:
   import gzip

   __lowerCAmelCase  :      List[Any]    =						str(tmp_path_factory.mktemp("""data"""					) / """dataset.jsonl.gz"""					)
   with open(_a ,       """rb"""					) as orig_file:
      with gzip.open(_a ,       """wb"""					) as zipped_file:
         zipped_file.writelines(_a					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						List[Any] ,       __A					:						List[str] ,       __A					:						Optional[Any]					)      ->     Optional[int]:
   __lowerCAmelCase  :      Tuple    =						tmp_path_factory.mktemp("""data"""					) / """dataset.jsonl.zip"""
   with zipfile.ZipFile(_a ,       """w"""					) as f:
      f.write(_a ,       arcname=os.path.basename(_a					)					)
      f.write(_a ,       arcname=os.path.basename(_a					)					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						Union[str, Any] ,       __A					:						List[Any] ,       __A					:						List[Any] ,       __A					:						Any					)      ->     Optional[int]:
   __lowerCAmelCase  :      Any    =						tmp_path_factory.mktemp("""data"""					) / """dataset_nested.jsonl.zip"""
   with zipfile.ZipFile(_a ,       """w"""					) as f:
      f.write(_a ,       arcname=os.path.join("""nested""" ,       os.path.basename(_a					)					)					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						int ,       __A					:						Optional[Any] ,       __A					:						Any					)      ->     int:
   __lowerCAmelCase  :      Optional[int]    =						tmp_path_factory.mktemp("""data"""					) / """dataset_with_dir.jsonl.zip"""
   with zipfile.ZipFile(_a ,       """w"""					) as f:
      f.write(_a ,       arcname=os.path.join("""main_dir""" ,       os.path.basename(_a					)					)					)
      f.write(_a ,       arcname=os.path.join("""main_dir""" ,       os.path.basename(_a					)					)					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						Optional[Any] ,       __A					:						Any ,       __A					:						Union[str, Any]					)      ->     List[Any]:
   __lowerCAmelCase  :      Tuple    =						tmp_path_factory.mktemp("""data"""					) / """dataset.jsonl.tar"""
   with tarfile.TarFile(_a ,       """w"""					) as f:
      f.add(_a ,       arcname=os.path.basename(_a					)					)
      f.add(_a ,       arcname=os.path.basename(_a					)					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						Optional[int] ,       __A					:						List[Any] ,       __A					:						List[Any] ,       __A					:						str					)      ->     Optional[int]:
   __lowerCAmelCase  :      Dict    =						tmp_path_factory.mktemp("""data"""					) / """dataset_nested.jsonl.tar"""
   with tarfile.TarFile(_a ,       """w"""					) as f:
      f.add(_a ,       arcname=os.path.join("""nested""" ,       os.path.basename(_a					)					)					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						Optional[int]					)      ->     Any:
   __lowerCAmelCase  :      Union[str, Any]    =						["""0""", """1""", """2""", """3"""]
   __lowerCAmelCase  :      Optional[Any]    =						str(tmp_path_factory.mktemp("""data"""					) / """dataset.txt"""					)
   with open(_a ,       """w"""					) as f:
      for item in data:
         f.write(item + """\n"""					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						Any					)      ->     Any:
   __lowerCAmelCase  :      Union[str, Any]    =						["""0""", """1""", """2""", """3"""]
   __lowerCAmelCase  :      List[Any]    =						str(tmp_path_factory.mktemp("""data"""					) / """dataset2.txt"""					)
   with open(_a ,       """w"""					) as f:
      for item in data:
         f.write(item + """\n"""					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						Dict					)      ->     List[str]:
   __lowerCAmelCase  :      List[str]    =						["""0""", """1""", """2""", """3"""]
   __lowerCAmelCase  :      List[str]    =						tmp_path_factory.mktemp("""data"""					) / """dataset.abc"""
   with open(_a ,       """w"""					) as f:
      for item in data:
         f.write(item + """\n"""					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						Any ,       __A					:						Dict ,       __A					:						Union[str, Any]					)      ->     List[Any]:
   __lowerCAmelCase  :      Tuple    =						tmp_path_factory.mktemp("""data"""					) / """dataset.text.zip"""
   with zipfile.ZipFile(_a ,       """w"""					) as f:
      f.write(_a ,       arcname=os.path.basename(_a					)					)
      f.write(_a ,       arcname=os.path.basename(_a					)					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						str ,       __A					:						str ,       __A					:						Optional[Any]					)      ->     Any:
   __lowerCAmelCase  :      str    =						tmp_path_factory.mktemp("""data"""					) / """dataset_with_dir.text.zip"""
   with zipfile.ZipFile(_a ,       """w"""					) as f:
      f.write(_a ,       arcname=os.path.join("""main_dir""" ,       os.path.basename(_a					)					)					)
      f.write(_a ,       arcname=os.path.join("""main_dir""" ,       os.path.basename(_a					)					)					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						Optional[Any] ,       __A					:						Optional[int] ,       __A					:						Union[str, Any]					)      ->     Any:
   __lowerCAmelCase  :      Union[str, Any]    =						tmp_path_factory.mktemp("""data"""					) / """dataset.ext.zip"""
   with zipfile.ZipFile(_a ,       """w"""					) as f:
      f.write(_a ,       arcname=os.path.basename("""unsupported.ext"""					)					)
      f.write(_a ,       arcname=os.path.basename("""unsupported_2.ext"""					)					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						Optional[Any]					)      ->     Tuple:
   __lowerCAmelCase  :      Dict    =						"""\n""".join(["""First""", """Second\u2029with Unicode new line""", """Third"""]					)
   __lowerCAmelCase  :      str    =						str(tmp_path_factory.mktemp("""data"""					) / """dataset_with_unicode_new_lines.txt"""					)
   with open(_a ,       """w""" ,       encoding="""utf-8"""					) as f:
      f.write(_a					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				()      ->     Dict:
   return os.path.join("""tests""" ,       """features""" ,       """data""" ,       """test_image_rgb.jpg"""					)





@pytest.fixture(scope="""session"""					)
def      snake_case_				()      ->     str:
   return os.path.join("""tests""" ,       """features""" ,       """data""" ,       """test_audio_44100.wav"""					)





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						Optional[Any] ,       __A					:						Tuple					)      ->     Dict:
   __lowerCAmelCase  :      List[str]    =						tmp_path_factory.mktemp("""data"""					) / """dataset.img.zip"""
   with zipfile.ZipFile(_a ,       """w"""					) as f:
      f.write(_a ,       arcname=os.path.basename(_a					)					)
      f.write(_a ,       arcname=os.path.basename(_a					).replace(""".jpg""" ,       """2.jpg"""					)					)
   return path





@pytest.fixture(scope="""session"""					)
def      snake_case_				(__A					:						Dict					)      ->     Optional[int]:
   __lowerCAmelCase  :      str    =						tmp_path_factory.mktemp("""data_dir"""					)

   (data_dir / "subdir").mkdir()
   with open(data_dir / """subdir""" / """train.txt""" ,       """w"""					) as f:
      f.write("""foo\n""" * 1_0					)
   with open(data_dir / """subdir""" / """test.txt""" ,       """w"""					) as f:
      f.write("""bar\n""" * 1_0					)
   # hidden file
   with open(data_dir / """subdir""" / """.test.txt""" ,       """w"""					) as f:
      f.write("""bar\n""" * 1_0					)

   # hidden directory
   (data_dir / ".subdir").mkdir()
   with open(data_dir / """.subdir""" / """train.txt""" ,       """w"""					) as f:
      f.write("""foo\n""" * 1_0					)
   with open(data_dir / """.subdir""" / """test.txt""" ,       """w"""					) as f:
      f.write("""bar\n""" * 1_0					)

   return data_dir





from __future__ import annotations

import unittest

from transformers import AutoTokenizer, MBartConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
   import tensorflow as tf

   from transformers import TFAutoModelForSeqaSeqLM, TFMBartForConditionalGeneration, TFMBartModel



@require_tf
class SCREAMING_SNAKE_CASE     :
  """simple docstring"""






  lowerCamelCase   :      Union[str, Any]										=MBartConfig
  lowerCamelCase   :      Optional[Any]										={}
  lowerCamelCase   :      Dict										="gelu"







  def __init__(      self			:   str	,							lowerCAmelCase			:   Any	,							lowerCAmelCase			:   List[Any]=13	,							lowerCAmelCase			:   List[str]=7	,							lowerCAmelCase			:   List[str]=True	,							lowerCAmelCase			:   Optional[Any]=False	,							lowerCAmelCase			:   Union[str, Any]=99	,							lowerCAmelCase			:   List[str]=32	,							lowerCAmelCase			:   List[Any]=2	,							lowerCAmelCase			:   Tuple=4	,							lowerCAmelCase			:   Any=37	,							lowerCAmelCase			:   Optional[int]=0.1	,							lowerCAmelCase			:   List[Any]=0.1	,							lowerCAmelCase			:   Dict=20	,							lowerCAmelCase			:   Any=2	,							lowerCAmelCase			:   Union[str, Any]=1	,							lowerCAmelCase			:   str=0	,							)							->     Any:






     """simple docstring"""







     __lowerCAmelCase  :      str    =						parent
     __lowerCAmelCase  :      int    =						batch_size
     __lowerCAmelCase  :      int    =						seq_length
     __lowerCAmelCase  :      Tuple    =						is_training
     __lowerCAmelCase  :      Optional[int]    =						use_labels
     __lowerCAmelCase  :      str    =						vocab_size
     __lowerCAmelCase  :      List[str]    =						hidden_size
     __lowerCAmelCase  :      Dict    =						num_hidden_layers
     __lowerCAmelCase  :      int    =						num_attention_heads
     __lowerCAmelCase  :      Any    =						intermediate_size
     __lowerCAmelCase  :      Dict    =						hidden_dropout_prob
     __lowerCAmelCase  :      List[Any]    =						attention_probs_dropout_prob
     __lowerCAmelCase  :      Tuple    =						max_position_embeddings
     __lowerCAmelCase  :      Union[str, Any]    =						eos_token_id
     __lowerCAmelCase  :      Optional[Any]    =						pad_token_id
     __lowerCAmelCase  :      int    =						bos_token_id







  def 	SCREAMING_SNAKE_CASE   (      self			:   List[Any]   )							->     List[Any]:






     """simple docstring"""







     __lowerCAmelCase  :      List[Any]    =						ids_tensor([self.batch_size, self.seq_length - 1]	,							self.vocab_size   )
     __lowerCAmelCase  :      Optional[Any]    =						tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size   )	,							1   )
     __lowerCAmelCase  :      int    =						tf.concat([input_ids, eos_tensor]	,							axis=1   )

     __lowerCAmelCase  :      Any    =						ids_tensor([self.batch_size, self.seq_length]	,							self.vocab_size   )

     __lowerCAmelCase  :      Dict    =						self.config_cls(
         vocab_size=self.vocab_size	,							d_model=self.hidden_size	,							encoder_layers=self.num_hidden_layers	,							decoder_layers=self.num_hidden_layers	,							encoder_attention_heads=self.num_attention_heads	,							decoder_attention_heads=self.num_attention_heads	,							encoder_ffn_dim=self.intermediate_size	,							decoder_ffn_dim=self.intermediate_size	,							dropout=self.hidden_dropout_prob	,							attention_dropout=self.attention_probs_dropout_prob	,							max_position_embeddings=self.max_position_embeddings	,							eos_token_ids=[2]	,							bos_token_id=self.bos_token_id	,							pad_token_id=self.pad_token_id	,							decoder_start_token_id=self.pad_token_id	,							**self.config_updates	,							)
     __lowerCAmelCase  :      Tuple    =						prepare_mbart_inputs_dict(lowerCAmelCase	,							lowerCAmelCase	,							lowerCAmelCase   )
     return config, inputs_dict







  def 	SCREAMING_SNAKE_CASE   (      self			:   List[str]	,							lowerCAmelCase			:   Any	,							lowerCAmelCase			:   str   )							->     Optional[Any]:






     """simple docstring"""







     __lowerCAmelCase  :      List[str]    =						TFMBartModel(config=lowerCAmelCase   ).get_decoder()
     __lowerCAmelCase  :      Tuple    =						inputs_dict["""input_ids"""]

     __lowerCAmelCase  :      Optional[Any]    =						input_ids[:1, :]
     __lowerCAmelCase  :      Union[str, Any]    =						inputs_dict["""attention_mask"""][:1, :]
     __lowerCAmelCase  :      Tuple    =						inputs_dict["""head_mask"""]
     __lowerCAmelCase  :      Any    =						1

     # first forward pass
     __lowerCAmelCase  :      List[Any]    =						model(lowerCAmelCase	,							attention_mask=lowerCAmelCase	,							head_mask=lowerCAmelCase	,							use_cache=lowerCAmelCase   )

     __lowerCAmelCase   ,__lowerCAmelCase  :      List[str]    =						outputs.to_tuple()
     __lowerCAmelCase  :      Union[str, Any]    =						past_key_values[1]






def      snake_case_				(__A					:						str ,       __A					:						Union[str, Any] ,       __A					:						Tuple ,       __A					:						Tuple=None ,       __A					:						Optional[Any]=None ,       __A					:						Optional[Any]=None ,       __A					:						Optional[int]=None ,       __A					:						Optional[Any]=None ,       )      ->     int:
   if attention_mask is None:
      __lowerCAmelCase  :      Dict    =						tf.cast(tf.math.not_equal(__A ,       config.pad_token_id					) ,       tf.inta					)
   if decoder_attention_mask is None:
      __lowerCAmelCase  :      str    =						tf.concat(
          [
              tf.ones(decoder_input_ids[:, :1].shape ,       dtype=tf.inta					),
              tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:] ,       config.pad_token_id					) ,       tf.inta					),
          ] ,       axis=-1 ,       )
   if head_mask is None:
      __lowerCAmelCase  :      Tuple    =						tf.ones((config.encoder_layers, config.encoder_attention_heads)					)
   if decoder_head_mask is None:
      __lowerCAmelCase  :      List[Any]    =						tf.ones((config.decoder_layers, config.decoder_attention_heads)					)
   if cross_attn_head_mask is None:
      __lowerCAmelCase  :      Any    =						tf.ones((config.decoder_layers, config.decoder_attention_heads)					)
   return {
       "input_ids": input_ids,
       "decoder_input_ids": decoder_input_ids,
       "attention_mask": attention_mask,
       "decoder_attention_mask": decoder_attention_mask,
       "head_mask": head_mask,
       "decoder_head_mask": decoder_head_mask,
       "cross_attn_head_mask": cross_attn_head_mask,
   }



@require_tf
class SCREAMING_SNAKE_CASE     (  a_       ,							a_       ,							unittest.TestCase			):
  """simple docstring"""






  lowerCamelCase   :      Optional[int]										=(TFMBartForConditionalGeneration, TFMBartModel) if is_tf_available() else ()
  lowerCamelCase   :      List[str]										=(TFMBartForConditionalGeneration,) if is_tf_available() else ()
  lowerCamelCase   :      Union[str, Any]										=(
      {
          "conversational": TFMBartForConditionalGeneration,
          "feature-extraction": TFMBartModel,
          "summarization": TFMBartForConditionalGeneration,
          "text2text-generation": TFMBartForConditionalGeneration,
          "translation": TFMBartForConditionalGeneration,
      }
      if is_tf_available()
      else {}
  )
  lowerCamelCase   :      str										=True
  lowerCamelCase   :      Tuple										=False
  lowerCamelCase   :      Dict										=False







  def 	SCREAMING_SNAKE_CASE   (      self			:   str	,							lowerCAmelCase			:   Optional[Any]	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   List[str]	,							lowerCAmelCase			:   Any	,							lowerCAmelCase			:   Optional[Any]   )							->     Optional[int]:






     """simple docstring"""







     if pipeline_test_casse_name != "FeatureExtractionPipelineTests":
        # Exception encountered when calling layer '...'
        return True

     return False







  def 	SCREAMING_SNAKE_CASE   (      self			:   Tuple   )							->     Optional[int]:






     """simple docstring"""







     __lowerCAmelCase  :      Union[str, Any]    =						TFMBartModelTester(self   )
     __lowerCAmelCase  :      Union[str, Any]    =						ConfigTester(self	,							config_class=lowerCAmelCase   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   List[str]   )							->     List[Any]:






     """simple docstring"""







     self.config_tester.run_common_tests()







  def 	SCREAMING_SNAKE_CASE   (      self			:   Tuple   )							->     List[str]:






     """simple docstring"""







     __lowerCAmelCase  :      List[Any]    =						self.model_tester.prepare_config_and_inputs_for_common()
     self.model_tester.check_decoder_model_past_large_inputs(*lowerCAmelCase   )






@require_sentencepiece
@require_tokenizers
@require_tf
class SCREAMING_SNAKE_CASE     (  unittest.TestCase			):
  """simple docstring"""






  lowerCamelCase   :      Union[str, Any]										=[
      " UN Chief Says There Is No Military Solution in Syria",
  ]
  lowerCamelCase   :      Tuple										=[
      "Şeful ONU declară că nu există o soluţie militară în Siria",
  ]
  lowerCamelCase   :      List[Any]										="facebook/mbart-large-en-ro"







  @cached_property
  def 	SCREAMING_SNAKE_CASE   (      self			:   List[str]   )							->     Any:






     """simple docstring"""







     return AutoTokenizer.from_pretrained(self.model_name   )







  @cached_property
  def 	SCREAMING_SNAKE_CASE   (      self			:   List[str]   )							->     Union[str, Any]:






     """simple docstring"""







     __lowerCAmelCase  :      int    =						TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name   )
     return model







  def 	SCREAMING_SNAKE_CASE   (      self			:   Optional[Any]	,							**lowerCAmelCase			:   Dict   )							->     Union[str, Any]:






     """simple docstring"""







     __lowerCAmelCase  :      Union[str, Any]    =						self.translate_src_text(**lowerCAmelCase   )
     self.assertListEqual(self.expected_text	,							lowerCAmelCase   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Union[str, Any]	,							**lowerCAmelCase			:   int   )							->     str:






     """simple docstring"""







     __lowerCAmelCase  :      Dict    =						self.tokenizer(self.src_text	,							**lowerCAmelCase	,							return_tensors="""tf"""   )
     __lowerCAmelCase  :      Union[str, Any]    =						self.model.generate(
         model_inputs.input_ids	,							attention_mask=model_inputs.attention_mask	,							num_beams=2   )
     __lowerCAmelCase  :      List[str]    =						self.tokenizer.batch_decode(lowerCAmelCase	,							skip_special_tokens=lowerCAmelCase   )
     return generated_words







  @slow
  def 	SCREAMING_SNAKE_CASE   (      self			:   Optional[int]   )							->     Any:






     """simple docstring"""







     self._assert_generated_batch_equal_expected()


