




import json
import os
import unittest

from transformers import DebertaTokenizer, DebertaTokenizerFast
from transformers.models.deberta.tokenization_deberta import VOCAB_FILES_NAMES
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin
class 						_A (  _lowerCamelCase      ,       unittest.TestCase				):
  _UpperCamelCase       :      List[str]			=      DebertaTokenizer
  _UpperCamelCase       :      int			=      True
  _UpperCamelCase       :      Dict			=      DebertaTokenizerFast






  def  __a      (  self		:     Tuple				)		->   List[Any]:






       """simple docstring"""







       super().setUp()

       # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
       lowercase :       List[Any]       =			[
           '''l''',
           '''o''',
           '''w''',
           '''e''',
           '''r''',
           '''s''',
           '''t''',
           '''i''',
           '''d''',
           '''n''',
           '''\u0120''',
           '''\u0120l''',
           '''\u0120n''',
           '''\u0120lo''',
           '''\u0120low''',
           '''er''',
           '''\u0120lowest''',
           '''\u0120newer''',
           '''\u0120wider''',
           '''[UNK]''',
       ]
       lowercase :       Dict       =			dict(zip(_A				,						range(len(_A				)				)				)				)
       lowercase :       int       =			['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
       lowercase :       Optional[Any]       =			{'''unk_token''': '''[UNK]'''}

       lowercase :       Any       =			os.path.join(self.tmpdirname				,						VOCAB_FILES_NAMES['''vocab_file''']				)
       lowercase :       str       =			os.path.join(self.tmpdirname				,						VOCAB_FILES_NAMES['''merges_file''']				)
       with open(self.vocab_file				,						'''w'''				,						encoding='''utf-8'''				) as fp:
            fp.write(json.dumps(_A				) + '''\n'''				)
       with open(self.merges_file				,						'''w'''				,						encoding='''utf-8'''				) as fp:
            fp.write('''\n'''.join(_A				)				)






  def  __a      (  self		:     Optional[int]				,						**_A		:     Tuple				)		->   Tuple:






       """simple docstring"""







       kwargs.update(self.special_tokens_map				)
       return self.tokenizer_class.from_pretrained(self.tmpdirname				,						**_A				)






  def  __a      (  self		:     List[Any]				,						_A		:     List[Any]				)		->   int:






       """simple docstring"""







       lowercase :       Tuple       =			'''lower newer'''
       lowercase :       List[Any]       =			'''lower newer'''
       return input_text, output_text






  def  __a      (  self		:     Tuple				)		->   Any:






       """simple docstring"""







       lowercase :       Dict       =			self.get_tokenizer()
       lowercase :       Optional[Any]       =			'''lower newer'''
       lowercase :       List[str]       =			['''l''', '''o''', '''w''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
       lowercase :       Union[str, Any]       =			tokenizer.tokenize(_A				)
       self.assertListEqual(_A				,						_A				)

       lowercase :       Any       =			tokens + [tokenizer.unk_token]
       lowercase :       int       =			[0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(_A				)				,						_A				)






  def  __a      (  self		:     int				)		->   List[Any]:






       """simple docstring"""







       lowercase :       Tuple       =			self.get_tokenizer()
       lowercase :       List[str]       =			tokenizer('''Hello'''				,						'''World'''				)
       lowercase :       Optional[int]       =			[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
       self.assertListEqual(tokd['''token_type_ids''']				,						_A				)






  @slow
  def  __a      (  self		:     Optional[int]				)		->   Union[str, Any]:






       """simple docstring"""







       lowercase :       Dict       =			self.tokenizer_class.from_pretrained('''microsoft/deberta-base'''				)

       lowercase :       int       =			tokenizer.encode('''sequence builders'''				,						add_special_tokens=_A				)
       lowercase :       int       =			tokenizer.encode('''multi-sequence build'''				,						add_special_tokens=_A				)

       lowercase :       Dict       =			tokenizer.encode(
           '''sequence builders'''				,						add_special_tokens=_A				,						add_prefix_space=_A				)
       lowercase :       str       =			tokenizer.encode(
           '''sequence builders'''				,						'''multi-sequence build'''				,						add_special_tokens=_A				,						add_prefix_space=_A				)

       lowercase :       Optional[Any]       =			tokenizer.build_inputs_with_special_tokens(_A				)
       lowercase :       int       =			tokenizer.build_inputs_with_special_tokens(_A				,						_A				)

       assert encoded_sentence == encoded_text_from_decode
       assert encoded_pair == encoded_pair_from_decode






  @slow
  def  __a      (  self		:     int				)		->   List[str]:






       """simple docstring"""







       lowercase :       Union[str, Any]       =			[self.tokenizer_class]
       if self.test_rust_tokenizer:
            tokenizer_classes.append(self.rust_tokenizer_class				)

       for tokenizer_class in tokenizer_classes:
            lowercase :       int       =			tokenizer_class.from_pretrained('''microsoft/deberta-base'''				)

            lowercase :       Optional[Any]       =			[
                '''ALBERT: A Lite BERT for Self-supervised Learning of Language Representations''',
                '''ALBERT incorporates two parameter reduction techniques''',
                '''The first one is a factorized embedding parameterization. By decomposing the large vocabulary'''
                ''' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'''
                ''' vocabulary embedding.''',
            ]

            lowercase :       Optional[int]       =			tokenizer(_A				,						padding=_A				)
            lowercase :       Any       =			[tokenizer.decode(_A				,						skip_special_tokens=_A				) for seq in encoding['''input_ids''']]

            # fmt: off
            lowercase :       Optional[int]       =			{
                '''input_ids''': [
                    [1, 2_118, 11_126, 565, 35, 83, 25_191, 163, 18_854, 13, 12_156, 12, 16_101, 25_376, 13_807, 9, 22_205, 27_893, 1_635, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                    [1, 2_118, 11_126, 565, 24_536, 80, 43_797, 4_878, 7_373, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                    [1, 133, 78, 65, 16, 10, 3_724, 1_538, 33_183, 11_303, 43_797, 1_938, 4, 870, 24_165, 29_105, 5, 739, 32_644, 33_183, 11_303, 36_173, 88, 80, 650, 7_821, 45_940, 6, 52, 2_559, 5, 1_836, 9, 5, 7_397, 13_171, 31, 5, 1_836, 9, 32_644, 33_183, 11_303, 4, 2]
                ],
                '''token_type_ids''': [
                    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
                ],
                '''attention_mask''': [
                    [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                    [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                    [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
                ]
            }
            # fmt: on

            lowercase :       List[str]       =			[
                '''ALBERT: A Lite BERT for Self-supervised Learning of Language Representations''',
                '''ALBERT incorporates two parameter reduction techniques''',
                '''The first one is a factorized embedding parameterization. By decomposing the large vocabulary'''
                ''' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'''
                ''' vocabulary embedding.''',
            ]

            self.assertDictEqual(encoding.data				,						_A				)

            for expected, decoded in zip(_A				,						_A				):
                 self.assertEqual(_A				,						_A				)





import os
from pathlib import Path
from unittest.mock import patch

import pytest
import zstandard as zstd

from datasets.download.download_config import DownloadConfig
from datasets.utils.file_utils import (
    OfflineModeIsEnabled,
    cached_path,
    fsspec_get,
    fsspec_head,
    ftp_get,
    ftp_head,
    get_from_cache,
    http_get,
    http_head,
)


lowerCAmelCase_						=			'\\n    Text data.\n    Second line of data.'

lowerCAmelCase_						=			'file'






@pytest.fixture(scope='''session'''   )
def  snake_case(    __magic_name__   )			->							List[str]:





     '''simple docstring'''


     lowercase :       Any       =			tmp_path_factory.mktemp('''data'''   ) / (FILE_PATH + '''.zstd''')
     lowercase :       List[Any]       =			bytes(__magic_name__			,		'''utf-8'''   )
     with zstd.open(__magic_name__			,		'''wb'''   ) as f:
          f.write(__magic_name__   )
     return path






@pytest.fixture
def  snake_case(    __magic_name__   )			->							List[Any]:





     '''simple docstring'''


     with open(os.path.join(tmpfs.local_root_dir			,		__magic_name__   )			,		'''w'''   ) as f:
          f.write(__magic_name__   )
     return FILE_PATH






@pytest.mark.parametrize('''compression_format'''			,		['''gzip''', '''xz''', '''zstd''']   )
def  snake_case(    __magic_name__			,		__magic_name__			,		__magic_name__			,		__magic_name__			,		__magic_name__			,		__magic_name__   )			->							List[str]:





     '''simple docstring'''


     lowercase :       Optional[int]       =			{'''gzip''': gz_file, '''xz''': xz_file, '''zstd''': zstd_path}
     lowercase :       int       =			input_paths[compression_format]
     lowercase :       Any       =			tmp_path / '''cache'''
     lowercase :       int       =			DownloadConfig(cache_dir=__magic_name__			,		extract_compressed_file=__magic_name__   )
     lowercase :       Optional[Any]       =			cached_path(__magic_name__			,		download_config=__magic_name__   )
     with open(__magic_name__   ) as f:
          lowercase :       Optional[int]       =			f.read()
     with open(__magic_name__   ) as f:
          lowercase :       str       =			f.read()
     assert extracted_file_content == expected_file_content






@pytest.mark.parametrize('''default_extracted'''			,		[True, False]   )
@pytest.mark.parametrize('''default_cache_dir'''			,		[True, False]   )
def  snake_case(    __magic_name__			,		__magic_name__			,		__magic_name__			,		__magic_name__			,		__magic_name__   )			->							List[str]:





     '''simple docstring'''


     lowercase :       Dict       =			'''custom_cache'''
     lowercase :       Union[str, Any]       =			'''custom_extracted_dir'''
     lowercase :       str       =			tmp_path / '''custom_extracted_path'''
     if default_extracted:
          lowercase :       Tuple       =			('''downloads''' if default_cache_dir else custom_cache_dir, '''extracted''')
     else:
          monkeypatch.setattr('''datasets.config.EXTRACTED_DATASETS_DIR'''			,		__magic_name__   )
          monkeypatch.setattr('''datasets.config.EXTRACTED_DATASETS_PATH'''			,		str(__magic_name__   )   )
          lowercase :       int       =			custom_extracted_path.parts[-2:] if default_cache_dir else (custom_cache_dir, custom_extracted_dir)

     lowercase :       List[str]       =			xz_file
     lowercase :       Any       =			(
         DownloadConfig(extract_compressed_file=__magic_name__   )
         if default_cache_dir
         else DownloadConfig(cache_dir=tmp_path / custom_cache_dir			,		extract_compressed_file=__magic_name__   )
     )
     lowercase :       Optional[int]       =			cached_path(__magic_name__			,		download_config=__magic_name__   )
     assert Path(__magic_name__   ).parent.parts[-2:] == expected






def  snake_case(    __magic_name__   )			->							List[Any]:





     '''simple docstring'''


     lowercase :       Any       =			str(Path(__magic_name__   ).resolve()   )
     assert cached_path(__magic_name__   ) == text_file
     # relative path
     lowercase :       Union[str, Any]       =			str(Path(__magic_name__   ).resolve().relative_to(Path(os.getcwd()   )   )   )
     assert cached_path(__magic_name__   ) == text_file






def  snake_case(    __magic_name__   )			->							Union[str, Any]:





     '''simple docstring'''


     lowercase :       List[Any]       =			str(tmp_path.resolve() / '''__missing_file__.txt'''   )
     with pytest.raises(__magic_name__   ):
          cached_path(__magic_name__   )
     # relative path
     lowercase :       Optional[int]       =			'''./__missing_file__.txt'''
     with pytest.raises(__magic_name__   ):
          cached_path(__magic_name__   )






def  snake_case(    __magic_name__   )			->							List[str]:





     '''simple docstring'''


     lowercase :       List[str]       =			get_from_cache(F"""tmp://{tmpfs_file}"""   )
     with open(__magic_name__   ) as f:
          lowercase :       List[Any]       =			f.read()
     assert output_file_content == FILE_CONTENT






@patch('''datasets.config.HF_DATASETS_OFFLINE'''			,		__magic_name__   )
def  snake_case(    )			->							List[Any]:





     '''simple docstring'''


     with pytest.raises(__magic_name__   ):
          cached_path('''https://huggingface.co'''   )






@patch('''datasets.config.HF_DATASETS_OFFLINE'''			,		__magic_name__   )
def  snake_case(    __magic_name__   )			->							Tuple:





     '''simple docstring'''


     lowercase :       Any       =			tmp_path_factory.mktemp('''data'''   ) / '''file.html'''
     with pytest.raises(__magic_name__   ):
          http_get('''https://huggingface.co'''			,		temp_file=__magic_name__   )
     with pytest.raises(__magic_name__   ):
          http_head('''https://huggingface.co'''   )






@patch('''datasets.config.HF_DATASETS_OFFLINE'''			,		__magic_name__   )
def  snake_case(    __magic_name__   )			->							Optional[Any]:





     '''simple docstring'''


     lowercase :       Optional[Any]       =			tmp_path_factory.mktemp('''data'''   ) / '''file.html'''
     with pytest.raises(__magic_name__   ):
          ftp_get('''ftp://huggingface.co'''			,		temp_file=__magic_name__   )
     with pytest.raises(__magic_name__   ):
          ftp_head('''ftp://huggingface.co'''   )






@patch('''datasets.config.HF_DATASETS_OFFLINE'''			,		__magic_name__   )
def  snake_case(    __magic_name__   )			->							List[str]:





     '''simple docstring'''


     lowercase :       str       =			tmp_path_factory.mktemp('''data'''   ) / '''file.html'''
     with pytest.raises(__magic_name__   ):
          fsspec_get('''s3://huggingface.co'''			,		temp_file=__magic_name__   )
     with pytest.raises(__magic_name__   ):
          fsspec_head('''s3://huggingface.co'''   )