import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
    from test_module.custom_tokenization_fast import CustomTokenizerFast


class      lowerCamelCase_					(       unittest.TestCase			):






    '''simple docstring'''






    def       UpperCamelCase__       (     self)	->     str:
        # A mock response for an HTTP head request to emulate server down
        __UpperCamelCase					:Optional[Any]        =		mock.Mock()
        __UpperCamelCase					:int        =		500
        __UpperCamelCase					:List[Any]        =		{}
        __UpperCamelCase					:List[str]        =		HTTPError
        __UpperCamelCase					:List[str]        =		{}

        # Download this model to make sure it's in the cache.
        __UpperCamelCase					:str        =		BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert''')

        # Under the mock environment we get a 500 error when trying to reach the tokenizer.
        with mock.patch('''requests.Session.request'''			,			return_value=__lowercase) as mock_head:
            __UpperCamelCase					:Optional[int]        =		BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert''')
            # This check we did call the fake head request
            mock_head.assert_called()







    @require_tokenizers
    def       UpperCamelCase__       (     self)	->     Optional[Any]:
        # A mock response for an HTTP head request to emulate server down
        __UpperCamelCase					:List[Any]        =		mock.Mock()
        __UpperCamelCase					:List[Any]        =		500
        __UpperCamelCase					:int        =		{}
        __UpperCamelCase					:List[Any]        =		HTTPError
        __UpperCamelCase					:Union[str, Any]        =		{}

        # Download this model to make sure it's in the cache.
        __UpperCamelCase					:Optional[int]        =		GPTaTokenizerFast.from_pretrained('''gpt2''')

        # Under the mock environment we get a 500 error when trying to reach the tokenizer.
        with mock.patch('''requests.Session.request'''			,			return_value=__lowercase) as mock_head:
            __UpperCamelCase					:Optional[int]        =		GPTaTokenizerFast.from_pretrained('''gpt2''')
            # This check we did call the fake head request
            mock_head.assert_called()







    def       UpperCamelCase__       (     self)	->     Tuple:
        # This test is for deprecated behavior and can be removed in v5
        try:
            __UpperCamelCase					:Any        =		tempfile.mktemp()
            with open(__lowercase			,			'''wb''') as f:
                http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''			,			__lowercase)

            __UpperCamelCase					:Dict        =		AlbertTokenizer.from_pretrained(__lowercase)
        finally:
            os.remove(__lowercase)

        # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
        # the current folder and have the right name.
        if os.path.isfile('''tokenizer.json'''):
            # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
            return
        try:
            with open('''tokenizer.json'''			,			'''wb''') as f:
                http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'''			,			__lowercase)
            __UpperCamelCase					:int        =		AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2''')
            # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
            self.assertEqual(tokenizer.vocab_size			,			1_000)
            # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

        finally:
            os.remove('''tokenizer.json''')







    def       UpperCamelCase__       (     self)	->     Optional[int]:
        # This test is for deprecated behavior and can be removed in v5
        __UpperCamelCase					:Any        =		AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''')


@is_staging_test
class      lowerCamelCase_					(       unittest.TestCase			):






    '''simple docstring'''






    a__     :						Optional[int]      =				["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """bla""", """blou"""]







    @classmethod
    def       UpperCamelCase__       (     cls)	->     Dict:
        __UpperCamelCase					:Union[str, Any]        =		TOKEN
        HfFolder.save_token(__lowercase)







    @classmethod
    def       UpperCamelCase__       (     cls)	->     str:
        try:
            delete_repo(token=cls._token			,			repo_id='''test-tokenizer''')
        except HTTPError:
            pass

        try:
            delete_repo(token=cls._token			,			repo_id='''valid_org/test-tokenizer-org''')
        except HTTPError:
            pass

        try:
            delete_repo(token=cls._token			,			repo_id='''test-dynamic-tokenizer''')
        except HTTPError:
            pass







    def       UpperCamelCase__       (     self)	->     Optional[Any]:
        with tempfile.TemporaryDirectory() as tmp_dir:
            __UpperCamelCase					:List[Any]        =		os.path.join(__lowercase			,			'''vocab.txt''')
            with open(__lowercase			,			'''w'''			,			encoding='''utf-8''') as vocab_writer:
                vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]))
            __UpperCamelCase					:Optional[Any]        =		BertTokenizer(__lowercase)

        tokenizer.push_to_hub('''test-tokenizer'''			,			use_auth_token=self._token)
        __UpperCamelCase					:Dict        =		BertTokenizer.from_pretrained(f"""{USER}/test-tokenizer""")
        self.assertDictEqual(new_tokenizer.vocab			,			tokenizer.vocab)

        # Reset repo
        delete_repo(token=self._token			,			repo_id='''test-tokenizer''')

        # Push to hub via save_pretrained
        with tempfile.TemporaryDirectory() as tmp_dir:
            tokenizer.save_pretrained(__lowercase			,			repo_id='''test-tokenizer'''			,			push_to_hub=__lowercase			,			use_auth_token=self._token)

        __UpperCamelCase					:List[str]        =		BertTokenizer.from_pretrained(f"""{USER}/test-tokenizer""")
        self.assertDictEqual(new_tokenizer.vocab			,			tokenizer.vocab)







    def       UpperCamelCase__       (     self)	->     Tuple:
        with tempfile.TemporaryDirectory() as tmp_dir:
            __UpperCamelCase					:Optional[Any]        =		os.path.join(__lowercase			,			'''vocab.txt''')
            with open(__lowercase			,			'''w'''			,			encoding='''utf-8''') as vocab_writer:
                vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]))
            __UpperCamelCase					:int        =		BertTokenizer(__lowercase)

        tokenizer.push_to_hub('''valid_org/test-tokenizer-org'''			,			use_auth_token=self._token)
        __UpperCamelCase					:Optional[int]        =		BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org''')
        self.assertDictEqual(new_tokenizer.vocab			,			tokenizer.vocab)

        # Reset repo
        delete_repo(token=self._token			,			repo_id='''valid_org/test-tokenizer-org''')

        # Push to hub via save_pretrained
        with tempfile.TemporaryDirectory() as tmp_dir:
            tokenizer.save_pretrained(
                __lowercase			,			repo_id='''valid_org/test-tokenizer-org'''			,			push_to_hub=__lowercase			,			use_auth_token=self._token)

        __UpperCamelCase					:List[str]        =		BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org''')
        self.assertDictEqual(new_tokenizer.vocab			,			tokenizer.vocab)







    @require_tokenizers
    def       UpperCamelCase__       (     self)	->     List[Any]:
        CustomTokenizer.register_for_auto_class()
        with tempfile.TemporaryDirectory() as tmp_dir:
            __UpperCamelCase					:Optional[int]        =		os.path.join(__lowercase			,			'''vocab.txt''')
            with open(__lowercase			,			'''w'''			,			encoding='''utf-8''') as vocab_writer:
                vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]))
            __UpperCamelCase					:Optional[Any]        =		CustomTokenizer(__lowercase)

        # No fast custom tokenizer
        tokenizer.push_to_hub('''test-dynamic-tokenizer'''			,			use_auth_token=self._token)

        __UpperCamelCase					:Union[str, Any]        =		AutoTokenizer.from_pretrained(f"""{USER}/test-dynamic-tokenizer"""			,			trust_remote_code=__lowercase)
        # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
        self.assertEqual(tokenizer.__class__.__name__			,			'''CustomTokenizer''')

        # Fast and slow custom tokenizer
        CustomTokenizerFast.register_for_auto_class()
        with tempfile.TemporaryDirectory() as tmp_dir:
            __UpperCamelCase					:int        =		os.path.join(__lowercase			,			'''vocab.txt''')
            with open(__lowercase			,			'''w'''			,			encoding='''utf-8''') as vocab_writer:
                vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]))

            __UpperCamelCase					:Tuple        =		BertTokenizerFast.from_pretrained(__lowercase)
            bert_tokenizer.save_pretrained(__lowercase)
            __UpperCamelCase					:Optional[int]        =		CustomTokenizerFast.from_pretrained(__lowercase)

        tokenizer.push_to_hub('''test-dynamic-tokenizer'''			,			use_auth_token=self._token)

        __UpperCamelCase					:Any        =		AutoTokenizer.from_pretrained(f"""{USER}/test-dynamic-tokenizer"""			,			trust_remote_code=__lowercase)
        # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
        self.assertEqual(tokenizer.__class__.__name__			,			'''CustomTokenizerFast''')
        __UpperCamelCase					:List[Any]        =		AutoTokenizer.from_pretrained(
            f"""{USER}/test-dynamic-tokenizer"""			,			use_fast=__lowercase			,			trust_remote_code=__lowercase)
        # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
        self.assertEqual(tokenizer.__class__.__name__			,			'''CustomTokenizer''')


class      lowerCamelCase_					(       unittest.TestCase			):






    '''simple docstring'''






    def       UpperCamelCase__       (     self)	->     Union[str, Any]:
        __UpperCamelCase					:Dict        =		Trie()
        trie.add('''Hello 友達''')
        self.assertEqual(trie.data			,			{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}})
        trie.add('''Hello''')
        trie.data
        self.assertEqual(trie.data			,			{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}})







    def       UpperCamelCase__       (     self)	->     List[Any]:
        __UpperCamelCase					:Any        =		Trie()
        self.assertEqual(trie.split('''[CLS] This is a extra_id_100''')			,			['''[CLS] This is a extra_id_100'''])
        trie.add('''[CLS]''')
        trie.add('''extra_id_1''')
        trie.add('''extra_id_100''')
        self.assertEqual(trie.split('''[CLS] This is a extra_id_100''')			,			['''[CLS]''', ''' This is a ''', '''extra_id_100'''])







    def       UpperCamelCase__       (     self)	->     List[Any]:
        __UpperCamelCase					:str        =		Trie()
        trie.add('''A''')
        self.assertEqual(trie.split('''ABC''')			,			['''A''', '''BC'''])
        self.assertEqual(trie.split('''BCA''')			,			['''BC''', '''A'''])







    def       UpperCamelCase__       (     self)	->     Optional[Any]:
        __UpperCamelCase					:Union[str, Any]        =		Trie()
        trie.add('''TOKEN]''')
        trie.add('''[SPECIAL_TOKEN]''')
        self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]''')			,			['''This is something ''', '''[SPECIAL_TOKEN]'''])







    def       UpperCamelCase__       (     self)	->     int:
        __UpperCamelCase					:Optional[Any]        =		Trie()
        trie.add('''A''')
        trie.add('''P''')
        trie.add('''[SPECIAL_TOKEN]''')
        self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]''')			,			['''This is something ''', '''[SPECIAL_TOKEN]'''])







    def       UpperCamelCase__       (     self)	->     int:
        __UpperCamelCase					:Tuple        =		Trie()
        trie.add('''AB''')
        trie.add('''B''')
        trie.add('''C''')
        self.assertEqual(trie.split('''ABC''')			,			['''AB''', '''C'''])







    def       UpperCamelCase__       (     self)	->     Optional[Any]:
        __UpperCamelCase					:str        =		Trie()
        trie.add('''ABC''')
        trie.add('''B''')
        trie.add('''CD''')
        self.assertEqual(trie.split('''ABCD''')			,			['''ABC''', '''D'''])







    def       UpperCamelCase__       (     self)	->     int:
        # Even if the offsets are wrong, we necessarily output correct string
        # parts.
        __UpperCamelCase					:Dict        =		Trie()
        __UpperCamelCase					:Optional[int]        =		trie.cut_text('''ABC'''			,			[0, 0, 2, 1, 2, 3])
        self.assertEqual(__lowercase			,			['''AB''', '''C'''])






import unittest

from transformers import BertGenerationTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


__lowercase										= '''▁'''

__lowercase										= get_tests_dir('''fixtures/test_sentencepiece.model''')


@require_sentencepiece
class      lowerCamelCase_					(       UpperCAmelCase_						,     unittest.TestCase			):






    '''simple docstring'''






    a__     :						int      =				BertGenerationTokenizer
    a__     :						Dict      =				False
    a__     :						str      =				True







    def       UpperCamelCase__       (     self)	->     List[str]:
        super().setUp()

        __UpperCamelCase					:Optional[Any]        =		BertGenerationTokenizer(__lowercase			,			keep_accents=__lowercase)
        tokenizer.save_pretrained(self.tmpdirname)







    def       UpperCamelCase__       (     self)	->     int:
        __UpperCamelCase					:Dict        =		'''<s>'''
        __UpperCamelCase					:Dict        =		1

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowercase)			,			__lowercase)
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowercase)			,			__lowercase)







    def       UpperCamelCase__       (     self)	->     Optional[int]:
        __UpperCamelCase					:Optional[Any]        =		list(self.get_tokenizer().get_vocab().keys())

        self.assertEqual(vocab_keys[0]			,			'''<unk>''')
        self.assertEqual(vocab_keys[1]			,			'''<s>''')
        self.assertEqual(vocab_keys[-1]			,			'''<pad>''')
        self.assertEqual(len(__lowercase)			,			1_002)







    def       UpperCamelCase__       (     self)	->     Dict:
        self.assertEqual(self.get_tokenizer().vocab_size			,			1_000)







    def       UpperCamelCase__       (     self)	->     Optional[Any]:
        __UpperCamelCase					:str        =		BertGenerationTokenizer(__lowercase			,			keep_accents=__lowercase)

        __UpperCamelCase					:Optional[Any]        =		tokenizer.tokenize('''This is a test''')
        self.assertListEqual(__lowercase			,			['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''])

        self.assertListEqual(
            tokenizer.convert_tokens_to_ids(__lowercase)			,			[285, 46, 10, 170, 382]			,			)

        __UpperCamelCase					:List[Any]        =		tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
        self.assertListEqual(
            __lowercase			,			[
                SPIECE_UNDERLINE + '''I''',
                SPIECE_UNDERLINE + '''was''',
                SPIECE_UNDERLINE + '''b''',
                '''or''',
                '''n''',
                SPIECE_UNDERLINE + '''in''',
                SPIECE_UNDERLINE + '''''',
                '''9''',
                '''2''',
                '''0''',
                '''0''',
                '''0''',
                ''',''',
                SPIECE_UNDERLINE + '''and''',
                SPIECE_UNDERLINE + '''this''',
                SPIECE_UNDERLINE + '''is''',
                SPIECE_UNDERLINE + '''f''',
                '''al''',
                '''s''',
                '''é''',
                '''.''',
            ]			,			)
        __UpperCamelCase					:Any        =		tokenizer.convert_tokens_to_ids(__lowercase)
        self.assertListEqual(
            __lowercase			,			[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]			,			)

        __UpperCamelCase					:Tuple        =		tokenizer.convert_ids_to_tokens(__lowercase)
        self.assertListEqual(
            __lowercase			,			[
                SPIECE_UNDERLINE + '''I''',
                SPIECE_UNDERLINE + '''was''',
                SPIECE_UNDERLINE + '''b''',
                '''or''',
                '''n''',
                SPIECE_UNDERLINE + '''in''',
                SPIECE_UNDERLINE + '''''',
                '''<unk>''',
                '''2''',
                '''0''',
                '''0''',
                '''0''',
                ''',''',
                SPIECE_UNDERLINE + '''and''',
                SPIECE_UNDERLINE + '''this''',
                SPIECE_UNDERLINE + '''is''',
                SPIECE_UNDERLINE + '''f''',
                '''al''',
                '''s''',
                '''<unk>''',
                '''.''',
            ]			,			)







    @cached_property
    def       UpperCamelCase__       (     self)	->     int:
        return BertGenerationTokenizer.from_pretrained('''google/bert_for_seq_generation_L-24_bbc_encoder''')







    @slow
    def       UpperCamelCase__       (     self)	->     List[str]:
        __UpperCamelCase					:Optional[Any]        =		'''Hello World!'''
        __UpperCamelCase					:Optional[int]        =		[18_536, 2_260, 101]

        self.assertListEqual(__lowercase			,			self.big_tokenizer.encode(__lowercase))







    @slow
    def       UpperCamelCase__       (     self)	->     List[str]:
        __UpperCamelCase					:Optional[Any]        =		(
            '''This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'''
            ''' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'''
        )
        __UpperCamelCase					:Union[str, Any]        =		[
            871,
            419,
            358,
            946,
            991,
            2_521,
            452,
            358,
            1_357,
            387,
            7_751,
            3_536,
            112,
            985,
            456,
            126,
            865,
            938,
            5_400,
            5_734,
            458,
            1_368,
            467,
            786,
            2_462,
            5_246,
            1_159,
            633,
            865,
            4_519,
            457,
            582,
            852,
            2_557,
            427,
            916,
            508,
            405,
            34_324,
            497,
            391,
            408,
            11_342,
            1_244,
            385,
            100,
            938,
            985,
            456,
            574,
            362,
            12_597,
            3_200,
            3_129,
            1_172,
        ]

        self.assertListEqual(__lowercase			,			self.big_tokenizer.encode(__lowercase))







    @require_torch
    @slow
    def       UpperCamelCase__       (     self)	->     Optional[int]:
        import torch

        from transformers import BertGenerationConfig, BertGenerationEncoder

        # Build sequence
        __UpperCamelCase					:Optional[Any]        =		list(self.big_tokenizer.get_vocab().keys())[:10]
        __UpperCamelCase					:Optional[int]        =		''' '''.join(__lowercase)
        __UpperCamelCase					:Optional[int]        =		self.big_tokenizer.encode_plus(__lowercase			,			return_tensors='''pt'''			,			return_token_type_ids=__lowercase)
        __UpperCamelCase					:List[str]        =		self.big_tokenizer.batch_encode_plus(
            [sequence + ''' ''' + sequence]			,			return_tensors='''pt'''			,			return_token_type_ids=__lowercase)

        __UpperCamelCase					:List[Any]        =		BertGenerationConfig()
        __UpperCamelCase					:Optional[Any]        =		BertGenerationEncoder(__lowercase)

        assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

        with torch.no_grad():
            model(**__lowercase)
            model(**__lowercase)







    @slow
    def       UpperCamelCase__       (     self)	->     Dict:
        # fmt: off
        __UpperCamelCase					:List[str]        =		{'''input_ids''': [[39_286, 458, 36_335, 2_001, 456, 13_073, 13_266, 455, 113, 7_746, 1_741, 11_157, 391, 13_073, 13_266, 455, 113, 3_967, 35_412, 113, 4_936, 109, 3_870, 2_377, 113, 30_084, 45_720, 458, 134, 17_496, 112, 503, 11_672, 113, 118, 112, 5_665, 13_347, 38_687, 112, 1_496, 31_389, 112, 3_268, 47_264, 134, 962, 112, 16_377, 8_035, 23_130, 430, 12_169, 15_518, 28_592, 458, 146, 41_697, 109, 391, 12_169, 15_518, 16_689, 458, 146, 41_358, 109, 452, 726, 4_034, 111, 763, 35_412, 5_082, 388, 1_903, 111, 9_051, 391, 2_870, 48_918, 1_900, 1_123, 550, 998, 112, 9_586, 15_985, 455, 391, 410, 22_955, 37_636, 114], [448, 17_496, 419, 3_663, 385, 763, 113, 27_533, 2_870, 3_283, 13_043, 1_639, 24_713, 523, 656, 24_013, 18_550, 2_521, 517, 27_014, 21_244, 420, 1_212, 1_465, 391, 927, 4_833, 388, 578, 11_786, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [484, 2_169, 7_687, 21_932, 18_146, 726, 363, 17_032, 3_391, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
        # fmt: on

        self.tokenizer_integration_test_util(
            expected_encoding=__lowercase			,			model_name='''google/bert_for_seq_generation_L-24_bbc_encoder'''			,			revision='''c817d1fd1be2ffa69431227a1fe320544943d4db'''			,			)





