"""simple docstring"""




import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
     _a				:		Optional[Any]	   =       'pt'
elif is_tf_available():
     _a				:		Optional[int]	   =       'tf'
else:
     _a				:		List[Any]	   =       'jax'
class 						__A							(			lowercase__   ,     unittest.TestCase		):
     _UpperCamelCase							:      Any		 =		ByTaTokenizer
     _UpperCamelCase							:      Optional[int]		 =		False






     def    __A  (		self  ):
       super().setUp()
       _lowerCAmelCase						:	Tuple        =					ByTaTokenizer()
       tokenizer.save_pretrained(self.tmpdirname  )






     @cached_property
     def    __A  (		self  ):
       return ByTaTokenizer.from_pretrained("""google/byt5-small"""  )






     def    __A  (		self    ,					**a__  ):
       return self.tokenizer_class.from_pretrained(self.tmpdirname    ,					**_UpperCamelCase  )






     def    __A  (		self    ,					a__    ,					a__=False    ,					a__=20    ,					a__=5  ):
       # XXX The default common tokenizer tests assume that every ID is decodable on its own.
       # This assumption is invalid for ByT5 because single bytes might not be
       # valid utf-8 (byte 128 for instance).
       # Here we're overriding the smallest possible method to provide
       # a clean sequence without making the same assumption.

       _lowerCAmelCase						:	List[Any]        =					[]
       for i in range(len(_UpperCamelCase  )  ):
         try:
           _lowerCAmelCase						:	Any        =					tokenizer.decode([i]    ,					clean_up_tokenization_spaces=_UpperCamelCase  )
         except UnicodeDecodeError:
           pass
         toks.append((i, tok)  )

       _lowerCAmelCase						:	Optional[int]        =					list(filter(lambda a__  : re.match(r"""^[ a-zA-Z]+$"""    ,					t[1]  )    ,					_UpperCamelCase  )  )
       _lowerCAmelCase						:	int        =					list(filter(lambda a__  : [t[0]] == tokenizer.encode(t[1]    ,					add_special_tokens=_UpperCamelCase  )    ,					_UpperCamelCase  )  )
       if max_length is not None and len(_UpperCamelCase  ) > max_length:
         _lowerCAmelCase						:	Dict        =					toks[:max_length]
       if min_length is not None and len(_UpperCamelCase  ) < min_length and len(_UpperCamelCase  ) > 0:
         while len(_UpperCamelCase  ) < min_length:
           _lowerCAmelCase						:	Any        =					toks + toks
        # toks_str = [t[1] for t in toks]
       _lowerCAmelCase						:	Tuple        =					[t[0] for t in toks]

       # Ensure consistency
       _lowerCAmelCase						:	Optional[Any]        =					tokenizer.decode(_UpperCamelCase    ,					clean_up_tokenization_spaces=_UpperCamelCase  )
       if " " not in output_txt and len(_UpperCamelCase  ) > 1:
         _lowerCAmelCase						:	Dict        =					(
             tokenizer.decode([toks_ids[0]]    ,					clean_up_tokenization_spaces=_UpperCamelCase  )
             + """ """
             + tokenizer.decode(toks_ids[1:]    ,					clean_up_tokenization_spaces=_UpperCamelCase  )
         )
       if with_prefix_space:
         _lowerCAmelCase						:	str        =					""" """ + output_txt
       _lowerCAmelCase						:	str        =					tokenizer.encode(_UpperCamelCase    ,					add_special_tokens=_UpperCamelCase  )
       return output_txt, output_ids






     def    __A  (		self  ):
       _lowerCAmelCase						:	int        =					self.ta_base_tokenizer
       _lowerCAmelCase						:	Dict        =					tokenizer(["""hi</s>""", """I went to the gym</s>""", """</s>"""]  )
       _lowerCAmelCase						:	Union[str, Any]        =					tokenizer(["""hi""", """I went to the gym""", """"""]  )
       self.assertListEqual(batch_with_eos_added["""input_ids"""]    ,					batch_without_eos_added["""input_ids"""]  )






     def    __A  (		self  ):
       _lowerCAmelCase						:	str        =					self.ta_base_tokenizer
       _lowerCAmelCase						:	List[Any]        =					"""Unicode €."""
       _lowerCAmelCase						:	Any        =					tokenizer(_UpperCamelCase  )
       _lowerCAmelCase						:	int        =					[88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
       self.assertEqual(encoded["""input_ids"""]    ,					_UpperCamelCase  )

       # decoding
       _lowerCAmelCase						:	Union[str, Any]        =					tokenizer.decode(_UpperCamelCase  )
       self.assertEqual(_UpperCamelCase    ,					"""Unicode €.</s>"""  )

       _lowerCAmelCase						:	Optional[int]        =					tokenizer("""e è é ê ë"""  )
       _lowerCAmelCase						:	List[Any]        =					[104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
       self.assertEqual(encoded["""input_ids"""]    ,					_UpperCamelCase  )
       # decoding
       _lowerCAmelCase						:	Optional[int]        =					tokenizer.decode(_UpperCamelCase  )
       self.assertEqual(_UpperCamelCase    ,					"""e è é ê ë</s>"""  )

       # encode/decode, but with `encode` instead of `__call__`
       self.assertEqual(tokenizer.decode(tokenizer.encode("""e è é ê ë"""  )  )    ,					"""e è é ê ë</s>"""  )






     def    __A  (		self  ):
       _lowerCAmelCase						:	List[Any]        =					self.ta_base_tokenizer
       _lowerCAmelCase						:	int        =					["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
       # fmt: off
       _lowerCAmelCase						:	List[Any]        =					[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
       # fmt: on
       _lowerCAmelCase						:	Union[str, Any]        =					tokenizer(_UpperCamelCase    ,					padding=_UpperCamelCase    ,					return_tensors=_UpperCamelCase  )
       self.assertIsInstance(_UpperCamelCase    ,					_UpperCamelCase  )

       if FRAMEWORK != "jax":
         _lowerCAmelCase						:	List[str]        =					list(batch.input_ids.numpy()[0]  )
       else:
         _lowerCAmelCase						:	List[str]        =					list(batch.input_ids.tolist()[0]  )

       self.assertListEqual(_UpperCamelCase    ,					_UpperCamelCase  )

       self.assertEqual((2, 37)    ,					batch.input_ids.shape  )
       self.assertEqual((2, 37)    ,					batch.attention_mask.shape  )






     def    __A  (		self  ):
       _lowerCAmelCase						:	Any        =					self.ta_base_tokenizer
       _lowerCAmelCase						:	Union[str, Any]        =					["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
       _lowerCAmelCase						:	Dict        =					tokenizer(_UpperCamelCase    ,					padding=_UpperCamelCase    ,					return_tensors=_UpperCamelCase  )
       # check if input_ids are returned and no decoder_input_ids
       self.assertIn("""input_ids"""    ,					_UpperCamelCase  )
       self.assertIn("""attention_mask"""    ,					_UpperCamelCase  )
       self.assertNotIn("""decoder_input_ids"""    ,					_UpperCamelCase  )
       self.assertNotIn("""decoder_attention_mask"""    ,					_UpperCamelCase  )






     def    __A  (		self  ):
       _lowerCAmelCase						:	Optional[Any]        =					self.ta_base_tokenizer
       _lowerCAmelCase						:	str        =					[
           """Summary of the text.""",
           """Another summary.""",
       ]
       _lowerCAmelCase						:	int        =					tokenizer(
           text_target=_UpperCamelCase    ,					max_length=32    ,					padding="""max_length"""    ,					truncation=_UpperCamelCase    ,					return_tensors=_UpperCamelCase  )
       self.assertEqual(32    ,					targets["""input_ids"""].shape[1]  )






     def    __A  (		self  ):
       _lowerCAmelCase						:	int        =					self.ta_base_tokenizer
       _lowerCAmelCase						:	int        =					["""A long paragraph for summarization. </s>"""]
       _lowerCAmelCase						:	Any        =					["""Summary of the text. </s>"""]
       # fmt: off
       _lowerCAmelCase						:	List[str]        =					[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
       _lowerCAmelCase						:	List[str]        =					[86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
       # fmt: on

       _lowerCAmelCase						:	Dict        =					tokenizer(_UpperCamelCase    ,					text_target=_UpperCamelCase  )

       self.assertEqual(_UpperCamelCase    ,					batch["""input_ids"""][0]  )
       self.assertEqual(_UpperCamelCase    ,					batch["""labels"""][0]  )






     def    __A  (		self  ):
       # safety check on max_len default value so we are sure the test works
       _lowerCAmelCase						:	int        =					self.get_tokenizers()
       for tokenizer in tokenizers:
         with self.subTest(F"{tokenizer.__class__.__name__}"  ):
           self.assertNotEqual(tokenizer.model_max_length    ,					42  )

        # Now let's start the test
       _lowerCAmelCase						:	Union[str, Any]        =					self.get_tokenizers()
       for tokenizer in tokenizers:
         with self.subTest(F"{tokenizer.__class__.__name__}"  ):
           # Isolate this from the other tests because we save additional tokens/etc
           _lowerCAmelCase						:	str        =					tempfile.mkdtemp()

           _lowerCAmelCase						:	int        =					""" He is very happy, UNwant\u00E9d,running"""
           _lowerCAmelCase						:	str        =					tokenizer.encode(_UpperCamelCase    ,					add_special_tokens=_UpperCamelCase  )
           tokenizer.save_pretrained(_UpperCamelCase  )

           _lowerCAmelCase						:	List[str]        =					tokenizer.__class__.from_pretrained(_UpperCamelCase  )
           _lowerCAmelCase						:	Union[str, Any]        =					after_tokenizer.encode(_UpperCamelCase    ,					add_special_tokens=_UpperCamelCase  )
           self.assertListEqual(_UpperCamelCase    ,					_UpperCamelCase  )

           shutil.rmtree(_UpperCamelCase  )

       _lowerCAmelCase						:	Tuple        =					self.get_tokenizers(model_max_length=42  )
       for tokenizer in tokenizers:
         with self.subTest(F"{tokenizer.__class__.__name__}"  ):
           # Isolate this from the other tests because we save additional tokens/etc
           _lowerCAmelCase						:	Optional[int]        =					tempfile.mkdtemp()

           _lowerCAmelCase						:	Dict        =					""" He is very happy, UNwant\u00E9d,running"""
           tokenizer.add_tokens(["""bim""", """bambam"""]  )
           _lowerCAmelCase						:	Optional[int]        =					tokenizer.additional_special_tokens
           additional_special_tokens.append("""new_additional_special_token"""  )
           tokenizer.add_special_tokens({"""additional_special_tokens""": additional_special_tokens}  )
           _lowerCAmelCase						:	str        =					tokenizer.encode(_UpperCamelCase    ,					add_special_tokens=_UpperCamelCase  )
           tokenizer.save_pretrained(_UpperCamelCase  )

           _lowerCAmelCase						:	Optional[int]        =					tokenizer.__class__.from_pretrained(_UpperCamelCase  )
           _lowerCAmelCase						:	Optional[Any]        =					after_tokenizer.encode(_UpperCamelCase    ,					add_special_tokens=_UpperCamelCase  )
           self.assertListEqual(_UpperCamelCase    ,					_UpperCamelCase  )
           self.assertIn("""new_additional_special_token"""    ,					after_tokenizer.additional_special_tokens  )
           self.assertEqual(after_tokenizer.model_max_length    ,					42  )

           _lowerCAmelCase						:	List[str]        =					tokenizer.__class__.from_pretrained(_UpperCamelCase    ,					model_max_length=43  )
           self.assertEqual(tokenizer.model_max_length    ,					43  )

           shutil.rmtree(_UpperCamelCase  )






     def    __A  (		self  ):
       _lowerCAmelCase						:	Tuple        =					[]
       if self.test_slow_tokenizer:
         tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())  )

       if self.test_rust_tokenizer:
         tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())  )

       for tokenizer_class, tokenizer_utils in tokenizer_list:
         with tempfile.TemporaryDirectory() as tmp_dir:
           tokenizer_utils.save_pretrained(_UpperCamelCase  )

           with open(os.path.join(_UpperCamelCase    ,					"""special_tokens_map.json"""  )    ,					encoding="""utf-8"""  ) as json_file:
             _lowerCAmelCase						:	str        =					json.load(_UpperCamelCase  )

           with open(os.path.join(_UpperCamelCase    ,					"""tokenizer_config.json"""  )    ,					encoding="""utf-8"""  ) as json_file:
             _lowerCAmelCase						:	Any        =					json.load(_UpperCamelCase  )

           _lowerCAmelCase						:	Any        =					[F"<extra_id_{i}>" for i in range(125  )]

           _lowerCAmelCase						:	Optional[Any]        =					added_tokens_extra_ids + [
               """an_additional_special_token"""
           ]
           _lowerCAmelCase						:	List[str]        =					added_tokens_extra_ids + [
               """an_additional_special_token"""
           ]

           with open(os.path.join(_UpperCamelCase    ,					"""special_tokens_map.json"""  )    ,					"""w"""    ,					encoding="""utf-8"""  ) as outfile:
             json.dump(_UpperCamelCase    ,					_UpperCamelCase  )
           with open(os.path.join(_UpperCamelCase    ,					"""tokenizer_config.json"""  )    ,					"""w"""    ,					encoding="""utf-8"""  ) as outfile:
             json.dump(_UpperCamelCase    ,					_UpperCamelCase  )

           # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
           # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
           # "special_tokens_map.json" files
           _lowerCAmelCase						:	str        =					tokenizer_class.from_pretrained(
               _UpperCamelCase    ,					)
           self.assertIn(
               """an_additional_special_token"""    ,					tokenizer_without_change_in_init.additional_special_tokens  )
           # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
           self.assertEqual(
               ["""an_additional_special_token"""]    ,					tokenizer_without_change_in_init.convert_ids_to_tokens(
                   tokenizer_without_change_in_init.convert_tokens_to_ids(["""an_additional_special_token"""]  )  )    ,					)

           # Now we test that we can change the value of additional_special_tokens in the from_pretrained
           _lowerCAmelCase						:	List[Any]        =					added_tokens_extra_ids + [AddedToken("""a_new_additional_special_token"""    ,					lstrip=_UpperCamelCase  )]
           _lowerCAmelCase						:	Optional[int]        =					tokenizer_class.from_pretrained(
               _UpperCamelCase    ,					additional_special_tokens=_UpperCamelCase    ,					)

           self.assertIn("""a_new_additional_special_token"""    ,					tokenizer.additional_special_tokens  )
           self.assertEqual(
               ["""a_new_additional_special_token"""]    ,					tokenizer.convert_ids_to_tokens(
                   tokenizer.convert_tokens_to_ids(["""a_new_additional_special_token"""]  )  )    ,					)






     def    __A  (		self  ):
       _lowerCAmelCase						:	Tuple        =					[]
       if self.test_slow_tokenizer:
         tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())  )

       if self.test_rust_tokenizer:
         tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())  )

       for tokenizer_class, tokenizer_utils in tokenizer_list:
         with tempfile.TemporaryDirectory() as tmp_dir:
           tokenizer_utils.save_pretrained(_UpperCamelCase  )

           _lowerCAmelCase						:	Optional[Any]        =					tokenizer_class.from_pretrained(_UpperCamelCase  )

           self.assertTrue(tokenizer.decode([255]  ) == """"""  )






     def    __A  (		self  ):
       pass






     def    __A  (		self  ):
       pass






     def    __A  (		self  ):
       pass






     def    __A  (		self  ):
       pass






     def    __A  (		self  ):
       # The default common tokenizer tests uses invalid tokens for ByT5 that can only accept one-character strings
       # and special added tokens as tokens
       _lowerCAmelCase						:	List[Any]        =					self.get_tokenizers(fast=_UpperCamelCase    ,					do_lower_case=_UpperCamelCase  )
       for tokenizer in tokenizers:
         with self.subTest(F"{tokenizer.__class__.__name__}"  ):
           _lowerCAmelCase						:	Tuple        =					["""t""", """h""", """i""", """s""", """ """, """i""", """s""", """ """, """a""", """ """, """t""", """e""", """x""", """t""", """</s>"""]
           _lowerCAmelCase						:	Optional[int]        =					tokenizer.convert_tokens_to_string(_UpperCamelCase  )

           self.assertIsInstance(_UpperCamelCase    ,					_UpperCamelCase  )






     def    __A  (		self  ):
       _lowerCAmelCase						:	Any        =					self.get_tokenizers()
       for tokenizer in tokenizers:
         with self.subTest(F"{tokenizer.__class__.__name__}"  ):
           _lowerCAmelCase						:	List[Any]        =					[
               """bos_token""",
               """eos_token""",
               """unk_token""",
               """sep_token""",
               """pad_token""",
               """cls_token""",
               """mask_token""",
           ]

           _lowerCAmelCase						:	Tuple        =					0
           _lowerCAmelCase						:	int        =					tokenizer.convert_ids_to_tokens(
               _UpperCamelCase    ,					skip_special_tokens=_UpperCamelCase  )

           for attr in attributes_list:
             setattr(_UpperCamelCase    ,					attr + """_id"""    ,					_UpperCamelCase  )
             self.assertEqual(getattr(_UpperCamelCase    ,					_UpperCamelCase  )    ,					_UpperCamelCase  )
             self.assertEqual(getattr(_UpperCamelCase    ,					attr + """_id"""  )    ,					_UpperCamelCase  )

             setattr(_UpperCamelCase    ,					attr + """_id"""    ,					_UpperCamelCase  )
             self.assertEqual(getattr(_UpperCamelCase    ,					_UpperCamelCase  )    ,					_UpperCamelCase  )
             self.assertEqual(getattr(_UpperCamelCase    ,					attr + """_id"""  )    ,					_UpperCamelCase  )

           setattr(_UpperCamelCase    ,					"""additional_special_tokens_ids"""    ,					[]  )
           self.assertListEqual(getattr(_UpperCamelCase    ,					"""additional_special_tokens"""  )    ,					[]  )
           self.assertListEqual(getattr(_UpperCamelCase    ,					"""additional_special_tokens_ids"""  )    ,					[]  )

           setattr(_UpperCamelCase    ,					"""additional_special_tokens_ids"""    ,					[token_id_to_test_setters]  )
           self.assertListEqual(getattr(_UpperCamelCase    ,					"""additional_special_tokens"""  )    ,					[token_to_test_setters]  )
           self.assertListEqual(getattr(_UpperCamelCase    ,					"""additional_special_tokens_ids"""  )    ,					[token_id_to_test_setters]  )





"""simple docstring"""




from ...configuration_utils import PretrainedConfig
from ...utils import logging


_a				:		Any	   =       logging.get_logger(__name__)

_a				:		Any	   =       {
    'uclanlp/visualbert-vqa': 'https://huggingface.co/uclanlp/visualbert-vqa/resolve/main/config.json',
    'uclanlp/visualbert-vqa-pre': 'https://huggingface.co/uclanlp/visualbert-vqa-pre/resolve/main/config.json',
    'uclanlp/visualbert-vqa-coco-pre': (
        'https://huggingface.co/uclanlp/visualbert-vqa-coco-pre/resolve/main/config.json'
    ),
    'uclanlp/visualbert-vcr': 'https://huggingface.co/uclanlp/visualbert-vcr/resolve/main/config.json',
    'uclanlp/visualbert-vcr-pre': 'https://huggingface.co/uclanlp/visualbert-vcr-pre/resolve/main/config.json',
    'uclanlp/visualbert-vcr-coco-pre': (
        'https://huggingface.co/uclanlp/visualbert-vcr-coco-pre/resolve/main/config.json'
    ),
    'uclanlp/visualbert-nlvr2': 'https://huggingface.co/uclanlp/visualbert-nlvr2/resolve/main/config.json',
    'uclanlp/visualbert-nlvr2-pre': 'https://huggingface.co/uclanlp/visualbert-nlvr2-pre/resolve/main/config.json',
    'uclanlp/visualbert-nlvr2-coco-pre': (
        'https://huggingface.co/uclanlp/visualbert-nlvr2-coco-pre/resolve/main/config.json'
    )
    # See all VisualBERT models at https://huggingface.co/models?filter=visual_bert
}
class 						__A							(			SCREAMING_SNAKE_CASE_		):
     _UpperCamelCase							:      List[Any]		 =		"visual_bert"






     def __init__(		self    ,					a__=30522    ,					a__=768    ,					a__=512    ,					a__=12    ,					a__=12    ,					a__=3072    ,					a__="gelu"    ,					a__=0.1    ,					a__=0.1    ,					a__=512    ,					a__=2    ,					a__=0.0_2    ,					a__=1e-12    ,					a__=False    ,					a__=True    ,					a__=1    ,					a__=0    ,					a__=2    ,					**a__    ,					):
       super().__init__(pad_token_id=a__    ,					bos_token_id=a__    ,					eos_token_id=a__    ,					**a__  )

       _lowerCAmelCase						:	Optional[Any]        =					vocab_size
       _lowerCAmelCase						:	int        =					max_position_embeddings
       _lowerCAmelCase						:	Optional[Any]        =					hidden_size
       _lowerCAmelCase						:	Optional[int]        =					visual_embedding_dim
       _lowerCAmelCase						:	Optional[int]        =					num_hidden_layers
       _lowerCAmelCase						:	List[Any]        =					num_attention_heads
       _lowerCAmelCase						:	Any        =					intermediate_size
       _lowerCAmelCase						:	List[Any]        =					hidden_act
       _lowerCAmelCase						:	int        =					hidden_dropout_prob
       _lowerCAmelCase						:	Optional[Any]        =					attention_probs_dropout_prob
       _lowerCAmelCase						:	Any        =					initializer_range
       _lowerCAmelCase						:	List[Any]        =					type_vocab_size
       _lowerCAmelCase						:	Union[str, Any]        =					layer_norm_eps
       _lowerCAmelCase						:	Optional[Any]        =					bypass_transformer
       _lowerCAmelCase						:	List[Any]        =					special_visual_initialize




