


import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class 	_a							(      UpperCamelCase__					,     unittest.TestCase      ):
							_lowercase     :      Union[str, Any]  				=			CLIPTokenizer
							_lowercase     :      int  				=			CLIPTokenizerFast
							_lowercase     :      Any  				=			True
							_lowercase     :      List[Any]  				=			{}
							_lowercase     :      Union[str, Any]  				=			False

							def 				lowerCamelCase_			(							self:     str							)      ->       List[Any]:





									"""simple docstring"""







									super().setUp()

									# fmt: off
									lowercase__						=				['''l''', '''o''', '''w''', '''e''', '''r''', '''s''', '''t''', '''i''', '''d''', '''n''', '''lo''', '''l</w>''', '''w</w>''', '''r</w>''', '''t</w>''', '''low</w>''', '''er</w>''', '''lowest</w>''', '''newer</w>''', '''wider''', '''<unk>''', '''<|startoftext|>''', '''<|endoftext|>''']
									# fmt: on
									lowercase__						=				dict(zip(UpperCamelCase_   ,						range(len(UpperCamelCase_							)							)							)							)
									lowercase__						=				['''#version: 0.2''', '''l o''', '''lo w</w>''', '''e r</w>''']
									lowercase__						=				{'''unk_token''': '''<unk>'''}

									lowercase__						=				os.path.join(self.tmpdirname   ,						VOCAB_FILES_NAMES['''vocab_file''']							)
									lowercase__						=				os.path.join(self.tmpdirname   ,						VOCAB_FILES_NAMES['''merges_file''']							)
									with open(self.vocab_file   ,						'''w'''   ,						encoding='''utf-8'''							) as fp:
											fp.write(json.dumps(UpperCamelCase_							) + '''\n'''							)
									with open(self.merges_file   ,						'''w'''   ,						encoding='''utf-8'''							) as fp:
											fp.write('''\n'''.join(UpperCamelCase_							)							)

							def 				lowerCamelCase_			(							self:     Union[str, Any]   ,						**UpperCamelCase_:     int							)      ->       Dict:





									"""simple docstring"""







									kwargs.update(self.special_tokens_map							)
									return CLIPTokenizer.from_pretrained(self.tmpdirname   ,						**UpperCamelCase_							)

							def 				lowerCamelCase_			(							self:     Dict   ,						**UpperCamelCase_:     Tuple							)      ->       List[Any]:





									"""simple docstring"""







									kwargs.update(self.special_tokens_map							)
									return CLIPTokenizerFast.from_pretrained(self.tmpdirname   ,						**UpperCamelCase_							)

							def 				lowerCamelCase_			(							self:     Any   ,						UpperCamelCase_:     int							)      ->       Dict:





									"""simple docstring"""







									lowercase__						=				'''lower newer'''
									lowercase__						=				'''lower newer'''
									return input_text, output_text

							def 				lowerCamelCase_			(							self:     Tuple							)      ->       List[Any]:





									"""simple docstring"""







									lowercase__						=				CLIPTokenizer(self.vocab_file   ,						self.merges_file   ,						**self.special_tokens_map							)
									lowercase__						=				'''lower newer'''
									lowercase__						=				['''lo''', '''w''', '''er</w>''', '''n''', '''e''', '''w''', '''er</w>''']
									lowercase__						=				tokenizer.tokenize(UpperCamelCase_							)
									self.assertListEqual(UpperCamelCase_   ,						UpperCamelCase_							)

									lowercase__						=				tokens + [tokenizer.unk_token]
									lowercase__						=				[10, 2, 16, 9, 3, 2, 16, 20]
									self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase_							)   ,						UpperCamelCase_							)

							@require_ftfy
							def 				lowerCamelCase_			(							self:     Dict							)      ->       Any:





									"""simple docstring"""







									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'							):
													lowercase__						=				self.tokenizer_class.from_pretrained(UpperCamelCase_   ,						**UpperCamelCase_							)
													lowercase__						=				self.rust_tokenizer_class.from_pretrained(UpperCamelCase_   ,						**UpperCamelCase_							)

													lowercase__						=				'''A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'''
													lowercase__						=				tokenizer_s.tokenize(UpperCamelCase_							)
													lowercase__						=				tokenizer_r.tokenize(UpperCamelCase_							)

													self.assertListEqual(UpperCamelCase_   ,						UpperCamelCase_							)

													# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
													# with Tilde) encoded in 2 different ways
													lowercase__						=				'''xa\u0303y''' + ''' ''' + '''x\xe3y'''
													lowercase__						=				tokenizer_s.tokenize(UpperCamelCase_							)
													lowercase__						=				tokenizer_r.tokenize(UpperCamelCase_							)

													self.assertListEqual(UpperCamelCase_   ,						UpperCamelCase_							)

													# Test that the tokenization is identical on unicode of space type
													lowercase__						=				[
													    '''\u0009''',  # (horizontal tab, '\t')
													    '''\u000B''',  # (vertical tab)
													    '''\u000C''',  # (form feed)
													    '''\u0020''',  # (space, ' ')
													    '''\u200E''',  # (left-to-right mark):w
													    '''\u200F''',  # (right-to-left mark)
													]
													for unicode_seq in spaces_unicodes:
															lowercase__						=				tokenizer_s.tokenize(UpperCamelCase_							)
															lowercase__						=				tokenizer_r.tokenize(UpperCamelCase_							)

															self.assertListEqual(UpperCamelCase_   ,						UpperCamelCase_							)

													# Test that the tokenization is identical on unicode of line break type
													lowercase__						=				[
													    '''\u000A''',  # (line feed, '\n')
													    '''\r\n''',  # (carriage return and line feed, '\r\n')
													    '''\u000D''',  # (carriage return, '\r')
													    '''\r''',  # (carriage return, '\r')
													    '''\u000D''',  # (carriage return, '\r')
													    '''\u2028''',  # (line separator)
													    '''\u2029''',  # (paragraph separator)
													    # "\u0085", # (next line)
													]

													# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
													# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
													# space (and thus into an empty list).

													for unicode_seq in line_break_unicodes:
															lowercase__						=				tokenizer_s.tokenize(UpperCamelCase_							)
															lowercase__						=				tokenizer_r.tokenize(UpperCamelCase_							)

															self.assertListEqual(UpperCamelCase_   ,						UpperCamelCase_							)

							def 				lowerCamelCase_			(							self:     Optional[Any]							)      ->       int:





									"""simple docstring"""







									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'							):
													lowercase__						=				'''hello'''  # `hello` is a token in the vocabulary of `pretrained_name`
													lowercase__						=				f'{text_of_1_token} {text_of_1_token}'

													lowercase__						=				self.rust_tokenizer_class.from_pretrained(
													    UpperCamelCase_   ,						use_fast=UpperCamelCase_   ,						)
													lowercase__						=				tokenizer_r(UpperCamelCase_   ,						return_offsets_mapping=UpperCamelCase_   ,						add_special_tokens=UpperCamelCase_							)
													self.assertEqual(encoding.offset_mapping[0]   ,						(0, len(UpperCamelCase_							))							)
													self.assertEqual(
													    encoding.offset_mapping[1]   ,						(len(UpperCamelCase_							) + 1, len(UpperCamelCase_							) + 1 + len(UpperCamelCase_							))   ,						)

													lowercase__						=				f' {text}'

													lowercase__						=				self.rust_tokenizer_class.from_pretrained(
													    UpperCamelCase_   ,						use_fast=UpperCamelCase_   ,						)
													lowercase__						=				tokenizer_r(UpperCamelCase_   ,						return_offsets_mapping=UpperCamelCase_   ,						add_special_tokens=UpperCamelCase_							)
													self.assertEqual(encoding.offset_mapping[0]   ,						(1, 1 + len(UpperCamelCase_							))							)
													self.assertEqual(
													    encoding.offset_mapping[1]   ,						(1 + len(UpperCamelCase_							) + 1, 1 + len(UpperCamelCase_							) + 1 + len(UpperCamelCase_							))   ,						)

							def 				lowerCamelCase_			(							self:     Tuple							)      ->       Dict:





									"""simple docstring"""







									with self.assertRaises(UpperCamelCase_							) as context:
											self.rust_tokenizer_class.from_pretrained('''robot-test/old-clip-tokenizer'''							)

									self.assertTrue(
									    context.exception.args[0].startswith(
									        '''The `backend_tokenizer` provided does not match the expected format.'''							)							)

							@require_ftfy
							def 				lowerCamelCase_			(							self:     str							)      ->       int:





									"""simple docstring"""







									super().test_tokenization_python_rust_equals()


							def 				lowerCamelCase_			(							self:     Tuple							)      ->       str:





									"""simple docstring"""







									pass






from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase	=							logging.get_logger(__name__)

lowerCAmelCase	=							{
    'unc-nlp/lxmert-base-uncased': 'https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/config.json',
}


class 	_a							(      UpperCamelCase__      ):
							_lowercase     :      List[Any]  				=			'''lxmert'''
							_lowercase     :      Any  				=			{}

							def __init__(							self:     Any   ,						UpperCamelCase_:     List[Any]=30_522   ,						UpperCamelCase_:     int=768   ,						UpperCamelCase_:     Optional[Any]=12   ,						UpperCamelCase_:     Dict=9_500   ,						UpperCamelCase_:     List[Any]=1_600   ,						UpperCamelCase_:     List[Any]=400   ,						UpperCamelCase_:     List[str]=3_072   ,						UpperCamelCase_:     int="gelu"   ,						UpperCamelCase_:     Any=0.1   ,						UpperCamelCase_:     int=0.1   ,						UpperCamelCase_:     List[str]=512   ,						UpperCamelCase_:     Optional[int]=2   ,						UpperCamelCase_:     Union[str, Any]=0.02   ,						UpperCamelCase_:     Dict=1E-1_2   ,						UpperCamelCase_:     List[Any]=9   ,						UpperCamelCase_:     List[Any]=5   ,						UpperCamelCase_:     Optional[Any]=5   ,						UpperCamelCase_:     str=2_048   ,						UpperCamelCase_:     Dict=4   ,						UpperCamelCase_:     Any=6.67   ,						UpperCamelCase_:     Dict=True   ,						UpperCamelCase_:     List[str]=True   ,						UpperCamelCase_:     Dict=True   ,						UpperCamelCase_:     Any=True   ,						UpperCamelCase_:     Tuple=True   ,						UpperCamelCase_:     Optional[Any]=True   ,						UpperCamelCase_:     Any=True   ,						**UpperCamelCase_:     Union[str, Any]   ,						)      ->       Union[str, Any]:





									"""simple docstring"""







									lowercase__						=				vocab_size
									lowercase__						=				hidden_size
									lowercase__						=				num_attention_heads
									lowercase__						=				hidden_act
									lowercase__						=				intermediate_size
									lowercase__						=				hidden_dropout_prob
									lowercase__						=				attention_probs_dropout_prob
									lowercase__						=				max_position_embeddings
									lowercase__						=				type_vocab_size
									lowercase__						=				initializer_range
									lowercase__						=				layer_norm_eps
									lowercase__						=				num_qa_labels
									lowercase__						=				num_object_labels
									lowercase__						=				num_attr_labels
									lowercase__						=				l_layers
									lowercase__						=				x_layers
									lowercase__						=				r_layers
									lowercase__						=				visual_feat_dim
									lowercase__						=				visual_pos_dim
									lowercase__						=				visual_loss_normalizer
									lowercase__						=				task_matched
									lowercase__						=				task_mask_lm
									lowercase__						=				task_obj_predict
									lowercase__						=				task_qa
									lowercase__						=				visual_obj_loss
									lowercase__						=				visual_attr_loss
									lowercase__						=				visual_feat_loss
									lowercase__						=				{'''vision''': r_layers, '''cross_encoder''': x_layers, '''language''': l_layers}
									super().__init__(**UpperCamelCase_							)


