


import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class 	snake_case_       (SCREAMING_SNAKE_CASE__     ,						unittest.TestCase				):
					UpperCAmelCase__ :   Optional[Any]   				=      CLIPTokenizer
					UpperCAmelCase__ :   int   				=      CLIPTokenizerFast
					UpperCAmelCase__ :   str   				=      True
					UpperCAmelCase__ :   Dict   				=      {}
					UpperCAmelCase__ :   Any   				=      False
					def 			lowerCamelCase__(				self       :Optional[int]				) ->   List[Any]:
												super().setUp()

												# fmt: off
												a__					=					['l', 'o', 'w', 'e', 'r', 's', 't', 'i', 'd', 'n', 'lo', 'l</w>', 'w</w>', 'r</w>', 't</w>', 'low</w>', 'er</w>', 'lowest</w>', 'newer</w>', 'wider', '<unk>', '<|startoftext|>', '<|endoftext|>']
												# fmt: on
												a__					=					dict(zip(A__      ,range(len(A__				)				)				)				)
												a__					=					['#version: 0.2', 'l o', 'lo w</w>', 'e r</w>']
												a__					=					{'unk_token': '<unk>'}

												a__					=					os.path.join(self.tmpdirname      ,VOCAB_FILES_NAMES['vocab_file']				)
												a__					=					os.path.join(self.tmpdirname      ,VOCAB_FILES_NAMES['merges_file']				)
												with open(self.vocab_file      ,'w'      ,encoding='utf-8'				) as fp:
																			fp.write(json.dumps(A__				) + '\n'				)
												with open(self.merges_file      ,'w'      ,encoding='utf-8'				) as fp:
																			fp.write('\n'.join(A__				)				)
					def 			lowerCamelCase__(				self       :Tuple      ,**__snake_case       :List[Any]				) ->   Tuple:
												kwargs.update(self.special_tokens_map				)
												return CLIPTokenizer.from_pretrained(self.tmpdirname      ,**A__				)
					def 			lowerCamelCase__(				self       :Tuple      ,**__snake_case       :Optional[int]				) ->   int:
												kwargs.update(self.special_tokens_map				)
												return CLIPTokenizerFast.from_pretrained(self.tmpdirname      ,**A__				)
					def 			lowerCamelCase__(				self       :List[Any]      ,__snake_case       :Optional[Any]				) ->   Optional[Any]:
												a__					=					'lower newer'
												a__					=					'lower newer'
												return input_text, output_text
					def 			lowerCamelCase__(				self       :int				) ->   List[str]:
												a__					=					CLIPTokenizer(self.vocab_file      ,self.merges_file      ,**self.special_tokens_map				)
												a__					=					'lower newer'
												a__					=					['lo', 'w', 'er</w>', 'n', 'e', 'w', 'er</w>']
												a__					=					tokenizer.tokenize(A__				)
												self.assertListEqual(A__      ,A__				)

												a__					=					tokens + [tokenizer.unk_token]
												a__					=					[10, 2, 16, 9, 3, 2, 16, 20]
												self.assertListEqual(tokenizer.convert_tokens_to_ids(A__				)      ,A__				)
					@require_ftfy
					def 			lowerCamelCase__(				self       :Any				) ->   Any:
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																			with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'				):
																										a__					=					self.tokenizer_class.from_pretrained(A__      ,**A__				)
																										a__					=					self.rust_tokenizer_class.from_pretrained(A__      ,**A__				)

																										a__					=					'A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'
																										a__					=					tokenizer_s.tokenize(A__				)
																										a__					=					tokenizer_r.tokenize(A__				)

																										self.assertListEqual(A__      ,A__				)

																										# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
																										# with Tilde) encoded in 2 different ways
																										a__					=					'xa\u0303y' + ' ' + 'x\xe3y'
																										a__					=					tokenizer_s.tokenize(A__				)
																										a__					=					tokenizer_r.tokenize(A__				)

																										self.assertListEqual(A__      ,A__				)

																										# Test that the tokenization is identical on unicode of space type
																										a__					=					[
																										    '\u0009',  # (horizontal tab, '\t')
																										    '\u000B',  # (vertical tab)
																										    '\u000C',  # (form feed)
																										    '\u0020',  # (space, ' ')
																										    '\u200E',  # (left-to-right mark):w
																										    '\u200F',  # (right-to-left mark)
																										]
																										for unicode_seq in spaces_unicodes:
																																	a__					=					tokenizer_s.tokenize(A__				)
																																	a__					=					tokenizer_r.tokenize(A__				)

																																	self.assertListEqual(A__      ,A__				)

																										# Test that the tokenization is identical on unicode of line break type
																										a__					=					[
																										    '\u000A',  # (line feed, '\n')
																										    '\r\n',  # (carriage return and line feed, '\r\n')
																										    '\u000D',  # (carriage return, '\r')
																										    '\r',  # (carriage return, '\r')
																										    '\u000D',  # (carriage return, '\r')
																										    '\u2028',  # (line separator)
																										    '\u2029',  # (paragraph separator)
																										    # "\u0085", # (next line)
																										]

																										# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
																										# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
																										# space (and thus into an empty list).

																										for unicode_seq in line_break_unicodes:
																																	a__					=					tokenizer_s.tokenize(A__				)
																																	a__					=					tokenizer_r.tokenize(A__				)

																																	self.assertListEqual(A__      ,A__				)
					def 			lowerCamelCase__(				self       :Union[str, Any]				) ->   List[str]:
												# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																			with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'				):
																										a__					=					'hello'  # `hello` is a token in the vocabulary of `pretrained_name`
																										a__					=					F'{text_of_1_token} {text_of_1_token}'

																										a__					=					self.rust_tokenizer_class.from_pretrained(
																										    A__      ,use_fast=A__      ,)
																										a__					=					tokenizer_r(A__      ,return_offsets_mapping=A__      ,add_special_tokens=A__				)
																										self.assertEqual(encoding.offset_mapping[0]      ,(0, len(A__				))				)
																										self.assertEqual(
																										    encoding.offset_mapping[1]      ,(len(A__				) + 1, len(A__				) + 1 + len(A__				))      ,)

																										a__					=					F' {text}'

																										a__					=					self.rust_tokenizer_class.from_pretrained(
																										    A__      ,use_fast=A__      ,)
																										a__					=					tokenizer_r(A__      ,return_offsets_mapping=A__      ,add_special_tokens=A__				)
																										self.assertEqual(encoding.offset_mapping[0]      ,(1, 1 + len(A__				))				)
																										self.assertEqual(
																										    encoding.offset_mapping[1]      ,(1 + len(A__				) + 1, 1 + len(A__				) + 1 + len(A__				))      ,)
					def 			lowerCamelCase__(				self       :List[Any]				) ->   List[str]:
												# Test related to the breaking change introduced in transformers v4.17.0
												# We need to check that an error in raised when the user try to load a previous version of the tokenizer.
												with self.assertRaises(A__				) as context:
																			self.rust_tokenizer_class.from_pretrained('robot-test/old-clip-tokenizer'				)

												self.assertTrue(
												    context.exception.args[0].startswith(
												        'The `backend_tokenizer` provided does not match the expected format.'				)				)
					@require_ftfy
					def 			lowerCamelCase__(				self       :Union[str, Any]				) ->   List[Any]:
												super().test_tokenization_python_rust_equals()



					def 			lowerCamelCase__(				self       :Any				) ->   List[Any]:
												# CLIP always lower cases letters
												pass








from .configuration_bert_masked import MaskedBertConfig
from .modeling_bert_masked import (
    MaskedBertForMultipleChoice,
    MaskedBertForQuestionAnswering,
    MaskedBertForSequenceClassification,
    MaskedBertForTokenClassification,
    MaskedBertModel,
)
from .modules import *




