

'''simple docstring'''



import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class lowerCamelCase			(  lowercase_	,						unittest.TestCase						):



			'''simple docstring'''






			__snake_case        =	CLIPTokenizer
			__snake_case        =	CLIPTokenizerFast
			__snake_case        =	True
			__snake_case        =	{}
			__snake_case        =	False


			def lowercase__						(				self	:						Dict			)      ->		str:





									'''simple docstring'''
									super().setUp()

									# fmt: off
									A__					:       List[Any]     =["""l""", """o""", """w""", """e""", """r""", """s""", """t""", """i""", """d""", """n""", """lo""", """l</w>""", """w</w>""", """r</w>""", """t</w>""", """low</w>""", """er</w>""", """lowest</w>""", """newer</w>""", """wider""", """<unk>""", """<|startoftext|>""", """<|endoftext|>"""]
									# fmt: on
									A__					:       Union[str, Any]     =dict(zip(lowerCAmelCase_		,      range(len(lowerCAmelCase_			)			)			)			)
									A__					:       str     =["""#version: 0.2""", """l o""", """lo w</w>""", """e r</w>"""]
									A__					:       int     ={"""unk_token""": """<unk>"""}

									A__					:       Union[str, Any]     =os.path.join(self.tmpdirname		,      VOCAB_FILES_NAMES["""vocab_file"""]			)
									A__					:       Optional[Any]     =os.path.join(self.tmpdirname		,      VOCAB_FILES_NAMES["""merges_file"""]			)
									with open(self.vocab_file		,      """w"""		,      encoding="""utf-8"""			) as fp:
															fp.write(json.dumps(lowerCAmelCase_			) + """\n"""			)
									with open(self.merges_file		,      """w"""		,      encoding="""utf-8"""			) as fp:
															fp.write("""\n""".join(lowerCAmelCase_			)			)


			def lowercase__						(				self	:						int		,      **lowerCAmelCase_	:						List[Any]			)      ->		List[str]:





									'''simple docstring'''
									kwargs.update(self.special_tokens_map			)
									return CLIPTokenizer.from_pretrained(self.tmpdirname		,      **lowerCAmelCase_			)


			def lowercase__						(				self	:						Optional[Any]		,      **lowerCAmelCase_	:						Optional[int]			)      ->		List[str]:





									'''simple docstring'''
									kwargs.update(self.special_tokens_map			)
									return CLIPTokenizerFast.from_pretrained(self.tmpdirname		,      **lowerCAmelCase_			)


			def lowercase__						(				self	:						Optional[Any]		,      lowerCAmelCase_	:						str			)      ->		str:





									'''simple docstring'''
									A__					:       Any     ="""lower newer"""
									A__					:       List[Any]     ="""lower newer"""
									return input_text, output_text


			def lowercase__						(				self	:						Optional[int]			)      ->		List[Any]:





									'''simple docstring'''
									A__					:       Dict     =CLIPTokenizer(self.vocab_file		,      self.merges_file		,      **self.special_tokens_map			)
									A__					:       List[Any]     ="""lower newer"""
									A__					:       Optional[Any]     =["""lo""", """w""", """er</w>""", """n""", """e""", """w""", """er</w>"""]
									A__					:       Optional[int]     =tokenizer.tokenize(lowerCAmelCase_			)
									self.assertListEqual(lowerCAmelCase_		,      lowerCAmelCase_			)

									A__					:       Optional[Any]     =tokens + [tokenizer.unk_token]
									A__					:       Union[str, Any]     =[10, 2, 16, 9, 3, 2, 16, 20]
									self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_			)		,      lowerCAmelCase_			)


			@require_ftfy
			def lowercase__						(				self	:						List[Any]			)      ->		List[str]:





									'''simple docstring'''
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"			):
																					A__					:       Optional[int]     =self.tokenizer_class.from_pretrained(lowerCAmelCase_		,      **lowerCAmelCase_			)
																					A__					:       Union[str, Any]     =self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_		,      **lowerCAmelCase_			)

																					A__					:       str     ="""A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."""
																					A__					:       Optional[int]     =tokenizer_s.tokenize(lowerCAmelCase_			)
																					A__					:       str     =tokenizer_r.tokenize(lowerCAmelCase_			)

																					self.assertListEqual(lowerCAmelCase_		,      lowerCAmelCase_			)

																					# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
																					# with Tilde) encoded in 2 different ways
																					A__					:       Optional[Any]     ="""xa\u0303y""" + """ """ + """x\xe3y"""
																					A__					:       List[Any]     =tokenizer_s.tokenize(lowerCAmelCase_			)
																					A__					:       List[Any]     =tokenizer_r.tokenize(lowerCAmelCase_			)

																					self.assertListEqual(lowerCAmelCase_		,      lowerCAmelCase_			)

																					# Test that the tokenization is identical on unicode of space type
																					A__					:       int     =[
																					    """\u0009""",  # (horizontal tab, '\t')
																					    """\u000B""",  # (vertical tab)
																					    """\u000C""",  # (form feed)
																					    """\u0020""",  # (space, ' ')
																					    """\u200E""",  # (left-to-right mark):w
																					    """\u200F""",  # (right-to-left mark)
																					]
																					for unicode_seq in spaces_unicodes:
																											A__					:       Tuple     =tokenizer_s.tokenize(lowerCAmelCase_			)
																											A__					:       str     =tokenizer_r.tokenize(lowerCAmelCase_			)

																											self.assertListEqual(lowerCAmelCase_		,      lowerCAmelCase_			)

																					# Test that the tokenization is identical on unicode of line break type
																					A__					:       int     =[
																					    """\u000A""",  # (line feed, '\n')
																					    """\r\n""",  # (carriage return and line feed, '\r\n')
																					    """\u000D""",  # (carriage return, '\r')
																					    """\r""",  # (carriage return, '\r')
																					    """\u000D""",  # (carriage return, '\r')
																					    """\u2028""",  # (line separator)
																					    """\u2029""",  # (paragraph separator)
																					    # "\u0085", # (next line)
																					]

																					# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
																					# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
																					# space (and thus into an empty list).

																					for unicode_seq in line_break_unicodes:
																											A__					:       List[Any]     =tokenizer_s.tokenize(lowerCAmelCase_			)
																											A__					:       Tuple     =tokenizer_r.tokenize(lowerCAmelCase_			)

																											self.assertListEqual(lowerCAmelCase_		,      lowerCAmelCase_			)


			def lowercase__						(				self	:						Dict			)      ->		str:





									'''simple docstring'''
									# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"			):
																					A__					:       Dict     ="""hello"""  # `hello` is a token in the vocabulary of `pretrained_name`
																					A__					:       Tuple     =f"{text_of_1_token} {text_of_1_token}"

																					A__					:       Optional[Any]     =self.rust_tokenizer_class.from_pretrained(
																					    lowerCAmelCase_		,      use_fast=lowerCAmelCase_		,      )
																					A__					:       Optional[int]     =tokenizer_r(lowerCAmelCase_		,      return_offsets_mapping=lowerCAmelCase_		,      add_special_tokens=lowerCAmelCase_			)
																					self.assertEqual(encoding.offset_mapping[0]		,      (0, len(lowerCAmelCase_			))			)
																					self.assertEqual(
																					    encoding.offset_mapping[1]		,      (len(lowerCAmelCase_			) + 1, len(lowerCAmelCase_			) + 1 + len(lowerCAmelCase_			))		,      )

																					A__					:       Tuple     =f" {text}"

																					A__					:       str     =self.rust_tokenizer_class.from_pretrained(
																					    lowerCAmelCase_		,      use_fast=lowerCAmelCase_		,      )
																					A__					:       List[Any]     =tokenizer_r(lowerCAmelCase_		,      return_offsets_mapping=lowerCAmelCase_		,      add_special_tokens=lowerCAmelCase_			)
																					self.assertEqual(encoding.offset_mapping[0]		,      (1, 1 + len(lowerCAmelCase_			))			)
																					self.assertEqual(
																					    encoding.offset_mapping[1]		,      (1 + len(lowerCAmelCase_			) + 1, 1 + len(lowerCAmelCase_			) + 1 + len(lowerCAmelCase_			))		,      )


			def lowercase__						(				self	:						str			)      ->		Any:





									'''simple docstring'''
									# Test related to the breaking change introduced in transformers v4.17.0
									# We need to check that an error in raised when the user try to load a previous version of the tokenizer.
									with self.assertRaises(lowerCAmelCase_			) as context:
															self.rust_tokenizer_class.from_pretrained("""robot-test/old-clip-tokenizer"""			)

									self.assertTrue(
									    context.exception.args[0].startswith(
									        """The `backend_tokenizer` provided does not match the expected format."""			)			)


			@require_ftfy
			def lowercase__						(				self	:						Any			)      ->		Any:





									'''simple docstring'''
									super().test_tokenization_python_rust_equals()



			def lowercase__						(				self	:						Dict			)      ->		Optional[int]:





									'''simple docstring'''
									# CLIP always lower cases letters
									pass




'''simple docstring'''



from typing import Dict

from .base import GenericTensor, Pipeline



class lowerCamelCase			(  lowercase_						):



			'''simple docstring'''






			def lowercase__						(				self	:						List[str]		,      lowerCAmelCase_	:						Tuple=None		,      lowerCAmelCase_	:						Dict=None		,      lowerCAmelCase_	:						Any=None		,      **lowerCAmelCase_	:						Union[str, Any]			)      ->		Optional[Any]:





									'''simple docstring'''
									if tokenize_kwargs is None:
															A__					:       List[Any]     ={}

									if truncation is not None:
															if "truncation" in tokenize_kwargs:
																					raise ValueError(
																					    """truncation parameter defined twice (given as keyword argument as well as in tokenize_kwargs)"""			)
															A__					:       str     =truncation

									A__					:       Optional[int]     =tokenize_kwargs

									A__					:       List[Any]     ={}
									if return_tensors is not None:
															A__					:       Any     =return_tensors

									return preprocess_params, {}, postprocess_params


			def lowercase__						(				self	:						int		,      lowerCAmelCase_	:						Union[str, Any]		,      **lowerCAmelCase_	:						Dict			)      ->		Dict[str, GenericTensor]:





									'''simple docstring'''
									A__					:       List[str]     =self.framework
									A__					:       Union[str, Any]     =self.tokenizer(lowerCAmelCase_		,      return_tensors=lowerCAmelCase_		,      **lowerCAmelCase_			)
									return model_inputs


			def lowercase__						(				self	:						Optional[int]		,      lowerCAmelCase_	:						Dict			)      ->		Optional[Any]:





									'''simple docstring'''
									A__					:       Union[str, Any]     =self.model(**lowerCAmelCase_			)
									return model_outputs


			def lowercase__						(				self	:						Tuple		,      lowerCAmelCase_	:						Tuple		,      lowerCAmelCase_	:						Any=False			)      ->		List[Any]:





									'''simple docstring'''
									# [0] is the first available tensor, logits or last_hidden_state.
									if return_tensors:
															return model_outputs[0]
									if self.framework == "pt":
															return model_outputs[0].tolist()
									elif self.framework == "tf":
															return model_outputs[0].numpy().tolist()



			def __call__(				self	:						int		,      *lowerCAmelCase_	:						Any		,      **lowerCAmelCase_	:						Union[str, Any]			)      ->		List[str]:





									'''simple docstring'''
									return super().__call__(*lowerCAmelCase_		,      **lowerCAmelCase_			)

