





import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class SCREAMING_SNAKE_CASE_  (   __SCREAMING_SNAKE_CASE    ,			unittest.TestCase   ):
						__magic_name__:     Union[str, Any]	 =    CLIPTokenizer
						__magic_name__:     int	 =    CLIPTokenizerFast
						__magic_name__:     Optional[Any]	 =    True
						__magic_name__:     Any	 =    {}
						__magic_name__:     Dict	 =    False






						def UpperCAmelCase_			( self : Dict		)							->      Any:







							"""simple docstring"""


							super().setUp()

							# fmt: off
							snake_case_						:     List[Any]     =		["""l""", """o""", """w""", """e""", """r""", """s""", """t""", """i""", """d""", """n""", """lo""", """l</w>""", """w</w>""", """r</w>""", """t</w>""", """low</w>""", """er</w>""", """lowest</w>""", """newer</w>""", """wider""", """<unk>""", """<|startoftext|>""", """<|endoftext|>"""]
							# fmt: on
							snake_case_						:     Tuple     =		dict(zip(__UpperCAmelCase						,				range(len(__UpperCAmelCase		)		)		)		)
							snake_case_						:     str     =		["""#version: 0.2""", """l o""", """lo w</w>""", """e r</w>"""]
							snake_case_						:     Optional[int]     =		{"""unk_token""": """<unk>"""}

							snake_case_						:     str     =		os.path.join(self.tmpdirname						,				VOCAB_FILES_NAMES['vocab_file']		)
							snake_case_						:     Optional[Any]     =		os.path.join(self.tmpdirname						,				VOCAB_FILES_NAMES['merges_file']		)
							with open(self.vocab_file						,				'w'						,				encoding='utf-8'		) as fp:
								fp.write(json.dumps(__UpperCAmelCase		) + '\n'		)
							with open(self.merges_file						,				'w'						,				encoding='utf-8'		) as fp:
								fp.write('\n'.join(__UpperCAmelCase		)		)






						def UpperCAmelCase_			( self : int						,				**_A : Any		)							->      Tuple:







							"""simple docstring"""


							kwargs.update(self.special_tokens_map		)
							return CLIPTokenizer.from_pretrained(self.tmpdirname						,				**__UpperCAmelCase		)






						def UpperCAmelCase_			( self : Any						,				**_A : str		)							->      int:







							"""simple docstring"""


							kwargs.update(self.special_tokens_map		)
							return CLIPTokenizerFast.from_pretrained(self.tmpdirname						,				**__UpperCAmelCase		)






						def UpperCAmelCase_			( self : str						,				_A : int		)							->      Optional[int]:







							"""simple docstring"""


							snake_case_						:     Optional[int]     =		"""lower newer"""
							snake_case_						:     Tuple     =		"""lower newer"""
							return input_text, output_text






						def UpperCAmelCase_			( self : int		)							->      Dict:







							"""simple docstring"""


							snake_case_						:     Optional[int]     =		CLIPTokenizer(self.vocab_file						,				self.merges_file						,				**self.special_tokens_map		)
							snake_case_						:     Union[str, Any]     =		"""lower newer"""
							snake_case_						:     int     =		["""lo""", """w""", """er</w>""", """n""", """e""", """w""", """er</w>"""]
							snake_case_						:     int     =		tokenizer.tokenize(__UpperCAmelCase		)
							self.assertListEqual(__UpperCAmelCase						,				__UpperCAmelCase		)

							snake_case_						:     Dict     =		tokens + [tokenizer.unk_token]
							snake_case_						:     Tuple     =		[10, 2, 16, 9, 3, 2, 16, 20]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase		)						,				__UpperCAmelCase		)






						@require_ftfy
						def UpperCAmelCase_			( self : List[Any]		)							->      Tuple:







							"""simple docstring"""


							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
									snake_case_						:     Optional[Any]     =		self.tokenizer_class.from_pretrained(__UpperCAmelCase						,				**__UpperCAmelCase		)
									snake_case_						:     Tuple     =		self.rust_tokenizer_class.from_pretrained(__UpperCAmelCase						,				**__UpperCAmelCase		)

									snake_case_						:     Optional[int]     =		"""A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."""
									snake_case_						:     Optional[Any]     =		tokenizer_s.tokenize(__UpperCAmelCase		)
									snake_case_						:     Optional[Any]     =		tokenizer_r.tokenize(__UpperCAmelCase		)

									self.assertListEqual(__UpperCAmelCase						,				__UpperCAmelCase		)

									# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
									# with Tilde) encoded in 2 different ways
									snake_case_						:     Dict     =		"""xa\u0303y""" + """ """ + """x\xe3y"""
									snake_case_						:     int     =		tokenizer_s.tokenize(__UpperCAmelCase		)
									snake_case_						:     Dict     =		tokenizer_r.tokenize(__UpperCAmelCase		)

									self.assertListEqual(__UpperCAmelCase						,				__UpperCAmelCase		)

									# Test that the tokenization is identical on unicode of space type
									snake_case_						:     int     =		[
									    """\u0009""",  # (horizontal tab, '\t')
									    """\u000B""",  # (vertical tab)
									    """\u000C""",  # (form feed)
									    """\u0020""",  # (space, ' ')
									    """\u200E""",  # (left-to-right mark):w
									    """\u200F""",  # (right-to-left mark)
									]
									for unicode_seq in spaces_unicodes:
										snake_case_						:     Optional[int]     =		tokenizer_s.tokenize(__UpperCAmelCase		)
										snake_case_						:     Any     =		tokenizer_r.tokenize(__UpperCAmelCase		)

										self.assertListEqual(__UpperCAmelCase						,				__UpperCAmelCase		)

									# Test that the tokenization is identical on unicode of line break type
									snake_case_						:     Union[str, Any]     =		[
									    """\u000A""",  # (line feed, '\n')
									    """\r\n""",  # (carriage return and line feed, '\r\n')
									    """\u000D""",  # (carriage return, '\r')
									    """\r""",  # (carriage return, '\r')
									    """\u000D""",  # (carriage return, '\r')
									    """\u2028""",  # (line separator)
									    """\u2029""",  # (paragraph separator)
									    # "\u0085", # (next line)
									]

									# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
									# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
									# space (and thus into an empty list).

									for unicode_seq in line_break_unicodes:
										snake_case_						:     str     =		tokenizer_s.tokenize(__UpperCAmelCase		)
										snake_case_						:     int     =		tokenizer_r.tokenize(__UpperCAmelCase		)

										self.assertListEqual(__UpperCAmelCase						,				__UpperCAmelCase		)






						def UpperCAmelCase_			( self : List[Any]		)							->      Tuple:







							"""simple docstring"""


							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
									snake_case_						:     Optional[Any]     =		"""hello"""  # `hello` is a token in the vocabulary of `pretrained_name`
									snake_case_						:     str     =		F"""{text_of_1_token} {text_of_1_token}"""

									snake_case_						:     Dict     =		self.rust_tokenizer_class.from_pretrained(
									    __UpperCAmelCase						,				use_fast=__UpperCAmelCase						,				)
									snake_case_						:     Optional[Any]     =		tokenizer_r(__UpperCAmelCase						,				return_offsets_mapping=__UpperCAmelCase						,				add_special_tokens=__UpperCAmelCase		)
									self.assertEqual(encoding.offset_mapping[0]						,				(0, len(__UpperCAmelCase		))		)
									self.assertEqual(
									    encoding.offset_mapping[1]						,				(len(__UpperCAmelCase		) + 1, len(__UpperCAmelCase		) + 1 + len(__UpperCAmelCase		))						,				)

									snake_case_						:     Any     =		F""" {text}"""

									snake_case_						:     Optional[int]     =		self.rust_tokenizer_class.from_pretrained(
									    __UpperCAmelCase						,				use_fast=__UpperCAmelCase						,				)
									snake_case_						:     Dict     =		tokenizer_r(__UpperCAmelCase						,				return_offsets_mapping=__UpperCAmelCase						,				add_special_tokens=__UpperCAmelCase		)
									self.assertEqual(encoding.offset_mapping[0]						,				(1, 1 + len(__UpperCAmelCase		))		)
									self.assertEqual(
									    encoding.offset_mapping[1]						,				(1 + len(__UpperCAmelCase		) + 1, 1 + len(__UpperCAmelCase		) + 1 + len(__UpperCAmelCase		))						,				)






						def UpperCAmelCase_			( self : int		)							->      Tuple:







							"""simple docstring"""


							with self.assertRaises(__UpperCAmelCase		) as context:
								self.rust_tokenizer_class.from_pretrained('robot-test/old-clip-tokenizer'		)

							self.assertTrue(
							    context.exception.args[0].startswith(
							        'The `backend_tokenizer` provided does not match the expected format.'		)		)






						@require_ftfy
						def UpperCAmelCase_			( self : int		)							->      Optional[int]:







							"""simple docstring"""


							super().test_tokenization_python_rust_equals()






						def UpperCAmelCase_			( self : Union[str, Any]		)							->      List[str]:







							"""simple docstring"""


							pass








_SCREAMING_SNAKE_CASE 		=						{
    """Pillow""": """Pillow""",
    """accelerate""": """accelerate>=0.11.0""",
    """compel""": """compel==0.1.8""",
    """black""": """black~=23.1""",
    """datasets""": """datasets""",
    """filelock""": """filelock""",
    """flax""": """flax>=0.4.1""",
    """hf-doc-builder""": """hf-doc-builder>=0.3.0""",
    """huggingface-hub""": """huggingface-hub>=0.13.2""",
    """requests-mock""": """requests-mock==1.10.0""",
    """importlib_metadata""": """importlib_metadata""",
    """invisible-watermark""": """invisible-watermark""",
    """isort""": """isort>=5.5.4""",
    """jax""": """jax>=0.2.8,!=0.3.2""",
    """jaxlib""": """jaxlib>=0.1.65""",
    """Jinja2""": """Jinja2""",
    """k-diffusion""": """k-diffusion>=0.0.12""",
    """torchsde""": """torchsde""",
    """note_seq""": """note_seq""",
    """librosa""": """librosa""",
    """numpy""": """numpy""",
    """omegaconf""": """omegaconf""",
    """parameterized""": """parameterized""",
    """protobuf""": """protobuf>=3.20.3,<4""",
    """pytest""": """pytest""",
    """pytest-timeout""": """pytest-timeout""",
    """pytest-xdist""": """pytest-xdist""",
    """ruff""": """ruff>=0.0.241""",
    """safetensors""": """safetensors""",
    """sentencepiece""": """sentencepiece>=0.1.91,!=0.1.92""",
    """scipy""": """scipy""",
    """onnx""": """onnx""",
    """regex""": """regex!=2019.12.17""",
    """requests""": """requests""",
    """tensorboard""": """tensorboard""",
    """torch""": """torch>=1.4""",
    """torchvision""": """torchvision""",
    """transformers""": """transformers>=4.25.1""",
    """urllib3""": """urllib3<=2.0.0""",
}

