




import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class        _a    (						__lowerCAmelCase , unittest.TestCase							):
     SCREAMING_SNAKE_CASE_							:   Optional[Any]       =						CLIPTokenizer
     SCREAMING_SNAKE_CASE_							:   Optional[int]       =						CLIPTokenizerFast
     SCREAMING_SNAKE_CASE_							:   Tuple       =						True
     SCREAMING_SNAKE_CASE_							:   List[str]       =						{}
     SCREAMING_SNAKE_CASE_							:   List[str]       =						False






     def     _lowercase     (			self   )							->      List[Any]:
          super().setUp()

          # fmt: off
          _snake_case							=				["l", "o", "w", "e", "r", "s", "t", "i", "d", "n", "lo", "l</w>", "w</w>", "r</w>", "t</w>", "low</w>", "er</w>", "lowest</w>", "newer</w>", "wider", "<unk>", "<|startoftext|>", "<|endoftext|>"]
          # fmt: on
          _snake_case							=				dict(zip(_SCREAMING_SNAKE_CASE ,range(len(_SCREAMING_SNAKE_CASE   )   )   )   )
          _snake_case							=				["#version: 0.2", "l o", "lo w</w>", "e r</w>"]
          _snake_case							=				{"unk_token": "<unk>"}

          _snake_case							=				os.path.join(self.tmpdirname ,VOCAB_FILES_NAMES["vocab_file"]   )
          _snake_case							=				os.path.join(self.tmpdirname ,VOCAB_FILES_NAMES["merges_file"]   )
          with open(self.vocab_file ,"w" ,encoding="utf-8"   ) as fp:
               fp.write(json.dumps(_SCREAMING_SNAKE_CASE   ) + "\n"   )
          with open(self.merges_file ,"w" ,encoding="utf-8"   ) as fp:
               fp.write("\n".join(_SCREAMING_SNAKE_CASE   )   )






     def     _lowercase     (			self ,**_SCREAMING_SNAKE_CASE   )							->      Union[str, Any]:
          kwargs.update(self.special_tokens_map   )
          return CLIPTokenizer.from_pretrained(self.tmpdirname ,**_SCREAMING_SNAKE_CASE   )






     def     _lowercase     (			self ,**_SCREAMING_SNAKE_CASE   )							->      Dict:
          kwargs.update(self.special_tokens_map   )
          return CLIPTokenizerFast.from_pretrained(self.tmpdirname ,**_SCREAMING_SNAKE_CASE   )






     def     _lowercase     (			self ,_SCREAMING_SNAKE_CASE   )							->      str:
          _snake_case							=				"lower newer"
          _snake_case							=				"lower newer"
          return input_text, output_text






     def     _lowercase     (			self   )							->      Union[str, Any]:
          _snake_case							=				CLIPTokenizer(self.vocab_file ,self.merges_file ,**self.special_tokens_map   )
          _snake_case							=				"lower newer"
          _snake_case							=				["lo", "w", "er</w>", "n", "e", "w", "er</w>"]
          _snake_case							=				tokenizer.tokenize(_SCREAMING_SNAKE_CASE   )
          self.assertListEqual(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE   )

          _snake_case							=				tokens + [tokenizer.unk_token]
          _snake_case							=				[10, 2, 16, 9, 3, 2, 16, 20]
          self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE   ) ,_SCREAMING_SNAKE_CASE   )






     @require_ftfy
     def     _lowercase     (			self   )							->      Union[str, Any]:
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""   ):
                    _snake_case							=				self.tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE ,**_SCREAMING_SNAKE_CASE   )
                    _snake_case							=				self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE ,**_SCREAMING_SNAKE_CASE   )

                    _snake_case							=				"A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."
                    _snake_case							=				tokenizer_s.tokenize(_SCREAMING_SNAKE_CASE   )
                    _snake_case							=				tokenizer_r.tokenize(_SCREAMING_SNAKE_CASE   )

                    self.assertListEqual(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE   )

                    # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
                    # with Tilde) encoded in 2 different ways
                    _snake_case							=				"xa\u0303y" + " " + "x\xe3y"
                    _snake_case							=				tokenizer_s.tokenize(_SCREAMING_SNAKE_CASE   )
                    _snake_case							=				tokenizer_r.tokenize(_SCREAMING_SNAKE_CASE   )

                    self.assertListEqual(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE   )

                    # Test that the tokenization is identical on unicode of space type
                    _snake_case							=				[
                        "\u0009",  # (horizontal tab, '\t')
                        "\u000B",  # (vertical tab)
                        "\u000C",  # (form feed)
                        "\u0020",  # (space, ' ')
                        "\u200E",  # (left-to-right mark):w
                        "\u200F",  # (right-to-left mark)
                    ]
                    for unicode_seq in spaces_unicodes:
                         _snake_case							=				tokenizer_s.tokenize(_SCREAMING_SNAKE_CASE   )
                         _snake_case							=				tokenizer_r.tokenize(_SCREAMING_SNAKE_CASE   )

                         self.assertListEqual(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE   )

                    # Test that the tokenization is identical on unicode of line break type
                    _snake_case							=				[
                        "\u000A",  # (line feed, '\n')
                        "\r\n",  # (carriage return and line feed, '\r\n')
                        "\u000D",  # (carriage return, '\r')
                        "\r",  # (carriage return, '\r')
                        "\u000D",  # (carriage return, '\r')
                        "\u2028",  # (line separator)
                        "\u2029",  # (paragraph separator)
                        # "\u0085", # (next line)
                    ]

                    # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
                    # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
                    # space (and thus into an empty list).

                    for unicode_seq in line_break_unicodes:
                         _snake_case							=				tokenizer_s.tokenize(_SCREAMING_SNAKE_CASE   )
                         _snake_case							=				tokenizer_r.tokenize(_SCREAMING_SNAKE_CASE   )

                         self.assertListEqual(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE   )






     def     _lowercase     (			self   )							->      Optional[Any]:
          # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""   ):
                    _snake_case							=				"hello"  # `hello` is a token in the vocabulary of `pretrained_name`
                    _snake_case							=				f"""{text_of_1_token} {text_of_1_token}"""

                    _snake_case							=				self.rust_tokenizer_class.from_pretrained(
                        _SCREAMING_SNAKE_CASE ,use_fast=_SCREAMING_SNAKE_CASE ,)
                    _snake_case							=				tokenizer_r(_SCREAMING_SNAKE_CASE ,return_offsets_mapping=_SCREAMING_SNAKE_CASE ,add_special_tokens=_SCREAMING_SNAKE_CASE   )
                    self.assertEqual(encoding.offset_mapping[0] ,(0, len(_SCREAMING_SNAKE_CASE   ))   )
                    self.assertEqual(
                        encoding.offset_mapping[1] ,(len(_SCREAMING_SNAKE_CASE   ) + 1, len(_SCREAMING_SNAKE_CASE   ) + 1 + len(_SCREAMING_SNAKE_CASE   )) ,)

                    _snake_case							=				f""" {text}"""

                    _snake_case							=				self.rust_tokenizer_class.from_pretrained(
                        _SCREAMING_SNAKE_CASE ,use_fast=_SCREAMING_SNAKE_CASE ,)
                    _snake_case							=				tokenizer_r(_SCREAMING_SNAKE_CASE ,return_offsets_mapping=_SCREAMING_SNAKE_CASE ,add_special_tokens=_SCREAMING_SNAKE_CASE   )
                    self.assertEqual(encoding.offset_mapping[0] ,(1, 1 + len(_SCREAMING_SNAKE_CASE   ))   )
                    self.assertEqual(
                        encoding.offset_mapping[1] ,(1 + len(_SCREAMING_SNAKE_CASE   ) + 1, 1 + len(_SCREAMING_SNAKE_CASE   ) + 1 + len(_SCREAMING_SNAKE_CASE   )) ,)






     def     _lowercase     (			self   )							->      str:
          # Test related to the breaking change introduced in transformers v4.17.0
          # We need to check that an error in raised when the user try to load a previous version of the tokenizer.
          with self.assertRaises(_SCREAMING_SNAKE_CASE   ) as context:
               self.rust_tokenizer_class.from_pretrained("robot-test/old-clip-tokenizer"   )

          self.assertTrue(
              context.exception.args[0].startswith(
                  "The `backend_tokenizer` provided does not match the expected format."   )   )






     @require_ftfy
     def     _lowercase     (			self   )							->      Dict:
          super().test_tokenization_python_rust_equals()






     def     _lowercase     (			self   )							->      Optional[int]:
          # CLIP always lower cases letters
          pass








'''simple docstring'''





from manim import *

class        _a    (						__lowerCAmelCase							):






     def     _lowercase     (			self   )							->      Optional[int]:
          _snake_case							=				Rectangle(height=0.5 ,width=0.5   )
          _snake_case							=				Rectangle(height=0.4_6 ,width=0.4_6   ).set_stroke(width=0   )

          _snake_case							=				[mem.copy() for i in range(6   )]
          _snake_case							=				[mem.copy() for i in range(6   )]
          _snake_case							=				VGroup(*_SCREAMING_SNAKE_CASE   ).arrange(_SCREAMING_SNAKE_CASE ,buff=0   )
          _snake_case							=				VGroup(*_SCREAMING_SNAKE_CASE   ).arrange(_SCREAMING_SNAKE_CASE ,buff=0   )
          _snake_case							=				VGroup(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE   ).arrange(_SCREAMING_SNAKE_CASE ,buff=0   )
          _snake_case							=				Text("CPU" ,font_size=24   )
          _snake_case							=				Group(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE   ).arrange(_SCREAMING_SNAKE_CASE ,buff=0.5 ,aligned_edge=_SCREAMING_SNAKE_CASE   )
          cpu.move_to([-2.5, -0.5, 0]   )
          self.add(_SCREAMING_SNAKE_CASE   )

          _snake_case							=				[mem.copy() for i in range(4   )]
          _snake_case							=				VGroup(*_SCREAMING_SNAKE_CASE   ).arrange(_SCREAMING_SNAKE_CASE ,buff=0   )
          _snake_case							=				Text("GPU" ,font_size=24   )
          _snake_case							=				Group(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE   ).arrange(_SCREAMING_SNAKE_CASE ,buff=0.5 ,aligned_edge=_SCREAMING_SNAKE_CASE   )
          gpu.move_to([-1, -1, 0]   )
          self.add(_SCREAMING_SNAKE_CASE   )

          _snake_case							=				[mem.copy() for i in range(6   )]
          _snake_case							=				VGroup(*_SCREAMING_SNAKE_CASE   ).arrange(_SCREAMING_SNAKE_CASE ,buff=0   )

          _snake_case							=				Text("Model" ,font_size=24   )
          _snake_case							=				Group(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE   ).arrange(_SCREAMING_SNAKE_CASE ,buff=0.5 ,aligned_edge=_SCREAMING_SNAKE_CASE   )
          model.move_to([3, -1.0, 0]   )
          self.add(_SCREAMING_SNAKE_CASE   )

          _snake_case							=				[]
          for i, rect in enumerate(_SCREAMING_SNAKE_CASE   ):
               rect.set_stroke(_SCREAMING_SNAKE_CASE   )
               # target = fill.copy().set_fill(YELLOW, opacity=0.7)
               # target.move_to(rect)
               # self.add(target)

               _snake_case							=				Rectangle(height=0.4_6 / 4 ,width=0.4_6 / 3   ).set_stroke(width=0.0   ).set_fill(_SCREAMING_SNAKE_CASE ,opacity=0.7   )

               if i == 0:
                    cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT   ) ,buff=0.0_2 ,direction=_SCREAMING_SNAKE_CASE   )
                    cpu_target.set_x(cpu_target.get_x() + 0.1   )
               elif i == 3:
                    cpu_target.next_to(cpu_targs[0] ,direction=_SCREAMING_SNAKE_CASE ,buff=0.0   )
               else:
                    cpu_target.next_to(cpu_targs[i - 1] ,direction=_SCREAMING_SNAKE_CASE ,buff=0.0   )
               self.add(_SCREAMING_SNAKE_CASE   )
               cpu_targs.append(_SCREAMING_SNAKE_CASE   )

          _snake_case							=				[mem.copy() for i in range(6   )]
          _snake_case							=				VGroup(*_SCREAMING_SNAKE_CASE   ).arrange(_SCREAMING_SNAKE_CASE ,buff=0   )

          _snake_case							=				Text("Loaded Checkpoint" ,font_size=24   )
          _snake_case							=				Group(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE   ).arrange(_SCREAMING_SNAKE_CASE ,aligned_edge=_SCREAMING_SNAKE_CASE ,buff=0.4   )
          checkpoint.move_to([3, 0.5, 0]   )

          _snake_case							=				Square(side_length=2.2   )
          key.move_to([-5, 2, 0]   )

          _snake_case							=				MarkupText(
              f"""<b>Key:</b>\n\n<span fgcolor='{YELLOW}'>●</span> Empty Model""" ,font_size=18 ,)

          key_text.move_to([-5, 2.4, 0]   )

          self.add(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE   )

          _snake_case							=				MarkupText(
              f"""<span fgcolor='{BLUE}'>●</span> Checkpoint""" ,font_size=18 ,)

          blue_text.next_to(_SCREAMING_SNAKE_CASE ,DOWN * 2.4 ,aligned_edge=key_text.get_left()   )

          _snake_case							=				MarkupText(
              f"""Next, a <i><span fgcolor=\"{BLUE}\">second</span></i> model is loaded into memory,\nwith the weights of a <span fgcolor=\"{BLUE}\">single shard</span>.""" ,font_size=24 ,)
          step_a.move_to([2, 2, 0]   )
          self.play(Write(_SCREAMING_SNAKE_CASE   ) ,Write(_SCREAMING_SNAKE_CASE   )   )

          self.play(Write(_SCREAMING_SNAKE_CASE ,run_time=1   ) ,Create(_SCREAMING_SNAKE_CASE ,run_time=1   )   )

          _snake_case							=				[]
          _snake_case							=				[]
          for i, rect in enumerate(_SCREAMING_SNAKE_CASE   ):
               _snake_case							=				fill.copy().set_fill(_SCREAMING_SNAKE_CASE ,opacity=0.7   )
               target.move_to(_SCREAMING_SNAKE_CASE   )
               first_animations.append(GrowFromCenter(_SCREAMING_SNAKE_CASE ,run_time=1   )   )

               _snake_case							=				target.copy()
               cpu_target.generate_target()
               if i < 5:
                    cpu_target.target.move_to(cpu_left_col_base[i + 1]   )
               else:
                    cpu_target.target.move_to(cpu_right_col_base[i - 5]   )
               second_animations.append(MoveToTarget(_SCREAMING_SNAKE_CASE ,run_time=1.5   )   )

          self.play(*_SCREAMING_SNAKE_CASE   )
          self.play(*_SCREAMING_SNAKE_CASE   )
          self.wait()


