


"""simple docstring"""







import json
import os
import unittest

from transformers.models.xlm.tokenization_xlm import VOCAB_FILES_NAMES, XLMTokenizer
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin


class 				SCREAMING_SNAKE_CASE__    (		lowercase       ,   unittest.TestCase    ):
       """simple docstring"""


       a  :    List[str]       =XLMTokenizer
       a  :    Dict       =False

       def 	lowercase__							(						self						):






         """simple docstring"""





         super().setUp()

         # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
         lowerCAmelCase					:					int								=							[
             "l",
             "o",
             "w",
             "e",
             "r",
             "s",
             "t",
             "i",
             "d",
             "n",
             "w</w>",
             "r</w>",
             "t</w>",
             "lo",
             "low",
             "er</w>",
             "low</w>",
             "lowest</w>",
             "newer</w>",
             "wider</w>",
             "<unk>",
         ]
         lowerCAmelCase					:					Union[str, Any]								=							dict(zip(snake_case__		,					range(len(snake_case__						)						)						)						)
         lowerCAmelCase					:					Union[str, Any]								=							["l o 123", "lo w 1456", "e r</w> 1789", ""]

         lowerCAmelCase					:					Union[str, Any]								=							os.path.join(self.tmpdirname		,					VOCAB_FILES_NAMES["vocab_file"]						)
         lowerCAmelCase					:					Optional[Any]								=							os.path.join(self.tmpdirname		,					VOCAB_FILES_NAMES["merges_file"]						)
         with open(self.vocab_file		,					"w"						) as fp:
           fp.write(json.dumps(snake_case__						)						)
         with open(self.merges_file		,					"w"						) as fp:
           fp.write("\n".join(snake_case__						)						)

       def 	lowercase__							(						self		,					snake_case__						):






         """simple docstring"""





         lowerCAmelCase					:					Union[str, Any]								=							"lower newer"
         lowerCAmelCase					:					Any								=							"lower newer"
         return input_text, output_text

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					int								=							XLMTokenizer(self.vocab_file		,					self.merges_file						)

         lowerCAmelCase					:					str								=							"lower"
         lowerCAmelCase					:					Union[str, Any]								=							["low", "er</w>"]
         lowerCAmelCase					:					Any								=							tokenizer.tokenize(snake_case__						)
         self.assertListEqual(snake_case__		,					snake_case__						)

         lowerCAmelCase					:					Optional[int]								=							tokens + ["<unk>"]
         lowerCAmelCase					:					List[Any]								=							[14, 15, 20]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__						)		,					snake_case__						)







       @slow
       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					Tuple								=							XLMTokenizer.from_pretrained("xlm-mlm-en-2048"						)

         lowerCAmelCase					:					Optional[int]								=							tokenizer.encode("sequence builders"		,					add_special_tokens=snake_case__						)
         lowerCAmelCase					:					List[str]								=							tokenizer.encode("multi-sequence build"		,					add_special_tokens=snake_case__						)

         lowerCAmelCase					:					List[Any]								=							tokenizer.build_inputs_with_special_tokens(snake_case__						)
         lowerCAmelCase					:					Any								=							tokenizer.build_inputs_with_special_tokens(snake_case__		,					snake_case__						)

         assert encoded_sentence == [0] + text + [1]
         assert encoded_pair == [0] + text + [1] + text_a + [1]









"""simple docstring"""







import unittest

from diffusers.pipelines.pipeline_utils import is_safetensors_compatible


class 				SCREAMING_SNAKE_CASE__    (		unittest.TestCase    ):
       """simple docstring"""


       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					int								=							[
             "safety_checker/pytorch_model.bin",
             "safety_checker/model.safetensors",
             "vae/diffusion_pytorch_model.bin",
             "vae/diffusion_pytorch_model.safetensors",
             "text_encoder/pytorch_model.bin",
             "text_encoder/model.safetensors",
             "unet/diffusion_pytorch_model.bin",
             "unet/diffusion_pytorch_model.safetensors",
         ]
         self.assertTrue(is_safetensors_compatible(snake_case__						)						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					Dict								=							[
             "unet/diffusion_pytorch_model.bin",
             "unet/diffusion_pytorch_model.safetensors",
         ]
         self.assertTrue(is_safetensors_compatible(snake_case__						)						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					Optional[Any]								=							[
             "safety_checker/pytorch_model.bin",
             "safety_checker/model.safetensors",
             "vae/diffusion_pytorch_model.bin",
             "vae/diffusion_pytorch_model.safetensors",
             "text_encoder/pytorch_model.bin",
             "text_encoder/model.safetensors",
             "unet/diffusion_pytorch_model.bin",
             # Removed: 'unet/diffusion_pytorch_model.safetensors',
         ]
         self.assertFalse(is_safetensors_compatible(snake_case__						)						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					List[str]								=							[
             "text_encoder/pytorch_model.bin",
             "text_encoder/model.safetensors",
         ]
         self.assertTrue(is_safetensors_compatible(snake_case__						)						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					Tuple								=							[
             "safety_checker/pytorch_model.bin",
             "safety_checker/model.safetensors",
             "vae/diffusion_pytorch_model.bin",
             "vae/diffusion_pytorch_model.safetensors",
             "text_encoder/pytorch_model.bin",
             # Removed: 'text_encoder/model.safetensors',
             "unet/diffusion_pytorch_model.bin",
             "unet/diffusion_pytorch_model.safetensors",
         ]
         self.assertFalse(is_safetensors_compatible(snake_case__						)						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					List[Any]								=							[
             "safety_checker/pytorch_model.fp16.bin",
             "safety_checker/model.fp16.safetensors",
             "vae/diffusion_pytorch_model.fp16.bin",
             "vae/diffusion_pytorch_model.fp16.safetensors",
             "text_encoder/pytorch_model.fp16.bin",
             "text_encoder/model.fp16.safetensors",
             "unet/diffusion_pytorch_model.fp16.bin",
             "unet/diffusion_pytorch_model.fp16.safetensors",
         ]
         lowerCAmelCase					:					Union[str, Any]								=							"fp16"
         self.assertTrue(is_safetensors_compatible(snake_case__		,					variant=snake_case__						)						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					Optional[int]								=							[
             "unet/diffusion_pytorch_model.fp16.bin",
             "unet/diffusion_pytorch_model.fp16.safetensors",
         ]
         lowerCAmelCase					:					Tuple								=							"fp16"
         self.assertTrue(is_safetensors_compatible(snake_case__		,					variant=snake_case__						)						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					List[Any]								=							[
             "unet/diffusion_pytorch_model.bin",
             "unet/diffusion_pytorch_model.safetensors",
         ]
         lowerCAmelCase					:					Union[str, Any]								=							"fp16"
         self.assertTrue(is_safetensors_compatible(snake_case__		,					variant=snake_case__						)						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					List[Any]								=							[
             "safety_checker/pytorch_model.fp16.bin",
             "safety_checker/model.fp16.safetensors",
             "vae/diffusion_pytorch_model.fp16.bin",
             "vae/diffusion_pytorch_model.fp16.safetensors",
             "text_encoder/pytorch_model.fp16.bin",
             "text_encoder/model.fp16.safetensors",
             "unet/diffusion_pytorch_model.fp16.bin",
             # Removed: 'unet/diffusion_pytorch_model.fp16.safetensors',
         ]
         lowerCAmelCase					:					Dict								=							"fp16"
         self.assertFalse(is_safetensors_compatible(snake_case__		,					variant=snake_case__						)						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					str								=							[
             "text_encoder/pytorch_model.fp16.bin",
             "text_encoder/model.fp16.safetensors",
         ]
         lowerCAmelCase					:					int								=							"fp16"
         self.assertTrue(is_safetensors_compatible(snake_case__		,					variant=snake_case__						)						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					str								=							[
             "text_encoder/pytorch_model.bin",
             "text_encoder/model.safetensors",
         ]
         lowerCAmelCase					:					Tuple								=							"fp16"
         self.assertTrue(is_safetensors_compatible(snake_case__		,					variant=snake_case__						)						)







       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					int								=							[
             "safety_checker/pytorch_model.fp16.bin",
             "safety_checker/model.fp16.safetensors",
             "vae/diffusion_pytorch_model.fp16.bin",
             "vae/diffusion_pytorch_model.fp16.safetensors",
             "text_encoder/pytorch_model.fp16.bin",
             # 'text_encoder/model.fp16.safetensors',
             "unet/diffusion_pytorch_model.fp16.bin",
             "unet/diffusion_pytorch_model.fp16.safetensors",
         ]
         lowerCAmelCase					:					List[Any]								=							"fp16"
         self.assertFalse(is_safetensors_compatible(snake_case__		,					variant=snake_case__						)						)





