'''simple docstring'''





import tempfile
import unittest

import numpy as np

from diffusers import (
    DDIMScheduler,
    DPMSolverMultistepScheduler,
    EulerAncestralDiscreteScheduler,
    EulerDiscreteScheduler,
    LMSDiscreteScheduler,
    OnnxStableDiffusionPipeline,
    PNDMScheduler,
)
from diffusers.utils.testing_utils import is_onnx_available, nightly, require_onnxruntime, require_torch_gpu

from ..test_pipelines_onnx_common import OnnxPipelineTesterMixin


if is_onnx_available():
				import onnxruntime as ort




class 	__A				(			__lowercase		,     unittest.TestCase     ):
				a__	:						Optional[int]	       =  """hf-internal-testing/tiny-random-OnnxStableDiffusionPipeline"""



				def  _lowercase						(self   :   Any       ,      __a   :   Optional[Any]=0			):
									UpperCAmelCase_	   =       np.random.RandomState(_a			)
									UpperCAmelCase_	   =       {
									    "prompt": "A painting of a squirrel eating a burger",
									    "generator": generator,
									    "num_inference_steps": 2,
									    "guidance_scale": 7.5,
									    "output_type": "numpy",
									}
									return inputs



				def  _lowercase						(self   :   str			):
									UpperCAmelCase_	   =       OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint       ,      provider="CPUExecutionProvider"			)
									pipe.set_progress_bar_config(disable=_a			)

									UpperCAmelCase_	   =       self.get_dummy_inputs()
									UpperCAmelCase_	   =       pipe(**_a			).images
									UpperCAmelCase_	   =       image[0, -3:, -3:, -1]

									assert image.shape == (1, 128, 128, 3)
									UpperCAmelCase_	   =       np.array([0.6_50_72, 0.5_84_92, 0.4_82_19, 0.5_55_21, 0.5_31_80, 0.5_59_39, 0.5_06_97, 0.3_98_00, 0.4_64_55]			)

									assert np.abs(image_slice.flatten() - expected_slice			).max() < 1E-2



				def  _lowercase						(self   :   Any			):
									UpperCAmelCase_	   =       OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint       ,      provider="CPUExecutionProvider"			)
									UpperCAmelCase_	   =       PNDMScheduler.from_config(pipe.scheduler.config       ,      skip_prk_steps=_a			)
									pipe.set_progress_bar_config(disable=_a			)

									UpperCAmelCase_	   =       self.get_dummy_inputs()
									UpperCAmelCase_	   =       pipe(**_a			).images
									UpperCAmelCase_	   =       image[0, -3:, -3:, -1]

									assert image.shape == (1, 128, 128, 3)
									UpperCAmelCase_	   =       np.array([0.6_58_63, 0.5_94_25, 0.4_93_26, 0.5_63_13, 0.5_38_75, 0.5_66_27, 0.5_10_65, 0.3_97_77, 0.4_63_30]			)

									assert np.abs(image_slice.flatten() - expected_slice			).max() < 1E-2



				def  _lowercase						(self   :   Union[str, Any]			):
									UpperCAmelCase_	   =       OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint       ,      provider="CPUExecutionProvider"			)
									UpperCAmelCase_	   =       LMSDiscreteScheduler.from_config(pipe.scheduler.config			)
									pipe.set_progress_bar_config(disable=_a			)

									UpperCAmelCase_	   =       self.get_dummy_inputs()
									UpperCAmelCase_	   =       pipe(**_a			).images
									UpperCAmelCase_	   =       image[0, -3:, -3:, -1]

									assert image.shape == (1, 128, 128, 3)
									UpperCAmelCase_	   =       np.array([0.5_37_55, 0.6_07_86, 0.4_74_02, 0.4_94_88, 0.5_18_69, 0.4_98_19, 0.4_79_85, 0.3_89_57, 0.4_42_79]			)

									assert np.abs(image_slice.flatten() - expected_slice			).max() < 1E-2



				def  _lowercase						(self   :   List[Any]			):
									UpperCAmelCase_	   =       OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint       ,      provider="CPUExecutionProvider"			)
									UpperCAmelCase_	   =       EulerDiscreteScheduler.from_config(pipe.scheduler.config			)
									pipe.set_progress_bar_config(disable=_a			)

									UpperCAmelCase_	   =       self.get_dummy_inputs()
									UpperCAmelCase_	   =       pipe(**_a			).images
									UpperCAmelCase_	   =       image[0, -3:, -3:, -1]

									assert image.shape == (1, 128, 128, 3)
									UpperCAmelCase_	   =       np.array([0.5_37_55, 0.6_07_86, 0.4_74_02, 0.4_94_88, 0.5_18_69, 0.4_98_19, 0.4_79_85, 0.3_89_57, 0.4_42_79]			)

									assert np.abs(image_slice.flatten() - expected_slice			).max() < 1E-2



				def  _lowercase						(self   :   Optional[int]			):
									UpperCAmelCase_	   =       OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint       ,      provider="CPUExecutionProvider"			)
									UpperCAmelCase_	   =       EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config			)
									pipe.set_progress_bar_config(disable=_a			)

									UpperCAmelCase_	   =       self.get_dummy_inputs()
									UpperCAmelCase_	   =       pipe(**_a			).images
									UpperCAmelCase_	   =       image[0, -3:, -3:, -1]

									assert image.shape == (1, 128, 128, 3)
									UpperCAmelCase_	   =       np.array([0.5_38_17, 0.6_08_12, 0.4_73_84, 0.4_95_30, 0.5_18_94, 0.4_98_14, 0.4_79_84, 0.3_89_58, 0.4_42_71]			)

									assert np.abs(image_slice.flatten() - expected_slice			).max() < 1E-2



				def  _lowercase						(self   :   Any			):
									UpperCAmelCase_	   =       OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint       ,      provider="CPUExecutionProvider"			)
									UpperCAmelCase_	   =       DPMSolverMultistepScheduler.from_config(pipe.scheduler.config			)
									pipe.set_progress_bar_config(disable=_a			)

									UpperCAmelCase_	   =       self.get_dummy_inputs()
									UpperCAmelCase_	   =       pipe(**_a			).images
									UpperCAmelCase_	   =       image[0, -3:, -3:, -1]

									assert image.shape == (1, 128, 128, 3)
									UpperCAmelCase_	   =       np.array([0.5_38_95, 0.6_08_08, 0.4_79_33, 0.4_96_08, 0.5_18_86, 0.4_99_50, 0.4_80_53, 0.3_89_57, 0.4_42_00]			)

									assert np.abs(image_slice.flatten() - expected_slice			).max() < 1E-2



				def  _lowercase						(self   :   Optional[Any]			):
									UpperCAmelCase_	   =       OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint       ,      provider="CPUExecutionProvider"			)
									pipe.set_progress_bar_config(disable=_a			)

									UpperCAmelCase_	   =       self.get_dummy_inputs()
									UpperCAmelCase_	   =       3 * [inputs["prompt"]]

									# forward
									UpperCAmelCase_	   =       pipe(**_a			)
									UpperCAmelCase_	   =       output.images[0, -3:, -3:, -1]

									UpperCAmelCase_	   =       self.get_dummy_inputs()
									UpperCAmelCase_	   =       3 * [inputs.pop("prompt"			)]

									UpperCAmelCase_	   =       pipe.tokenizer(
									    _a       ,      padding="max_length"       ,      max_length=pipe.tokenizer.model_max_length       ,      truncation=_a       ,      return_tensors="np"       ,      )
									UpperCAmelCase_	   =       text_inputs["input_ids"]

									UpperCAmelCase_	   =       pipe.text_encoder(input_ids=text_inputs.astype(np.intaa			)			)[0]

									UpperCAmelCase_	   =       prompt_embeds

									# forward
									UpperCAmelCase_	   =       pipe(**_a			)
									UpperCAmelCase_	   =       output.images[0, -3:, -3:, -1]

									assert np.abs(image_slice_a.flatten() - image_slice_a.flatten()			).max() < 1E-4






				def  _lowercase						(self   :   List[Any]			):
									UpperCAmelCase_	   =       OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint       ,      provider="CPUExecutionProvider"			)
									pipe.set_progress_bar_config(disable=_a			)

									UpperCAmelCase_	   =       self.get_dummy_inputs()
									UpperCAmelCase_	   =       3 * ["this is a negative prompt"]
									UpperCAmelCase_	   =       negative_prompt
									UpperCAmelCase_	   =       3 * [inputs["prompt"]]

									# forward
									UpperCAmelCase_	   =       pipe(**_a			)
									UpperCAmelCase_	   =       output.images[0, -3:, -3:, -1]

									UpperCAmelCase_	   =       self.get_dummy_inputs()
									UpperCAmelCase_	   =       3 * [inputs.pop("prompt"			)]

									UpperCAmelCase_	   =       []
									for p in [prompt, negative_prompt]:
														UpperCAmelCase_	   =       pipe.tokenizer(
														    _a       ,      padding="max_length"       ,      max_length=pipe.tokenizer.model_max_length       ,      truncation=_a       ,      return_tensors="np"       ,      )
														UpperCAmelCase_	   =       text_inputs["input_ids"]

														embeds.append(pipe.text_encoder(input_ids=text_inputs.astype(np.intaa			)			)[0]			)

									UpperCAmelCase_   ,			UpperCAmelCase_	   =       embeds

									# forward
									UpperCAmelCase_	   =       pipe(**_a			)
									UpperCAmelCase_	   =       output.images[0, -3:, -3:, -1]

									assert np.abs(image_slice_a.flatten() - image_slice_a.flatten()			).max() < 1E-4




@nightly
@require_onnxruntime
@require_torch_gpu
class 	__A				(			unittest.TestCase     ):



				@property
				def  _lowercase						(self   :   Dict			):
									return (
									    "CUDAExecutionProvider",
									    {
									        "gpu_mem_limit": "15000000000",  # 15GB
									        "arena_extend_strategy": "kSameAsRequested",
									    },
									)



				@property
				def  _lowercase						(self   :   Union[str, Any]			):
									UpperCAmelCase_	   =       ort.SessionOptions()
									UpperCAmelCase_	   =       False
									return options



				def  _lowercase						(self   :   List[Any]			):
									# using the PNDM scheduler by default
									UpperCAmelCase_	   =       OnnxStableDiffusionPipeline.from_pretrained(
									    "CompVis/stable-diffusion-v1-4"       ,      revision="onnx"       ,      safety_checker=_a       ,      feature_extractor=_a       ,      provider=self.gpu_provider       ,      sess_options=self.gpu_options       ,      )
									sd_pipe.set_progress_bar_config(disable=_a			)

									UpperCAmelCase_	   =       "A painting of a squirrel eating a burger"
									np.random.seed(0			)
									UpperCAmelCase_	   =       sd_pipe([prompt]       ,      guidance_scale=6.0       ,      num_inference_steps=10       ,      output_type="np"			)
									UpperCAmelCase_	   =       output.images

									UpperCAmelCase_	   =       image[0, -3:, -3:, -1]

									assert image.shape == (1, 512, 512, 3)
									UpperCAmelCase_	   =       np.array([0.04_52, 0.03_90, 0.00_87, 0.03_50, 0.06_17, 0.03_64, 0.05_44, 0.05_23, 0.07_20]			)

									assert np.abs(image_slice.flatten() - expected_slice			).max() < 1E-3



				def  _lowercase						(self   :   Tuple			):
									UpperCAmelCase_	   =       DDIMScheduler.from_pretrained(
									    "runwayml/stable-diffusion-v1-5"       ,      subfolder="scheduler"       ,      revision="onnx"			)
									UpperCAmelCase_	   =       OnnxStableDiffusionPipeline.from_pretrained(
									    "runwayml/stable-diffusion-v1-5"       ,      revision="onnx"       ,      scheduler=_a       ,      safety_checker=_a       ,      feature_extractor=_a       ,      provider=self.gpu_provider       ,      sess_options=self.gpu_options       ,      )
									sd_pipe.set_progress_bar_config(disable=_a			)

									UpperCAmelCase_	   =       "open neural network exchange"
									UpperCAmelCase_	   =       np.random.RandomState(0			)
									UpperCAmelCase_	   =       sd_pipe([prompt]       ,      guidance_scale=7.5       ,      num_inference_steps=10       ,      generator=_a       ,      output_type="np"			)
									UpperCAmelCase_	   =       output.images
									UpperCAmelCase_	   =       image[0, -3:, -3:, -1]

									assert image.shape == (1, 512, 512, 3)
									UpperCAmelCase_	   =       np.array([0.28_67, 0.19_74, 0.14_81, 0.72_94, 0.72_51, 0.66_67, 0.41_94, 0.56_42, 0.64_86]			)

									assert np.abs(image_slice.flatten() - expected_slice			).max() < 1E-3



				def  _lowercase						(self   :   str			):
									UpperCAmelCase_	   =       LMSDiscreteScheduler.from_pretrained(
									    "runwayml/stable-diffusion-v1-5"       ,      subfolder="scheduler"       ,      revision="onnx"			)
									UpperCAmelCase_	   =       OnnxStableDiffusionPipeline.from_pretrained(
									    "runwayml/stable-diffusion-v1-5"       ,      revision="onnx"       ,      scheduler=_a       ,      safety_checker=_a       ,      feature_extractor=_a       ,      provider=self.gpu_provider       ,      sess_options=self.gpu_options       ,      )
									sd_pipe.set_progress_bar_config(disable=_a			)

									UpperCAmelCase_	   =       "open neural network exchange"
									UpperCAmelCase_	   =       np.random.RandomState(0			)
									UpperCAmelCase_	   =       sd_pipe([prompt]       ,      guidance_scale=7.5       ,      num_inference_steps=10       ,      generator=_a       ,      output_type="np"			)
									UpperCAmelCase_	   =       output.images
									UpperCAmelCase_	   =       image[0, -3:, -3:, -1]

									assert image.shape == (1, 512, 512, 3)
									UpperCAmelCase_	   =       np.array([0.23_06, 0.19_59, 0.15_93, 0.65_49, 0.63_94, 0.54_08, 0.50_65, 0.60_10, 0.61_61]			)

									assert np.abs(image_slice.flatten() - expected_slice			).max() < 1E-3



				def  _lowercase						(self   :   int			):
									UpperCAmelCase_	   =       0

									def test_callback_fn(__a   :   Tuple       ,      __a   :   Tuple       ,      __a   :   List[str]			) -> None:
														UpperCAmelCase_	   =       True
														nonlocal number_of_steps
														number_of_steps += 1
														if step == 0:
																			assert latents.shape == (1, 4, 64, 64)
																			UpperCAmelCase_	   =       latents[0, -3:, -3:, -1]
																			UpperCAmelCase_	   =       np.array(
																			    [-0.67_72, -0.38_35, -1.24_56, 0.19_05, -1.09_74, 0.69_67, -1.93_53, 0.01_78, 1.01_67]			)

																			assert np.abs(latents_slice.flatten() - expected_slice			).max() < 1E-3
														elif step == 5:
																			assert latents.shape == (1, 4, 64, 64)
																			UpperCAmelCase_	   =       latents[0, -3:, -3:, -1]
																			UpperCAmelCase_	   =       np.array(
																			    [-0.33_51, 0.22_41, -0.18_37, -0.23_25, -0.65_77, 0.33_93, -0.02_41, 0.58_99, 1.38_75]			)

																			assert np.abs(latents_slice.flatten() - expected_slice			).max() < 1E-3

									UpperCAmelCase_	   =       False

									UpperCAmelCase_	   =       OnnxStableDiffusionPipeline.from_pretrained(
									    "runwayml/stable-diffusion-v1-5"       ,      revision="onnx"       ,      safety_checker=_a       ,      feature_extractor=_a       ,      provider=self.gpu_provider       ,      sess_options=self.gpu_options       ,      )
									pipe.set_progress_bar_config(disable=_a			)

									UpperCAmelCase_	   =       "Andromeda galaxy in a bottle"

									UpperCAmelCase_	   =       np.random.RandomState(0			)
									pipe(
									    prompt=_a       ,      num_inference_steps=5       ,      guidance_scale=7.5       ,      generator=_a       ,      callback=_a       ,      callback_steps=1       ,      )
									assert test_callback_fn.has_been_called
									assert number_of_steps == 6






				def  _lowercase						(self   :   str			):
									UpperCAmelCase_	   =       OnnxStableDiffusionPipeline.from_pretrained(
									    "runwayml/stable-diffusion-v1-5"       ,      revision="onnx"       ,      safety_checker=_a       ,      feature_extractor=_a       ,      provider=self.gpu_provider       ,      sess_options=self.gpu_options       ,      )
									assert isinstance(_a       ,      _a			)
									assert pipe.safety_checker is None

									UpperCAmelCase_	   =       pipe("example prompt"       ,      num_inference_steps=2			).images[0]
									assert image is not None

									# check that there's no error when saving a pipeline with one of the models being None
									with tempfile.TemporaryDirectory() as tmpdirname:
														pipe.save_pretrained(_a			)
														UpperCAmelCase_	   =       OnnxStableDiffusionPipeline.from_pretrained(_a			)

									# sanity check that the pipeline still works
									assert pipe.safety_checker is None
									UpperCAmelCase_	   =       pipe("example prompt"       ,      num_inference_steps=2			).images[0]
									assert image is not None





'''simple docstring'''





import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class 	__A				(			UpperCamelCase__		,     unittest.TestCase     ):
    a__	:						List[Any]	       =  MobileBertTokenizer
    a__	:						str	       =  MobileBertTokenizerFast
    a__	:						List[str]	       =  True
    a__	:						Dict	       =  True
    a__	:						Optional[int]	       =  filter_non_english
    a__	:						int	       =  """google/mobilebert-uncased"""



    def  _lowercase						(self   :   List[str]			):
         super().setUp()

         UpperCAmelCase_	   =       [
             "[UNK]",
             "[CLS]",
             "[SEP]",
             "[PAD]",
             "[MASK]",
             "want",
             "##want",
             "##ed",
             "wa",
             "un",
             "runn",
             "##ing",
             ",",
             "low",
             "lowest",
         ]
         UpperCAmelCase_	   =       os.path.join(self.tmpdirname       ,      VOCAB_FILES_NAMES["vocab_file"]			)
         with open(self.vocab_file       ,      "w"       ,      encoding="utf-8"			) as vocab_writer:
              vocab_writer.write("".join([x + "\n" for x in vocab_tokens]			)			)

         UpperCAmelCase_	   =       [
             (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
             for tokenizer_def in self.tokenizers_list
         ]



    def  _lowercase						(self   :   Tuple       ,      __a   :   str			):
         UpperCAmelCase_	   =       "UNwant\u00E9d,running"
         UpperCAmelCase_	   =       "unwanted, running"
         return input_text, output_text



    def  _lowercase						(self   :   List[Any]			):
         UpperCAmelCase_	   =       self.tokenizer_class(self.vocab_file			)

         UpperCAmelCase_	   =       tokenizer.tokenize("UNwant\u00E9d,running"			)
         self.assertListEqual(__a       ,      ["un", "##want", "##ed", ",", "runn", "##ing"]			)
         self.assertListEqual(tokenizer.convert_tokens_to_ids(__a			)       ,      [9, 6, 7, 12, 10, 11]			)



    def  _lowercase						(self   :   Dict			):
         if not self.test_rust_tokenizer:
              return

         UpperCAmelCase_	   =       self.get_tokenizer()
         UpperCAmelCase_	   =       self.get_rust_tokenizer()

         UpperCAmelCase_	   =       "UNwant\u00E9d,running"

         UpperCAmelCase_	   =       tokenizer.tokenize(__a			)
         UpperCAmelCase_	   =       rust_tokenizer.tokenize(__a			)
         self.assertListEqual(__a       ,      __a			)

         UpperCAmelCase_	   =       tokenizer.encode(__a       ,      add_special_tokens=__a			)
         UpperCAmelCase_	   =       rust_tokenizer.encode(__a       ,      add_special_tokens=__a			)
         self.assertListEqual(__a       ,      __a			)

         UpperCAmelCase_	   =       self.get_rust_tokenizer()
         UpperCAmelCase_	   =       tokenizer.encode(__a			)
         UpperCAmelCase_	   =       rust_tokenizer.encode(__a			)
         self.assertListEqual(__a       ,      __a			)

         # With lower casing
         UpperCAmelCase_	   =       self.get_tokenizer(do_lower_case=__a			)
         UpperCAmelCase_	   =       self.get_rust_tokenizer(do_lower_case=__a			)

         UpperCAmelCase_	   =       "UNwant\u00E9d,running"

         UpperCAmelCase_	   =       tokenizer.tokenize(__a			)
         UpperCAmelCase_	   =       rust_tokenizer.tokenize(__a			)
         self.assertListEqual(__a       ,      __a			)

         UpperCAmelCase_	   =       tokenizer.encode(__a       ,      add_special_tokens=__a			)
         UpperCAmelCase_	   =       rust_tokenizer.encode(__a       ,      add_special_tokens=__a			)
         self.assertListEqual(__a       ,      __a			)

         UpperCAmelCase_	   =       self.get_rust_tokenizer()
         UpperCAmelCase_	   =       tokenizer.encode(__a			)
         UpperCAmelCase_	   =       rust_tokenizer.encode(__a			)
         self.assertListEqual(__a       ,      __a			)



    def  _lowercase						(self   :   Dict			):
         UpperCAmelCase_	   =       BasicTokenizer()

         self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"			)       ,      ["ah", "\u535A", "\u63A8", "zz"]			)



    def  _lowercase						(self   :   Optional[int]			):
         UpperCAmelCase_	   =       BasicTokenizer(do_lower_case=__a			)

         self.assertListEqual(
             tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "			)       ,      ["hello", "!", "how", "are", "you", "?"]			)
         self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)       ,      ["hello"]			)



    def  _lowercase						(self   :   Any			):
         UpperCAmelCase_	   =       BasicTokenizer(do_lower_case=__a       ,      strip_accents=__a			)

         self.assertListEqual(
             tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)       ,      ["hällo", "!", "how", "are", "you", "?"]			)
         self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)       ,      ["h\u00E9llo"]			)



    def  _lowercase						(self   :   Optional[int]			):
         UpperCAmelCase_	   =       BasicTokenizer(do_lower_case=__a       ,      strip_accents=__a			)

         self.assertListEqual(
             tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)       ,      ["hallo", "!", "how", "are", "you", "?"]			)
         self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)       ,      ["hello"]			)



    def  _lowercase						(self   :   Optional[int]			):
         UpperCAmelCase_	   =       BasicTokenizer(do_lower_case=__a			)

         self.assertListEqual(
             tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)       ,      ["hallo", "!", "how", "are", "you", "?"]			)
         self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)       ,      ["hello"]			)



    def  _lowercase						(self   :   Optional[Any]			):
         UpperCAmelCase_	   =       BasicTokenizer(do_lower_case=__a			)

         self.assertListEqual(
             tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "			)       ,      ["HeLLo", "!", "how", "Are", "yoU", "?"]			)



    def  _lowercase						(self   :   Optional[int]			):
         UpperCAmelCase_	   =       BasicTokenizer(do_lower_case=__a       ,      strip_accents=__a			)

         self.assertListEqual(
             tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)       ,      ["HäLLo", "!", "how", "Are", "yoU", "?"]			)



    def  _lowercase						(self   :   Tuple			):
         UpperCAmelCase_	   =       BasicTokenizer(do_lower_case=__a       ,      strip_accents=__a			)

         self.assertListEqual(
             tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)       ,      ["HaLLo", "!", "how", "Are", "yoU", "?"]			)



    def  _lowercase						(self   :   Tuple			):
         UpperCAmelCase_	   =       BasicTokenizer(do_lower_case=__a       ,      never_split=["[UNK]"]			)

         self.assertListEqual(
             tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"			)       ,      ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]			)



    def  _lowercase						(self   :   Any			):
         UpperCAmelCase_	   =       ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

         UpperCAmelCase_	   =       {}
         for i, token in enumerate(__a			):
              UpperCAmelCase_	   =       i
         UpperCAmelCase_	   =       WordpieceTokenizer(vocab=__a       ,      unk_token="[UNK]"			)

         self.assertListEqual(tokenizer.tokenize(""			)       ,      []			)

         self.assertListEqual(tokenizer.tokenize("unwanted running"			)       ,      ["un", "##want", "##ed", "runn", "##ing"]			)

         self.assertListEqual(tokenizer.tokenize("unwantedX running"			)       ,      ["[UNK]", "runn", "##ing"]			)



    def  _lowercase						(self   :   Optional[int]			):
         self.assertTrue(_is_whitespace(" "			)			)
         self.assertTrue(_is_whitespace("\t"			)			)
         self.assertTrue(_is_whitespace("\r"			)			)
         self.assertTrue(_is_whitespace("\n"			)			)
         self.assertTrue(_is_whitespace("\u00A0"			)			)

         self.assertFalse(_is_whitespace("A"			)			)
         self.assertFalse(_is_whitespace("-"			)			)



    def  _lowercase						(self   :   str			):
         self.assertTrue(_is_control("\u0005"			)			)

         self.assertFalse(_is_control("A"			)			)
         self.assertFalse(_is_control(" "			)			)
         self.assertFalse(_is_control("\t"			)			)
         self.assertFalse(_is_control("\r"			)			)



    def  _lowercase						(self   :   Any			):
         self.assertTrue(_is_punctuation("-"			)			)
         self.assertTrue(_is_punctuation("$"			)			)
         self.assertTrue(_is_punctuation("`"			)			)
         self.assertTrue(_is_punctuation("."			)			)

         self.assertFalse(_is_punctuation("A"			)			)
         self.assertFalse(_is_punctuation(" "			)			)



    def  _lowercase						(self   :   Any			):
         UpperCAmelCase_	   =       self.get_tokenizer()
         UpperCAmelCase_	   =       self.get_rust_tokenizer()

         # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
         self.assertListEqual([tokenizer.tokenize(__a			) for t in ["Test", "\xad", "test"]]       ,      [["[UNK]"], [], ["[UNK]"]]			)

         self.assertListEqual(
             [rust_tokenizer.tokenize(__a			) for t in ["Test", "\xad", "test"]]       ,      [["[UNK]"], [], ["[UNK]"]]			)



    @slow
    def  _lowercase						(self   :   Dict			):
         UpperCAmelCase_	   =       self.tokenizer_class.from_pretrained("google/mobilebert-uncased"			)

         UpperCAmelCase_	   =       tokenizer.encode("sequence builders"       ,      add_special_tokens=__a			)
         UpperCAmelCase_	   =       tokenizer.encode("multi-sequence build"       ,      add_special_tokens=__a			)

         UpperCAmelCase_	   =       tokenizer.build_inputs_with_special_tokens(__a			)
         UpperCAmelCase_	   =       tokenizer.build_inputs_with_special_tokens(__a       ,      __a			)

         assert encoded_sentence == [101] + text + [102]
         assert encoded_pair == [101] + text + [102] + text_a + [102]



    def  _lowercase						(self   :   Dict			):
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
                   UpperCAmelCase_	   =       self.rust_tokenizer_class.from_pretrained(__a       ,      **__a			)

                   UpperCAmelCase_	   =       f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
                   UpperCAmelCase_	   =       tokenizer_r.encode_plus(
                       __a       ,      return_attention_mask=__a       ,      return_token_type_ids=__a       ,      return_offsets_mapping=__a       ,      add_special_tokens=__a       ,      )

                   UpperCAmelCase_	   =       tokenizer_r.do_lower_case if hasattr(__a       ,      "do_lower_case"			) else False
                   UpperCAmelCase_	   =       (
                       [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), "A"),
                           ((1, 2), ","),
                           ((3, 5), "na"),
                           ((5, 6), "##ï"),
                           ((6, 8), "##ve"),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), "Allen"),
                           ((21, 23), "##NL"),
                           ((23, 24), "##P"),
                           ((25, 33), "sentence"),
                           ((33, 34), "."),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                       if not do_lower_case
                       else [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), "a"),
                           ((1, 2), ","),
                           ((3, 8), "naive"),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), "allen"),
                           ((21, 23), "##nl"),
                           ((23, 24), "##p"),
                           ((25, 33), "sentence"),
                           ((33, 34), "."),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                   )

                   self.assertEqual(
                       [e[1] for e in expected_results]       ,      tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]			)			)
                   self.assertEqual([e[0] for e in expected_results]       ,      tokens["offset_mapping"]			)






    def  _lowercase						(self   :   Optional[int]			):
         UpperCAmelCase_	   =       ["的", "人", "有"]
         UpperCAmelCase_	   =       "".join(__a			)
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
                   UpperCAmelCase_	   =       True
                   UpperCAmelCase_	   =       self.tokenizer_class.from_pretrained(__a       ,      **__a			)
                   UpperCAmelCase_	   =       self.rust_tokenizer_class.from_pretrained(__a       ,      **__a			)

                   UpperCAmelCase_	   =       tokenizer_p.encode(__a       ,      add_special_tokens=__a			)
                   UpperCAmelCase_	   =       tokenizer_r.encode(__a       ,      add_special_tokens=__a			)

                   UpperCAmelCase_	   =       tokenizer_r.convert_ids_to_tokens(__a			)
                   UpperCAmelCase_	   =       tokenizer_p.convert_ids_to_tokens(__a			)

                   # it is expected that each Chinese character is not preceded by "##"
                   self.assertListEqual(__a       ,      __a			)
                   self.assertListEqual(__a       ,      __a			)

                   UpperCAmelCase_	   =       False
                   UpperCAmelCase_	   =       self.rust_tokenizer_class.from_pretrained(__a       ,      **__a			)
                   UpperCAmelCase_	   =       self.tokenizer_class.from_pretrained(__a       ,      **__a			)

                   UpperCAmelCase_	   =       tokenizer_r.encode(__a       ,      add_special_tokens=__a			)
                   UpperCAmelCase_	   =       tokenizer_p.encode(__a       ,      add_special_tokens=__a			)

                   UpperCAmelCase_	   =       tokenizer_r.convert_ids_to_tokens(__a			)
                   UpperCAmelCase_	   =       tokenizer_p.convert_ids_to_tokens(__a			)

                   # it is expected that only the first Chinese character is not preceded by "##".
                   UpperCAmelCase_	   =       [
                       f"""##{token}""" if idx != 0 else token for idx, token in enumerate(__a			)
                   ]
                   self.assertListEqual(__a       ,      __a			)
                   self.assertListEqual(__a       ,      __a			)




