'''simple docstring'''





from typing import Optional

import torch
import torch.utils.checkpoint
from torch import Tensor, nn
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss

from ...activations import ACTaFN
from ...file_utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward
from ...modeling_outputs import (
    BaseModelOutputWithNoAttention,
    BaseModelOutputWithPoolingAndNoAttention,
    ImageClassifierOutputWithNoAttention,
)
from ...modeling_utils import PreTrainedModel
from ...utils import logging
from .configuration_regnet import RegNetConfig


SCREAMING_SNAKE_CASE_:					Optional[Any]  =logging.get_logger(__name__)

# General docstring
SCREAMING_SNAKE_CASE_:					Tuple  ='RegNetConfig'

# Base docstring
SCREAMING_SNAKE_CASE_:					Union[str, Any]  ='facebook/regnet-y-040'
SCREAMING_SNAKE_CASE_:					str  =[1, 10_88, 7, 7]

# Image classification docstring
SCREAMING_SNAKE_CASE_:					Any  ='facebook/regnet-y-040'
SCREAMING_SNAKE_CASE_:					Dict  ='tabby, tabby cat'

SCREAMING_SNAKE_CASE_:					Union[str, Any]  =[
    'facebook/regnet-y-040',
    # See all regnet models at https://huggingface.co/models?filter=regnet
]




class 	__A				(			nn.Module     ):



				def __init__(self   :   Dict       ,      __a   :   int       ,      __a   :   int       ,      __a   :   int = 3       ,      __a   :   int = 1       ,      __a   :   int = 1       ,      __a   :   Optional[str] = "relu"       ,      ):
									super().__init__()
									UpperCAmelCase_	   =       nn.Convad(
									    __lowercase       ,      __lowercase       ,      kernel_size=__lowercase       ,      stride=__lowercase       ,      padding=kernel_size // 2       ,      groups=__lowercase       ,      bias=__lowercase       ,      )
									UpperCAmelCase_	   =       nn.BatchNormad(__lowercase			)
									UpperCAmelCase_	   =       ACTaFN[activation] if activation is not None else nn.Identity()






				def  _lowercase						(self   :   str       ,      __a   :   Optional[int]			):
									UpperCAmelCase_	   =       self.convolution(__lowercase			)
									UpperCAmelCase_	   =       self.normalization(__lowercase			)
									UpperCAmelCase_	   =       self.activation(__lowercase			)
									return hidden_state




class 	__A				(			nn.Module     ):



				def __init__(self   :   Any       ,      __a   :   RegNetConfig			):
									super().__init__()
									UpperCAmelCase_	   =       RegNetConvLayer(
									    config.num_channels       ,      config.embedding_size       ,      kernel_size=3       ,      stride=2       ,      activation=config.hidden_act			)
									UpperCAmelCase_	   =       config.num_channels






				def  _lowercase						(self   :   Union[str, Any]       ,      __a   :   str			):
									UpperCAmelCase_	   =       pixel_values.shape[1]
									if num_channels != self.num_channels:
														raise ValueError(
														    "Make sure that the channel dimension of the pixel values match with the one set in the configuration."			)
									UpperCAmelCase_	   =       self.embedder(__lowercase			)
									return hidden_state




class 	__A				(			nn.Module     ):



				def __init__(self   :   List[Any]       ,      __a   :   int       ,      __a   :   int       ,      __a   :   int = 2			):
									super().__init__()
									UpperCAmelCase_	   =       nn.Convad(__lowercase       ,      __lowercase       ,      kernel_size=1       ,      stride=__lowercase       ,      bias=__lowercase			)
									UpperCAmelCase_	   =       nn.BatchNormad(__lowercase			)






				def  _lowercase						(self   :   Tuple       ,      __a   :   Tensor			):
									UpperCAmelCase_	   =       self.convolution(__lowercase			)
									UpperCAmelCase_	   =       self.normalization(__lowercase			)
									return hidden_state




class 	__A				(			nn.Module     ):



				def __init__(self   :   Union[str, Any]       ,      __a   :   int       ,      __a   :   int			):
									super().__init__()

									UpperCAmelCase_	   =       nn.AdaptiveAvgPoolad((1, 1)			)
									UpperCAmelCase_	   =       nn.Sequential(
									    nn.Convad(__lowercase       ,      __lowercase       ,      kernel_size=1			)       ,      nn.ReLU()       ,      nn.Convad(__lowercase       ,      __lowercase       ,      kernel_size=1			)       ,      nn.Sigmoid()       ,      )






				def  _lowercase						(self   :   str       ,      __a   :   Optional[Any]			):
									# b c h w -> b c 1 1
									UpperCAmelCase_	   =       self.pooler(__lowercase			)
									UpperCAmelCase_	   =       self.attention(__lowercase			)
									UpperCAmelCase_	   =       hidden_state * attention
									return hidden_state




class 	__A				(			nn.Module     ):



				def __init__(self   :   Any       ,      __a   :   RegNetConfig       ,      __a   :   int       ,      __a   :   int       ,      __a   :   int = 1			):
									super().__init__()
									UpperCAmelCase_	   =       in_channels != out_channels or stride != 1
									UpperCAmelCase_	   =       max(1       ,      out_channels // config.groups_width			)
									UpperCAmelCase_	   =       (
									    RegNetShortCut(__lowercase       ,      __lowercase       ,      stride=__lowercase			) if should_apply_shortcut else nn.Identity()
									)
									UpperCAmelCase_	   =       nn.Sequential(
									    RegNetConvLayer(__lowercase       ,      __lowercase       ,      kernel_size=1       ,      activation=config.hidden_act			)       ,      RegNetConvLayer(__lowercase       ,      __lowercase       ,      stride=__lowercase       ,      groups=__lowercase       ,      activation=config.hidden_act			)       ,      RegNetConvLayer(__lowercase       ,      __lowercase       ,      kernel_size=1       ,      activation=__lowercase			)       ,      )
									UpperCAmelCase_	   =       ACTaFN[config.hidden_act]






				def  _lowercase						(self   :   Tuple       ,      __a   :   Optional[Any]			):
									UpperCAmelCase_	   =       hidden_state
									UpperCAmelCase_	   =       self.layer(__lowercase			)
									UpperCAmelCase_	   =       self.shortcut(__lowercase			)
									hidden_state += residual
									UpperCAmelCase_	   =       self.activation(__lowercase			)
									return hidden_state




class 	__A				(			nn.Module     ):



				def __init__(self   :   List[Any]       ,      __a   :   RegNetConfig       ,      __a   :   int       ,      __a   :   int       ,      __a   :   int = 1			):
									super().__init__()
									UpperCAmelCase_	   =       in_channels != out_channels or stride != 1
									UpperCAmelCase_	   =       max(1       ,      out_channels // config.groups_width			)
									UpperCAmelCase_	   =       (
									    RegNetShortCut(__lowercase       ,      __lowercase       ,      stride=__lowercase			) if should_apply_shortcut else nn.Identity()
									)
									UpperCAmelCase_	   =       nn.Sequential(
									    RegNetConvLayer(__lowercase       ,      __lowercase       ,      kernel_size=1       ,      activation=config.hidden_act			)       ,      RegNetConvLayer(__lowercase       ,      __lowercase       ,      stride=__lowercase       ,      groups=__lowercase       ,      activation=config.hidden_act			)       ,      RegNetSELayer(__lowercase       ,      reduced_channels=int(round(in_channels / 4			)			)			)       ,      RegNetConvLayer(__lowercase       ,      __lowercase       ,      kernel_size=1       ,      activation=__lowercase			)       ,      )
									UpperCAmelCase_	   =       ACTaFN[config.hidden_act]






				def  _lowercase						(self   :   List[str]       ,      __a   :   Union[str, Any]			):
									UpperCAmelCase_	   =       hidden_state
									UpperCAmelCase_	   =       self.layer(__lowercase			)
									UpperCAmelCase_	   =       self.shortcut(__lowercase			)
									hidden_state += residual
									UpperCAmelCase_	   =       self.activation(__lowercase			)
									return hidden_state




class 	__A				(			nn.Module     ):



				def __init__(self   :   Optional[Any]       ,      __a   :   RegNetConfig       ,      __a   :   int       ,      __a   :   int       ,      __a   :   int = 2       ,      __a   :   int = 2       ,      ):
									super().__init__()

									UpperCAmelCase_	   =       RegNetXLayer if config.layer_type == '''x''' else RegNetYLayer

									UpperCAmelCase_	   =       nn.Sequential(
									    # downsampling is done in the first layer with stride of 2
									    layer(
									        __lowercase       ,      __lowercase       ,      __lowercase       ,      stride=__lowercase       ,      )       ,      *[layer(__lowercase       ,      __lowercase       ,      __lowercase			) for _ in range(depth - 1			)]       ,      )






				def  _lowercase						(self   :   str       ,      __a   :   Dict			):
									UpperCAmelCase_	   =       self.layers(__lowercase			)
									return hidden_state




class 	__A				(			nn.Module     ):



				def __init__(self   :   Optional[int]       ,      __a   :   RegNetConfig			):
									super().__init__()
									UpperCAmelCase_	   =       nn.ModuleList([]			)
									# based on `downsample_in_first_stage`, the first layer of the first stage may or may not downsample the input
									self.stages.append(
									    RegNetStage(
									        __lowercase       ,      config.embedding_size       ,      config.hidden_sizes[0]       ,      stride=2 if config.downsample_in_first_stage else 1       ,      depth=config.depths[0]       ,      )			)
									UpperCAmelCase_	   =       zip(config.hidden_sizes       ,      config.hidden_sizes[1:]			)
									for (in_channels, out_channels), depth in zip(__lowercase       ,      config.depths[1:]			):
														self.stages.append(RegNetStage(__lowercase       ,      __lowercase       ,      __lowercase       ,      depth=__lowercase			)			)






				def  _lowercase						(self   :   Union[str, Any]       ,      __a   :   Tensor       ,      __a   :   bool = False       ,      __a   :   bool = True			):
									UpperCAmelCase_	   =       () if output_hidden_states else None

									for stage_module in self.stages:
														if output_hidden_states:
																			UpperCAmelCase_	   =       hidden_states + (hidden_state,)

														UpperCAmelCase_	   =       stage_module(__lowercase			)

									if output_hidden_states:
														UpperCAmelCase_	   =       hidden_states + (hidden_state,)

									if not return_dict:
														return tuple(v for v in [hidden_state, hidden_states] if v is not None			)

									return BaseModelOutputWithNoAttention(last_hidden_state=__lowercase       ,      hidden_states=__lowercase			)




class 	__A				(			__lowerCamelCase     ):
				a__	:						Optional[Any]	       =  RegNetConfig
				a__	:						int	       =  """regnet"""
				a__	:						Union[str, Any]	       =  """pixel_values"""
				a__	:						int	       =  True



				def  _lowercase						(self   :   Tuple       ,      __a   :   int			):
									if isinstance(__lowercase       ,      nn.Convad			):
														nn.init.kaiming_normal_(module.weight       ,      mode="fan_out"       ,      nonlinearity="relu"			)
									elif isinstance(__lowercase       ,      (nn.BatchNormad, nn.GroupNorm)			):
														nn.init.constant_(module.weight       ,      1			)
														nn.init.constant_(module.bias       ,      0			)






				def  _lowercase						(self   :   List[Any]       ,      __a   :   Optional[int]       ,      __a   :   List[str]=False			):
									if isinstance(__lowercase       ,      __lowercase			):
														UpperCAmelCase_	   =       value




SCREAMING_SNAKE_CASE_:					Union[str, Any]  =r'\n    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it\n    as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and\n    behavior.\n\n    Parameters:\n        config ([`RegNetConfig`]): Model configuration class with all the parameters of the model.\n            Initializing with a config file does not load the weights associated with the model, only the\n            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.\n'

SCREAMING_SNAKE_CASE_:					Optional[Any]  =r'\n    Args:\n        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):\n            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See\n            [`ConvNextImageProcessor.__call__`] for details.\n\n        output_hidden_states (`bool`, *optional*):\n            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for\n            more detail.\n        return_dict (`bool`, *optional*):\n            Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple.\n'




@add_start_docstrings(
    """The bare RegNet model outputting raw features without any specific head on top."""		,     __lowerCamelCase		,     )
# Copied from transformers.models.resnet.modeling_resnet.ResNetModel with RESNET->REGNET,ResNet->RegNet
class 	__A				(			__lowerCamelCase     ):



				def __init__(self   :   List[Any]       ,      __a   :   str			):
									super().__init__(__lowercase			)
									UpperCAmelCase_	   =       config
									UpperCAmelCase_	   =       RegNetEmbeddings(__lowercase			)
									UpperCAmelCase_	   =       RegNetEncoder(__lowercase			)
									UpperCAmelCase_	   =       nn.AdaptiveAvgPoolad((1, 1)			)
									# Initialize weights and apply final processing
									self.post_init()






				@add_start_docstrings_to_model_forward(__lowercase			)
				@add_code_sample_docstrings(
				    checkpoint=_CHECKPOINT_FOR_DOC       ,      output_type=__lowercase       ,      config_class=_CONFIG_FOR_DOC       ,      modality="vision"       ,      expected_output=_EXPECTED_OUTPUT_SHAPE       ,      )
				def  _lowercase						(self   :   List[str]       ,      __a   :   Tensor       ,      __a   :   Optional[bool] = None       ,      __a   :   Optional[bool] = None			):
									UpperCAmelCase_	   =       (
									    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
									)
									UpperCAmelCase_	   =       return_dict if return_dict is not None else self.config.use_return_dict

									UpperCAmelCase_	   =       self.embedder(__lowercase			)

									UpperCAmelCase_	   =       self.encoder(
									    __lowercase       ,      output_hidden_states=__lowercase       ,      return_dict=__lowercase			)

									UpperCAmelCase_	   =       encoder_outputs[0]

									UpperCAmelCase_	   =       self.pooler(__lowercase			)

									if not return_dict:
														return (last_hidden_state, pooled_output) + encoder_outputs[1:]

									return BaseModelOutputWithPoolingAndNoAttention(
									    last_hidden_state=__lowercase       ,      pooler_output=__lowercase       ,      hidden_states=encoder_outputs.hidden_states       ,      )




@add_start_docstrings(
    """
    RegNet Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for
    ImageNet.
    """		,     __lowerCamelCase		,     )
# Copied from transformers.models.resnet.modeling_resnet.ResNetForImageClassification with RESNET->REGNET,ResNet->RegNet,resnet->regnet
class 	__A				(			__lowerCamelCase     ):



				def __init__(self   :   int       ,      __a   :   Union[str, Any]			):
									super().__init__(__lowercase			)
									UpperCAmelCase_	   =       config.num_labels
									UpperCAmelCase_	   =       RegNetModel(__lowercase			)
									# classification head
									UpperCAmelCase_	   =       nn.Sequential(
									    nn.Flatten()       ,      nn.Linear(config.hidden_sizes[-1]       ,      config.num_labels			) if config.num_labels > 0 else nn.Identity()       ,      )
									# initialize weights and apply final processing
									self.post_init()






				@add_start_docstrings_to_model_forward(__lowercase			)
				@add_code_sample_docstrings(
				    checkpoint=_IMAGE_CLASS_CHECKPOINT       ,      output_type=__lowercase       ,      config_class=_CONFIG_FOR_DOC       ,      expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT       ,      )
				def  _lowercase						(self   :   Dict       ,      __a   :   Optional[torch.FloatTensor] = None       ,      __a   :   Optional[torch.LongTensor] = None       ,      __a   :   Optional[bool] = None       ,      __a   :   Optional[bool] = None       ,      ):
									UpperCAmelCase_	   =       return_dict if return_dict is not None else self.config.use_return_dict

									UpperCAmelCase_	   =       self.regnet(__lowercase       ,      output_hidden_states=__lowercase       ,      return_dict=__lowercase			)

									UpperCAmelCase_	   =       outputs.pooler_output if return_dict else outputs[1]

									UpperCAmelCase_	   =       self.classifier(__lowercase			)

									UpperCAmelCase_	   =       None

									if labels is not None:
														if self.config.problem_type is None:
																			if self.num_labels == 1:
																								UpperCAmelCase_	   =       '''regression'''
																			elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
																								UpperCAmelCase_	   =       '''single_label_classification'''
																			else:
																								UpperCAmelCase_	   =       '''multi_label_classification'''
														if self.config.problem_type == "regression":
																			UpperCAmelCase_	   =       MSELoss()
																			if self.num_labels == 1:
																								UpperCAmelCase_	   =       loss_fct(logits.squeeze()       ,      labels.squeeze()			)
																			else:
																								UpperCAmelCase_	   =       loss_fct(__lowercase       ,      __lowercase			)
														elif self.config.problem_type == "single_label_classification":
																			UpperCAmelCase_	   =       CrossEntropyLoss()
																			UpperCAmelCase_	   =       loss_fct(logits.view(-1       ,      self.num_labels			)       ,      labels.view(-1			)			)
														elif self.config.problem_type == "multi_label_classification":
																			UpperCAmelCase_	   =       BCEWithLogitsLoss()
																			UpperCAmelCase_	   =       loss_fct(__lowercase       ,      __lowercase			)

									if not return_dict:
														UpperCAmelCase_	   =       (logits,) + outputs[2:]
														return (loss,) + output if loss is not None else output

									return ImageClassifierOutputWithNoAttention(loss=__lowercase       ,      logits=__lowercase       ,      hidden_states=outputs.hidden_states			)





'''simple docstring'''





import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class 	__A				(			UpperCamelCase__		,     unittest.TestCase     ):
    a__	:						List[Any]	       =  MobileBertTokenizer
    a__	:						str	       =  MobileBertTokenizerFast
    a__	:						List[str]	       =  True
    a__	:						Dict	       =  True
    a__	:						Optional[int]	       =  filter_non_english
    a__	:						int	       =  """google/mobilebert-uncased"""



    def  _lowercase						(self   :   List[str]			):
         super().setUp()

         UpperCAmelCase_	   =       [
             "[UNK]",
             "[CLS]",
             "[SEP]",
             "[PAD]",
             "[MASK]",
             "want",
             "##want",
             "##ed",
             "wa",
             "un",
             "runn",
             "##ing",
             ",",
             "low",
             "lowest",
         ]
         UpperCAmelCase_	   =       os.path.join(self.tmpdirname       ,      VOCAB_FILES_NAMES["vocab_file"]			)
         with open(self.vocab_file       ,      "w"       ,      encoding="utf-8"			) as vocab_writer:
              vocab_writer.write("".join([x + "\n" for x in vocab_tokens]			)			)

         UpperCAmelCase_	   =       [
             (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
             for tokenizer_def in self.tokenizers_list
         ]



    def  _lowercase						(self   :   Tuple       ,      __a   :   str			):
         UpperCAmelCase_	   =       "UNwant\u00E9d,running"
         UpperCAmelCase_	   =       "unwanted, running"
         return input_text, output_text



    def  _lowercase						(self   :   List[Any]			):
         UpperCAmelCase_	   =       self.tokenizer_class(self.vocab_file			)

         UpperCAmelCase_	   =       tokenizer.tokenize("UNwant\u00E9d,running"			)
         self.assertListEqual(__a       ,      ["un", "##want", "##ed", ",", "runn", "##ing"]			)
         self.assertListEqual(tokenizer.convert_tokens_to_ids(__a			)       ,      [9, 6, 7, 12, 10, 11]			)



    def  _lowercase						(self   :   Dict			):
         if not self.test_rust_tokenizer:
              return

         UpperCAmelCase_	   =       self.get_tokenizer()
         UpperCAmelCase_	   =       self.get_rust_tokenizer()

         UpperCAmelCase_	   =       "UNwant\u00E9d,running"

         UpperCAmelCase_	   =       tokenizer.tokenize(__a			)
         UpperCAmelCase_	   =       rust_tokenizer.tokenize(__a			)
         self.assertListEqual(__a       ,      __a			)

         UpperCAmelCase_	   =       tokenizer.encode(__a       ,      add_special_tokens=__a			)
         UpperCAmelCase_	   =       rust_tokenizer.encode(__a       ,      add_special_tokens=__a			)
         self.assertListEqual(__a       ,      __a			)

         UpperCAmelCase_	   =       self.get_rust_tokenizer()
         UpperCAmelCase_	   =       tokenizer.encode(__a			)
         UpperCAmelCase_	   =       rust_tokenizer.encode(__a			)
         self.assertListEqual(__a       ,      __a			)

         # With lower casing
         UpperCAmelCase_	   =       self.get_tokenizer(do_lower_case=__a			)
         UpperCAmelCase_	   =       self.get_rust_tokenizer(do_lower_case=__a			)

         UpperCAmelCase_	   =       "UNwant\u00E9d,running"

         UpperCAmelCase_	   =       tokenizer.tokenize(__a			)
         UpperCAmelCase_	   =       rust_tokenizer.tokenize(__a			)
         self.assertListEqual(__a       ,      __a			)

         UpperCAmelCase_	   =       tokenizer.encode(__a       ,      add_special_tokens=__a			)
         UpperCAmelCase_	   =       rust_tokenizer.encode(__a       ,      add_special_tokens=__a			)
         self.assertListEqual(__a       ,      __a			)

         UpperCAmelCase_	   =       self.get_rust_tokenizer()
         UpperCAmelCase_	   =       tokenizer.encode(__a			)
         UpperCAmelCase_	   =       rust_tokenizer.encode(__a			)
         self.assertListEqual(__a       ,      __a			)



    def  _lowercase						(self   :   Dict			):
         UpperCAmelCase_	   =       BasicTokenizer()

         self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"			)       ,      ["ah", "\u535A", "\u63A8", "zz"]			)



    def  _lowercase						(self   :   Optional[int]			):
         UpperCAmelCase_	   =       BasicTokenizer(do_lower_case=__a			)

         self.assertListEqual(
             tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "			)       ,      ["hello", "!", "how", "are", "you", "?"]			)
         self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)       ,      ["hello"]			)



    def  _lowercase						(self   :   Any			):
         UpperCAmelCase_	   =       BasicTokenizer(do_lower_case=__a       ,      strip_accents=__a			)

         self.assertListEqual(
             tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)       ,      ["hällo", "!", "how", "are", "you", "?"]			)
         self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)       ,      ["h\u00E9llo"]			)



    def  _lowercase						(self   :   Optional[int]			):
         UpperCAmelCase_	   =       BasicTokenizer(do_lower_case=__a       ,      strip_accents=__a			)

         self.assertListEqual(
             tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)       ,      ["hallo", "!", "how", "are", "you", "?"]			)
         self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)       ,      ["hello"]			)



    def  _lowercase						(self   :   Optional[int]			):
         UpperCAmelCase_	   =       BasicTokenizer(do_lower_case=__a			)

         self.assertListEqual(
             tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)       ,      ["hallo", "!", "how", "are", "you", "?"]			)
         self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)       ,      ["hello"]			)



    def  _lowercase						(self   :   Optional[Any]			):
         UpperCAmelCase_	   =       BasicTokenizer(do_lower_case=__a			)

         self.assertListEqual(
             tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "			)       ,      ["HeLLo", "!", "how", "Are", "yoU", "?"]			)



    def  _lowercase						(self   :   Optional[int]			):
         UpperCAmelCase_	   =       BasicTokenizer(do_lower_case=__a       ,      strip_accents=__a			)

         self.assertListEqual(
             tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)       ,      ["HäLLo", "!", "how", "Are", "yoU", "?"]			)



    def  _lowercase						(self   :   Tuple			):
         UpperCAmelCase_	   =       BasicTokenizer(do_lower_case=__a       ,      strip_accents=__a			)

         self.assertListEqual(
             tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)       ,      ["HaLLo", "!", "how", "Are", "yoU", "?"]			)



    def  _lowercase						(self   :   Tuple			):
         UpperCAmelCase_	   =       BasicTokenizer(do_lower_case=__a       ,      never_split=["[UNK]"]			)

         self.assertListEqual(
             tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"			)       ,      ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]			)



    def  _lowercase						(self   :   Any			):
         UpperCAmelCase_	   =       ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

         UpperCAmelCase_	   =       {}
         for i, token in enumerate(__a			):
              UpperCAmelCase_	   =       i
         UpperCAmelCase_	   =       WordpieceTokenizer(vocab=__a       ,      unk_token="[UNK]"			)

         self.assertListEqual(tokenizer.tokenize(""			)       ,      []			)

         self.assertListEqual(tokenizer.tokenize("unwanted running"			)       ,      ["un", "##want", "##ed", "runn", "##ing"]			)

         self.assertListEqual(tokenizer.tokenize("unwantedX running"			)       ,      ["[UNK]", "runn", "##ing"]			)



    def  _lowercase						(self   :   Optional[int]			):
         self.assertTrue(_is_whitespace(" "			)			)
         self.assertTrue(_is_whitespace("\t"			)			)
         self.assertTrue(_is_whitespace("\r"			)			)
         self.assertTrue(_is_whitespace("\n"			)			)
         self.assertTrue(_is_whitespace("\u00A0"			)			)

         self.assertFalse(_is_whitespace("A"			)			)
         self.assertFalse(_is_whitespace("-"			)			)



    def  _lowercase						(self   :   str			):
         self.assertTrue(_is_control("\u0005"			)			)

         self.assertFalse(_is_control("A"			)			)
         self.assertFalse(_is_control(" "			)			)
         self.assertFalse(_is_control("\t"			)			)
         self.assertFalse(_is_control("\r"			)			)



    def  _lowercase						(self   :   Any			):
         self.assertTrue(_is_punctuation("-"			)			)
         self.assertTrue(_is_punctuation("$"			)			)
         self.assertTrue(_is_punctuation("`"			)			)
         self.assertTrue(_is_punctuation("."			)			)

         self.assertFalse(_is_punctuation("A"			)			)
         self.assertFalse(_is_punctuation(" "			)			)



    def  _lowercase						(self   :   Any			):
         UpperCAmelCase_	   =       self.get_tokenizer()
         UpperCAmelCase_	   =       self.get_rust_tokenizer()

         # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
         self.assertListEqual([tokenizer.tokenize(__a			) for t in ["Test", "\xad", "test"]]       ,      [["[UNK]"], [], ["[UNK]"]]			)

         self.assertListEqual(
             [rust_tokenizer.tokenize(__a			) for t in ["Test", "\xad", "test"]]       ,      [["[UNK]"], [], ["[UNK]"]]			)



    @slow
    def  _lowercase						(self   :   Dict			):
         UpperCAmelCase_	   =       self.tokenizer_class.from_pretrained("google/mobilebert-uncased"			)

         UpperCAmelCase_	   =       tokenizer.encode("sequence builders"       ,      add_special_tokens=__a			)
         UpperCAmelCase_	   =       tokenizer.encode("multi-sequence build"       ,      add_special_tokens=__a			)

         UpperCAmelCase_	   =       tokenizer.build_inputs_with_special_tokens(__a			)
         UpperCAmelCase_	   =       tokenizer.build_inputs_with_special_tokens(__a       ,      __a			)

         assert encoded_sentence == [101] + text + [102]
         assert encoded_pair == [101] + text + [102] + text_a + [102]



    def  _lowercase						(self   :   Dict			):
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
                   UpperCAmelCase_	   =       self.rust_tokenizer_class.from_pretrained(__a       ,      **__a			)

                   UpperCAmelCase_	   =       f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
                   UpperCAmelCase_	   =       tokenizer_r.encode_plus(
                       __a       ,      return_attention_mask=__a       ,      return_token_type_ids=__a       ,      return_offsets_mapping=__a       ,      add_special_tokens=__a       ,      )

                   UpperCAmelCase_	   =       tokenizer_r.do_lower_case if hasattr(__a       ,      "do_lower_case"			) else False
                   UpperCAmelCase_	   =       (
                       [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), "A"),
                           ((1, 2), ","),
                           ((3, 5), "na"),
                           ((5, 6), "##ï"),
                           ((6, 8), "##ve"),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), "Allen"),
                           ((21, 23), "##NL"),
                           ((23, 24), "##P"),
                           ((25, 33), "sentence"),
                           ((33, 34), "."),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                       if not do_lower_case
                       else [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), "a"),
                           ((1, 2), ","),
                           ((3, 8), "naive"),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), "allen"),
                           ((21, 23), "##nl"),
                           ((23, 24), "##p"),
                           ((25, 33), "sentence"),
                           ((33, 34), "."),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                   )

                   self.assertEqual(
                       [e[1] for e in expected_results]       ,      tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]			)			)
                   self.assertEqual([e[0] for e in expected_results]       ,      tokens["offset_mapping"]			)






    def  _lowercase						(self   :   Optional[int]			):
         UpperCAmelCase_	   =       ["的", "人", "有"]
         UpperCAmelCase_	   =       "".join(__a			)
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
                   UpperCAmelCase_	   =       True
                   UpperCAmelCase_	   =       self.tokenizer_class.from_pretrained(__a       ,      **__a			)
                   UpperCAmelCase_	   =       self.rust_tokenizer_class.from_pretrained(__a       ,      **__a			)

                   UpperCAmelCase_	   =       tokenizer_p.encode(__a       ,      add_special_tokens=__a			)
                   UpperCAmelCase_	   =       tokenizer_r.encode(__a       ,      add_special_tokens=__a			)

                   UpperCAmelCase_	   =       tokenizer_r.convert_ids_to_tokens(__a			)
                   UpperCAmelCase_	   =       tokenizer_p.convert_ids_to_tokens(__a			)

                   # it is expected that each Chinese character is not preceded by "##"
                   self.assertListEqual(__a       ,      __a			)
                   self.assertListEqual(__a       ,      __a			)

                   UpperCAmelCase_	   =       False
                   UpperCAmelCase_	   =       self.rust_tokenizer_class.from_pretrained(__a       ,      **__a			)
                   UpperCAmelCase_	   =       self.tokenizer_class.from_pretrained(__a       ,      **__a			)

                   UpperCAmelCase_	   =       tokenizer_r.encode(__a       ,      add_special_tokens=__a			)
                   UpperCAmelCase_	   =       tokenizer_p.encode(__a       ,      add_special_tokens=__a			)

                   UpperCAmelCase_	   =       tokenizer_r.convert_ids_to_tokens(__a			)
                   UpperCAmelCase_	   =       tokenizer_p.convert_ids_to_tokens(__a			)

                   # it is expected that only the first Chinese character is not preceded by "##".
                   UpperCAmelCase_	   =       [
                       f"""##{token}""" if idx != 0 else token for idx, token in enumerate(__a			)
                   ]
                   self.assertListEqual(__a       ,      __a			)
                   self.assertListEqual(__a       ,      __a			)




