





from __future__ import annotations

import copy
import inspect
import json
import math
import os
import tempfile
import unittest
from importlib import import_module

import numpy as np

from transformers import ViTMAEConfig
from transformers.file_utils import cached_property, is_tf_available, is_vision_available
from transformers.testing_utils import require_tf, require_vision, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
					import tensorflow as tf

					from transformers import TFViTMAEForPreTraining, TFViTMAEModel


if is_vision_available():
					from PIL import Image

					from transformers import ViTImageProcessor

class     __SCREAMING_SNAKE_CASE   :






							def __init__(  self					,	__lowerCAmelCase					,	__lowerCAmelCase=13					,	__lowerCAmelCase=30					,	__lowerCAmelCase=2					,	__lowerCAmelCase=3					,	__lowerCAmelCase=True					,	__lowerCAmelCase=True					,	__lowerCAmelCase=32					,	__lowerCAmelCase=2					,	__lowerCAmelCase=4					,	__lowerCAmelCase=37					,	__lowerCAmelCase="gelu"					,	__lowerCAmelCase=0.1					,	__lowerCAmelCase=0.1					,	__lowerCAmelCase=10					,	__lowerCAmelCase=0.02					,	__lowerCAmelCase=3					,	__lowerCAmelCase=0.6					,	__lowerCAmelCase=None					,	):
														UpperCamelCase__			       =     parent
														UpperCamelCase__			       =     batch_size
														UpperCamelCase__			       =     image_size
														UpperCamelCase__			       =     patch_size
														UpperCamelCase__			       =     num_channels
														UpperCamelCase__			       =     is_training
														UpperCamelCase__			       =     use_labels
														UpperCamelCase__			       =     hidden_size
														UpperCamelCase__			       =     num_hidden_layers
														UpperCamelCase__			       =     num_attention_heads
														UpperCamelCase__			       =     intermediate_size
														UpperCamelCase__			       =     hidden_act
														UpperCamelCase__			       =     hidden_dropout_prob
														UpperCamelCase__			       =     attention_probs_dropout_prob
														UpperCamelCase__			       =     type_sequence_label_size
														UpperCamelCase__			       =     initializer_range
														UpperCamelCase__			       =     mask_ratio
														UpperCamelCase__			       =     scope

														# in ViTMAE, the expected sequence length = (num_patches + 1) * (1 - config.mask_ratio), rounded above
														# (we add 1 for the [CLS] token)
														UpperCamelCase__			       =     (image_size // patch_size) ** 2
														UpperCamelCase__			       =     int(math.ceil((1 - mask_ratio) * (num_patches + 1)					)					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]					)

														UpperCamelCase__			       =     None
														if self.use_labels:
																					UpperCamelCase__			       =     ids_tensor([self.batch_size]					,	self.type_sequence_label_size					)

														UpperCamelCase__			       =     self.get_config()

														return config, pixel_values, labels






							def 			_lowerCamelCase						(  self					):
														return ViTMAEConfig(
														    image_size=self.image_size					,	patch_size=self.patch_size					,	num_channels=self.num_channels					,	hidden_size=self.hidden_size					,	num_hidden_layers=self.num_hidden_layers					,	num_attention_heads=self.num_attention_heads					,	intermediate_size=self.intermediate_size					,	decoder_hidden_size=self.hidden_size					,	decoder_num_hidden_layers=self.num_hidden_layers					,	decoder_num_attention_heads=self.num_attention_heads					,	decoder_intermediate_size=self.intermediate_size					,	hidden_act=self.hidden_act					,	hidden_dropout_prob=self.hidden_dropout_prob					,	attention_probs_dropout_prob=self.attention_probs_dropout_prob					,	is_decoder=_a					,	initializer_range=self.initializer_range					,	mask_ratio=self.mask_ratio					,	)






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase					,	__lowerCAmelCase					):
														UpperCamelCase__			       =     TFViTMAEModel(config=_a					)
														UpperCamelCase__			       =     model(_a					,	training=_a					)
														self.parent.assertEqual(result.last_hidden_state.shape					,	(self.batch_size, self.seq_length, self.hidden_size)					)






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase					,	__lowerCAmelCase					):
														UpperCamelCase__			       =     TFViTMAEForPreTraining(_a					)
														UpperCamelCase__			       =     model(_a					,	training=_a					)
														# expected sequence length = num_patches
														UpperCamelCase__			       =     (self.image_size // self.patch_size) ** 2
														UpperCamelCase__			       =     self.patch_size**2 * self.num_channels
														self.parent.assertEqual(result.logits.shape					,	(self.batch_size, num_patches, expected_num_channels)					)

														# test greyscale images
														UpperCamelCase__			       =     1
														UpperCamelCase__			       =     TFViTMAEForPreTraining(_a					)

														UpperCamelCase__			       =     floats_tensor([self.batch_size, 1, self.image_size, self.image_size]					)
														UpperCamelCase__			       =     model(_a					,	training=_a					)
														UpperCamelCase__			       =     self.patch_size**2
														self.parent.assertEqual(result.logits.shape					,	(self.batch_size, num_patches, expected_num_channels)					)







							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     self.prepare_config_and_inputs()
														(UpperCamelCase__)			       =     config_and_inputs
														UpperCamelCase__			       =     {"pixel_values": pixel_values}
														return config, inputs_dict






@require_tf
class     __SCREAMING_SNAKE_CASE   (       _a				,						_a				,						unittest.TestCase    ):
							snake_case       : List[Any]         =    (TFViTMAEModel, TFViTMAEForPreTraining) if is_tf_available() else ()
							snake_case       : List[str]         =    {"""feature-extraction""": TFViTMAEModel} if is_tf_available() else {}

							snake_case       : Optional[Any]         =    False
							snake_case       : List[Any]         =    False
							snake_case       : Any         =    False
							snake_case       : Tuple         =    False






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     TFViTMAEModelTester(self					)
														UpperCamelCase__			       =     ConfigTester(self					,	config_class=_a					,	has_text_modality=_a					,	hidden_size=37					)






							def 			_lowerCamelCase						(  self					):
														self.config_tester.run_common_tests()






							@unittest.skip(reason="""ViTMAE does not use inputs_embeds"""					)
							def 			_lowerCamelCase						(  self					):
														pass






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     self.model_tester.prepare_config_and_inputs_for_common()

														for model_class in self.all_model_classes:
																					UpperCamelCase__			       =     model_class(_a					)
																					self.assertIsInstance(model.get_input_embeddings()					,	(tf.keras.layers.Layer)					)
																					UpperCamelCase__			       =     model.get_output_embeddings()
																					self.assertTrue(x is None or isinstance(_a					,	tf.keras.layers.Layer					)					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     self.model_tester.prepare_config_and_inputs_for_common()

														for model_class in self.all_model_classes:
																					UpperCamelCase__			       =     model_class(_a					)
																					UpperCamelCase__			       =     inspect.signature(model.call					)
																					# signature.parameters is an OrderedDict => so arg_names order is deterministic
																					UpperCamelCase__			       =     [*signature.parameters.keys()]

																					UpperCamelCase__			       =     ["pixel_values"]
																					self.assertListEqual(arg_names[:1]					,	_a					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_model(*_a					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_pretraining(*_a					)






							def 			_lowerCamelCase						(  self					):
														# make the mask reproducible
														np.random.seed(2					)

														UpperCamelCase__			       =     self.model_tester.prepare_config_and_inputs_for_common()

														UpperCamelCase__			       =     int((config.image_size // config.patch_size) ** 2					)
														UpperCamelCase__			       =     np.random.uniform(size=(self.model_tester.batch_size, num_patches)					)

														for model_class in self.all_model_classes:
																					UpperCamelCase__			       =     model_class(_a					)
																					UpperCamelCase__			       =     self._prepare_for_class(_a					,	_a					)

																					UpperCamelCase__			       =     model(_a					,	noise=_a					)

																					UpperCamelCase__			       =     copy.deepcopy(self._prepare_for_class(_a					,	_a					)					)
																					UpperCamelCase__			       =     model(**_a					,	noise=_a					)
																					UpperCamelCase__			       =     outputs_dict[0].numpy()
																					UpperCamelCase__			       =     outputs_keywords[0].numpy()

																					self.assertLess(np.sum(np.abs(output_dict - output_keywords					)					)					,	1E-6					)






							def 			_lowerCamelCase						(  self					):
														# make the mask reproducible
														np.random.seed(2					)

														UpperCamelCase__			       =     self.model_tester.prepare_config_and_inputs_for_common()

														UpperCamelCase__			       =     int((config.image_size // config.patch_size) ** 2					)
														UpperCamelCase__			       =     np.random.uniform(size=(self.model_tester.batch_size, num_patches)					)

														def prepare_numpy_arrays(__lowerCAmelCase					):
																					UpperCamelCase__			       =     {}
																					for k, v in inputs_dict.items():
																												if tf.is_tensor(_a					):
																																			UpperCamelCase__			       =     v.numpy()
																												else:
																																			UpperCamelCase__			       =     np.array(_a					)

																					return inputs_np_dict

														for model_class in self.all_model_classes:
																					UpperCamelCase__			       =     model_class(_a					)

																					UpperCamelCase__			       =     self._prepare_for_class(_a					,	_a					)
																					UpperCamelCase__			       =     prepare_numpy_arrays(_a					)

																					UpperCamelCase__			       =     model(_a					,	noise=_a					)
																					UpperCamelCase__			       =     model(**_a					,	noise=_a					)
																					self.assert_outputs_same(_a					,	_a					)






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase					,	__lowerCAmelCase					):
														# make masks reproducible
														np.random.seed(2					)

														UpperCamelCase__			       =     int((tf_model.config.image_size // tf_model.config.patch_size) ** 2					)
														UpperCamelCase__			       =     np.random.uniform(size=(self.model_tester.batch_size, num_patches)					)
														UpperCamelCase__			       =     tf.constant(_a					)

														# Add `noise` argument.
														# PT inputs will be prepared in `super().check_pt_tf_models()` with this added `noise` argument
														UpperCamelCase__			       =     tf_noise

														super().check_pt_tf_models(_a					,	_a					,	_a					)






							def 			_lowerCamelCase						(  self					):
														# make mask reproducible
														np.random.seed(2					)

														UpperCamelCase__			       =     self.model_tester.prepare_config_and_inputs_for_common()

														UpperCamelCase__			       =     {
														    module_member
														    for model_class in self.all_model_classes
														    for module in (import_module(model_class.__module__					),)
														    for module_member_name in dir(_a					)
														    if module_member_name.endswith("""MainLayer"""					)
														    # This condition is required, since `modeling_tf_clip.py` has 3 classes whose names end with `MainLayer`.
														    and module_member_name[: -len("""MainLayer"""					)] == model_class.__name__[: -len("""Model"""					)]
														    for module_member in (getattr(_a					,	_a					),)
														    if isinstance(_a					,	_a					)
														    and tf.keras.layers.Layer in module_member.__bases__
														    and getattr(_a					,	"""_keras_serializable"""					,	_a					)
														}

														UpperCamelCase__			       =     int((config.image_size // config.patch_size) ** 2					)
														UpperCamelCase__			       =     np.random.uniform(size=(self.model_tester.batch_size, num_patches)					)
														UpperCamelCase__			       =     tf.convert_to_tensor(_a					)
														inputs_dict.update({"""noise""": noise}					)

														for main_layer_class in tf_main_layer_classes:
																					UpperCamelCase__			       =     main_layer_class(_a					)

																					UpperCamelCase__			       =     {
																					    name: tf.keras.Input(tensor.shape[1:]					,	dtype=tensor.dtype					) for name, tensor in inputs_dict.items()
																					}

																					UpperCamelCase__			       =     tf.keras.Model(_a					,	outputs=main_layer(_a					)					)
																					UpperCamelCase__			       =     model(_a					)

																					with tempfile.TemporaryDirectory() as tmpdirname:
																												UpperCamelCase__			       =     os.path.join(_a					,	"""keras_model.h5"""					)
																												model.save(_a					)
																												UpperCamelCase__			       =     tf.keras.models.load_model(
																												    _a					,	custom_objects={main_layer_class.__name__: main_layer_class}					)
																												assert isinstance(_a					,	tf.keras.Model					)
																												UpperCamelCase__			       =     model(_a					)
																												self.assert_outputs_same(_a					,	_a					)






							@slow
							def 			_lowerCamelCase						(  self					):
														# make mask reproducible
														np.random.seed(2					)

														UpperCamelCase__			       =     self.model_tester.prepare_config_and_inputs_for_common()

														UpperCamelCase__			       =     int((config.image_size // config.patch_size) ** 2					)
														UpperCamelCase__			       =     np.random.uniform(size=(self.model_tester.batch_size, num_patches)					)

														for model_class in self.all_model_classes:
																					UpperCamelCase__			       =     model_class(_a					)
																					UpperCamelCase__			       =     self._prepare_for_class(_a					,	_a					)
																					UpperCamelCase__			       =     model(_a					,	noise=_a					)

																					if model_class.__name__ == "TFViTMAEModel":
																												UpperCamelCase__			       =     outputs.last_hidden_state.numpy()
																												UpperCamelCase__			       =     0
																					else:
																												UpperCamelCase__			       =     outputs.logits.numpy()
																												UpperCamelCase__			       =     0

																					with tempfile.TemporaryDirectory() as tmpdirname:
																												model.save_pretrained(_a					,	saved_model=_a					)
																												UpperCamelCase__			       =     model_class.from_pretrained(_a					)
																												UpperCamelCase__			       =     model(_a					,	noise=_a					)

																												if model_class.__name__ == "TFViTMAEModel":
																																			UpperCamelCase__			       =     after_outputs["last_hidden_state"].numpy()
																																			UpperCamelCase__			       =     0
																												else:
																																			UpperCamelCase__			       =     after_outputs["logits"].numpy()
																																			UpperCamelCase__			       =     0

																												UpperCamelCase__			       =     np.amax(np.abs(out_a - out_a					)					)
																												self.assertLessEqual(_a					,	1E-5					)






							def 			_lowerCamelCase						(  self					):
														# make mask reproducible
														np.random.seed(2					)

														UpperCamelCase__			       =     self.model_tester.prepare_config_and_inputs_for_common()

														UpperCamelCase__			       =     int((config.image_size // config.patch_size) ** 2					)
														UpperCamelCase__			       =     np.random.uniform(size=(self.model_tester.batch_size, num_patches)					)

														for model_class in self.all_model_classes:
																					UpperCamelCase__			       =     model_class(_a					)
																					UpperCamelCase__			       =     self._prepare_for_class(_a					,	_a					)

																					UpperCamelCase__			       =     model(_a					,	noise=_a					)
																					UpperCamelCase__			       =     model.get_config()
																					# make sure that returned config is jsonifiable, which is required by keras
																					json.dumps(_a					)
																					UpperCamelCase__			       =     model_class.from_config(model.get_config()					)
																					# make sure it also accepts a normal config
																					UpperCamelCase__			       =     model_class.from_config(model.config					)
																					UpperCamelCase__			       =     new_model(_a					)  # Build model
																					new_model.set_weights(model.get_weights()					)
																					UpperCamelCase__			       =     new_model(_a					,	noise=_a					)

																					self.assert_outputs_same(_a					,	_a					)






							@unittest.skip(
							    reason="""ViTMAE returns a random mask + ids_restore in each forward pass. See test_save_load\n    to get deterministic results."""					)
							def 			_lowerCamelCase						(  self					):
														pass






							@unittest.skip(reason="""ViTMAE returns a random mask + ids_restore in each forward pass. See test_save_load"""					)
							def 			_lowerCamelCase						(  self					):
														pass







							@slow
							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     TFViTMAEModel.from_pretrained("""google/vit-base-patch16-224"""					)
														self.assertIsNotNone(_a					)






def       _UpperCamelCase       ():






							"""simple docstring"""





							UpperCamelCase__			       =     Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""    )
							return image

@require_tf
@require_vision
class     __SCREAMING_SNAKE_CASE   (       unittest.TestCase    ):






							@cached_property
							def 			_lowerCamelCase						(  self					):
														return ViTImageProcessor.from_pretrained("""facebook/vit-mae-base"""					) if is_vision_available() else None







							@slow
							def 			_lowerCamelCase						(  self					):
														# make random mask reproducible across the PT and TF model
														np.random.seed(2					)

														UpperCamelCase__			       =     TFViTMAEForPreTraining.from_pretrained("""facebook/vit-mae-base"""					)

														UpperCamelCase__			       =     self.default_image_processor
														UpperCamelCase__			       =     prepare_img()
														UpperCamelCase__			       =     image_processor(images=_a					,	return_tensors="""tf"""					)

														# prepare a noise vector that will be also used for testing the TF model
														# (this way we can ensure that the PT and TF models operate on the same inputs)
														UpperCamelCase__			       =     ViTMAEConfig()
														UpperCamelCase__			       =     int((vit_mae_config.image_size // vit_mae_config.patch_size) ** 2					)
														UpperCamelCase__			       =     np.random.uniform(size=(1, num_patches)					)

														# forward pass
														UpperCamelCase__			       =     model(**_a					,	noise=_a					)

														# verify the logits
														UpperCamelCase__			       =     tf.convert_to_tensor([1, 196, 768]					)
														self.assertEqual(outputs.logits.shape					,	_a					)

														UpperCamelCase__			       =     tf.convert_to_tensor(
														    [[-0.0548, -1.7023, -0.9325], [0.3721, -0.5670, -0.2233], [0.8235, -1.3878, -0.3524]]					)

														tf.debugging.assert_near(outputs.logits[0, :3, :3]					,	_a					,	atol=1E-4					)











import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
					import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCamelCase__     =	logging.get_logger(__name__)

UpperCamelCase__     =	{"vocab_file": "vocab.txt"}

UpperCamelCase__     =	{
    "vocab_file": {
        "openbmb/cpm-ant-10b": "https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt",
    },
}

UpperCamelCase__     =	{
    "openbmb/cpm-ant-10b": 1_024,
}





def       _UpperCamelCase       (a__				:Tuple    ):






							"""simple docstring"""





							UpperCamelCase__			       =     collections.OrderedDict()
							with open(a__				,     """r"""				,     encoding="""utf-8"""    ) as reader:
														UpperCamelCase__			       =     reader.readlines()
							for index, token in enumerate(a__    ):
														UpperCamelCase__			       =     token.rstrip("""\n"""    )
														UpperCamelCase__			       =     index
							return vocab

class     __SCREAMING_SNAKE_CASE   (       _a    ):






							def __init__(  self					,	__lowerCAmelCase					,	__lowerCAmelCase="<unk>"					,	__lowerCAmelCase=200					):
														UpperCamelCase__			       =     vocab
														UpperCamelCase__			       =     unk_token
														UpperCamelCase__			       =     max_input_chars_per_word







							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					):
														UpperCamelCase__			       =     list(__lowerCAmelCase					)
														if len(__lowerCAmelCase					) > self.max_input_chars_per_word:
																					return [self.unk_token]

														UpperCamelCase__			       =     0
														UpperCamelCase__			       =     []
														while start < len(__lowerCAmelCase					):
																					UpperCamelCase__			       =     len(__lowerCAmelCase					)
																					UpperCamelCase__			       =     None
																					while start < end:
																												UpperCamelCase__			       =     """""".join(chars[start:end]					)
																												if substr in self.vocab:
																																			UpperCamelCase__			       =     substr
																																			break
																												end -= 1
																					if cur_substr is None:
																												sub_tokens.append(self.unk_token					)
																												start += 1
																					else:
																												sub_tokens.append(__lowerCAmelCase					)
																												UpperCamelCase__			       =     end

														return sub_tokens






class     __SCREAMING_SNAKE_CASE   (       _a    ):
							snake_case       : Optional[int]         =    VOCAB_FILES_NAMES
							snake_case       : Optional[Any]         =    PRETRAINED_VOCAB_FILES_MAP
							snake_case       : List[Any]         =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							snake_case       : Optional[int]         =    ["""input_ids""", """attention_mask"""]
							snake_case       : int         =    False






							def __init__(  self					,	__lowerCAmelCase					,	__lowerCAmelCase="<d>"					,	__lowerCAmelCase="</d>"					,	__lowerCAmelCase="<s>"					,	__lowerCAmelCase="</s>"					,	__lowerCAmelCase="<pad>"					,	__lowerCAmelCase="<unk>"					,	__lowerCAmelCase="</n>"					,	__lowerCAmelCase="</_>"					,	__lowerCAmelCase="left"					,	**__lowerCAmelCase					,	):
														requires_backends(self					,	["""jieba"""]					)
														super().__init__(
														    bod_token=__lowerCAmelCase					,	eod_token=__lowerCAmelCase					,	bos_token=__lowerCAmelCase					,	eos_token=__lowerCAmelCase					,	pad_token=__lowerCAmelCase					,	unk_token=__lowerCAmelCase					,	line_token=__lowerCAmelCase					,	space_token=__lowerCAmelCase					,	padding_side=__lowerCAmelCase					,	**__lowerCAmelCase					,	)
														UpperCamelCase__			       =     bod_token
														UpperCamelCase__			       =     eod_token
														UpperCamelCase__			       =     load_vocab(__lowerCAmelCase					)
														UpperCamelCase__			       =     self.encoder[space_token]
														UpperCamelCase__			       =     self.encoder[line_token]

														del self.encoder[space_token]
														del self.encoder[line_token]

														UpperCamelCase__			       =     collections.OrderedDict(sorted(self.encoder.items()					,	key=lambda __lowerCAmelCase					: x[1]					)					)
														UpperCamelCase__			       =     {v: k for k, v in self.encoder.items()}

														UpperCamelCase__			       =     WordpieceTokenizer(vocab=self.encoder					,	unk_token=self.unk_token					)






							@property
							def 			_lowerCamelCase						(  self					):
														return self.encoder[self.bod_token]






							@property
							def 			_lowerCamelCase						(  self					):
														return self.encoder[self.eod_token]






							@property
							def 			_lowerCamelCase						(  self					):
														return self.encoder["\n"]






							@property
							def 			_lowerCamelCase						(  self					):
														return len(self.encoder					)






							def 			_lowerCamelCase						(  self					):
														return dict(self.encoder					,	**self.added_tokens_encoder					)






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					):
														UpperCamelCase__			       =     []
														for x in jieba.cut(__lowerCAmelCase					,	cut_all=__lowerCAmelCase					):
																					output_tokens.extend(self.wordpiece_tokenizer.tokenize(__lowerCAmelCase					)					)
														return output_tokens






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	**__lowerCAmelCase					):
														UpperCamelCase__			       =     [i for i in token_ids if i >= 0]
														UpperCamelCase__			       =     [
														    x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
														]
														return super()._decode(__lowerCAmelCase					,	**__lowerCAmelCase					)






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					):
														return token in self.encoder






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					):
														return "".join(__lowerCAmelCase					)






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					):
														return self.encoder.get(__lowerCAmelCase					,	self.encoder.get(self.unk_token					)					)






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					):
														return self.decoder.get(__lowerCAmelCase					,	self.unk_token					)






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase = None					):
														if os.path.isdir(__lowerCAmelCase					):
																					UpperCamelCase__			       =     os.path.join(
																					    __lowerCAmelCase					,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]					)
														else:
																					UpperCamelCase__			       =     (filename_prefix + """-""" if filename_prefix else """""") + save_directory
														UpperCamelCase__			       =     0
														if " " in self.encoder:
																					UpperCamelCase__			       =     self.encoder[""" """]
																					del self.encoder[" "]
														if "\n" in self.encoder:
																					UpperCamelCase__			       =     self.encoder["""\n"""]
																					del self.encoder["\n"]
														UpperCamelCase__			       =     collections.OrderedDict(sorted(self.encoder.items()					,	key=lambda __lowerCAmelCase					: x[1]					)					)
														with open(__lowerCAmelCase					,	"""w"""					,	encoding="""utf-8"""					) as writer:
																					for token, token_index in self.encoder.items():
																												if index != token_index:
																																			logger.warning(
																																			    f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
																																			    """ Please check that the vocabulary is not corrupted!"""					)
																																			UpperCamelCase__			       =     token_index
																												writer.write(token + """\n"""					)
																												index += 1
														return (vocab_file,)






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase = None					):
														if token_ids_a is None:
																					return [self.bos_token_id] + token_ids_a
														return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a







							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase = None					,	__lowerCAmelCase = False					):

														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=__lowerCAmelCase					,	token_ids_a=__lowerCAmelCase					,	already_has_special_tokens=__lowerCAmelCase					)

														if token_ids_a is not None:
																					return [1] + ([0] * len(__lowerCAmelCase					)) + [1] + ([0] * len(__lowerCAmelCase					))
														return [1] + ([0] * len(__lowerCAmelCase					))




