



import os
import re
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase__		:					str							=logging.get_logger(__name__)

lowerCAmelCase__		:					Union[str, Any]							={'''vocab_file''': '''spiece.model'''}

lowerCAmelCase__		:					Dict							={
    '''vocab_file''': {
        '''google/bigbird-roberta-base''': '''https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model''',
        '''google/bigbird-roberta-large''': (
            '''https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model'''
        ),
        '''google/bigbird-base-trivia-itc''': (
            '''https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model'''
        ),
    }
}

lowerCAmelCase__		:					Tuple							={
    '''google/bigbird-roberta-base''': 4096,
    '''google/bigbird-roberta-large''': 4096,
    '''google/bigbird-base-trivia-itc''': 4096,
}

class        UpperCAmelCase_  (       A_	):

						'''simple docstring'''




						UpperCamelCase__      :			Any      =  VOCAB_FILES_NAMES
						UpperCamelCase__      :			Union[str, Any]      =  PRETRAINED_VOCAB_FILES_MAP
						UpperCamelCase__      :			Tuple      =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						UpperCamelCase__      :			List[str]      =  ["input_ids", "attention_mask"]
						UpperCamelCase__      :			List[int]      =  []






						def __init__(   self				,							_A				,							_A="<unk>"				,							_A="<s>"				,							_A="</s>"				,							_A="<pad>"				,							_A="[SEP]"				,							_A="[MASK]"				,							_A="[CLS]"				,							_A = None				,							**_A				,							):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= AddedToken(snake_case__				,							lstrip=snake_case__				,							rstrip=snake_case__		) if isinstance(snake_case__				,							snake_case__		) else bos_token
													__SCREAMING_SNAKE_CASE									= AddedToken(snake_case__				,							lstrip=snake_case__				,							rstrip=snake_case__		) if isinstance(snake_case__				,							snake_case__		) else eos_token
													__SCREAMING_SNAKE_CASE									= AddedToken(snake_case__				,							lstrip=snake_case__				,							rstrip=snake_case__		) if isinstance(snake_case__				,							snake_case__		) else unk_token
													__SCREAMING_SNAKE_CASE									= AddedToken(snake_case__				,							lstrip=snake_case__				,							rstrip=snake_case__		) if isinstance(snake_case__				,							snake_case__		) else pad_token
													__SCREAMING_SNAKE_CASE									= AddedToken(snake_case__				,							lstrip=snake_case__				,							rstrip=snake_case__		) if isinstance(snake_case__				,							snake_case__		) else cls_token
													__SCREAMING_SNAKE_CASE									= AddedToken(snake_case__				,							lstrip=snake_case__				,							rstrip=snake_case__		) if isinstance(snake_case__				,							snake_case__		) else sep_token

													# Mask token behave like a normal word, i.e. include the space before it
													__SCREAMING_SNAKE_CASE									= AddedToken(snake_case__				,							lstrip=snake_case__				,							rstrip=snake_case__		) if isinstance(snake_case__				,							snake_case__		) else mask_token

													__SCREAMING_SNAKE_CASE									= {} if sp_model_kwargs is None else sp_model_kwargs

													super().__init__(
													    bos_token=snake_case__				,							eos_token=snake_case__				,							unk_token=snake_case__				,							pad_token=snake_case__				,							sep_token=snake_case__				,							mask_token=snake_case__				,							cls_token=snake_case__				,							sp_model_kwargs=self.sp_model_kwargs				,							**snake_case__				,							)

													__SCREAMING_SNAKE_CASE									= vocab_file

													__SCREAMING_SNAKE_CASE									= spm.SentencePieceProcessor(**self.sp_model_kwargs		)
													self.sp_model.Load(snake_case__		)






						@property
						def 	_A     (   self		):







													'''simple docstring'''

													return self.sp_model.get_piece_size()






						def 	_A     (   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= {self.convert_ids_to_tokens(snake_case__		): i for i in range(self.vocab_size		)}
													vocab.update(self.added_tokens_encoder		)
													return vocab






						def __getstate__(   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= self.__dict__.copy()
													__SCREAMING_SNAKE_CASE									= None
													return state






						def __setstate__(   self				,							_A		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= d

													# for backward compatibility
													if not hasattr(self				,							'sp_model_kwargs'		):
																				__SCREAMING_SNAKE_CASE									= {}

													__SCREAMING_SNAKE_CASE									= spm.SentencePieceProcessor(**self.sp_model_kwargs		)
													self.sp_model.Load(self.vocab_file		)






						def 	_A     (   self				,							_A		):







													'''simple docstring'''

													return self.sp_model.encode(snake_case__				,							out_type=snake_case__		)






						def 	_A     (   self				,							_A		):







													'''simple docstring'''

													return self.sp_model.piece_to_id(snake_case__		)






						def 	_A     (   self				,							_A		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= self.sp_model.IdToPiece(snake_case__		)
													return token






						def 	_A     (   self				,							_A		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= []
													__SCREAMING_SNAKE_CASE									= ""
													__SCREAMING_SNAKE_CASE									= False
													for token in tokens:
																				# make sure that special tokens are not decoded using sentencepiece model
																				if token in self.all_special_tokens:
																											if not prev_is_special:
																																		out_string += " "
																											out_string += self.sp_model.decode(snake_case__		) + token
																											__SCREAMING_SNAKE_CASE									= True
																											__SCREAMING_SNAKE_CASE									= []
																				else:
																											current_sub_tokens.append(snake_case__		)
																											__SCREAMING_SNAKE_CASE									= False
													out_string += self.sp_model.decode(snake_case__		)
													return out_string.strip()






						def 	_A     (   self				,							_A				,							_A = False				,							_A = None				,							_A = True				,							**_A				,							):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= kwargs.pop('use_source_tokenizer'				,							snake_case__		)

													__SCREAMING_SNAKE_CASE									= self.convert_ids_to_tokens(snake_case__				,							skip_special_tokens=snake_case__		)

													# To avoid mixing byte-level and unicode for byte-level BPT
													# we need to build string separately for added tokens and byte-level tokens
													# cf. https://github.com/huggingface/transformers/issues/1133
													__SCREAMING_SNAKE_CASE									= []
													__SCREAMING_SNAKE_CASE									= []
													for token in filtered_tokens:
																				if skip_special_tokens and token in self.all_special_ids:
																											continue
																				if token in self.added_tokens_encoder:
																											if current_sub_text:
																																		sub_texts.append(self.convert_tokens_to_string(snake_case__		)		)
																																		__SCREAMING_SNAKE_CASE									= []
																											sub_texts.append(snake_case__		)
																				else:
																											current_sub_text.append(snake_case__		)
													if current_sub_text:
																				sub_texts.append(self.convert_tokens_to_string(snake_case__		)		)

													# Mimic the behavior of the Rust tokenizer:
													# No space before [MASK] and [SEP]
													if spaces_between_special_tokens:
																				__SCREAMING_SNAKE_CASE									= re.sub(r' (\[(MASK|SEP)\])'				,							r'\1'				,							' '.join(snake_case__		)		)
													else:
																				__SCREAMING_SNAKE_CASE									= "".join(snake_case__		)

													__SCREAMING_SNAKE_CASE									= (
													    clean_up_tokenization_spaces
													    if clean_up_tokenization_spaces is not None
													    else self.clean_up_tokenization_spaces
													)
													if clean_up_tokenization_spaces:
																				__SCREAMING_SNAKE_CASE									= self.clean_up_tokenization(snake_case__		)
																				return clean_text
													else:
																				return text






						def 	_A     (   self				,							_A				,							_A = None		):







													'''simple docstring'''

													if not os.path.isdir(snake_case__		):
																				logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""		)
																				return
													__SCREAMING_SNAKE_CASE									= os.path.join(
													    snake_case__				,							(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']		)

													if os.path.abspath(self.vocab_file		) != os.path.abspath(snake_case__		) and os.path.isfile(self.vocab_file		):
																				copyfile(self.vocab_file				,							snake_case__		)
													elif not os.path.isfile(self.vocab_file		):
																				with open(snake_case__				,							'wb'		) as fi:
																											__SCREAMING_SNAKE_CASE									= self.sp_model.serialized_model_proto()
																											fi.write(snake_case__		)

													return (out_vocab_file,)






						def 	_A     (   self				,							_A				,							_A = None		):







													'''simple docstring'''

													if token_ids_a is None:
																				return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
													__SCREAMING_SNAKE_CASE									= [self.cls_token_id]
													__SCREAMING_SNAKE_CASE									= [self.sep_token_id]
													return cls + token_ids_a + sep + token_ids_a + sep






						def 	_A     (   self				,							_A				,							_A = None				,							_A = False		):







													'''simple docstring'''

													if already_has_special_tokens:
																				return super().get_special_tokens_mask(
																				    token_ids_a=snake_case__				,							token_ids_a=snake_case__				,							already_has_special_tokens=snake_case__		)

													if token_ids_a is None:
																				return [1] + ([0] * len(snake_case__		)) + [1]
													return [1] + ([0] * len(snake_case__		)) + [1] + ([0] * len(snake_case__		)) + [1]






						def 	_A     (   self				,							_A				,							_A = None		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= [self.sep_token_id]
													__SCREAMING_SNAKE_CASE									= [self.cls_token_id]
													if token_ids_a is None:
																				return len(cls + token_ids_a + sep		) * [0]
													return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]











from __future__ import annotations

import unittest

import numpy as np

from transformers import BlipTextConfig
from transformers.testing_utils import require_tf, slow
from transformers.utils import is_tf_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask


if is_tf_available():
			import tensorflow as tf

			from transformers import TFBlipTextModel
			from transformers.models.blip.modeling_tf_blip import TF_BLIP_PRETRAINED_MODEL_ARCHIVE_LIST

class        UpperCAmelCase_  :

						'''simple docstring'''




						def __init__(   self				,							_A				,							_A=12				,							_A=7				,							_A=True				,							_A=True				,							_A=True				,							_A=99				,							_A=32				,							_A=32				,							_A=2				,							_A=4				,							_A=37				,							_A=0.1				,							_A=0.1				,							_A=512				,							_A=0.0_2				,							_A=0				,							_A=None				,							):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= parent
													__SCREAMING_SNAKE_CASE									= batch_size
													__SCREAMING_SNAKE_CASE									= seq_length
													__SCREAMING_SNAKE_CASE									= is_training
													__SCREAMING_SNAKE_CASE									= use_input_mask
													__SCREAMING_SNAKE_CASE									= use_labels
													__SCREAMING_SNAKE_CASE									= vocab_size
													__SCREAMING_SNAKE_CASE									= hidden_size
													__SCREAMING_SNAKE_CASE									= projection_dim
													__SCREAMING_SNAKE_CASE									= num_hidden_layers
													__SCREAMING_SNAKE_CASE									= num_attention_heads
													__SCREAMING_SNAKE_CASE									= intermediate_size
													__SCREAMING_SNAKE_CASE									= dropout
													__SCREAMING_SNAKE_CASE									= attention_dropout
													__SCREAMING_SNAKE_CASE									= max_position_embeddings
													__SCREAMING_SNAKE_CASE									= initializer_range
													__SCREAMING_SNAKE_CASE									= scope
													__SCREAMING_SNAKE_CASE									= bos_token_id






						def 	_A     (   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= ids_tensor([self.batch_size, self.seq_length]				,							self.vocab_size		)

													__SCREAMING_SNAKE_CASE									= None
													if self.use_input_mask:
																				__SCREAMING_SNAKE_CASE									= random_attention_mask([self.batch_size, self.seq_length]		)

													if input_mask is not None:
																				__SCREAMING_SNAKE_CASE									= input_mask.numpy()
																				__SCREAMING_SNAKE_CASE							,	__SCREAMING_SNAKE_CASE									= input_mask.shape
																				__SCREAMING_SNAKE_CASE									= np.random.randint(1				,							seq_length - 1				,							size=(batch_size,)		)
																				for batch_idx, start_index in enumerate(_A		):
																											__SCREAMING_SNAKE_CASE									= 1
																											__SCREAMING_SNAKE_CASE									= 0

													__SCREAMING_SNAKE_CASE									= self.get_config()

													return config, input_ids, tf.convert_to_tensor(_A		)






						def 	_A     (   self		):







													'''simple docstring'''

													return BlipTextConfig(
													    vocab_size=self.vocab_size				,							hidden_size=self.hidden_size				,							projection_dim=self.projection_dim				,							num_hidden_layers=self.num_hidden_layers				,							num_attention_heads=self.num_attention_heads				,							intermediate_size=self.intermediate_size				,							dropout=self.dropout				,							attention_dropout=self.attention_dropout				,							max_position_embeddings=self.max_position_embeddings				,							initializer_range=self.initializer_range				,							bos_token_id=self.bos_token_id				,							)






						def 	_A     (   self				,							_A				,							_A				,							_A		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= TFBlipTextModel(config=_A		)
													__SCREAMING_SNAKE_CASE									= model(_A				,							attention_mask=_A				,							training=_A		)
													__SCREAMING_SNAKE_CASE									= model(_A				,							training=_A		)
													self.parent.assertEqual(result.last_hidden_state.shape				,							(self.batch_size, self.seq_length, self.hidden_size)		)
													self.parent.assertEqual(result.pooler_output.shape				,							(self.batch_size, self.hidden_size)		)






						def 	_A     (   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= self.prepare_config_and_inputs()
													__SCREAMING_SNAKE_CASE							,	__SCREAMING_SNAKE_CASE							,	__SCREAMING_SNAKE_CASE									= config_and_inputs
													__SCREAMING_SNAKE_CASE									= {'input_ids': input_ids, 'attention_mask': input_mask}
													return config, inputs_dict







@require_tf
class        UpperCAmelCase_  (       UpperCamelCase_   ,   unittest.TestCase	):

						'''simple docstring'''




						UpperCamelCase__      :			str      =  (TFBlipTextModel,) if is_tf_available() else ()
						UpperCamelCase__      :			int      =  False
						UpperCamelCase__      :			Optional[Any]      =  False
						UpperCamelCase__      :			Tuple      =  False






						def 	_A     (   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= BlipTextModelTester(self		)
													__SCREAMING_SNAKE_CASE									= ConfigTester(self				,							config_class=_A				,							hidden_size=37		)






						def 	_A     (   self		):







													'''simple docstring'''

													self.config_tester.run_common_tests()






						def 	_A     (   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_model(*_A		)






						def 	_A     (   self		):







													'''simple docstring'''

													pass






						def 	_A     (   self		):







													'''simple docstring'''

													pass






						@unittest.skip(reason='Blip does not use inputs_embeds'		)
						def 	_A     (   self		):







													'''simple docstring'''

													pass






						@unittest.skip(reason='BlipTextModel has no base class and is not available in MODEL_MAPPING'		)
						def 	_A     (   self		):







													'''simple docstring'''

													pass






						@unittest.skip(reason='BlipTextModel has no base class and is not available in MODEL_MAPPING'		)
						def 	_A     (   self		):







													'''simple docstring'''

													pass






						@slow
						def 	_A     (   self		):







													'''simple docstring'''

													for model_name in TF_BLIP_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																				__SCREAMING_SNAKE_CASE									= TFBlipTextModel.from_pretrained(_A		)
																				self.assertIsNotNone(_A		)






						def 	_A     (   self				,							_A=True		):







													'''simple docstring'''

													super().test_pt_tf_model_equivalence(allow_missing_keys=_A		)






