






from __future__ import annotations

import unittest

import numpy as np

from transformers import OPTConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
		import tensorflow as tf

		from transformers import GPTaTokenizer, TFOPTForCausalLM, TFOPTModel




def 					lowercase__				(					__snake_case   :							List[str]					,	__snake_case   :							Optional[Any]					,	__snake_case   :							str=None					,	__snake_case   :							int=None  ):


					'''simple docstring'''


					if attention_mask is None:
										UpperCAmelCase_   :				Dict						=						tf.cast(tf.math.not_equal(__snake_case					,	config.pad_token_id  )					,	tf.inta  )
					return {"input_ids": input_ids, "attention_mask": attention_mask}



@require_tf
class   lowerCamelCase				:







		'''simple docstring'''



		_snake_case   :   str							=  OPTConfig
		_snake_case   :   str							=  {}
		_snake_case   :   str							=  '''gelu'''



		def __init__(							self    ,		_UpperCamelCase    ,		_UpperCamelCase=1_3    ,		_UpperCamelCase=7    ,		_UpperCamelCase=True    ,		_UpperCamelCase=False    ,		_UpperCamelCase=9_9    ,		_UpperCamelCase=1_6    ,		_UpperCamelCase=2    ,		_UpperCamelCase=4    ,		_UpperCamelCase=4    ,		_UpperCamelCase="gelu"    ,		_UpperCamelCase=0.1    ,		_UpperCamelCase=0.1    ,		_UpperCamelCase=2_0    ,		_UpperCamelCase=2    ,		_UpperCamelCase=1    ,		_UpperCamelCase=0    ,		_UpperCamelCase=1_6    ,		_UpperCamelCase=1_6    ,		)     ->							Union[str, Any]:
							UpperCAmelCase_   :				Union[str, Any]						=						parent
							UpperCAmelCase_   :				List[Any]						=						batch_size
							UpperCAmelCase_   :				Optional[Any]						=						seq_length
							UpperCAmelCase_   :				Any						=						is_training
							UpperCAmelCase_   :				Optional[Any]						=						use_labels
							UpperCAmelCase_   :				Optional[Any]						=						vocab_size
							UpperCAmelCase_   :				Tuple						=						hidden_size
							UpperCAmelCase_   :				Optional[int]						=						num_hidden_layers
							UpperCAmelCase_   :				Tuple						=						num_attention_heads
							UpperCAmelCase_   :				Tuple						=						intermediate_size
							UpperCAmelCase_   :				Tuple						=						hidden_act
							UpperCAmelCase_   :				Tuple						=						hidden_dropout_prob
							UpperCAmelCase_   :				Union[str, Any]						=						attention_probs_dropout_prob
							UpperCAmelCase_   :				Union[str, Any]						=						max_position_embeddings
							UpperCAmelCase_   :				str						=						eos_token_id
							UpperCAmelCase_   :				Dict						=						pad_token_id
							UpperCAmelCase_   :				List[Any]						=						bos_token_id
							UpperCAmelCase_   :				List[str]						=						embed_dim
							UpperCAmelCase_   :				Union[str, Any]						=						word_embed_proj_dim
							UpperCAmelCase_   :				str						=						False



		def  __UpperCAmelCase      (							self       )     ->							str:
							UpperCAmelCase_   :				Optional[Any]						=						ids_tensor([self.batch_size, self.seq_length - 1]    ,		self.vocab_size       )
							UpperCAmelCase_   :				List[str]						=						tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size       )    ,		1       )
							UpperCAmelCase_   :				Union[str, Any]						=						tf.concat([input_ids, eos_tensor]    ,		axis=1       )

							UpperCAmelCase_   :				Any						=						self.config_cls(
							    vocab_size=self.vocab_size    ,		hidden_size=self.hidden_size    ,		num_hidden_layers=self.num_hidden_layers    ,		num_attention_heads=self.num_attention_heads    ,		ffn_dim=self.intermediate_size    ,		dropout=self.hidden_dropout_prob    ,		attention_dropout=self.attention_probs_dropout_prob    ,		max_position_embeddings=self.max_position_embeddings    ,		eos_token_id=self.eos_token_id    ,		bos_token_id=self.bos_token_id    ,		pad_token_id=self.pad_token_id    ,		embed_dim=self.embed_dim    ,		word_embed_proj_dim=self.word_embed_proj_dim    ,		is_encoder_decoder=_UpperCamelCase    ,		**self.config_updates    ,		)
							UpperCAmelCase_   :				Optional[Any]						=						prepare_opt_inputs_dict(_UpperCamelCase    ,		_UpperCamelCase       )
							return config, inputs_dict



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase       )     ->							Any:
							UpperCAmelCase_   :				Union[str, Any]						=						TFOPTModel(config=_UpperCamelCase       )
							UpperCAmelCase_   :				Union[str, Any]						=						inputs_dict['input_ids']

							UpperCAmelCase_   :				str						=						input_ids[:1, :]
							UpperCAmelCase_   :				List[Any]						=						inputs_dict['attention_mask'][:1, :]
							UpperCAmelCase_   :				Optional[Any]						=						1

							# first forward pass
							UpperCAmelCase_   :				str						=						model(_UpperCamelCase    ,		attention_mask=_UpperCamelCase    ,		use_cache=_UpperCamelCase       )

							UpperCAmelCase_	,  UpperCAmelCase_   :				Optional[Any]						=						outputs.to_tuple()

							# create hypothetical next token and extent to next_input_ids
							UpperCAmelCase_   :				Dict						=						ids_tensor((self.batch_size, 3)    ,		config.vocab_size       )
							UpperCAmelCase_   :				int						=						tf.cast(ids_tensor((self.batch_size, 3)    ,		2       )    ,		tf.inta       )

							# append to next input_ids and
							UpperCAmelCase_   :				int						=						tf.concat([input_ids, next_tokens]    ,		axis=-1       )
							UpperCAmelCase_   :				List[Any]						=						tf.concat([attention_mask, next_attn_mask]    ,		axis=-1       )

							UpperCAmelCase_   :				str						=						model(_UpperCamelCase    ,		attention_mask=_UpperCamelCase       )[0]
							UpperCAmelCase_   :				Optional[int]						=						model(_UpperCamelCase    ,		attention_mask=_UpperCamelCase    ,		past_key_values=_UpperCamelCase       )[0]

							self.parent.assertEqual(next_tokens.shape[1]    ,		output_from_past.shape[1]       )

							# select random slice
							UpperCAmelCase_   :				Optional[Any]						=						int(ids_tensor((1,)    ,		output_from_past.shape[-1]       )       )
							UpperCAmelCase_   :				Tuple						=						output_from_no_past[:, -3:, random_slice_idx]
							UpperCAmelCase_   :				List[str]						=						output_from_past[:, :, random_slice_idx]

							# test that outputs are equal for slice
							tf.debugging.assert_near(_UpperCamelCase    ,		_UpperCamelCase    ,		rtol=1E-3       )





@require_tf
class   lowerCamelCase				(_snake_case			,  _snake_case			,  unittest.TestCase      ):







		'''simple docstring'''



		_snake_case   :   Union[str, Any]							=  (TFOPTModel, TFOPTForCausalLM) if is_tf_available() else ()
		_snake_case   :   Tuple							=  (TFOPTForCausalLM,) if is_tf_available() else ()
		_snake_case   :   Union[str, Any]							=  (
		    {'''feature-extraction''': TFOPTModel, '''text-generation''': TFOPTForCausalLM} if is_tf_available() else {}
		)
		_snake_case   :   int							=  False
		_snake_case   :   str							=  False
		_snake_case   :   Dict							=  False
		_snake_case   :   Union[str, Any]							=  1_0



		def  __UpperCAmelCase      (							self       )     ->							Optional[Any]:
							UpperCAmelCase_   :				Optional[Any]						=						TFOPTModelTester(self       )
							UpperCAmelCase_   :				Dict						=						ConfigTester(self    ,		config_class=_UpperCamelCase       )



		def  __UpperCAmelCase      (							self       )     ->							str:
							self.config_tester.run_common_tests()



		def  __UpperCAmelCase      (							self       )     ->							Union[str, Any]:
							UpperCAmelCase_   :				Dict						=						self.model_tester.prepare_config_and_inputs_for_common()
							self.model_tester.check_decoder_model_past_large_inputs(*_UpperCamelCase       )



		def  __UpperCAmelCase      (							self       )     ->							Optional[Any]:
							UpperCAmelCase_	,  UpperCAmelCase_   :				str						=						self.model_tester.prepare_config_and_inputs_for_common()

							def _get_word_embedding_weight(_UpperCamelCase    ,		_UpperCamelCase       ):
												if hasattr(_UpperCamelCase    ,		'weight'       ):
																	return embedding_layer.weight
												else:
																	# Here we build the word embeddings weights if not exists.
																	# And then we retry to get the attribute once built.
																	model.build()
																	if hasattr(_UpperCamelCase    ,		'weight'       ):
																						return embedding_layer.weight
																	else:
																						return None

							for model_class in self.all_model_classes:
												for size in [config.vocab_size - 1_0, config.vocab_size + 1_0]:
																	# build the embeddings
																	UpperCAmelCase_   :				Dict						=						model_class(config=_UpperCamelCase       )
																	UpperCAmelCase_   :				List[str]						=						_get_word_embedding_weight(_UpperCamelCase    ,		model.get_input_embeddings()       )
																	UpperCAmelCase_   :				Tuple						=						_get_word_embedding_weight(_UpperCamelCase    ,		model.get_output_embeddings()       )

																	# reshape the embeddings
																	model.resize_token_embeddings(_UpperCamelCase       )
																	UpperCAmelCase_   :				Optional[int]						=						_get_word_embedding_weight(_UpperCamelCase    ,		model.get_input_embeddings()       )
																	UpperCAmelCase_   :				Dict						=						_get_word_embedding_weight(_UpperCamelCase    ,		model.get_output_embeddings()       )

																	# check that the resized embeddings size matches the desired size.
																	UpperCAmelCase_   :				List[str]						=						size if size is not None else config.vocab_size

																	self.assertEqual(new_input_embeddings.shape[0]    ,		_UpperCamelCase       )

																	# check that weights remain the same after resizing
																	UpperCAmelCase_   :				List[Any]						=						True
																	for pa, pa in zip(old_input_embeddings.value()    ,		new_input_embeddings.value()       ):
																						if tf.math.reduce_sum(tf.math.abs(pa - pa       )       ) > 0:
																											UpperCAmelCase_   :				str						=						False
																	self.assertTrue(_UpperCamelCase       )

																	if old_output_embeddings is not None and new_output_embeddings is not None:
																						self.assertEqual(new_output_embeddings.shape[0]    ,		_UpperCamelCase       )

																						UpperCAmelCase_   :				Dict						=						True
																						for pa, pa in zip(old_output_embeddings.value()    ,		new_output_embeddings.value()       ):
																											if tf.math.reduce_sum(tf.math.abs(pa - pa       )       ) > 0:
																																UpperCAmelCase_   :				str						=						False
																						self.assertTrue(_UpperCamelCase       )





def 					lowercase__				(					__snake_case   :							List[Any]  ):


					'''simple docstring'''


					return tf.constant(__snake_case					,	dtype=tf.intaa  )



@require_tf
class   lowerCamelCase				(unittest.TestCase      ):







		'''simple docstring'''



		_snake_case   :   List[Any]							=  9_9



		def  __UpperCAmelCase      (							self       )     ->							str:
							UpperCAmelCase_   :				Any						=						tf.ones((4, 1)    ,		dtype=tf.intaa       ) * 2
							UpperCAmelCase_   :				Dict						=						tf.concat([ids_tensor((4, 6)    ,		self.vocab_size - 3       ) + 3, eos_column_vector]    ,		axis=1       )
							UpperCAmelCase_   :				List[str]						=						input_ids.shape[0]
							UpperCAmelCase_   :				Union[str, Any]						=						OPTConfig(
							    vocab_size=self.vocab_size    ,		hidden_size=2_4    ,		num_hidden_layers=2    ,		num_attention_heads=2    ,		ffn_dim=3_2    ,		max_position_embeddings=4_8    ,		eos_token_id=2    ,		pad_token_id=1    ,		bos_token_id=0    ,		)
							return config, input_ids, batch_size





@require_sentencepiece
@require_tf
class   lowerCamelCase				(unittest.TestCase      ):







		'''simple docstring'''



		@slow
		def  __UpperCAmelCase      (							self       )     ->							Dict:
							UpperCAmelCase_   :				Optional[int]						=						TFOPTModel.from_pretrained('facebook/opt-350m'       )
							UpperCAmelCase_   :				Tuple						=						_long_tensor([[0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]]       )
							UpperCAmelCase_   :				List[str]						=						tf.not_equal(_UpperCamelCase    ,		model.config.pad_token_id       )
							with tf.GradientTape():
												UpperCAmelCase_   :				List[str]						=						model(input_ids=_UpperCamelCase    ,		attention_mask=_UpperCamelCase       ).last_hidden_state
							UpperCAmelCase_   :				Optional[int]						=						(1, 1_1, 5_1_2)
							self.assertEqual(output.shape    ,		_UpperCamelCase       )
							UpperCAmelCase_   :				Dict						=						tf.constant(
							    [[-0.28_73, -1.92_18, -0.30_33], [-1.27_10, -0.13_38, -0.19_02], [0.40_95, 0.12_14, -1.31_21]]       )
							self.assertTrue(np.allclose(output[:, :3, :3]    ,		_UpperCamelCase    ,		atol=4E-3       )       )

							UpperCAmelCase_   :				List[str]						=						tf.function(_UpperCamelCase    ,		jit_compile=_UpperCamelCase       )
							UpperCAmelCase_   :				Optional[Any]						=						xla_generate(_UpperCamelCase    ,		_UpperCamelCase       )[0]
							self.assertTrue(np.allclose(output[:, :3, :3]    ,		_UpperCamelCase    ,		atol=4E-2       )       )





@require_tf
@slow
class   lowerCamelCase				(unittest.TestCase      ):







		'''simple docstring'''



		def  __UpperCAmelCase      (							self       )     ->							List[str]:
							super().setUp()
							UpperCAmelCase_   :				Any						=						'facebook/opt-350m'



		def  __UpperCAmelCase      (							self       )     ->							Tuple:
							UpperCAmelCase_   :				Tuple						=						TFOPTForCausalLM.from_pretrained(self.path_model       )
							UpperCAmelCase_   :				int						=						GPTaTokenizer.from_pretrained(self.path_model       )

							UpperCAmelCase_   :				Tuple						=						[
							    'Today is a beautiful day and I want to',
							    'In the city of',
							    'Paris is the capital of France and',
							    'Computers and mobile phones have taken',
							]
							# verify that prompt without BOS token is identical to Metaseq -> add_special_tokens=False
							UpperCAmelCase_   :				List[Any]						=						tokenizer(_UpperCamelCase    ,		return_tensors='tf'    ,		padding=_UpperCamelCase    ,		add_special_tokens=_UpperCamelCase       )
							UpperCAmelCase_   :				int						=						tf.math.reduce_mean(model(inputs.input_ids    ,		attention_mask=inputs.attention_mask       )[0]    ,		axis=-1       )
							UpperCAmelCase_   :				Union[str, Any]						=						tf.constant(
							    [
							        [1.38_51, -13.89_23, -10.52_29, -10.75_33, -0.23_09, -10.23_84, -0.53_65, -9.09_47, -5.16_70],
							        [-4.70_73, -10.62_76, -3.94_15, -21.52_42, -0.28_22, -0.28_22, -0.28_22, -0.28_22, -0.28_22],
							        [0.62_47, -3.42_29, -8.91_79, -1.42_97, -14.16_50, 1.41_46, -9.02_18, -0.27_03, -0.27_03],
							        [6.47_83, -1.99_13, -10.79_26, -2.33_36, 1.50_92, -0.99_74, -6.82_13, 1.34_77, 1.34_77],
							    ]       )
							self.assertTrue(np.allclose(_UpperCamelCase    ,		_UpperCamelCase    ,		atol=1E-4       )       )

							UpperCAmelCase_   :				Optional[int]						=						tf.function(_UpperCamelCase    ,		jit_compile=_UpperCamelCase       )
							UpperCAmelCase_   :				Union[str, Any]						=						tf.math.reduce_mean(xla_generate(inputs.input_ids    ,		attention_mask=inputs.attention_mask       )[0]    ,		axis=-1       )
							self.assertTrue(np.allclose(_UpperCamelCase    ,		_UpperCamelCase    ,		atol=1E-4       )       )





@require_tf
@slow
class   lowerCamelCase				(unittest.TestCase      ):







		'''simple docstring'''



		@property
		def  __UpperCAmelCase      (							self       )     ->							int:
							return [
							    "Today is a beautiful day and I want",
							    "In the city of",
							    "Paris is the capital of France and",
							    "Computers and mobile phones have taken",
							]



		def  __UpperCAmelCase      (							self       )     ->							Union[str, Any]:
							UpperCAmelCase_   :				Any						=						'facebook/opt-125m'

							UpperCAmelCase_   :				Optional[Any]						=						[
							    'Today is a beautiful day and I want to',
							    'In the city of New York, the city',
							    'Paris is the capital of France and the capital',
							    'Computers and mobile phones have taken over the',
							]

							UpperCAmelCase_   :				List[str]						=						[]
							UpperCAmelCase_   :				Optional[Any]						=						GPTaTokenizer.from_pretrained(_UpperCamelCase       )
							UpperCAmelCase_   :				str						=						TFOPTForCausalLM.from_pretrained(_UpperCamelCase       )

							for prompt in self.prompts:
												UpperCAmelCase_   :				Any						=						tokenizer(_UpperCamelCase    ,		return_tensors='tf'       ).input_ids

												UpperCAmelCase_   :				int						=						model.generate(_UpperCamelCase    ,		max_length=1_0       )

												UpperCAmelCase_   :				Tuple						=						tokenizer.batch_decode(_UpperCamelCase    ,		skip_special_tokens=_UpperCamelCase       )
												predicted_outputs += generated_string

							self.assertListEqual(_UpperCamelCase    ,		_UpperCamelCase       )



		def  __UpperCAmelCase      (							self       )     ->							Optional[Any]:
							UpperCAmelCase_   :				str						=						'facebook/opt-350m'

							UpperCAmelCase_   :				Dict						=						GPTaTokenizer.from_pretrained(_UpperCamelCase       )
							UpperCAmelCase_   :				Union[str, Any]						=						TFOPTForCausalLM.from_pretrained(_UpperCamelCase       )

							UpperCAmelCase_   :				str						=						'left'

							# use different length sentences to test batching
							UpperCAmelCase_   :				Optional[Any]						=						[
							    'Hello, my dog is a little',
							    'Today, I',
							]

							UpperCAmelCase_   :				Dict						=						tokenizer(_UpperCamelCase    ,		return_tensors='tf'    ,		padding=_UpperCamelCase       )
							UpperCAmelCase_   :				Any						=						inputs['input_ids']

							UpperCAmelCase_   :				Union[str, Any]						=						model.generate(input_ids=_UpperCamelCase    ,		attention_mask=inputs['attention_mask']       )

							UpperCAmelCase_   :				Dict						=						tokenizer(sentences[0]    ,		return_tensors='tf'       ).input_ids
							UpperCAmelCase_   :				Union[str, Any]						=						model.generate(input_ids=_UpperCamelCase       )

							UpperCAmelCase_   :				Optional[Any]						=						inputs_non_padded.shape[-1] - tf.math.reduce_sum(
							    tf.cast(inputs['attention_mask'][-1]    ,		tf.intaa       )       )
							UpperCAmelCase_   :				Union[str, Any]						=						tokenizer(sentences[1]    ,		return_tensors='tf'       ).input_ids
							UpperCAmelCase_   :				Dict						=						model.generate(input_ids=_UpperCamelCase    ,		max_length=model.config.max_length - num_paddings       )

							UpperCAmelCase_   :				Any						=						tokenizer.batch_decode(_UpperCamelCase    ,		skip_special_tokens=_UpperCamelCase       )
							UpperCAmelCase_   :				Any						=						tokenizer.decode(output_non_padded[0]    ,		skip_special_tokens=_UpperCamelCase       )
							UpperCAmelCase_   :				List[Any]						=						tokenizer.decode(output_padded[0]    ,		skip_special_tokens=_UpperCamelCase       )

							UpperCAmelCase_   :				Tuple						=						[
							    'Hello, my dog is a little bit of a dork.\nI\'m a little bit',
							    'Today, I was in the middle of a conversation with a friend about the',
							]
							self.assertListEqual(_UpperCamelCase    ,		_UpperCamelCase       )
							self.assertListEqual(_UpperCamelCase    ,		[non_padded_sentence, padded_sentence]       )



		def  __UpperCAmelCase      (							self       )     ->							Union[str, Any]:
							UpperCAmelCase_   :				List[str]						=						'facebook/opt-350m'

							UpperCAmelCase_   :				Tuple						=						[
							    'Today is a beautiful day and I want to',
							    'In the city of San Francisco, the city',
							    'Paris is the capital of France and the capital',
							    'Computers and mobile phones have taken over the',
							]

							UpperCAmelCase_   :				Dict						=						[]
							UpperCAmelCase_   :				Dict						=						GPTaTokenizer.from_pretrained(_UpperCamelCase       )
							UpperCAmelCase_   :				List[str]						=						TFOPTForCausalLM.from_pretrained(_UpperCamelCase       )

							for prompt in self.prompts:
												UpperCAmelCase_   :				Optional[Any]						=						tokenizer(_UpperCamelCase    ,		return_tensors='tf'       ).input_ids

												UpperCAmelCase_   :				Optional[Any]						=						model.generate(_UpperCamelCase    ,		max_length=1_0       )

												UpperCAmelCase_   :				str						=						tokenizer.batch_decode(_UpperCamelCase    ,		skip_special_tokens=_UpperCamelCase       )
												predicted_outputs += generated_string

							self.assertListEqual(_UpperCamelCase    ,		_UpperCamelCase       )














import argparse

import torch

from transformers import RemBertConfig, RemBertModel, load_tf_weights_in_rembert
from transformers.utils import logging


logging.set_verbosity_info()




def 					lowercase__				(					__snake_case   :							Optional[Any]					,	__snake_case   :							Any					,	__snake_case   :							int  ):


					'''simple docstring'''


					UpperCAmelCase_   :				Union[str, Any]						=						RemBertConfig.from_json_file(__snake_case  )
					print('Building PyTorch model from configuration: {}'.format(str(__snake_case  )  )  )
					UpperCAmelCase_   :				Dict						=						RemBertModel(__snake_case  )

					# Load weights from tf checkpoint
					load_tf_weights_in_rembert(__snake_case					,	__snake_case					,	__snake_case  )

					# Save pytorch-model
					print('Save PyTorch model to {}'.format(__snake_case  )  )
					torch.save(model.state_dict()					,	__snake_case  )


if __name__ == "__main__":
		__UpperCAmelCase     							=						argparse.ArgumentParser()
		# Required parameters
		parser.add_argument(
		    '--tf_checkpoint_path', default=None, type=str, required=True, help='Path to the TensorFlow checkpoint path.'
		)
		parser.add_argument(
		    '--rembert_config_file',
		    default=None,
		    type=str,
		    required=True,
		    help=(
		        'The config json file corresponding to the pre-trained RemBERT model. \n'
		        'This specifies the model architecture.'
		    ),
		)
		parser.add_argument(
		    '--pytorch_dump_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
		)
		__UpperCAmelCase     							=						parser.parse_args()
		convert_rembert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, args.rembert_config_file, args.pytorch_dump_path)






