





from __future__ import annotations

import unittest

import numpy as np

from transformers import OPTConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
     import tensorflow as tf

     from transformers import GPTaTokenizer, TFOPTForCausalLM, TFOPTModel
def 					_A	(		SCREAMING_SNAKE_CASE					:       Union[str, Any]	,	SCREAMING_SNAKE_CASE					:       List[Any]	,	SCREAMING_SNAKE_CASE					:       Optional[Any]=None	,	SCREAMING_SNAKE_CASE					:       str=None						):





  """simple docstring"""



  if attention_mask is None:
    a__     : Optional[int]  =tf.cast(tf.math.not_equal(SCREAMING_SNAKE_CASE	,	config.pad_token_id						)	,	tf.inta						)
  return {"input_ids": input_ids, "attention_mask": attention_mask}







@require_tf
class     __lowerCAmelCase	:
      _lowercase    : List[str]            =      OPTConfig
      _lowercase    : int            =      {}
      _lowercase    : Optional[int]            =      """gelu"""







      def __init__(   self		,				lowerCAmelCase__		,				lowerCAmelCase__=1_3		,				lowerCAmelCase__=7		,				lowerCAmelCase__=True		,				lowerCAmelCase__=False		,				lowerCAmelCase__=9_9		,				lowerCAmelCase__=1_6		,				lowerCAmelCase__=2		,				lowerCAmelCase__=4		,				lowerCAmelCase__=4		,				lowerCAmelCase__="gelu"		,				lowerCAmelCase__=0.1		,				lowerCAmelCase__=0.1		,				lowerCAmelCase__=2_0		,				lowerCAmelCase__=2		,				lowerCAmelCase__=1		,				lowerCAmelCase__=0		,				lowerCAmelCase__=1_6		,				lowerCAmelCase__=1_6		,				)       ->						Optional[int]:



        '''simple docstring'''




        a__     : Tuple  =parent
        a__     : List[str]  =batch_size
        a__     : Tuple  =seq_length
        a__     : Dict  =is_training
        a__     : Tuple  =use_labels
        a__     : Tuple  =vocab_size
        a__     : Optional[int]  =hidden_size
        a__     : Union[str, Any]  =num_hidden_layers
        a__     : Optional[int]  =num_attention_heads
        a__     : int  =intermediate_size
        a__     : Any  =hidden_act
        a__     : int  =hidden_dropout_prob
        a__     : List[Any]  =attention_probs_dropout_prob
        a__     : List[Any]  =max_position_embeddings
        a__     : Optional[Any]  =eos_token_id
        a__     : List[str]  =pad_token_id
        a__     : Optional[Any]  =bos_token_id
        a__     : Tuple  =embed_dim
        a__     : Optional[Any]  =word_embed_proj_dim
        a__     : Union[str, Any]  =False







      def        _lowercase  (   self							)       ->						Any:



        '''simple docstring'''




        a__     : Tuple  =ids_tensor([self.batch_size, self.seq_length - 1]		,				self.vocab_size							)
        a__     : Dict  =tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size							)		,				1							)
        a__     : int  =tf.concat([input_ids, eos_tensor]		,				axis=1							)

        a__     : Tuple  =self.config_cls(
            vocab_size=self.vocab_size		,				hidden_size=self.hidden_size		,				num_hidden_layers=self.num_hidden_layers		,				num_attention_heads=self.num_attention_heads		,				ffn_dim=self.intermediate_size		,				dropout=self.hidden_dropout_prob		,				attention_dropout=self.attention_probs_dropout_prob		,				max_position_embeddings=self.max_position_embeddings		,				eos_token_id=self.eos_token_id		,				bos_token_id=self.bos_token_id		,				pad_token_id=self.pad_token_id		,				embed_dim=self.embed_dim		,				word_embed_proj_dim=self.word_embed_proj_dim		,				is_encoder_decoder=lowerCAmelCase__		,				**self.config_updates		,				)
        a__     : Tuple  =prepare_opt_inputs_dict(lowerCAmelCase__		,				lowerCAmelCase__							)
        return config, inputs_dict







      def        _lowercase  (   self		,				lowerCAmelCase__		,				lowerCAmelCase__							)       ->						Optional[int]:



        '''simple docstring'''




        a__     : List[str]  =TFOPTModel(config=lowerCAmelCase__							)
        a__     : Optional[Any]  =inputs_dict["input_ids"]

        a__     : List[str]  =input_ids[:1, :]
        a__     : List[str]  =inputs_dict["attention_mask"][:1, :]
        a__     : Union[str, Any]  =1

        # first forward pass
        a__     : Union[str, Any]  =model(lowerCAmelCase__		,				attention_mask=lowerCAmelCase__		,				use_cache=lowerCAmelCase__							)

        a__     : Union[str, Any]  =outputs.to_tuple()

        # create hypothetical next token and extent to next_input_ids
        a__     : Tuple  =ids_tensor((self.batch_size, 3)		,				config.vocab_size							)
        a__     : Union[str, Any]  =tf.cast(ids_tensor((self.batch_size, 3)		,				2							)		,				tf.inta							)

        # append to next input_ids and
        a__     : Tuple  =tf.concat([input_ids, next_tokens]		,				axis=-1							)
        a__     : Optional[Any]  =tf.concat([attention_mask, next_attn_mask]		,				axis=-1							)

        a__     : Optional[int]  =model(lowerCAmelCase__		,				attention_mask=lowerCAmelCase__							)[0]
        a__     : int  =model(lowerCAmelCase__		,				attention_mask=lowerCAmelCase__		,				past_key_values=lowerCAmelCase__							)[0]

        self.parent.assertEqual(next_tokens.shape[1]		,				output_from_past.shape[1]							)

        # select random slice
        a__     : int  =int(ids_tensor((1,)		,				output_from_past.shape[-1]							)							)
        a__     : str  =output_from_no_past[:, -3:, random_slice_idx]
        a__     : Optional[Any]  =output_from_past[:, :, random_slice_idx]

        # test that outputs are equal for slice
        tf.debugging.assert_near(lowerCAmelCase__		,				lowerCAmelCase__		,				rtol=1E-3							)







@require_tf
class     __lowerCAmelCase	(   UpperCamelCase__						,    UpperCamelCase__						,    unittest.TestCase):
      _lowercase    : str            =      (TFOPTModel, TFOPTForCausalLM) if is_tf_available() else ()
      _lowercase    : Dict            =      (TFOPTForCausalLM,) if is_tf_available() else ()
      _lowercase    : Union[str, Any]            =      (
          {"""feature-extraction""": TFOPTModel, """text-generation""": TFOPTForCausalLM} if is_tf_available() else {}
      )
      _lowercase    : Any            =      False
      _lowercase    : str            =      False
      _lowercase    : Any            =      False
      _lowercase    : Union[str, Any]            =      10







      def        _lowercase  (   self							)       ->						List[str]:



        '''simple docstring'''




        a__     : List[str]  =TFOPTModelTester(self							)
        a__     : Optional[Any]  =ConfigTester(self		,				config_class=lowerCAmelCase__							)







      def        _lowercase  (   self							)       ->						Optional[Any]:



        '''simple docstring'''




        self.config_tester.run_common_tests()







      def        _lowercase  (   self							)       ->						Tuple:



        '''simple docstring'''




        a__     : Union[str, Any]  =self.model_tester.prepare_config_and_inputs_for_common()
        self.model_tester.check_decoder_model_past_large_inputs(*lowerCAmelCase__							)







      def        _lowercase  (   self							)       ->						Optional[int]:



        '''simple docstring'''




        a__     : List[str]  =self.model_tester.prepare_config_and_inputs_for_common()

        def _get_word_embedding_weight(lowerCAmelCase__		,				lowerCAmelCase__							):
          if hasattr(lowerCAmelCase__		,				"weight"							):
            return embedding_layer.weight
          else:
            # Here we build the word embeddings weights if not exists.
            # And then we retry to get the attribute once built.
            model.build()
            if hasattr(lowerCAmelCase__		,				"weight"							):
              return embedding_layer.weight
            else:
              return None

        for model_class in self.all_model_classes:
          for size in [config.vocab_size - 1_0, config.vocab_size + 1_0]:
            # build the embeddings
            a__     : Union[str, Any]  =model_class(config=lowerCAmelCase__							)
            a__     : List[Any]  =_get_word_embedding_weight(lowerCAmelCase__		,				model.get_input_embeddings()							)
            a__     : Optional[Any]  =_get_word_embedding_weight(lowerCAmelCase__		,				model.get_output_embeddings()							)

            # reshape the embeddings
            model.resize_token_embeddings(lowerCAmelCase__							)
            a__     : Optional[int]  =_get_word_embedding_weight(lowerCAmelCase__		,				model.get_input_embeddings()							)
            a__     : Optional[Any]  =_get_word_embedding_weight(lowerCAmelCase__		,				model.get_output_embeddings()							)

            # check that the resized embeddings size matches the desired size.
            a__     : Dict  =size if size is not None else config.vocab_size

            self.assertEqual(new_input_embeddings.shape[0]		,				lowerCAmelCase__							)

            # check that weights remain the same after resizing
            a__     : int  =True
            for pa, pa in zip(old_input_embeddings.value()		,				new_input_embeddings.value()							):
              if tf.math.reduce_sum(tf.math.abs(pa - pa							)							) > 0:
                a__     : int  =False
            self.assertTrue(lowerCAmelCase__							)

            if old_output_embeddings is not None and new_output_embeddings is not None:
              self.assertEqual(new_output_embeddings.shape[0]		,				lowerCAmelCase__							)

              a__     : int  =True
              for pa, pa in zip(old_output_embeddings.value()		,				new_output_embeddings.value()							):
                if tf.math.reduce_sum(tf.math.abs(pa - pa							)							) > 0:
                  a__     : Union[str, Any]  =False
              self.assertTrue(lowerCAmelCase__							)
def 					_A	(		SCREAMING_SNAKE_CASE					:       int						):





  """simple docstring"""



  return tf.constant(SCREAMING_SNAKE_CASE	,	dtype=tf.intaa						)







@require_tf
class     __lowerCAmelCase	(   unittest.TestCase):
      _lowercase    : List[str]            =      99







      def        _lowercase  (   self							)       ->						Optional[int]:



        '''simple docstring'''




        a__     : Dict  =tf.ones((4, 1)		,				dtype=tf.intaa							) * 2
        a__     : Union[str, Any]  =tf.concat([ids_tensor((4, 6)		,				self.vocab_size - 3							) + 3, eos_column_vector]		,				axis=1							)
        a__     : List[str]  =input_ids.shape[0]
        a__     : List[str]  =OPTConfig(
            vocab_size=self.vocab_size		,				hidden_size=2_4		,				num_hidden_layers=2		,				num_attention_heads=2		,				ffn_dim=3_2		,				max_position_embeddings=4_8		,				eos_token_id=2		,				pad_token_id=1		,				bos_token_id=0		,				)
        return config, input_ids, batch_size







@require_sentencepiece
@require_tf
class     __lowerCAmelCase	(   unittest.TestCase):







      @slow
      def        _lowercase  (   self							)       ->						Optional[int]:



        '''simple docstring'''




        a__     : Dict  =TFOPTModel.from_pretrained("facebook/opt-350m"							)
        a__     : str  =_long_tensor([[0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]]							)
        a__     : int  =tf.not_equal(lowerCAmelCase__		,				model.config.pad_token_id							)
        with tf.GradientTape():
          a__     : Any  =model(input_ids=lowerCAmelCase__		,				attention_mask=lowerCAmelCase__							).last_hidden_state
        a__     : Tuple  =(1, 1_1, 5_1_2)
        self.assertEqual(output.shape		,				lowerCAmelCase__							)
        a__     : List[str]  =tf.constant(
            [[-0.28_73, -1.92_18, -0.30_33], [-1.27_10, -0.13_38, -0.19_02], [0.40_95, 0.12_14, -1.31_21]]							)
        self.assertTrue(np.allclose(output[:, :3, :3]		,				lowerCAmelCase__		,				atol=4E-3							)							)

        a__     : Any  =tf.function(lowerCAmelCase__		,				jit_compile=lowerCAmelCase__							)
        a__     : List[Any]  =xla_generate(lowerCAmelCase__		,				lowerCAmelCase__							)[0]
        self.assertTrue(np.allclose(output[:, :3, :3]		,				lowerCAmelCase__		,				atol=4E-2							)							)







@require_tf
@slow
class     __lowerCAmelCase	(   unittest.TestCase):







      def        _lowercase  (   self							)       ->						Optional[Any]:



        '''simple docstring'''




        super().setUp()
        a__     : Optional[int]  ="facebook/opt-350m"







      def        _lowercase  (   self							)       ->						Optional[Any]:



        '''simple docstring'''




        a__     : List[Any]  =TFOPTForCausalLM.from_pretrained(self.path_model							)
        a__     : str  =GPTaTokenizer.from_pretrained(self.path_model							)

        a__     : Dict  =[
            "Today is a beautiful day and I want to",
            "In the city of",
            "Paris is the capital of France and",
            "Computers and mobile phones have taken",
        ]
        # verify that prompt without BOS token is identical to Metaseq -> add_special_tokens=False
        a__     : Union[str, Any]  =tokenizer(lowerCAmelCase__		,				return_tensors="tf"		,				padding=lowerCAmelCase__		,				add_special_tokens=lowerCAmelCase__							)
        a__     : List[str]  =tf.math.reduce_mean(model(inputs.input_ids		,				attention_mask=inputs.attention_mask							)[0]		,				axis=-1							)
        a__     : Tuple  =tf.constant(
            [
                [1.38_51, -13.89_23, -10.52_29, -10.75_33, -0.23_09, -10.23_84, -0.53_65, -9.09_47, -5.16_70],
                [-4.70_73, -10.62_76, -3.94_15, -21.52_42, -0.28_22, -0.28_22, -0.28_22, -0.28_22, -0.28_22],
                [0.62_47, -3.42_29, -8.91_79, -1.42_97, -14.16_50, 1.41_46, -9.02_18, -0.27_03, -0.27_03],
                [6.47_83, -1.99_13, -10.79_26, -2.33_36, 1.50_92, -0.99_74, -6.82_13, 1.34_77, 1.34_77],
            ]							)
        self.assertTrue(np.allclose(lowerCAmelCase__		,				lowerCAmelCase__		,				atol=1E-4							)							)

        a__     : Union[str, Any]  =tf.function(lowerCAmelCase__		,				jit_compile=lowerCAmelCase__							)
        a__     : Optional[int]  =tf.math.reduce_mean(xla_generate(inputs.input_ids		,				attention_mask=inputs.attention_mask							)[0]		,				axis=-1							)
        self.assertTrue(np.allclose(lowerCAmelCase__		,				lowerCAmelCase__		,				atol=1E-4							)							)







@require_tf
@slow
class     __lowerCAmelCase	(   unittest.TestCase):







      @property
      def        _lowercase  (   self							)       ->						Optional[Any]:



        '''simple docstring'''




        return [
            "Today is a beautiful day and I want",
            "In the city of",
            "Paris is the capital of France and",
            "Computers and mobile phones have taken",
        ]







      def        _lowercase  (   self							)       ->						List[Any]:



        '''simple docstring'''




        a__     : Optional[int]  ="facebook/opt-125m"

        a__     : Dict  =[
            "Today is a beautiful day and I want to",
            "In the city of New York, the city",
            "Paris is the capital of France and the capital",
            "Computers and mobile phones have taken over the",
        ]

        a__     : Any  =[]
        a__     : Union[str, Any]  =GPTaTokenizer.from_pretrained(lowerCAmelCase__							)
        a__     : Tuple  =TFOPTForCausalLM.from_pretrained(lowerCAmelCase__							)

        for prompt in self.prompts:
          a__     : List[str]  =tokenizer(lowerCAmelCase__		,				return_tensors="tf"							).input_ids

          a__     : str  =model.generate(lowerCAmelCase__		,				max_length=1_0							)

          a__     : int  =tokenizer.batch_decode(lowerCAmelCase__		,				skip_special_tokens=lowerCAmelCase__							)
          predicted_outputs += generated_string

        self.assertListEqual(lowerCAmelCase__		,				lowerCAmelCase__							)







      def        _lowercase  (   self							)       ->						Union[str, Any]:



        '''simple docstring'''




        a__     : Union[str, Any]  ="facebook/opt-350m"

        a__     : str  =GPTaTokenizer.from_pretrained(lowerCAmelCase__							)
        a__     : Dict  =TFOPTForCausalLM.from_pretrained(lowerCAmelCase__							)

        a__     : List[str]  ="left"

        # use different length sentences to test batching
        a__     : Any  =[
            "Hello, my dog is a little",
            "Today, I",
        ]

        a__     : List[Any]  =tokenizer(lowerCAmelCase__		,				return_tensors="tf"		,				padding=lowerCAmelCase__							)
        a__     : Optional[Any]  =inputs["input_ids"]

        a__     : Optional[Any]  =model.generate(input_ids=lowerCAmelCase__		,				attention_mask=inputs["attention_mask"]							)

        a__     : int  =tokenizer(sentences[0]		,				return_tensors="tf"							).input_ids
        a__     : str  =model.generate(input_ids=lowerCAmelCase__							)

        a__     : List[Any]  =inputs_non_padded.shape[-1] - tf.math.reduce_sum(
            tf.cast(inputs["attention_mask"][-1]		,				tf.intaa							)							)
        a__     : Tuple  =tokenizer(sentences[1]		,				return_tensors="tf"							).input_ids
        a__     : Tuple  =model.generate(input_ids=lowerCAmelCase__		,				max_length=model.config.max_length - num_paddings							)

        a__     : List[Any]  =tokenizer.batch_decode(lowerCAmelCase__		,				skip_special_tokens=lowerCAmelCase__							)
        a__     : Optional[int]  =tokenizer.decode(output_non_padded[0]		,				skip_special_tokens=lowerCAmelCase__							)
        a__     : Tuple  =tokenizer.decode(output_padded[0]		,				skip_special_tokens=lowerCAmelCase__							)

        a__     : Dict  =[
            "Hello, my dog is a little bit of a dork.\nI'm a little bit",
            "Today, I was in the middle of a conversation with a friend about the",
        ]
        self.assertListEqual(lowerCAmelCase__		,				lowerCAmelCase__							)
        self.assertListEqual(lowerCAmelCase__		,				[non_padded_sentence, padded_sentence]							)







      def        _lowercase  (   self							)       ->						Union[str, Any]:



        '''simple docstring'''




        a__     : Dict  ="facebook/opt-350m"

        a__     : Tuple  =[
            "Today is a beautiful day and I want to",
            "In the city of San Francisco, the city",
            "Paris is the capital of France and the capital",
            "Computers and mobile phones have taken over the",
        ]

        a__     : Union[str, Any]  =[]
        a__     : List[str]  =GPTaTokenizer.from_pretrained(lowerCAmelCase__							)
        a__     : Any  =TFOPTForCausalLM.from_pretrained(lowerCAmelCase__							)

        for prompt in self.prompts:
          a__     : Dict  =tokenizer(lowerCAmelCase__		,				return_tensors="tf"							).input_ids

          a__     : Union[str, Any]  =model.generate(lowerCAmelCase__		,				max_length=1_0							)

          a__     : List[Any]  =tokenizer.batch_decode(lowerCAmelCase__		,				skip_special_tokens=lowerCAmelCase__							)
          predicted_outputs += generated_string

        self.assertListEqual(lowerCAmelCase__		,				lowerCAmelCase__							)













import unittest

from transformers import AutoTokenizer, NystromformerConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         NystromformerForMaskedLM,
         NystromformerForMultipleChoice,
         NystromformerForQuestionAnswering,
         NystromformerForSequenceClassification,
         NystromformerForTokenClassification,
         NystromformerModel,
     )
     from transformers.models.nystromformer.modeling_nystromformer import NYSTROMFORMER_PRETRAINED_MODEL_ARCHIVE_LIST







class     __lowerCAmelCase	:







      def __init__(   self		,				lowerCAmelCase__		,				lowerCAmelCase__=1_3		,				lowerCAmelCase__=7		,				lowerCAmelCase__=True		,				lowerCAmelCase__=True		,				lowerCAmelCase__=True		,				lowerCAmelCase__=True		,				lowerCAmelCase__=9_9		,				lowerCAmelCase__=3_2		,				lowerCAmelCase__=5		,				lowerCAmelCase__=4		,				lowerCAmelCase__=3_7		,				lowerCAmelCase__="gelu"		,				lowerCAmelCase__=0.1		,				lowerCAmelCase__=0.1		,				lowerCAmelCase__=5_1_2		,				lowerCAmelCase__=1_6		,				lowerCAmelCase__=2		,				lowerCAmelCase__=0.02		,				lowerCAmelCase__=3		,				lowerCAmelCase__=4		,				lowerCAmelCase__=None		,				)       ->						List[Any]:



        '''simple docstring'''




        a__     : Optional[Any]  =parent
        a__     : Tuple  =batch_size
        a__     : List[Any]  =seq_length
        a__     : Dict  =is_training
        a__     : Any  =use_input_mask
        a__     : int  =use_token_type_ids
        a__     : Optional[Any]  =use_labels
        a__     : Optional[Any]  =vocab_size
        a__     : List[str]  =hidden_size
        a__     : int  =num_hidden_layers
        a__     : Tuple  =num_attention_heads
        a__     : Union[str, Any]  =intermediate_size
        a__     : Optional[int]  =hidden_act
        a__     : int  =hidden_dropout_prob
        a__     : Union[str, Any]  =attention_probs_dropout_prob
        a__     : List[Any]  =max_position_embeddings
        a__     : str  =type_vocab_size
        a__     : Optional[Any]  =type_sequence_label_size
        a__     : Union[str, Any]  =initializer_range
        a__     : List[Any]  =num_labels
        a__     : str  =num_choices
        a__     : int  =scope







      def        _lowercase  (   self							)       ->						int:



        '''simple docstring'''




        a__     : List[Any]  =ids_tensor([self.batch_size, self.seq_length]		,				self.vocab_size							)

        a__     : str  =None
        if self.use_input_mask:
          a__     : List[Any]  =random_attention_mask([self.batch_size, self.seq_length]							)

        a__     : str  =None
        if self.use_token_type_ids:
          a__     : List[str]  =ids_tensor([self.batch_size, self.seq_length]		,				self.type_vocab_size							)

        a__     : Dict  =None
        a__     : str  =None
        a__     : str  =None
        if self.use_labels:
          a__     : List[str]  =ids_tensor([self.batch_size]		,				self.type_sequence_label_size							)
          a__     : Tuple  =ids_tensor([self.batch_size, self.seq_length]		,				self.num_labels							)
          a__     : Dict  =ids_tensor([self.batch_size]		,				self.num_choices							)

        a__     : Tuple  =self.get_config()

        return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







      def        _lowercase  (   self							)       ->						Tuple:



        '''simple docstring'''




        return NystromformerConfig(
            vocab_size=self.vocab_size		,				hidden_size=self.hidden_size		,				num_hidden_layers=self.num_hidden_layers		,				num_attention_heads=self.num_attention_heads		,				intermediate_size=self.intermediate_size		,				hidden_act=self.hidden_act		,				hidden_dropout_prob=self.hidden_dropout_prob		,				attention_probs_dropout_prob=self.attention_probs_dropout_prob		,				max_position_embeddings=self.max_position_embeddings		,				type_vocab_size=self.type_vocab_size		,				is_decoder=lowerCAmelCase__		,				initializer_range=self.initializer_range		,				)







      def        _lowercase  (   self		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__							)       ->						List[Any]:



        '''simple docstring'''




        a__     : Tuple  =NystromformerModel(config=lowerCAmelCase__							)
        model.to(lowerCAmelCase__							)
        model.eval()
        a__     : Optional[Any]  =model(lowerCAmelCase__		,				attention_mask=lowerCAmelCase__		,				token_type_ids=lowerCAmelCase__							)
        a__     : str  =model(lowerCAmelCase__		,				token_type_ids=lowerCAmelCase__							)
        a__     : Optional[int]  =model(lowerCAmelCase__							)
        self.parent.assertEqual(result.last_hidden_state.shape		,				(self.batch_size, self.seq_length, self.hidden_size)							)







      def        _lowercase  (   self		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__							)       ->						int:



        '''simple docstring'''




        a__     : int  =NystromformerForMaskedLM(config=lowerCAmelCase__							)
        model.to(lowerCAmelCase__							)
        model.eval()
        a__     : Dict  =model(lowerCAmelCase__		,				attention_mask=lowerCAmelCase__		,				token_type_ids=lowerCAmelCase__		,				labels=lowerCAmelCase__							)
        self.parent.assertEqual(result.logits.shape		,				(self.batch_size, self.seq_length, self.vocab_size)							)







      def        _lowercase  (   self		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__							)       ->						Any:



        '''simple docstring'''




        a__     : Optional[int]  =NystromformerForQuestionAnswering(config=lowerCAmelCase__							)
        model.to(lowerCAmelCase__							)
        model.eval()
        a__     : str  =model(
            lowerCAmelCase__		,				attention_mask=lowerCAmelCase__		,				token_type_ids=lowerCAmelCase__		,				start_positions=lowerCAmelCase__		,				end_positions=lowerCAmelCase__		,				)
        self.parent.assertEqual(result.start_logits.shape		,				(self.batch_size, self.seq_length)							)
        self.parent.assertEqual(result.end_logits.shape		,				(self.batch_size, self.seq_length)							)







      def        _lowercase  (   self		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__							)       ->						Optional[int]:



        '''simple docstring'''




        a__     : Optional[Any]  =self.num_labels
        a__     : Dict  =NystromformerForSequenceClassification(lowerCAmelCase__							)
        model.to(lowerCAmelCase__							)
        model.eval()
        a__     : List[str]  =model(lowerCAmelCase__		,				attention_mask=lowerCAmelCase__		,				token_type_ids=lowerCAmelCase__		,				labels=lowerCAmelCase__							)
        self.parent.assertEqual(result.logits.shape		,				(self.batch_size, self.num_labels)							)







      def        _lowercase  (   self		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__							)       ->						List[str]:



        '''simple docstring'''




        a__     : Tuple  =self.num_labels
        a__     : List[str]  =NystromformerForTokenClassification(config=lowerCAmelCase__							)
        model.to(lowerCAmelCase__							)
        model.eval()
        a__     : List[Any]  =model(lowerCAmelCase__		,				attention_mask=lowerCAmelCase__		,				token_type_ids=lowerCAmelCase__		,				labels=lowerCAmelCase__							)
        self.parent.assertEqual(result.logits.shape		,				(self.batch_size, self.seq_length, self.num_labels)							)







      def        _lowercase  (   self		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__							)       ->						Tuple:



        '''simple docstring'''




        a__     : List[Any]  =self.num_choices
        a__     : Optional[Any]  =NystromformerForMultipleChoice(config=lowerCAmelCase__							)
        model.to(lowerCAmelCase__							)
        model.eval()
        a__     : List[str]  =input_ids.unsqueeze(1							).expand(-1		,				self.num_choices		,				-1							).contiguous()
        a__     : List[Any]  =token_type_ids.unsqueeze(1							).expand(-1		,				self.num_choices		,				-1							).contiguous()
        a__     : List[Any]  =input_mask.unsqueeze(1							).expand(-1		,				self.num_choices		,				-1							).contiguous()
        a__     : Dict  =model(
            lowerCAmelCase__		,				attention_mask=lowerCAmelCase__		,				token_type_ids=lowerCAmelCase__		,				labels=lowerCAmelCase__		,				)
        self.parent.assertEqual(result.logits.shape		,				(self.batch_size, self.num_choices)							)







      def        _lowercase  (   self							)       ->						Tuple:



        '''simple docstring'''




        a__     : Optional[Any]  =self.prepare_config_and_inputs()
        (
            (
            a__
        ) ,							(
            a__
        ) ,							(
            a__
        ) ,							(
            a__
        ) ,							(
            a__
        ) ,							(
            a__
        ) ,							(
            a__
        ) ,							
        )     : List[str]  =config_and_inputs
        a__     : str  ={"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
        return config, inputs_dict







@require_torch
class     __lowerCAmelCase	(   UpperCamelCase__						,    UpperCamelCase__						,    unittest.TestCase):
      _lowercase    : int            =      (
          (
              NystromformerModel,
              NystromformerForMaskedLM,
              NystromformerForMultipleChoice,
              NystromformerForQuestionAnswering,
              NystromformerForSequenceClassification,
              NystromformerForTokenClassification,
          )
          if is_torch_available()
          else ()
      )
      _lowercase    : Union[str, Any]            =      (
          {
              """feature-extraction""": NystromformerModel,
              """fill-mask""": NystromformerForMaskedLM,
              """question-answering""": NystromformerForQuestionAnswering,
              """text-classification""": NystromformerForSequenceClassification,
              """token-classification""": NystromformerForTokenClassification,
              """zero-shot""": NystromformerForSequenceClassification,
          }
          if is_torch_available()
          else {}
      )
      _lowercase    : Union[str, Any]            =      False
      _lowercase    : Union[str, Any]            =      False







      def        _lowercase  (   self							)       ->						List[str]:



        '''simple docstring'''




        a__     : Optional[int]  =NystromformerModelTester(self							)
        a__     : Optional[int]  =ConfigTester(self		,				config_class=lowerCAmelCase__		,				hidden_size=3_7							)







      def        _lowercase  (   self							)       ->						int:



        '''simple docstring'''




        self.config_tester.run_common_tests()







      def        _lowercase  (   self							)       ->						Dict:



        '''simple docstring'''




        a__     : List[str]  =self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*lowerCAmelCase__							)







      def        _lowercase  (   self							)       ->						Optional[Any]:



        '''simple docstring'''




        a__     : Tuple  =self.model_tester.prepare_config_and_inputs()
        for type in ["absolute", "relative_key", "relative_key_query"]:
          a__     : int  =type
          self.model_tester.create_and_check_model(*lowerCAmelCase__							)







      def        _lowercase  (   self							)       ->						Optional[Any]:



        '''simple docstring'''




        a__     : Optional[Any]  =self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_masked_lm(*lowerCAmelCase__							)







      def        _lowercase  (   self							)       ->						Dict:



        '''simple docstring'''




        a__     : str  =self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_multiple_choice(*lowerCAmelCase__							)







      def        _lowercase  (   self							)       ->						List[str]:



        '''simple docstring'''




        a__     : Tuple  =self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_question_answering(*lowerCAmelCase__							)







      def        _lowercase  (   self							)       ->						Union[str, Any]:



        '''simple docstring'''




        a__     : Tuple  =self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_sequence_classification(*lowerCAmelCase__							)







      def        _lowercase  (   self							)       ->						Optional[Any]:



        '''simple docstring'''




        a__     : int  =self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_token_classification(*lowerCAmelCase__							)







      @slow
      def        _lowercase  (   self							)       ->						str:



        '''simple docstring'''




        for model_name in NYSTROMFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
          a__     : int  =NystromformerModel.from_pretrained(lowerCAmelCase__							)
          self.assertIsNotNone(lowerCAmelCase__							)







@require_torch
class     __lowerCAmelCase	(   unittest.TestCase):







      @slow
      def        _lowercase  (   self							)       ->						str:



        '''simple docstring'''




        a__     : str  =NystromformerModel.from_pretrained("uw-madison/nystromformer-512"							)
        a__     : int  =torch.tensor([[0, 1, 2, 3, 4, 5]]							)

        with torch.no_grad():
          a__     : Tuple  =model(lowerCAmelCase__							)[0]

        a__     : List[str]  =torch.Size((1, 6, 7_6_8)							)
        self.assertEqual(output.shape		,				lowerCAmelCase__							)

        a__     : int  =torch.tensor(
            [[[-0.45_32, -0.09_36, 0.51_37], [-0.26_76, 0.06_28, 0.61_86], [-0.36_29, -0.17_26, 0.47_16]]]							)

        self.assertTrue(torch.allclose(output[:, :3, :3]		,				lowerCAmelCase__		,				atol=1E-4							)							)







      @slow
      def        _lowercase  (   self							)       ->						Optional[Any]:



        '''simple docstring'''




        a__     : Union[str, Any]  ="the [MASK] of Belgium is Brussels"

        a__     : str  =AutoTokenizer.from_pretrained("uw-madison/nystromformer-512"							)
        a__     : int  =NystromformerForMaskedLM.from_pretrained("uw-madison/nystromformer-512"							)

        a__     : List[Any]  =tokenizer(lowerCAmelCase__		,				return_tensors="pt"							)

        with torch.no_grad():
          a__     : str  =model(encoding.input_ids							).logits

        a__     : List[str]  =token_logits[:, 2, :].argmax(-1							)[0]

        self.assertEqual(tokenizer.decode(lowerCAmelCase__							)		,				"capital"							)






