import unittest

from transformers import BertGenerationConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import BertGenerationDecoder, BertGenerationEncoder




class 	__A				:



    def __init__(self   :   Union[str, Any]       ,      __a   :   Any       ,      __a   :   List[Any]=13       ,      __a   :   Optional[Any]=7       ,      __a   :   List[Any]=True       ,      __a   :   Optional[Any]=True       ,      __a   :   Any=99       ,      __a   :   List[Any]=32       ,      __a   :   Optional[Any]=5       ,      __a   :   Optional[int]=4       ,      __a   :   Tuple=37       ,      __a   :   int="gelu"       ,      __a   :   Union[str, Any]=0.1       ,      __a   :   str=0.1       ,      __a   :   str=50       ,      __a   :   List[Any]=0.02       ,      __a   :   str=True       ,      __a   :   Tuple=None       ,      ):
         UpperCAmelCase_	   =       parent
         UpperCAmelCase_	   =       batch_size
         UpperCAmelCase_	   =       seq_length
         UpperCAmelCase_	   =       is_training
         UpperCAmelCase_	   =       use_input_mask
         UpperCAmelCase_	   =       vocab_size
         UpperCAmelCase_	   =       hidden_size
         UpperCAmelCase_	   =       num_hidden_layers
         UpperCAmelCase_	   =       num_attention_heads
         UpperCAmelCase_	   =       intermediate_size
         UpperCAmelCase_	   =       hidden_act
         UpperCAmelCase_	   =       hidden_dropout_prob
         UpperCAmelCase_	   =       attention_probs_dropout_prob
         UpperCAmelCase_	   =       max_position_embeddings
         UpperCAmelCase_	   =       initializer_range
         UpperCAmelCase_	   =       use_labels
         UpperCAmelCase_	   =       scope



    def  _lowercase						(self   :   Union[str, Any]			):
         UpperCAmelCase_	   =       ids_tensor([self.batch_size, self.seq_length]       ,      self.vocab_size			)

         UpperCAmelCase_	   =       None
         if self.use_input_mask:
              UpperCAmelCase_	   =       random_attention_mask([self.batch_size, self.seq_length]			)

         if self.use_labels:
              UpperCAmelCase_	   =       ids_tensor([self.batch_size, self.seq_length]       ,      self.vocab_size			)

         UpperCAmelCase_	   =       self.get_config()

         return config, input_ids, input_mask, token_labels



    def  _lowercase						(self   :   List[Any]			):
         return BertGenerationConfig(
             vocab_size=self.vocab_size       ,      hidden_size=self.hidden_size       ,      num_hidden_layers=self.num_hidden_layers       ,      num_attention_heads=self.num_attention_heads       ,      intermediate_size=self.intermediate_size       ,      hidden_act=self.hidden_act       ,      hidden_dropout_prob=self.hidden_dropout_prob       ,      attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,      max_position_embeddings=self.max_position_embeddings       ,      is_decoder=__a       ,      initializer_range=self.initializer_range       ,      )



    def  _lowercase						(self   :   Any			):
         (
             (
             UpperCAmelCase_			
         )   ,			(
             UpperCAmelCase_			
         )   ,			(
             UpperCAmelCase_			
         )   ,			(
             UpperCAmelCase_			
         )   ,						
         )	   =       self.prepare_config_and_inputs()

         UpperCAmelCase_	   =       True
         UpperCAmelCase_	   =       floats_tensor([self.batch_size, self.seq_length, self.hidden_size]			)
         UpperCAmelCase_	   =       ids_tensor([self.batch_size, self.seq_length]       ,      vocab_size=2			)

         return (
             config,
             input_ids,
             input_mask,
             token_labels,
             encoder_hidden_states,
             encoder_attention_mask,
         )



    def  _lowercase						(self   :   Any       ,      __a   :   Optional[int]       ,      __a   :   Any       ,      __a   :   Any       ,      __a   :   Tuple       ,      **__a   :   Any       ,      ):
         UpperCAmelCase_	   =       BertGenerationEncoder(config=__a			)
         model.to(__a			)
         model.eval()
         UpperCAmelCase_	   =       model(__a       ,      attention_mask=__a			)
         UpperCAmelCase_	   =       model(__a			)
         self.parent.assertEqual(result.last_hidden_state.shape       ,      (self.batch_size, self.seq_length, self.hidden_size)			)



    def  _lowercase						(self   :   Dict       ,      __a   :   Tuple       ,      __a   :   Optional[int]       ,      __a   :   Any       ,      __a   :   Optional[Any]       ,      __a   :   Optional[int]       ,      __a   :   int       ,      **__a   :   List[str]       ,      ):
         UpperCAmelCase_	   =       True
         UpperCAmelCase_	   =       BertGenerationEncoder(config=__a			)
         model.to(__a			)
         model.eval()
         UpperCAmelCase_	   =       model(
             __a       ,      attention_mask=__a       ,      encoder_hidden_states=__a       ,      encoder_attention_mask=__a       ,      )
         UpperCAmelCase_	   =       model(
             __a       ,      attention_mask=__a       ,      encoder_hidden_states=__a       ,      )
         self.parent.assertEqual(result.last_hidden_state.shape       ,      (self.batch_size, self.seq_length, self.hidden_size)			)



    def  _lowercase						(self   :   Any       ,      __a   :   Any       ,      __a   :   List[Any]       ,      __a   :   Union[str, Any]       ,      __a   :   Union[str, Any]       ,      __a   :   List[str]       ,      __a   :   Any       ,      **__a   :   Optional[Any]       ,      ):
         UpperCAmelCase_	   =       True
         UpperCAmelCase_	   =       True
         UpperCAmelCase_	   =       BertGenerationDecoder(config=__a			).to(__a			).eval()

         # first forward pass
         UpperCAmelCase_	   =       model(
             __a       ,      attention_mask=__a       ,      encoder_hidden_states=__a       ,      encoder_attention_mask=__a       ,      use_cache=__a       ,      )
         UpperCAmelCase_	   =       outputs.past_key_values

         # create hypothetical multiple next token and extent to next_input_ids
         UpperCAmelCase_	   =       ids_tensor((self.batch_size, 3)       ,      config.vocab_size			)
         UpperCAmelCase_	   =       ids_tensor((self.batch_size, 3)       ,      vocab_size=2			)

         # append to next input_ids and
         UpperCAmelCase_	   =       torch.cat([input_ids, next_tokens]       ,      dim=-1			)
         UpperCAmelCase_	   =       torch.cat([input_mask, next_mask]       ,      dim=-1			)

         UpperCAmelCase_	   =       model(
             __a       ,      attention_mask=__a       ,      encoder_hidden_states=__a       ,      encoder_attention_mask=__a       ,      output_hidden_states=__a       ,      )["hidden_states"][0]
         UpperCAmelCase_	   =       model(
             __a       ,      attention_mask=__a       ,      encoder_hidden_states=__a       ,      encoder_attention_mask=__a       ,      past_key_values=__a       ,      output_hidden_states=__a       ,      )["hidden_states"][0]

         # select random slice
         UpperCAmelCase_	   =       ids_tensor((1,)       ,      output_from_past.shape[-1]			).item()
         UpperCAmelCase_	   =       output_from_no_past[:, -3:, random_slice_idx].detach()
         UpperCAmelCase_	   =       output_from_past[:, :, random_slice_idx].detach()

         self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]			)

         # test that outputs are equal for slice
         self.parent.assertTrue(torch.allclose(__a       ,      __a       ,      atol=1E-3			)			)



    def  _lowercase						(self   :   Any       ,      __a   :   List[str]       ,      __a   :   Any       ,      __a   :   List[str]       ,      __a   :   Optional[int]       ,      *__a   :   Optional[Any]       ,      ):
         UpperCAmelCase_	   =       BertGenerationDecoder(__a			)
         model.to(__a			)
         model.eval()
         UpperCAmelCase_	   =       model(__a       ,      attention_mask=__a       ,      labels=__a			)
         self.parent.assertEqual(result.logits.shape       ,      (self.batch_size, self.seq_length, self.vocab_size)			)






    def  _lowercase						(self   :   Tuple			):
         UpperCAmelCase_   ,			UpperCAmelCase_   ,			UpperCAmelCase_   ,			UpperCAmelCase_	   =       self.prepare_config_and_inputs()
         UpperCAmelCase_	   =       {"input_ids": input_ids, "attention_mask": input_mask}
         return config, inputs_dict




@require_torch
class 	__A				(			UpperCamelCase__		,     UpperCamelCase__		,     UpperCamelCase__		,     unittest.TestCase     ):
    a__	:						int	       =  (BertGenerationEncoder, BertGenerationDecoder) if is_torch_available() else ()
    a__	:						Any	       =  (BertGenerationDecoder,) if is_torch_available() else ()
    a__	:						List[Any]	       =  (
        {"""feature-extraction""": BertGenerationEncoder, """text-generation""": BertGenerationDecoder}
        if is_torch_available()
        else {}
    )



    def  _lowercase						(self   :   List[str]			):
         UpperCAmelCase_	   =       BertGenerationEncoderTester(self			)
         UpperCAmelCase_	   =       ConfigTester(self       ,      config_class=__a       ,      hidden_size=37			)



    def  _lowercase						(self   :   int			):
         self.config_tester.run_common_tests()



    def  _lowercase						(self   :   Union[str, Any]			):
         UpperCAmelCase_	   =       self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*__a			)



    def  _lowercase						(self   :   Optional[Any]			):
         UpperCAmelCase_   ,			UpperCAmelCase_   ,			UpperCAmelCase_   ,			UpperCAmelCase_	   =       self.model_tester.prepare_config_and_inputs()
         UpperCAmelCase_	   =       "bert"
         self.model_tester.create_and_check_model(__a       ,      __a       ,      __a       ,      __a			)



    def  _lowercase						(self   :   Union[str, Any]			):
         UpperCAmelCase_	   =       self.model_tester.prepare_config_and_inputs_for_decoder()
         self.model_tester.create_and_check_model_as_decoder(*__a			)



    def  _lowercase						(self   :   List[Any]			):
         UpperCAmelCase_	   =       self.model_tester.prepare_config_and_inputs_for_decoder()
         self.model_tester.create_and_check_decoder_model_past_large_inputs(*__a			)



    def  _lowercase						(self   :   int			):
         # This regression test was failing with PyTorch < 1.3
         (
             (
             UpperCAmelCase_			
         )   ,			(
             UpperCAmelCase_			
         )   ,			(
             UpperCAmelCase_			
         )   ,			(
             UpperCAmelCase_			
         )   ,			(
             UpperCAmelCase_			
         )   ,			(
             UpperCAmelCase_			
         )   ,						
         )	   =       self.model_tester.prepare_config_and_inputs_for_decoder()

         UpperCAmelCase_	   =       None

         self.model_tester.create_and_check_model_as_decoder(
             __a       ,      __a       ,      __a       ,      __a       ,      __a       ,      __a       ,      )



    def  _lowercase						(self   :   Any			):
         UpperCAmelCase_	   =       self.model_tester.prepare_config_and_inputs_for_decoder()
         self.model_tester.create_and_check_for_causal_lm(*__a			)






    @slow
    def  _lowercase						(self   :   str			):
         UpperCAmelCase_	   =       BertGenerationEncoder.from_pretrained("google/bert_for_seq_generation_L-24_bbc_encoder"			)
         self.assertIsNotNone(__a			)




@require_torch
class 	__A				(			unittest.TestCase     ):



    @slow
    def  _lowercase						(self   :   Any			):
         UpperCAmelCase_	   =       BertGenerationEncoder.from_pretrained("google/bert_for_seq_generation_L-24_bbc_encoder"			)
         UpperCAmelCase_	   =       torch.tensor([[101, 7592, 1010, 2026, 3899, 2003, 10140, 102]]			)
         with torch.no_grad():
              UpperCAmelCase_	   =       model(__a			)[0]
         UpperCAmelCase_	   =       torch.Size([1, 8, 1024]			)
         self.assertEqual(output.shape       ,      __a			)
         UpperCAmelCase_	   =       torch.tensor(
             [[[0.17_75, 0.00_83, -0.03_21], [1.60_02, 0.12_87, 0.39_12], [2.14_73, 0.57_91, 0.60_66]]]			)
         self.assertTrue(torch.allclose(output[:, :3, :3]       ,      __a       ,      atol=1E-4			)			)




@require_torch
class 	__A				(			unittest.TestCase     ):



    @slow
    def  _lowercase						(self   :   Tuple			):
         UpperCAmelCase_	   =       BertGenerationDecoder.from_pretrained("google/bert_for_seq_generation_L-24_bbc_encoder"			)
         UpperCAmelCase_	   =       torch.tensor([[101, 7592, 1010, 2026, 3899, 2003, 10140, 102]]			)
         with torch.no_grad():
              UpperCAmelCase_	   =       model(__a			)[0]
         UpperCAmelCase_	   =       torch.Size([1, 8, 50358]			)
         self.assertEqual(output.shape       ,      __a			)
         UpperCAmelCase_	   =       torch.tensor(
             [[[-0.57_88, -2.59_94, -3.70_54], [0.04_38, 4.79_97, 1.87_95], [1.58_62, 6.64_09, 4.46_38]]]			)
         self.assertTrue(torch.allclose(output[:, :3, :3]       ,      __a       ,      atol=1E-4			)			)





'''simple docstring'''





from unittest import TestCase

from datasets import Dataset
from minhash_deduplication import deduplicate_dataset, make_duplicate_clusters





def      lowerCAmelCase_    ( )   ->				str:


     '''simple docstring'''




     UpperCAmelCase_	   =       {
         "repo_name": ["test_repo1", "test_repo2", "test_repo3"],
         "path": ["test_1.py", "test_2.py", "unit_test.py"],
         "content": ["a " * 20, "a " * 30, "b " * 7],
     }
     UpperCAmelCase_	   =       Dataset.from_dict(snake_case_      )
     return dataset






class 	__A				(			UpperCamelCase__     ):



    def  _lowercase						(self   :   str			):
         UpperCAmelCase_	   =       get_dataset()
         UpperCAmelCase_	   =       make_duplicate_clusters(__a       ,      0.85			)
         self.assertEqual(len(duplicate_clusters[0]			)       ,      2			)






    def  _lowercase						(self   :   Optional[Any]			):
         UpperCAmelCase_	   =       get_dataset()
         UpperCAmelCase_   ,			UpperCAmelCase_	   =       deduplicate_dataset(__a			)
         self.assertEqual(len(__a			)       ,      2			)
         print(__a			)
         self.assertEqual(duplicate_clusters[0][0]["copies"]       ,      2			)
         self.assertEqual(duplicate_clusters[0][0]["is_extreme"]       ,      __a			)




