

'''simple docstring'''
from unittest import TestCase

from datasets import Dataset
from minhash_deduplication import deduplicate_dataset, make_duplicate_clusters


def 	_lowerCAmelCase (						):
     """simple docstring"""




     _SCREAMING_SNAKE_CASE							   ={
         'repo_name': ['test_repo1', 'test_repo2', 'test_repo3'],
         'path': ['test_1.py', 'test_2.py', 'unit_test.py'],
         'content': ['a ' * 20, 'a ' * 30, 'b ' * 7],
     }
     _SCREAMING_SNAKE_CASE							   =Dataset.from_dict(snake_case_    )
     return dataset







class 							A__   (	UpperCamelCase__  ):





  def 						A  (		self				:     str			)     ->   Dict:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							   =get_dataset()
       _SCREAMING_SNAKE_CASE							   =make_duplicate_clusters(__a							, 0.85			)
       self.assertEqual(len(duplicate_clusters[0]			)							, 2			)





  def 						A  (		self				:     Optional[Any]			)     ->   str:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							   =get_dataset()
       _SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE							   =deduplicate_dataset(__a			)
       self.assertEqual(len(__a			)							, 2			)
       print(__a			)
       self.assertEqual(duplicate_clusters[0][0]['copies']							, 2			)
       self.assertEqual(duplicate_clusters[0][0]['is_extreme']							, __a			)



'''simple docstring'''
def 	_lowerCAmelCase (						_UpperCamelCase						:					str    )				->      bool:
     """simple docstring"""




     _SCREAMING_SNAKE_CASE							   =0
     for ch in input_str:
          _SCREAMING_SNAKE_CASE							   =ord(_UpperCamelCase    )
          _SCREAMING_SNAKE_CASE							   =pow(2							,			_UpperCamelCase    )

          # If we already turned on bit for current character's unicode
          if bitmap >> ch_unicode & 1 == 1:
               return False
          bitmap |= ch_bit_index_on
     return True


if __name__ == "__main__":
     import doctest

     doctest.testmod()
