

'''simple docstring'''




import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer






class   _SCREAMING_SNAKE_CASE   (   __snake_case     ):



	def 			snake_case__  (   self		:   List[str]      ):
						__magic_name__						       =    tempfile.mkdtemp()
						__magic_name__						       =    5

						# Realm tok
						__magic_name__						       =    [
						    '''[UNK]''',
						    '''[CLS]''',
						    '''[SEP]''',
						    '''[PAD]''',
						    '''[MASK]''',
						    '''test''',
						    '''question''',
						    '''this''',
						    '''is''',
						    '''the''',
						    '''first''',
						    '''second''',
						    '''third''',
						    '''fourth''',
						    '''fifth''',
						    '''record''',
						    '''want''',
						    '''##want''',
						    '''##ed''',
						    '''wa''',
						    '''un''',
						    '''runn''',
						    '''##ing''',
						    ''',''',
						    '''low''',
						    '''lowest''',
						]
						__magic_name__						       =    os.path.join(self.tmpdirname			,				'''realm_tokenizer'''      )
						os.makedirs(a__			,				exist_ok=a__      )
						__magic_name__						       =    os.path.join(a__			,				VOCAB_FILES_NAMES['''vocab_file''']      )
						with open(self.vocab_file			,				'''w'''			,				encoding='''utf-8'''      ) as vocab_writer:
											vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]      )      )

						__magic_name__						       =    os.path.join(self.tmpdirname			,				'''realm_block_records'''      )
						os.makedirs(a__			,				exist_ok=a__      )



	def 			snake_case__  (   self		:   Any      ):
						return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname			,				'''realm_tokenizer'''      )      )



	def 			snake_case__  (   self		:   int      ):
						shutil.rmtree(self.tmpdirname      )



	def 			snake_case__  (   self		:   Any      ):
						__magic_name__						       =    RealmConfig(num_block_records=self.num_block_records      )
						return config



	def 			snake_case__  (   self		:   Optional[int]      ):
						__magic_name__						       =    Dataset.from_dict(
						    {
						        '''id''': ['''0''', '''1'''],
						        '''question''': ['''foo''', '''bar'''],
						        '''answers''': [['''Foo''', '''Bar'''], ['''Bar''']],
						    }      )
						return dataset



	def 			snake_case__  (   self		:   Any      ):
						__magic_name__						       =    np.array(
						    [
						        B'''This is the first record''',
						        B'''This is the second record''',
						        B'''This is the third record''',
						        B'''This is the fourth record''',
						        B'''This is the fifth record''',
						        B'''This is a longer longer longer record''',
						    ]			,				dtype=a__			,				)
						return block_records



	def 			snake_case__  (   self		:   str      ):
						__magic_name__						       =    RealmRetriever(
						    block_records=self.get_dummy_block_records()			,				tokenizer=self.get_tokenizer()			,				)
						return retriever



	def 			snake_case__  (   self		:   Union[str, Any]      ):
						__magic_name__						       =    self.get_config()
						__magic_name__						       =    self.get_dummy_retriever()
						__magic_name__						       =    retriever.tokenizer

						__magic_name__						       =    np.array([0, 3]			,				dtype='''long'''      )
						__magic_name__						       =    tokenizer(['''Test question''']      ).input_ids
						__magic_name__						       =    tokenizer(
						    ['''the fourth''']			,				add_special_tokens=a__			,				return_token_type_ids=a__			,				return_attention_mask=a__			,				).input_ids
						__magic_name__						       =    config.reader_seq_len

						__magic_name__       ,  __magic_name__       ,  __magic_name__       ,  __magic_name__						       =    retriever(
						    a__			,				a__			,				answer_ids=a__			,				max_length=a__			,				return_tensors='''np'''      )

						self.assertEqual(len(a__      )			,				2      )
						self.assertEqual(len(a__      )			,				2      )
						self.assertEqual(len(a__      )			,				2      )
						self.assertEqual(concat_inputs.input_ids.shape			,				(2, 10)      )
						self.assertEqual(concat_inputs.attention_mask.shape			,				(2, 10)      )
						self.assertEqual(concat_inputs.token_type_ids.shape			,				(2, 10)      )
						self.assertEqual(concat_inputs.special_tokens_mask.shape			,				(2, 10)      )
						self.assertEqual(
						    tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0]      )			,				['''[CLS]''', '''test''', '''question''', '''[SEP]''', '''this''', '''is''', '''the''', '''first''', '''record''', '''[SEP]''']			,				)
						self.assertEqual(
						    tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1]      )			,				['''[CLS]''', '''test''', '''question''', '''[SEP]''', '''this''', '''is''', '''the''', '''fourth''', '''record''', '''[SEP]''']			,				)



	def 			snake_case__  (   self		:   Union[str, Any]      ):
						__magic_name__						       =    self.get_config()
						__magic_name__						       =    self.get_dummy_retriever()
						__magic_name__						       =    retriever.tokenizer

						__magic_name__						       =    np.array([0, 3, 5]			,				dtype='''long'''      )
						__magic_name__						       =    tokenizer(['''Test question''']      ).input_ids
						__magic_name__						       =    tokenizer(
						    ['''the fourth''', '''longer longer''']			,				add_special_tokens=a__			,				return_token_type_ids=a__			,				return_attention_mask=a__			,				).input_ids
						__magic_name__						       =    config.reader_seq_len

						__magic_name__       ,  __magic_name__       ,  __magic_name__       ,  __magic_name__						       =    retriever(
						    a__			,				a__			,				answer_ids=a__			,				max_length=a__			,				return_tensors='''np'''      )

						self.assertEqual([False, True, True]			,				a__      )
						self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]			,				a__      )
						self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]			,				a__      )



	def 			snake_case__  (   self		:   List[str]      ):
						__magic_name__						       =    self.get_dummy_retriever()
						retriever.save_pretrained(os.path.join(self.tmpdirname			,				'''realm_block_records'''      )      )

						# Test local path
						__magic_name__						       =    retriever.from_pretrained(os.path.join(self.tmpdirname			,				'''realm_block_records'''      )      )
						self.assertEqual(retriever.block_records[0]			,				B'''This is the first record'''      )

						# Test mocked remote path
						with patch('''transformers.models.realm.retrieval_realm.hf_hub_download'''      ) as mock_hf_hub_download:
											__magic_name__						       =    os.path.join(
											    os.path.join(self.tmpdirname			,				'''realm_block_records'''      )			,				_REALM_BLOCK_RECORDS_FILENAME      )
											__magic_name__						       =    RealmRetriever.from_pretrained('''google/realm-cc-news-pretrained-openqa'''      )

						self.assertEqual(retriever.block_records[0]			,				B'''This is the first record'''      )









'''simple docstring'''




import unittest

import numpy as np
from datasets import load_dataset

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
      import torch

if is_vision_available():
      from PIL import Image

      from transformers import BeitImageProcessor






class   _SCREAMING_SNAKE_CASE   (   unittest.TestCase     ):



 def __init__(   self		:   Tuple			,				a__		:   Union[str, Any]			,				a__		:   Union[str, Any]=7			,				a__		:   Dict=3			,				a__		:   Optional[Any]=18			,				a__		:   Optional[Any]=30			,				a__		:   Tuple=400			,				a__		:   Optional[int]=True			,				a__		:   int=None			,				a__		:   Union[str, Any]=True			,				a__		:   Optional[Any]=None			,				a__		:   str=True			,				a__		:   List[Any]=[0.5, 0.5, 0.5]			,				a__		:   Tuple=[0.5, 0.5, 0.5]			,				a__		:   Union[str, Any]=False			,				):
      __magic_name__						       =    size if size is not None else {'''height''': 20, '''width''': 20}
      __magic_name__						       =    crop_size if crop_size is not None else {'''height''': 18, '''width''': 18}
      __magic_name__						       =    parent
      __magic_name__						       =    batch_size
      __magic_name__						       =    num_channels
      __magic_name__						       =    image_size
      __magic_name__						       =    min_resolution
      __magic_name__						       =    max_resolution
      __magic_name__						       =    do_resize
      __magic_name__						       =    size
      __magic_name__						       =    do_center_crop
      __magic_name__						       =    crop_size
      __magic_name__						       =    do_normalize
      __magic_name__						       =    image_mean
      __magic_name__						       =    image_std
      __magic_name__						       =    do_reduce_labels



 def 			snake_case__  (   self		:   Optional[Any]      ):
      return {
          "do_resize": self.do_resize,
          "size": self.size,
          "do_center_crop": self.do_center_crop,
          "crop_size": self.crop_size,
          "do_normalize": self.do_normalize,
          "image_mean": self.image_mean,
          "image_std": self.image_std,
          "do_reduce_labels": self.do_reduce_labels,
      }




def 						UpperCamelCase	(			)				->     str:






     '''simple docstring'''
     __magic_name__						       =    load_dataset('''hf-internal-testing/fixtures_ade20k'''		,						split='''test'''			)

     __magic_name__						       =    Image.open(dataset[0]['''file''']			)
     __magic_name__						       =    Image.open(dataset[1]['''file''']			)

     return image, map




def 						UpperCamelCase	(			)				->     List[str]:






     '''simple docstring'''
     __magic_name__						       =    load_dataset('''hf-internal-testing/fixtures_ade20k'''		,						split='''test'''			)

     __magic_name__						       =    Image.open(ds[0]['''file''']			)
     __magic_name__						       =    Image.open(ds[1]['''file''']			)
     __magic_name__						       =    Image.open(ds[2]['''file''']			)
     __magic_name__						       =    Image.open(ds[3]['''file''']			)

     return [imagea, imagea], [mapa, mapa]






@require_torch
@require_vision
class   _SCREAMING_SNAKE_CASE   (   __a		,unittest.TestCase     ):
 __SCREAMING_SNAKE_CASE					:Optional[Any]      	=					BeitImageProcessor if is_vision_available() else None



 def 			snake_case__  (   self		:   Dict      ):
      __magic_name__						       =    BeitImageProcessingTester(self      )



 @property
 def 			snake_case__  (   self		:   Dict      ):
      return self.image_processor_tester.prepare_image_processor_dict()



 def 			snake_case__  (   self		:   Optional[Any]      ):
      __magic_name__						       =    self.image_processing_class(**self.image_processor_dict      )
      self.assertTrue(hasattr(a__			,				'''do_resize'''      )      )
      self.assertTrue(hasattr(a__			,				'''size'''      )      )
      self.assertTrue(hasattr(a__			,				'''do_center_crop'''      )      )
      self.assertTrue(hasattr(a__			,				'''center_crop'''      )      )
      self.assertTrue(hasattr(a__			,				'''do_normalize'''      )      )
      self.assertTrue(hasattr(a__			,				'''image_mean'''      )      )
      self.assertTrue(hasattr(a__			,				'''image_std'''      )      )



 def 			snake_case__  (   self		:   int      ):
      __magic_name__						       =    self.image_processing_class.from_dict(self.image_processor_dict      )
      self.assertEqual(image_processor.size			,				{'''height''': 20, '''width''': 20}      )
      self.assertEqual(image_processor.crop_size			,				{'''height''': 18, '''width''': 18}      )
      self.assertEqual(image_processor.do_reduce_labels			,				a__      )

      __magic_name__						       =    self.image_processing_class.from_dict(
          self.image_processor_dict			,				size=42			,				crop_size=84			,				reduce_labels=a__      )
      self.assertEqual(image_processor.size			,				{'''height''': 42, '''width''': 42}      )
      self.assertEqual(image_processor.crop_size			,				{'''height''': 84, '''width''': 84}      )
      self.assertEqual(image_processor.do_reduce_labels			,				a__      )



 def 			snake_case__  (   self		:   Optional[Any]      ):
      pass



 def 			snake_case__  (   self		:   Dict      ):
      # Initialize image_processing
      __magic_name__						       =    self.image_processing_class(**self.image_processor_dict      )
      # create random PIL images
      __magic_name__						       =    prepare_image_inputs(self.image_processor_tester			,				equal_resolution=a__      )
      for image in image_inputs:
           self.assertIsInstance(a__			,				Image.Image      )

      # Test not batched input
      __magic_name__						       =    image_processing(image_inputs[0]			,				return_tensors='''pt'''      ).pixel_values
      self.assertEqual(
          encoded_images.shape			,				(
              1,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.crop_size['''height'''],
              self.image_processor_tester.crop_size['''width'''],
          )			,				)

      # Test batched
      __magic_name__						       =    image_processing(a__			,				return_tensors='''pt'''      ).pixel_values
      self.assertEqual(
          encoded_images.shape			,				(
              self.image_processor_tester.batch_size,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.crop_size['''height'''],
              self.image_processor_tester.crop_size['''width'''],
          )			,				)



 def 			snake_case__  (   self		:   Optional[int]      ):
      # Initialize image_processing
      __magic_name__						       =    self.image_processing_class(**self.image_processor_dict      )
      # create random numpy tensors
      __magic_name__						       =    prepare_image_inputs(self.image_processor_tester			,				equal_resolution=a__			,				numpify=a__      )
      for image in image_inputs:
           self.assertIsInstance(a__			,				np.ndarray      )

      # Test not batched input
      __magic_name__						       =    image_processing(image_inputs[0]			,				return_tensors='''pt'''      ).pixel_values
      self.assertEqual(
          encoded_images.shape			,				(
              1,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.crop_size['''height'''],
              self.image_processor_tester.crop_size['''width'''],
          )			,				)

      # Test batched
      __magic_name__						       =    image_processing(a__			,				return_tensors='''pt'''      ).pixel_values
      self.assertEqual(
          encoded_images.shape			,				(
              self.image_processor_tester.batch_size,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.crop_size['''height'''],
              self.image_processor_tester.crop_size['''width'''],
          )			,				)



 def 			snake_case__  (   self		:   List[str]      ):
      # Initialize image_processing
      __magic_name__						       =    self.image_processing_class(**self.image_processor_dict      )
      # create random PyTorch tensors
      __magic_name__						       =    prepare_image_inputs(self.image_processor_tester			,				equal_resolution=a__			,				torchify=a__      )
      for image in image_inputs:
           self.assertIsInstance(a__			,				torch.Tensor      )

      # Test not batched input
      __magic_name__						       =    image_processing(image_inputs[0]			,				return_tensors='''pt'''      ).pixel_values
      self.assertEqual(
          encoded_images.shape			,				(
              1,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.crop_size['''height'''],
              self.image_processor_tester.crop_size['''width'''],
          )			,				)

      # Test batched
      __magic_name__						       =    image_processing(a__			,				return_tensors='''pt'''      ).pixel_values
      self.assertEqual(
          encoded_images.shape			,				(
              self.image_processor_tester.batch_size,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.crop_size['''height'''],
              self.image_processor_tester.crop_size['''width'''],
          )			,				)



 def 			snake_case__  (   self		:   Union[str, Any]      ):
      # Initialize image_processing
      __magic_name__						       =    self.image_processing_class(**self.image_processor_dict      )
      # create random PyTorch tensors
      __magic_name__						       =    prepare_image_inputs(self.image_processor_tester			,				equal_resolution=a__			,				torchify=a__      )
      __magic_name__						       =    []
      for image in image_inputs:
           self.assertIsInstance(a__			,				torch.Tensor      )
           maps.append(torch.zeros(image.shape[-2:]      ).long()      )

      # Test not batched input
      __magic_name__						       =    image_processing(image_inputs[0]			,				maps[0]			,				return_tensors='''pt'''      )
      self.assertEqual(
          encoding['''pixel_values'''].shape			,				(
              1,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.crop_size['''height'''],
              self.image_processor_tester.crop_size['''width'''],
          )			,				)
      self.assertEqual(
          encoding['''labels'''].shape			,				(
              1,
              self.image_processor_tester.crop_size['''height'''],
              self.image_processor_tester.crop_size['''width'''],
          )			,				)
      self.assertEqual(encoding['''labels'''].dtype			,				torch.long      )
      self.assertTrue(encoding['''labels'''].min().item() >= 0      )
      self.assertTrue(encoding['''labels'''].max().item() <= 255      )

      # Test batched
      __magic_name__						       =    image_processing(a__			,				a__			,				return_tensors='''pt'''      )
      self.assertEqual(
          encoding['''pixel_values'''].shape			,				(
              self.image_processor_tester.batch_size,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.crop_size['''height'''],
              self.image_processor_tester.crop_size['''width'''],
          )			,				)
      self.assertEqual(
          encoding['''labels'''].shape			,				(
              self.image_processor_tester.batch_size,
              self.image_processor_tester.crop_size['''height'''],
              self.image_processor_tester.crop_size['''width'''],
          )			,				)
      self.assertEqual(encoding['''labels'''].dtype			,				torch.long      )
      self.assertTrue(encoding['''labels'''].min().item() >= 0      )
      self.assertTrue(encoding['''labels'''].max().item() <= 255      )

      # Test not batched input (PIL images)
      __magic_name__       ,  __magic_name__						       =    prepare_semantic_single_inputs()

      __magic_name__						       =    image_processing(a__			,				a__			,				return_tensors='''pt'''      )
      self.assertEqual(
          encoding['''pixel_values'''].shape			,				(
              1,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.crop_size['''height'''],
              self.image_processor_tester.crop_size['''width'''],
          )			,				)
      self.assertEqual(
          encoding['''labels'''].shape			,				(
              1,
              self.image_processor_tester.crop_size['''height'''],
              self.image_processor_tester.crop_size['''width'''],
          )			,				)
      self.assertEqual(encoding['''labels'''].dtype			,				torch.long      )
      self.assertTrue(encoding['''labels'''].min().item() >= 0      )
      self.assertTrue(encoding['''labels'''].max().item() <= 255      )

      # Test batched input (PIL images)
      __magic_name__       ,  __magic_name__						       =    prepare_semantic_batch_inputs()

      __magic_name__						       =    image_processing(a__			,				a__			,				return_tensors='''pt'''      )
      self.assertEqual(
          encoding['''pixel_values'''].shape			,				(
              2,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.crop_size['''height'''],
              self.image_processor_tester.crop_size['''width'''],
          )			,				)
      self.assertEqual(
          encoding['''labels'''].shape			,				(
              2,
              self.image_processor_tester.crop_size['''height'''],
              self.image_processor_tester.crop_size['''width'''],
          )			,				)
      self.assertEqual(encoding['''labels'''].dtype			,				torch.long      )
      self.assertTrue(encoding['''labels'''].min().item() >= 0      )
      self.assertTrue(encoding['''labels'''].max().item() <= 255      )



 def 			snake_case__  (   self		:   Any      ):
      # Initialize image_processing
      __magic_name__						       =    self.image_processing_class(**self.image_processor_dict      )

      # ADE20k has 150 classes, and the background is included, so labels should be between 0 and 150
      __magic_name__       ,  __magic_name__						       =    prepare_semantic_single_inputs()
      __magic_name__						       =    image_processing(a__			,				a__			,				return_tensors='''pt'''      )
      self.assertTrue(encoding['''labels'''].min().item() >= 0      )
      self.assertTrue(encoding['''labels'''].max().item() <= 150      )

      __magic_name__						       =    True
      __magic_name__						       =    image_processing(a__			,				a__			,				return_tensors='''pt'''      )
      self.assertTrue(encoding['''labels'''].min().item() >= 0      )
      self.assertTrue(encoding['''labels'''].max().item() <= 255      )






