

'''simple docstring'''







import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pytest

from datasets.arrow_dataset import Dataset
from datasets.search import ElasticSearchIndex, FaissIndex, MissingIndex

from .utils import require_elasticsearch, require_faiss


__lowerCamelCase						    =      pytest.mark.integration


@require_faiss
class    A__ ( _snake_case     ):



    def    snake_case_     (   self    )  -> Tuple:





           '''simple docstring'''
           A_		   = Dataset.from_dict({"""filename""": ["""my_name-train""" + """_""" + str(UpperCamelCase__    ) for x in np.arange(30    ).tolist()]}    )
           return dset



    def    snake_case_     (   self    )  -> List[str]:





           '''simple docstring'''
           import faiss

           A_		   = self._create_dummy_dataset()
           A_		   = dset.map(
               lambda UpperCamelCase__						,			UpperCamelCase__    : {"vecs": i * np.ones(5						,			dtype=np.floataa    )}						,			with_indices=UpperCamelCase__						,			keep_in_memory=UpperCamelCase__    )
           A_		   = dset.add_faiss_index("""vecs"""						,			batch_size=100						,			metric_type=faiss.METRIC_INNER_PRODUCT    )
           A_   ,     A_		   = dset.get_nearest_examples("""vecs"""						,			np.ones(5						,			dtype=np.floataa    )    )
           self.assertEqual(examples["""filename"""][0]						,			"""my_name-train_29"""    )
           dset.drop_index("""vecs"""    )



    def    snake_case_     (   self    )  -> Union[str, Any]:





           '''simple docstring'''
           import faiss

           A_		   = self._create_dummy_dataset()
           dset.add_faiss_index_from_external_arrays(
               external_arrays=np.ones((30, 5)    ) * np.arange(30    ).reshape(-1						,			1    )						,			index_name="""vecs"""						,			batch_size=100						,			metric_type=faiss.METRIC_INNER_PRODUCT						,			)
           A_   ,     A_		   = dset.get_nearest_examples("""vecs"""						,			np.ones(5						,			dtype=np.floataa    )    )
           self.assertEqual(examples["""filename"""][0]						,			"""my_name-train_29"""    )



    def    snake_case_     (   self    )  -> Tuple:





           '''simple docstring'''
           import faiss

           A_		   = self._create_dummy_dataset()
           dset.add_faiss_index_from_external_arrays(
               external_arrays=np.ones((30, 5)    ) * np.arange(30    ).reshape(-1						,			1    )						,			index_name="""vecs"""						,			metric_type=faiss.METRIC_INNER_PRODUCT						,			)

           # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
           # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
           # see https://bugs.python.org/issue14243 and
           # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
           with tempfile.NamedTemporaryFile(delete=UpperCamelCase__    ) as tmp_file:
                  dset.save_faiss_index("""vecs"""						,			tmp_file.name    )
                  dset.load_faiss_index("""vecs2"""						,			tmp_file.name    )
           os.unlink(tmp_file.name    )

           A_   ,     A_		   = dset.get_nearest_examples("""vecs2"""						,			np.ones(5						,			dtype=np.floataa    )    )
           self.assertEqual(examples["""filename"""][0]						,			"""my_name-train_29"""    )



    def    snake_case_     (   self    )  -> Optional[int]:





           '''simple docstring'''
           A_		   = self._create_dummy_dataset()
           dset.add_faiss_index_from_external_arrays(
               external_arrays=np.ones((30, 5)    ) * np.arange(30    ).reshape(-1						,			1    )						,			index_name="""vecs"""    )
           dset.drop_index("""vecs"""    )
           self.assertRaises(UpperCamelCase__						,			partial(dset.get_nearest_examples						,			"""vecs2"""						,			np.ones(5						,			dtype=np.floataa    )    )    )







    def    snake_case_     (   self    )  -> Tuple:





           '''simple docstring'''
           from elasticsearch import Elasticsearch

           A_		   = self._create_dummy_dataset()
           with patch("""elasticsearch.Elasticsearch.search"""    ) as mocked_search, patch(
               """elasticsearch.client.IndicesClient.create"""    ) as mocked_index_create, patch("""elasticsearch.helpers.streaming_bulk"""    ) as mocked_bulk:
                  A_		   = {"""acknowledged""": True}
                  mocked_bulk.return_value([(True, None)] * 30    )
                  A_		   = {"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 29}]}}
                  A_		   = Elasticsearch()

                  dset.add_elasticsearch_index("""filename"""						,			es_client=UpperCamelCase__    )
                  A_   ,     A_		   = dset.get_nearest_examples("""filename"""						,			"""my_name-train_29"""    )
                  self.assertEqual(examples["""filename"""][0]						,			"""my_name-train_29"""    )


@require_faiss
class    A__ ( _snake_case     ):



    def    snake_case_     (   self    )  -> Tuple:





           '''simple docstring'''
           import faiss

           A_		   = FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT    )

           # add vectors
           index.add_vectors(np.eye(5						,			dtype=np.floataa    )    )
           self.assertIsNotNone(index.faiss_index    )
           self.assertEqual(index.faiss_index.ntotal						,			5    )
           index.add_vectors(np.zeros((5, 5)						,			dtype=np.floataa    )    )
           self.assertEqual(index.faiss_index.ntotal						,			10    )

           # single query
           A_		   = np.zeros(5						,			dtype=np.floataa    )
           A_		   = 1
           A_   ,     A_		   = index.search(UpperCamelCase__    )
           self.assertRaises(UpperCamelCase__						,			index.search						,			query.reshape(-1						,			1    )    )
           self.assertGreater(scores[0]						,			0    )
           self.assertEqual(indices[0]						,			1    )

           # batched queries
           A_		   = np.eye(5						,			dtype=np.floataa    )[::-1]
           A_   ,     A_		   = index.search_batch(UpperCamelCase__    )
           self.assertRaises(UpperCamelCase__						,			index.search_batch						,			queries[0]    )
           A_		   = [scores[0] for scores in total_scores]
           A_		   = [indices[0] for indices in total_indices]
           self.assertGreater(np.min(UpperCamelCase__    )						,			0    )
           self.assertListEqual([4, 3, 2, 1, 0]						,			UpperCamelCase__    )



    def    snake_case_     (   self    )  -> Dict:





           '''simple docstring'''
           import faiss

           A_		   = FaissIndex(string_factory="""Flat"""    )
           index.add_vectors(np.eye(5						,			dtype=np.floataa    )    )
           self.assertIsInstance(index.faiss_index						,			faiss.IndexFlat    )
           A_		   = FaissIndex(string_factory="""LSH"""    )
           index.add_vectors(np.eye(5						,			dtype=np.floataa    )    )
           self.assertIsInstance(index.faiss_index						,			faiss.IndexLSH    )
           with self.assertRaises(UpperCamelCase__    ):
                  A_		   = FaissIndex(string_factory="""Flat"""						,			custom_index=faiss.IndexFlat(5    )    )



    def    snake_case_     (   self    )  -> List[str]:





           '''simple docstring'''
           import faiss

           A_		   = faiss.IndexFlat(5    )
           A_		   = FaissIndex(custom_index=UpperCamelCase__    )
           index.add_vectors(np.eye(5						,			dtype=np.floataa    )    )
           self.assertIsInstance(index.faiss_index						,			faiss.IndexFlat    )







    def    snake_case_     (   self    )  -> Dict:





           '''simple docstring'''
           import faiss

           A_		   = FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT    )
           index.add_vectors(np.eye(5						,			dtype=np.floataa    )    )

           # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
           # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
           # see https://bugs.python.org/issue14243 and
           # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
           with tempfile.NamedTemporaryFile(delete=UpperCamelCase__    ) as tmp_file:
                  index.save(tmp_file.name    )
                  A_		   = FaissIndex.load(tmp_file.name    )
           os.unlink(tmp_file.name    )

           A_		   = np.zeros(5						,			dtype=np.floataa    )
           A_		   = 1
           A_   ,     A_		   = index.search(UpperCamelCase__    )
           self.assertGreater(scores[0]						,			0    )
           self.assertEqual(indices[0]						,			1    )

@require_faiss
def  UpperCAmelCase__      ( UpperCAmelCase__			)     ->     List[Any]:
       import faiss

       A_		   = FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT			)
       index.add_vectors(np.eye(5,    dtype=np.floataa			)			)

       A_		   = """index.faiss"""
       A_		   = F'''mock://{index_name}'''
       index.save(UpperCAmelCase__,    storage_options=mockfs.storage_options			)
       A_		   = FaissIndex.load(UpperCAmelCase__,    storage_options=mockfs.storage_options			)

       A_		   = np.zeros(5,    dtype=np.floataa			)
       A_		   = 1
       A_   ,     A_		   = index.search(UpperCAmelCase__			)
       assert scores[0] > 0
       assert indices[0] == 1







@require_elasticsearch
class    A__ ( _snake_case     ):



    def    snake_case_     (   self    )  -> Union[str, Any]:





           '''simple docstring'''
           from elasticsearch import Elasticsearch

           with patch("""elasticsearch.Elasticsearch.search"""    ) as mocked_search, patch(
               """elasticsearch.client.IndicesClient.create"""    ) as mocked_index_create, patch("""elasticsearch.helpers.streaming_bulk"""    ) as mocked_bulk:
                  A_		   = Elasticsearch()
                  A_		   = {"""acknowledged""": True}
                  A_		   = ElasticSearchIndex(es_client=UpperCamelCase__    )
                  mocked_bulk.return_value([(True, None)] * 3    )
                  index.add_documents(["""foo""", """bar""", """foobar"""]    )

                  # single query
                  A_		   = """foo"""
                  A_		   = {"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 0}]}}
                  A_   ,     A_		   = index.search(UpperCamelCase__    )
                  self.assertEqual(scores[0]						,			1    )
                  self.assertEqual(indices[0]						,			0    )

                  # single query with timeout
                  A_		   = """foo"""
                  A_		   = {"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 0}]}}
                  A_   ,     A_		   = index.search(UpperCamelCase__						,			request_timeout=30    )
                  self.assertEqual(scores[0]						,			1    )
                  self.assertEqual(indices[0]						,			0    )

                  # batched queries
                  A_		   = ["""foo""", """bar""", """foobar"""]
                  A_		   = {"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 1}]}}
                  A_   ,     A_		   = index.search_batch(UpperCamelCase__    )
                  A_		   = [scores[0] for scores in total_scores]
                  A_		   = [indices[0] for indices in total_indices]
                  self.assertGreater(np.min(UpperCamelCase__    )						,			0    )
                  self.assertListEqual([1, 1, 1]						,			UpperCamelCase__    )

                  # batched queries with timeout
                  A_		   = ["""foo""", """bar""", """foobar"""]
                  A_		   = {"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 1}]}}
                  A_   ,     A_		   = index.search_batch(UpperCamelCase__						,			request_timeout=30    )
                  A_		   = [scores[0] for scores in total_scores]
                  A_		   = [indices[0] for indices in total_indices]
                  self.assertGreater(np.min(UpperCamelCase__    )						,			0    )
                  self.assertListEqual([1, 1, 1]						,			UpperCamelCase__    )



'''simple docstring'''







import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
      from PIL import Image

      from transformers import CLIPImageProcessor, CLIPProcessor


@require_vision
class    A__ ( unittest.TestCase     ):



    def    snake_case_     (   self    )  -> Any:





           '''simple docstring'''
           A_		   = tempfile.mkdtemp()

           # fmt: off
           A_		   = ["""l""", """o""", """w""", """e""", """r""", """s""", """t""", """i""", """d""", """n""", """lo""", """l</w>""", """w</w>""", """r</w>""", """t</w>""", """low</w>""", """er</w>""", """lowest</w>""", """newer</w>""", """wider""", """<unk>""", """<|startoftext|>""", """<|endoftext|>"""]
           # fmt: on
           A_		   = dict(zip(UpperCamelCase__						,			range(len(UpperCamelCase__    )    )    )    )
           A_		   = ["""#version: 0.2""", """l o""", """lo w</w>""", """e r</w>""", """"""]
           A_		   = {"""unk_token""": """<unk>"""}

           A_		   = os.path.join(self.tmpdirname						,			VOCAB_FILES_NAMES["""vocab_file"""]    )
           A_		   = os.path.join(self.tmpdirname						,			VOCAB_FILES_NAMES["""merges_file"""]    )
           with open(self.vocab_file						,			"""w"""						,			encoding="""utf-8"""    ) as fp:
                  fp.write(json.dumps(UpperCamelCase__    ) + """\n"""    )
           with open(self.merges_file						,			"""w"""						,			encoding="""utf-8"""    ) as fp:
                  fp.write("""\n""".join(UpperCamelCase__    )    )

           A_		   = {
               """do_resize""": True,
               """size""": 20,
               """do_center_crop""": True,
               """crop_size""": 18,
               """do_normalize""": True,
               """image_mean""": [0.48145466, 0.4578275, 0.40821073],
               """image_std""": [0.26862954, 0.26130258, 0.27577711],
           }
           A_		   = os.path.join(self.tmpdirname						,			UpperCamelCase__    )
           with open(self.image_processor_file						,			"""w"""						,			encoding="""utf-8"""    ) as fp:
                  json.dump(UpperCamelCase__						,			UpperCamelCase__    )



    def    snake_case_     (   self						,			**UpperCamelCase__    )  -> str:





           '''simple docstring'''
           return CLIPTokenizer.from_pretrained(self.tmpdirname						,			**UpperCamelCase__    )



    def    snake_case_     (   self						,			**UpperCamelCase__    )  -> Any:





           '''simple docstring'''
           return CLIPTokenizerFast.from_pretrained(self.tmpdirname						,			**UpperCamelCase__    )



    def    snake_case_     (   self						,			**UpperCamelCase__    )  -> Optional[Any]:





           '''simple docstring'''
           return CLIPImageProcessor.from_pretrained(self.tmpdirname						,			**UpperCamelCase__    )



    def    snake_case_     (   self    )  -> List[Any]:





           '''simple docstring'''
           shutil.rmtree(self.tmpdirname    )



    def    snake_case_     (   self    )  -> Any:





           '''simple docstring'''
           A_		   = [np.random.randint(255						,			size=(3, 30, 400)						,			dtype=np.uinta    )]

           A_		   = [Image.fromarray(np.moveaxis(UpperCamelCase__						,			0						,			-1    )    ) for x in image_inputs]

           return image_inputs



    def    snake_case_     (   self    )  -> Optional[int]:





           '''simple docstring'''
           A_		   = self.get_tokenizer()
           A_		   = self.get_rust_tokenizer()
           A_		   = self.get_image_processor()

           A_		   = CLIPProcessor(tokenizer=UpperCamelCase__						,			image_processor=UpperCamelCase__    )
           processor_slow.save_pretrained(self.tmpdirname    )
           A_		   = CLIPProcessor.from_pretrained(self.tmpdirname						,			use_fast=UpperCamelCase__    )

           A_		   = CLIPProcessor(tokenizer=UpperCamelCase__						,			image_processor=UpperCamelCase__    )
           processor_fast.save_pretrained(self.tmpdirname    )
           A_		   = CLIPProcessor.from_pretrained(self.tmpdirname    )

           self.assertEqual(processor_slow.tokenizer.get_vocab()						,			tokenizer_slow.get_vocab()    )
           self.assertEqual(processor_fast.tokenizer.get_vocab()						,			tokenizer_fast.get_vocab()    )
           self.assertEqual(tokenizer_slow.get_vocab()						,			tokenizer_fast.get_vocab()    )
           self.assertIsInstance(processor_slow.tokenizer						,			UpperCamelCase__    )
           self.assertIsInstance(processor_fast.tokenizer						,			UpperCamelCase__    )

           self.assertEqual(processor_slow.image_processor.to_json_string()						,			image_processor.to_json_string()    )
           self.assertEqual(processor_fast.image_processor.to_json_string()						,			image_processor.to_json_string()    )
           self.assertIsInstance(processor_slow.image_processor						,			UpperCamelCase__    )
           self.assertIsInstance(processor_fast.image_processor						,			UpperCamelCase__    )



    def    snake_case_     (   self    )  -> str:





           '''simple docstring'''
           A_		   = CLIPProcessor(tokenizer=self.get_tokenizer()						,			image_processor=self.get_image_processor()    )
           processor.save_pretrained(self.tmpdirname    )

           A_		   = self.get_tokenizer(bos_token="""(BOS)"""						,			eos_token="""(EOS)"""    )
           A_		   = self.get_image_processor(do_normalize=UpperCamelCase__						,			padding_value=1.0    )

           A_		   = CLIPProcessor.from_pretrained(
               self.tmpdirname						,			bos_token="""(BOS)"""						,			eos_token="""(EOS)"""						,			do_normalize=UpperCamelCase__						,			padding_value=1.0    )

           self.assertEqual(processor.tokenizer.get_vocab()						,			tokenizer_add_kwargs.get_vocab()    )
           self.assertIsInstance(processor.tokenizer						,			UpperCamelCase__    )

           self.assertEqual(processor.image_processor.to_json_string()						,			image_processor_add_kwargs.to_json_string()    )
           self.assertIsInstance(processor.image_processor						,			UpperCamelCase__    )



    def    snake_case_     (   self    )  -> str:





           '''simple docstring'''
           A_		   = self.get_image_processor()
           A_		   = self.get_tokenizer()

           A_		   = CLIPProcessor(tokenizer=UpperCamelCase__						,			image_processor=UpperCamelCase__    )

           A_		   = self.prepare_image_inputs()

           A_		   = image_processor(UpperCamelCase__						,			return_tensors="""np"""    )
           A_		   = processor(images=UpperCamelCase__						,			return_tensors="""np"""    )

           for key in input_image_proc.keys():
                  self.assertAlmostEqual(input_image_proc[key].sum()						,			input_processor[key].sum()						,			delta=1e-2    )



    def    snake_case_     (   self    )  -> Optional[int]:





           '''simple docstring'''
           A_		   = self.get_image_processor()
           A_		   = self.get_tokenizer()

           A_		   = CLIPProcessor(tokenizer=UpperCamelCase__						,			image_processor=UpperCamelCase__    )

           A_		   = """lower newer"""

           A_		   = processor(text=UpperCamelCase__    )

           A_		   = tokenizer(UpperCamelCase__    )

           for key in encoded_tok.keys():
                  self.assertListEqual(encoded_tok[key]						,			encoded_processor[key]    )



    def    snake_case_     (   self    )  -> Dict:





           '''simple docstring'''
           A_		   = self.get_image_processor()
           A_		   = self.get_tokenizer()

           A_		   = CLIPProcessor(tokenizer=UpperCamelCase__						,			image_processor=UpperCamelCase__    )

           A_		   = """lower newer"""
           A_		   = self.prepare_image_inputs()

           A_		   = processor(text=UpperCamelCase__						,			images=UpperCamelCase__    )

           self.assertListEqual(list(inputs.keys()    )						,			["""input_ids""", """attention_mask""", """pixel_values"""]    )

           # test if it raises when no input is passed
           with pytest.raises(UpperCamelCase__    ):
                  processor()



    def    snake_case_     (   self    )  -> Union[str, Any]:





           '''simple docstring'''
           A_		   = self.get_image_processor()
           A_		   = self.get_tokenizer()

           A_		   = CLIPProcessor(tokenizer=UpperCamelCase__						,			image_processor=UpperCamelCase__    )

           A_		   = [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

           A_		   = processor.batch_decode(UpperCamelCase__    )
           A_		   = tokenizer.batch_decode(UpperCamelCase__    )

           self.assertListEqual(UpperCamelCase__						,			UpperCamelCase__    )







    def    snake_case_     (   self    )  -> Optional[int]:





           '''simple docstring'''
           A_		   = self.get_image_processor()
           A_		   = self.get_tokenizer()

           A_		   = CLIPProcessor(tokenizer=UpperCamelCase__						,			image_processor=UpperCamelCase__    )

           A_		   = """lower newer"""
           A_		   = self.prepare_image_inputs()

           A_		   = processor(text=UpperCamelCase__						,			images=UpperCamelCase__    )

           self.assertListEqual(list(inputs.keys()    )						,			processor.model_input_names    )
