





'''simple docstring'''

import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pytest

from datasets.arrow_dataset import Dataset
from datasets.search import ElasticSearchIndex, FaissIndex, MissingIndex

from .utils import require_elasticsearch, require_faiss


__lowerCAmelCase							:     Dict		     =pytest.mark.integration






@require_faiss
class        UpperCAmelCase     (	SCREAMING_SNAKE_CASE__    ):
 def 					UpperCAmelCase_							(    self				:int							)->							Any:
    A__    =      Dataset.from_dict({"filename": ["my_name-train" + "_" + str(A__							) for x in np.arange(30							).tolist()]}							)
    return dset
 def 					UpperCAmelCase_							(    self				:Optional[int]							)->							List[str]:
    import faiss

    A__    =      self._create_dummy_dataset()
    A__    =      dset.map(
        lambda lowercase_						,	lowercase_							: {"vecs": i * np.ones(5						,	dtype=np.floataa							)}						,	with_indices=A__						,	keep_in_memory=A__							)
    A__    =      dset.add_faiss_index("vecs"						,	batch_size=1_00						,	metric_type=faiss.METRIC_INNER_PRODUCT							)
    A__,     A__    =      dset.get_nearest_examples("vecs"						,	np.ones(5						,	dtype=np.floataa							)							)
    self.assertEqual(examples["filename"][0]						,	"my_name-train_29"							)
    dset.drop_index("vecs"							)
 def 					UpperCAmelCase_							(    self				:str							)->							int:
    import faiss

    A__    =      self._create_dummy_dataset()
    dset.add_faiss_index_from_external_arrays(
        external_arrays=np.ones((30, 5)							) * np.arange(30							).reshape(-1						,	1							)						,	index_name="vecs"						,	batch_size=1_00						,	metric_type=faiss.METRIC_INNER_PRODUCT						,	)
    A__,     A__    =      dset.get_nearest_examples("vecs"						,	np.ones(5						,	dtype=np.floataa							)							)
    self.assertEqual(examples["filename"][0]						,	"my_name-train_29"							)
 def 					UpperCAmelCase_							(    self				:Dict							)->							List[Any]:
    import faiss

    A__    =      self._create_dummy_dataset()
    dset.add_faiss_index_from_external_arrays(
        external_arrays=np.ones((30, 5)							) * np.arange(30							).reshape(-1						,	1							)						,	index_name="vecs"						,	metric_type=faiss.METRIC_INNER_PRODUCT						,	)

    # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
    # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
    # see https://bugs.python.org/issue14243 and
    # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
    with tempfile.NamedTemporaryFile(delete=A__							) as tmp_file:
       dset.save_faiss_index("vecs"						,	tmp_file.name							)
       dset.load_faiss_index("vecs2"						,	tmp_file.name							)
    os.unlink(tmp_file.name							)

    A__,     A__    =      dset.get_nearest_examples("vecs2"						,	np.ones(5						,	dtype=np.floataa							)							)
    self.assertEqual(examples["filename"][0]						,	"my_name-train_29"							)
 def 					UpperCAmelCase_							(    self				:int							)->							List[str]:
    A__    =      self._create_dummy_dataset()
    dset.add_faiss_index_from_external_arrays(
        external_arrays=np.ones((30, 5)							) * np.arange(30							).reshape(-1						,	1							)						,	index_name="vecs"							)
    dset.drop_index("vecs"							)
    self.assertRaises(A__						,	partial(dset.get_nearest_examples						,	"vecs2"						,	np.ones(5						,	dtype=np.floataa							)							)							)



 def 					UpperCAmelCase_							(    self				:Union[str, Any]							)->							Any:
    from elasticsearch import Elasticsearch

    A__    =      self._create_dummy_dataset()
    with patch("elasticsearch.Elasticsearch.search"							) as mocked_search, patch(
        "elasticsearch.client.IndicesClient.create"							) as mocked_index_create, patch("elasticsearch.helpers.streaming_bulk"							) as mocked_bulk:
       A__    =      {"acknowledged": True}
       mocked_bulk.return_value([(True, None)] * 30							)
       A__    =      {"hits": {"hits": [{"_score": 1, "_id": 29}]}}
       A__    =      Elasticsearch()

       dset.add_elasticsearch_index("filename"						,	es_client=A__							)
       A__,     A__    =      dset.get_nearest_examples("filename"						,	"my_name-train_29"							)
       self.assertEqual(examples["filename"][0]						,	"my_name-train_29"							)






@require_faiss
class        UpperCAmelCase     (	SCREAMING_SNAKE_CASE__    ):
 def 					UpperCAmelCase_							(    self				:List[Any]							)->							Optional[Any]:
    import faiss

    A__    =      FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT							)

    # add vectors
    index.add_vectors(np.eye(5						,	dtype=np.floataa							)							)
    self.assertIsNotNone(index.faiss_index							)
    self.assertEqual(index.faiss_index.ntotal						,	5							)
    index.add_vectors(np.zeros((5, 5)						,	dtype=np.floataa							)							)
    self.assertEqual(index.faiss_index.ntotal						,	10							)

    # single query
    A__    =      np.zeros(5						,	dtype=np.floataa							)
    A__    =      1
    A__,     A__    =      index.search(A__							)
    self.assertRaises(A__						,	index.search						,	query.reshape(-1						,	1							)							)
    self.assertGreater(scores[0]						,	0							)
    self.assertEqual(indices[0]						,	1							)

    # batched queries
    A__    =      np.eye(5						,	dtype=np.floataa							)[::-1]
    A__,     A__    =      index.search_batch(A__							)
    self.assertRaises(A__						,	index.search_batch						,	queries[0]							)
    A__    =      [scores[0] for scores in total_scores]
    A__    =      [indices[0] for indices in total_indices]
    self.assertGreater(np.min(A__							)						,	0							)
    self.assertListEqual([4, 3, 2, 1, 0]						,	A__							)
 def 					UpperCAmelCase_							(    self				:Tuple							)->							str:
    import faiss

    A__    =      FaissIndex(string_factory="Flat"							)
    index.add_vectors(np.eye(5						,	dtype=np.floataa							)							)
    self.assertIsInstance(index.faiss_index						,	faiss.IndexFlat							)
    A__    =      FaissIndex(string_factory="LSH"							)
    index.add_vectors(np.eye(5						,	dtype=np.floataa							)							)
    self.assertIsInstance(index.faiss_index						,	faiss.IndexLSH							)
    with self.assertRaises(A__							):
       A__    =      FaissIndex(string_factory="Flat"						,	custom_index=faiss.IndexFlat(5							)							)
 def 					UpperCAmelCase_							(    self				:Dict							)->							Any:
    import faiss

    A__    =      faiss.IndexFlat(5							)
    A__    =      FaissIndex(custom_index=A__							)
    index.add_vectors(np.eye(5						,	dtype=np.floataa							)							)
    self.assertIsInstance(index.faiss_index						,	faiss.IndexFlat							)



 def 					UpperCAmelCase_							(    self				:Optional[int]							)->							Tuple:
    import faiss

    A__    =      FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT							)
    index.add_vectors(np.eye(5						,	dtype=np.floataa							)							)

    # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
    # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
    # see https://bugs.python.org/issue14243 and
    # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
    with tempfile.NamedTemporaryFile(delete=A__							) as tmp_file:
       index.save(tmp_file.name							)
       A__    =      FaissIndex.load(tmp_file.name							)
    os.unlink(tmp_file.name							)

    A__    =      np.zeros(5						,	dtype=np.floataa							)
    A__    =      1
    A__,     A__    =      index.search(A__							)
    self.assertGreater(scores[0]						,	0							)
    self.assertEqual(indices[0]						,	1							)





@require_faiss
def 						UpperCamelCase    (							_lowerCamelCase : Union[str, Any]	):
   import faiss

   A__    =      FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT	)
   index.add_vectors(np.eye(5						,	dtype=np.floataa	)	)

   A__    =      "index.faiss"
   A__    =      F"mock://{index_name}"
   index.save(lowerCAmelCase__						,	storage_options=mockfs.storage_options	)
   A__    =      FaissIndex.load(lowerCAmelCase__						,	storage_options=mockfs.storage_options	)

   A__    =      np.zeros(5						,	dtype=np.floataa	)
   A__    =      1
   A__,     A__    =      index.search(lowerCAmelCase__	)
   assert scores[0] > 0
   assert indices[0] == 1






@require_elasticsearch
class        UpperCAmelCase     (	SCREAMING_SNAKE_CASE__    ):
 def 					UpperCAmelCase_							(    self				:str							)->							Optional[int]:
    from elasticsearch import Elasticsearch

    with patch("elasticsearch.Elasticsearch.search"							) as mocked_search, patch(
        "elasticsearch.client.IndicesClient.create"							) as mocked_index_create, patch("elasticsearch.helpers.streaming_bulk"							) as mocked_bulk:
       A__    =      Elasticsearch()
       A__    =      {"acknowledged": True}
       A__    =      ElasticSearchIndex(es_client=A__							)
       mocked_bulk.return_value([(True, None)] * 3							)
       index.add_documents(["foo", "bar", "foobar"]							)

       # single query
       A__    =      "foo"
       A__    =      {"hits": {"hits": [{"_score": 1, "_id": 0}]}}
       A__,     A__    =      index.search(A__							)
       self.assertEqual(scores[0]						,	1							)
       self.assertEqual(indices[0]						,	0							)

       # single query with timeout
       A__    =      "foo"
       A__    =      {"hits": {"hits": [{"_score": 1, "_id": 0}]}}
       A__,     A__    =      index.search(A__						,	request_timeout=30							)
       self.assertEqual(scores[0]						,	1							)
       self.assertEqual(indices[0]						,	0							)

       # batched queries
       A__    =      ["foo", "bar", "foobar"]
       A__    =      {"hits": {"hits": [{"_score": 1, "_id": 1}]}}
       A__,     A__    =      index.search_batch(A__							)
       A__    =      [scores[0] for scores in total_scores]
       A__    =      [indices[0] for indices in total_indices]
       self.assertGreater(np.min(A__							)						,	0							)
       self.assertListEqual([1, 1, 1]						,	A__							)

       # batched queries with timeout
       A__    =      ["foo", "bar", "foobar"]
       A__    =      {"hits": {"hits": [{"_score": 1, "_id": 1}]}}
       A__,     A__    =      index.search_batch(A__						,	request_timeout=30							)
       A__    =      [scores[0] for scores in total_scores]
       A__    =      [indices[0] for indices in total_indices]
       self.assertGreater(np.min(A__							)						,	0							)
       self.assertListEqual([1, 1, 1]						,	A__							)










'''simple docstring'''

import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class        UpperCAmelCase     (	UpperCamelCase__      ,	unittest.TestCase    ):
 __lowercase       		=    CLIPTokenizer
 __lowercase       		=    CLIPTokenizerFast
 __lowercase       		=    True
 __lowercase       		=    {}
 __lowercase       		=    False
 def 					UpperCAmelCase_							(    self				:Optional[int]							)->							Optional[Any]:
     super().setUp()

     # fmt: off
     A__    =      ["l", "o", "w", "e", "r", "s", "t", "i", "d", "n", "lo", "l</w>", "w</w>", "r</w>", "t</w>", "low</w>", "er</w>", "lowest</w>", "newer</w>", "wider", "<unk>", "<|startoftext|>", "<|endoftext|>"]
     # fmt: on
     A__    =      dict(zip(lowercase_						,	range(len(lowercase_							)							)							)							)
     A__    =      ["#version: 0.2", "l o", "lo w</w>", "e r</w>"]
     A__    =      {"unk_token": "<unk>"}

     A__    =      os.path.join(self.tmpdirname						,	VOCAB_FILES_NAMES["vocab_file"]							)
     A__    =      os.path.join(self.tmpdirname						,	VOCAB_FILES_NAMES["merges_file"]							)
     with open(self.vocab_file						,	"w"						,	encoding="utf-8"							) as fp:
         fp.write(json.dumps(lowercase_							) + "\n"							)
     with open(self.merges_file						,	"w"						,	encoding="utf-8"							) as fp:
         fp.write("\n".join(lowercase_							)							)
 def 					UpperCAmelCase_							(    self				:Optional[int]						,	**lowercase_				:List[str]							)->							str:
     kwargs.update(self.special_tokens_map							)
     return CLIPTokenizer.from_pretrained(self.tmpdirname						,	**lowercase_							)
 def 					UpperCAmelCase_							(    self				:Optional[int]						,	**lowercase_				:Optional[int]							)->							Tuple:
     kwargs.update(self.special_tokens_map							)
     return CLIPTokenizerFast.from_pretrained(self.tmpdirname						,	**lowercase_							)
 def 					UpperCAmelCase_							(    self				:Optional[int]						,	lowercase_				:List[Any]							)->							List[str]:
     A__    =      "lower newer"
     A__    =      "lower newer"
     return input_text, output_text
 def 					UpperCAmelCase_							(    self				:int							)->							List[str]:
     A__    =      CLIPTokenizer(self.vocab_file						,	self.merges_file						,	**self.special_tokens_map							)
     A__    =      "lower newer"
     A__    =      ["lo", "w", "er</w>", "n", "e", "w", "er</w>"]
     A__    =      tokenizer.tokenize(lowercase_							)
     self.assertListEqual(lowercase_						,	lowercase_							)

     A__    =      tokens + [tokenizer.unk_token]
     A__    =      [10, 2, 16, 9, 3, 2, 16, 20]
     self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase_							)						,	lowercase_							)
 @require_ftfy
 def 					UpperCAmelCase_							(    self				:Dict							)->							str:
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"							):
             A__    =      self.tokenizer_class.from_pretrained(lowercase_						,	**lowercase_							)
             A__    =      self.rust_tokenizer_class.from_pretrained(lowercase_						,	**lowercase_							)

             A__    =      "A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."
             A__    =      tokenizer_s.tokenize(lowercase_							)
             A__    =      tokenizer_r.tokenize(lowercase_							)

             self.assertListEqual(lowercase_						,	lowercase_							)

             # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
             # with Tilde) encoded in 2 different ways
             A__    =      "xa\u0303y" + " " + "x\xe3y"
             A__    =      tokenizer_s.tokenize(lowercase_							)
             A__    =      tokenizer_r.tokenize(lowercase_							)

             self.assertListEqual(lowercase_						,	lowercase_							)

             # Test that the tokenization is identical on unicode of space type
             A__    =      [
                 "\u0009",  # (horizontal tab, '\t')
                 "\u000B",  # (vertical tab)
                 "\u000C",  # (form feed)
                 "\u0020",  # (space, ' ')
                 "\u200E",  # (left-to-right mark):w
                 "\u200F",  # (right-to-left mark)
             ]
             for unicode_seq in spaces_unicodes:
                 A__    =      tokenizer_s.tokenize(lowercase_							)
                 A__    =      tokenizer_r.tokenize(lowercase_							)

                 self.assertListEqual(lowercase_						,	lowercase_							)

             # Test that the tokenization is identical on unicode of line break type
             A__    =      [
                 "\u000A",  # (line feed, '\n')
                 "\r\n",  # (carriage return and line feed, '\r\n')
                 "\u000D",  # (carriage return, '\r')
                 "\r",  # (carriage return, '\r')
                 "\u000D",  # (carriage return, '\r')
                 "\u2028",  # (line separator)
                 "\u2029",  # (paragraph separator)
                 # "\u0085", # (next line)
             ]

             # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
             # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
             # space (and thus into an empty list).

             for unicode_seq in line_break_unicodes:
                 A__    =      tokenizer_s.tokenize(lowercase_							)
                 A__    =      tokenizer_r.tokenize(lowercase_							)

                 self.assertListEqual(lowercase_						,	lowercase_							)
 def 					UpperCAmelCase_							(    self				:Optional[int]							)->							Optional[Any]:
     # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"							):
             A__    =      "hello"  # `hello` is a token in the vocabulary of `pretrained_name`
             A__    =      F"{text_of_1_token} {text_of_1_token}"

             A__    =      self.rust_tokenizer_class.from_pretrained(
                 lowercase_						,	use_fast=lowercase_						,	)
             A__    =      tokenizer_r(lowercase_						,	return_offsets_mapping=lowercase_						,	add_special_tokens=lowercase_							)
             self.assertEqual(encoding.offset_mapping[0]						,	(0, len(lowercase_							))							)
             self.assertEqual(
                 encoding.offset_mapping[1]						,	(len(lowercase_							) + 1, len(lowercase_							) + 1 + len(lowercase_							))						,	)

             A__    =      F" {text}"

             A__    =      self.rust_tokenizer_class.from_pretrained(
                 lowercase_						,	use_fast=lowercase_						,	)
             A__    =      tokenizer_r(lowercase_						,	return_offsets_mapping=lowercase_						,	add_special_tokens=lowercase_							)
             self.assertEqual(encoding.offset_mapping[0]						,	(1, 1 + len(lowercase_							))							)
             self.assertEqual(
                 encoding.offset_mapping[1]						,	(1 + len(lowercase_							) + 1, 1 + len(lowercase_							) + 1 + len(lowercase_							))						,	)
 def 					UpperCAmelCase_							(    self				:Dict							)->							Optional[Any]:
     # Test related to the breaking change introduced in transformers v4.17.0
     # We need to check that an error in raised when the user try to load a previous version of the tokenizer.
     with self.assertRaises(lowercase_							) as context:
         self.rust_tokenizer_class.from_pretrained("robot-test/old-clip-tokenizer"							)

     self.assertTrue(
         context.exception.args[0].startswith(
             "The `backend_tokenizer` provided does not match the expected format."							)							)
 @require_ftfy
 def 					UpperCAmelCase_							(    self				:str							)->							Any:
     super().test_tokenization_python_rust_equals()



 def 					UpperCAmelCase_							(    self				:Optional[int]							)->							Union[str, Any]:
     # CLIP always lower cases letters
     pass



