

import json
import os
import pickle
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers import is_faiss_available
from transformers.models.bart.configuration_bart import BartConfig
from transformers.models.bart.tokenization_bart import BartTokenizer
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
from transformers.models.dpr.configuration_dpr import DPRConfig
from transformers.models.dpr.tokenization_dpr import DPRContextEncoderTokenizer, DPRQuestionEncoderTokenizer
from transformers.models.rag.configuration_rag import RagConfig
from transformers.models.rag.retrieval_rag import CustomHFIndex, RagRetriever
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES
from transformers.testing_utils import require_faiss, require_sentencepiece, require_tokenizers, require_torch


if is_faiss_available():
	import faiss



@require_faiss
class    _a						(__magic_name__    ):


					'''simple docstring'''






					def   __A      (			self      ):
						A__							:   List[Any]					       =		tempfile.mkdtemp()
						A__							:   Any					       =		8

						# DPR tok
						A__							:   str					       =		[
						    """[UNK]""",
						    """[CLS]""",
						    """[SEP]""",
						    """[PAD]""",
						    """[MASK]""",
						    """want""",
						    """##want""",
						    """##ed""",
						    """wa""",
						    """un""",
						    """runn""",
						    """##ing""",
						    """,""",
						    """low""",
						    """lowest""",
						]
						A__							:   int					       =		os.path.join(self.tmpdirname   ,	"""dpr_tokenizer"""      )
						os.makedirs(A__   ,	exist_ok=A__      )
						A__							:   List[Any]					       =		os.path.join(A__   ,	DPR_VOCAB_FILES_NAMES["""vocab_file"""]      )
						with open(self.vocab_file   ,	"""w"""   ,	encoding="""utf-8"""      ) as vocab_writer:
							vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]      )      )

						# BART tok
						A__							:   Dict					       =		[
						    """l""",
						    """o""",
						    """w""",
						    """e""",
						    """r""",
						    """s""",
						    """t""",
						    """i""",
						    """d""",
						    """n""",
						    """\u0120""",
						    """\u0120l""",
						    """\u0120n""",
						    """\u0120lo""",
						    """\u0120low""",
						    """er""",
						    """\u0120lowest""",
						    """\u0120newer""",
						    """\u0120wider""",
						    """<unk>""",
						]
						A__							:   Any					       =		dict(zip(A__   ,	range(len(A__      )      )      )      )
						A__							:   Dict					       =		["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
						A__							:   Dict					       =		{"""unk_token""": """<unk>"""}

						A__							:   Dict					       =		os.path.join(self.tmpdirname   ,	"""bart_tokenizer"""      )
						os.makedirs(A__   ,	exist_ok=A__      )
						A__							:   Optional[Any]					       =		os.path.join(A__   ,	BART_VOCAB_FILES_NAMES["""vocab_file"""]      )
						A__							:   Optional[Any]					       =		os.path.join(A__   ,	BART_VOCAB_FILES_NAMES["""merges_file"""]      )
						with open(self.vocab_file   ,	"""w"""   ,	encoding="""utf-8"""      ) as fp:
							fp.write(json.dumps(A__      ) + """\n"""      )
						with open(self.merges_file   ,	"""w"""   ,	encoding="""utf-8"""      ) as fp:
							fp.write("""\n""".join(A__      )      )






					def   __A      (			self      ):
						return DPRQuestionEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname   ,	"""dpr_tokenizer"""      )      )






					def   __A      (			self      ):
						return DPRContextEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname   ,	"""dpr_tokenizer"""      )      )






					def   __A      (			self      ):
						return BartTokenizer.from_pretrained(os.path.join(self.tmpdirname   ,	"""bart_tokenizer"""      )      )






					def   __A      (			self      ):
						shutil.rmtree(self.tmpdirname      )






					def   __A      (			self      ):
						A__							:   List[str]					       =		Dataset.from_dict(
						    {
						        """id""": ["""0""", """1"""],
						        """text""": ["""foo""", """bar"""],
						        """title""": ["""Foo""", """Bar"""],
						        """embeddings""": [np.ones(self.retrieval_vector_size      ), 2 * np.ones(self.retrieval_vector_size      )],
						    }      )
						dataset.add_faiss_index("""embeddings"""   ,	string_factory="""Flat"""   ,	metric_type=faiss.METRIC_INNER_PRODUCT      )
						return dataset






					def   __A      (			self      ):
						A__							:   int					       =		self.get_dummy_dataset()
						A__							:   Optional[Any]					       =		RagConfig(
						    retrieval_vector_size=self.retrieval_vector_size   ,	question_encoder=DPRConfig().to_dict()   ,	generator=BartConfig().to_dict()   ,	)
						with patch("""transformers.models.rag.retrieval_rag.load_dataset"""      ) as mock_load_dataset:
							A__							:   Dict					       =		dataset
							A__							:   Tuple					       =		RagRetriever(
							    A__   ,	question_encoder_tokenizer=self.get_dpr_tokenizer()   ,	generator_tokenizer=self.get_bart_tokenizer()   ,	)
						return retriever






					def   __A      (			self   ,	A__      ):
						A__							:   Any					       =		self.get_dummy_dataset()
						A__							:   Dict					       =		RagConfig(
						    retrieval_vector_size=self.retrieval_vector_size   ,	question_encoder=DPRConfig().to_dict()   ,	generator=BartConfig().to_dict()   ,	index_name="""custom"""   ,	)
						if from_disk:
							A__							:   Optional[Any]					       =		os.path.join(self.tmpdirname   ,	"""dataset"""      )
							A__							:   Tuple					       =		os.path.join(self.tmpdirname   ,	"""index.faiss"""      )
							dataset.get_index("""embeddings"""      ).save(os.path.join(self.tmpdirname   ,	"""index.faiss"""      )      )
							dataset.drop_index("""embeddings"""      )
							dataset.save_to_disk(os.path.join(self.tmpdirname   ,	"""dataset"""      )      )
							del dataset
							A__							:   Optional[Any]					       =		RagRetriever(
							    A__   ,	question_encoder_tokenizer=self.get_dpr_tokenizer()   ,	generator_tokenizer=self.get_bart_tokenizer()   ,	)
						else:
							A__							:   Tuple					       =		RagRetriever(
							    A__   ,	question_encoder_tokenizer=self.get_dpr_tokenizer()   ,	generator_tokenizer=self.get_bart_tokenizer()   ,	index=CustomHFIndex(config.retrieval_vector_size   ,	A__      )   ,	)
						return retriever






					def   __A      (			self      ):
						A__							:   Union[str, Any]					       =		Dataset.from_dict(
						    {
						        """id""": ["""0""", """1"""],
						        """text""": ["""foo""", """bar"""],
						        """title""": ["""Foo""", """Bar"""],
						        """embeddings""": [np.ones(self.retrieval_vector_size + 1      ), 2 * np.ones(self.retrieval_vector_size + 1      )],
						    }      )
						dataset.add_faiss_index("""embeddings"""   ,	string_factory="""Flat"""   ,	metric_type=faiss.METRIC_INNER_PRODUCT      )

						A__							:   List[str]					       =		os.path.join(self.tmpdirname   ,	"""hf_bert_base.hnswSQ8_correct_phi_128.c_index"""      )
						dataset.save_faiss_index("""embeddings"""   ,	index_file_name + """.index.dpr"""      )
						pickle.dump(dataset["""id"""]   ,	open(index_file_name + """.index_meta.dpr"""   ,	"""wb"""      )      )

						A__							:   Tuple					       =		os.path.join(self.tmpdirname   ,	"""psgs_w100.tsv.pkl"""      )
						A__							:   Any					       =		{sample["""id"""]: [sample["""text"""], sample["""title"""]] for sample in dataset}
						pickle.dump(A__   ,	open(A__   ,	"""wb"""      )      )

						A__							:   int					       =		RagConfig(
						    retrieval_vector_size=self.retrieval_vector_size   ,	question_encoder=DPRConfig().to_dict()   ,	generator=BartConfig().to_dict()   ,	index_name="""legacy"""   ,	index_path=self.tmpdirname   ,	)
						A__							:   Any					       =		RagRetriever(
						    A__   ,	question_encoder_tokenizer=self.get_dpr_tokenizer()   ,	generator_tokenizer=self.get_bart_tokenizer()      )
						return retriever






					def   __A      (			self      ):
						A__							:   Optional[Any]					       =		1
						A__							:   Tuple					       =		self.get_dummy_canonical_hf_index_retriever()
						A__							:   Optional[int]					       =		np.array(
						    [np.ones(self.retrieval_vector_size      ), -np.ones(self.retrieval_vector_size      )]   ,	dtype=np.floataa      )
						A__       ,       A__       ,       A__							:   Dict					       =		retriever.retrieve(A__   ,	n_docs=A__      )
						self.assertEqual(retrieved_doc_embeds.shape   ,	(2, n_docs, self.retrieval_vector_size)      )
						self.assertEqual(len(A__      )   ,	2      )
						self.assertEqual(sorted(doc_dicts[0]      )   ,	["""embeddings""", """id""", """text""", """title"""]      )
						self.assertEqual(len(doc_dicts[0]["""id"""]      )   ,	A__      )
						self.assertEqual(doc_dicts[0]["""id"""][0]   ,	"""1"""      )  # max inner product is reached with second doc
						self.assertEqual(doc_dicts[1]["""id"""][0]   ,	"""0"""      )  # max inner product is reached with first doc
						self.assertListEqual(doc_ids.tolist()   ,	[[1], [0]]      )






					def   __A      (			self      ):
						A__							:   int					       =		self.get_dummy_canonical_hf_index_retriever()
						with tempfile.TemporaryDirectory() as tmp_dirname:
							with patch("""transformers.models.rag.retrieval_rag.load_dataset"""      ) as mock_load_dataset:
								A__							:   Optional[int]					       =		self.get_dummy_dataset()
								retriever.save_pretrained(A__      )
								A__							:   Optional[Any]					       =		RagRetriever.from_pretrained(A__      )
								self.assertIsInstance(A__   ,	A__      )
								A__							:   Tuple					       =		np.array(
								    [np.ones(self.retrieval_vector_size      ), -np.ones(self.retrieval_vector_size      )]   ,	dtype=np.floataa      )
								A__							:   Any					       =		retriever.retrieve(A__   ,	n_docs=1      )
								self.assertTrue(out is not None      )






					def   __A      (			self      ):
						A__							:   Tuple					       =		1
						A__							:   Optional[int]					       =		self.get_dummy_custom_hf_index_retriever(from_disk=A__      )
						A__							:   Any					       =		np.array(
						    [np.ones(self.retrieval_vector_size      ), -np.ones(self.retrieval_vector_size      )]   ,	dtype=np.floataa      )
						A__       ,       A__       ,       A__							:   List[str]					       =		retriever.retrieve(A__   ,	n_docs=A__      )
						self.assertEqual(retrieved_doc_embeds.shape   ,	(2, n_docs, self.retrieval_vector_size)      )
						self.assertEqual(len(A__      )   ,	2      )
						self.assertEqual(sorted(doc_dicts[0]      )   ,	["""embeddings""", """id""", """text""", """title"""]      )
						self.assertEqual(len(doc_dicts[0]["""id"""]      )   ,	A__      )
						self.assertEqual(doc_dicts[0]["""id"""][0]   ,	"""1"""      )  # max inner product is reached with second doc
						self.assertEqual(doc_dicts[1]["""id"""][0]   ,	"""0"""      )  # max inner product is reached with first doc
						self.assertListEqual(doc_ids.tolist()   ,	[[1], [0]]      )






					def   __A      (			self      ):
						A__							:   Optional[Any]					       =		self.get_dummy_custom_hf_index_retriever(from_disk=A__      )
						with tempfile.TemporaryDirectory() as tmp_dirname:
							retriever.save_pretrained(A__      )
							A__							:   Tuple					       =		RagRetriever.from_pretrained(A__      )
							self.assertIsInstance(A__   ,	A__      )
							A__							:   Optional[Any]					       =		np.array(
							    [np.ones(self.retrieval_vector_size      ), -np.ones(self.retrieval_vector_size      )]   ,	dtype=np.floataa      )
							A__							:   Dict					       =		retriever.retrieve(A__   ,	n_docs=1      )
							self.assertTrue(out is not None      )






					def   __A      (			self      ):
						A__							:   Tuple					       =		1
						A__							:   List[str]					       =		self.get_dummy_custom_hf_index_retriever(from_disk=A__      )
						A__							:   int					       =		np.array(
						    [np.ones(self.retrieval_vector_size      ), -np.ones(self.retrieval_vector_size      )]   ,	dtype=np.floataa      )
						A__       ,       A__       ,       A__							:   Tuple					       =		retriever.retrieve(A__   ,	n_docs=A__      )
						self.assertEqual(retrieved_doc_embeds.shape   ,	(2, n_docs, self.retrieval_vector_size)      )
						self.assertEqual(len(A__      )   ,	2      )
						self.assertEqual(sorted(doc_dicts[0]      )   ,	["""embeddings""", """id""", """text""", """title"""]      )
						self.assertEqual(len(doc_dicts[0]["""id"""]      )   ,	A__      )
						self.assertEqual(doc_dicts[0]["""id"""][0]   ,	"""1"""      )  # max inner product is reached with second doc
						self.assertEqual(doc_dicts[1]["""id"""][0]   ,	"""0"""      )  # max inner product is reached with first doc
						self.assertListEqual(doc_ids.tolist()   ,	[[1], [0]]      )






					def   __A      (			self      ):
						A__							:   Any					       =		self.get_dummy_custom_hf_index_retriever(from_disk=A__      )
						with tempfile.TemporaryDirectory() as tmp_dirname:
							retriever.save_pretrained(A__      )
							A__							:   Optional[int]					       =		RagRetriever.from_pretrained(A__      )
							self.assertIsInstance(A__   ,	A__      )
							A__							:   int					       =		np.array(
							    [np.ones(self.retrieval_vector_size      ), -np.ones(self.retrieval_vector_size      )]   ,	dtype=np.floataa      )
							A__							:   Union[str, Any]					       =		retriever.retrieve(A__   ,	n_docs=1      )
							self.assertTrue(out is not None      )






					def   __A      (			self      ):
						A__							:   Optional[int]					       =		1
						A__							:   List[Any]					       =		self.get_dummy_legacy_index_retriever()
						A__							:   Dict					       =		np.array(
						    [np.ones(self.retrieval_vector_size      ), -np.ones(self.retrieval_vector_size      )]   ,	dtype=np.floataa      )
						A__       ,       A__       ,       A__							:   Optional[int]					       =		retriever.retrieve(A__   ,	n_docs=A__      )
						self.assertEqual(retrieved_doc_embeds.shape   ,	(2, n_docs, self.retrieval_vector_size)      )
						self.assertEqual(len(A__      )   ,	2      )
						self.assertEqual(sorted(doc_dicts[0]      )   ,	["""text""", """title"""]      )
						self.assertEqual(len(doc_dicts[0]["""text"""]      )   ,	A__      )
						self.assertEqual(doc_dicts[0]["""text"""][0]   ,	"""bar"""      )  # max inner product is reached with second doc
						self.assertEqual(doc_dicts[1]["""text"""][0]   ,	"""foo"""      )  # max inner product is reached with first doc
						self.assertListEqual(doc_ids.tolist()   ,	[[1], [0]]      )






					def   __A      (			self      ):
						A__							:   List[Any]					       =		self.get_dummy_legacy_index_retriever()
						with tempfile.TemporaryDirectory() as tmp_dirname:
							retriever.save_pretrained(A__      )
							A__							:   Tuple					       =		RagRetriever.from_pretrained(A__      )
							self.assertIsInstance(A__   ,	A__      )
							A__							:   Tuple					       =		np.array(
							    [np.ones(self.retrieval_vector_size      ), -np.ones(self.retrieval_vector_size      )]   ,	dtype=np.floataa      )
							A__							:   Dict					       =		retriever.retrieve(A__   ,	n_docs=1      )
							self.assertTrue(out is not None      )






					@require_torch
					@require_tokenizers
					@require_sentencepiece
					def   __A      (			self      ):
						import torch

						A__							:   str					       =		1
						A__							:   Optional[int]					       =		self.get_dummy_canonical_hf_index_retriever()
						A__							:   Optional[Any]					       =		[[5, 7], [10, 11]]
						A__							:   Union[str, Any]					       =		np.array(
						    [np.ones(self.retrieval_vector_size      ), -np.ones(self.retrieval_vector_size      )]   ,	dtype=np.floataa      )
						A__							:   Tuple					       =		retriever(A__   ,	A__   ,	prefix=retriever.config.generator.prefix   ,	n_docs=A__      )
						A__       ,       A__       ,       A__							:   Tuple					       =		(
						    out["""context_input_ids"""],
						    out["""context_attention_mask"""],
						    out["""retrieved_doc_embeds"""],
						)
						self.assertEqual(retrieved_doc_embeds.shape   ,	(2, n_docs, self.retrieval_vector_size)      )
						self.assertIsInstance(A__   ,	A__      )
						self.assertIsInstance(A__   ,	A__      )
						self.assertIsInstance(A__   ,	np.ndarray      )

						A__							:   List[str]					       =		retriever(
						    A__   ,	A__   ,	prefix=retriever.config.generator.prefix   ,	n_docs=A__   ,	return_tensors="""pt"""   ,	)
						A__       ,       A__       ,       A__       ,       A__							:   Optional[Any]					       =		(  # noqa: F841
						    out["""context_input_ids"""],
						    out["""context_attention_mask"""],
						    out["""retrieved_doc_embeds"""],
						    out["""doc_ids"""],
						)
						self.assertEqual(retrieved_doc_embeds.shape   ,	(2, n_docs, self.retrieval_vector_size)      )
						self.assertIsInstance(A__   ,	torch.Tensor      )
						self.assertIsInstance(A__   ,	torch.Tensor      )
						self.assertIsInstance(A__   ,	torch.Tensor      )






					@require_torch
					@require_tokenizers
					@require_sentencepiece
					def   __A      (			self      ):
						A__							:   Any					       =		self.get_dpr_ctx_encoder_tokenizer()
						A__							:   Optional[int]					       =		1
						A__							:   Any					       =		self.get_dummy_custom_hf_index_retriever(from_disk=A__      )
						retriever.set_ctx_encoder_tokenizer(A__      )

						A__							:   List[str]					       =		[[5, 7], [10, 11]]
						A__							:   List[Any]					       =		np.array(
						    [np.ones(self.retrieval_vector_size      ), -np.ones(self.retrieval_vector_size      )]   ,	dtype=np.floataa      )
						A__							:   Tuple					       =		retriever(A__   ,	A__   ,	prefix=retriever.config.generator.prefix   ,	n_docs=A__      )

						self.assertEqual(
						    len(A__      )   ,	6      )  # check whether the retriever output consist of 6 attributes including tokenized docs
						self.assertEqual(
						    all(k in out for k in ("""tokenized_doc_ids""", """tokenized_doc_attention_mask""")      )   ,	A__      )  # check for doc token related keys in dictionary.






import warnings
from contextlib import contextmanager

from ....processing_utils import ProcessorMixin



class    _a						(__magic_name__    ):


					'''simple docstring'''






					UpperCAmelCase__:       Optional[Any]  							=	'''MCTCTFeatureExtractor'''
					UpperCAmelCase__:       Optional[int]  							=	'''AutoTokenizer'''






					def __init__(			self   ,	A__   ,	A__      ):
						super().__init__(A__   ,	A__      )
						A__							:   List[str]					       =		self.feature_extractor
						A__							:   Optional[int]					       =		False






					def __call__(			self   ,	*A__   ,	**A__      ):
						# For backward compatibility
						if self._in_target_context_manager:
							return self.current_processor(*A__   ,	**A__      )

						if "raw_speech" in kwargs:
							warnings.warn("""Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead."""      )
							A__							:   Dict					       =		kwargs.pop("""raw_speech"""      )
						else:
							A__							:   Tuple					       =		kwargs.pop("""audio"""   ,	A__      )
						A__							:   Union[str, Any]					       =		kwargs.pop("""sampling_rate"""   ,	A__      )
						A__							:   int					       =		kwargs.pop("""text"""   ,	A__      )
						if len(A__      ) > 0:
							A__							:   Optional[int]					       =		args[0]
							A__							:   Dict					       =		args[1:]

						if audio is None and text is None:
							raise ValueError("""You need to specify either an `audio` or `text` input to process."""      )

						if audio is not None:
							A__							:   List[str]					       =		self.feature_extractor(A__   ,	*A__   ,	sampling_rate=A__   ,	**A__      )
						if text is not None:
							A__							:   Optional[Any]					       =		self.tokenizer(A__   ,	**A__      )

						if text is None:
							return inputs
						elif audio is None:
							return encodings
						else:
							A__							:   List[Any]					       =		encodings["""input_ids"""]
							return inputs






					def   __A      (			self   ,	*A__   ,	**A__      ):
						return self.tokenizer.batch_decode(*A__   ,	**A__      )






					def   __A      (			self   ,	*A__   ,	**A__      ):
						# For backward compatibility
						if self._in_target_context_manager:
							return self.current_processor.pad(*A__   ,	**A__      )

						A__							:   Optional[Any]					       =		kwargs.pop("""input_features"""   ,	A__      )
						A__							:   Union[str, Any]					       =		kwargs.pop("""labels"""   ,	A__      )
						if len(A__      ) > 0:
							A__							:   List[Any]					       =		args[0]
							A__							:   Optional[int]					       =		args[1:]

						if input_features is not None:
							A__							:   Union[str, Any]					       =		self.feature_extractor.pad(A__   ,	*A__   ,	**A__      )
						if labels is not None:
							A__							:   List[Any]					       =		self.tokenizer.pad(A__   ,	**A__      )

						if labels is None:
							return input_features
						elif input_features is None:
							return labels
						else:
							A__							:   Dict					       =		labels["""input_ids"""]
							return input_features






					def   __A      (			self   ,	*A__   ,	**A__      ):
						return self.tokenizer.decode(*A__   ,	**A__      )






					@contextmanager
					def   __A      (			self      ):
						warnings.warn(
						    """`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your """
						    """labels by using the argument `text` of the regular `__call__` method (either in the same call as """
						    """your audio inputs, or in a separate call."""      )
						A__							:   int					       =		True
						A__							:   List[Any]					       =		self.tokenizer
						yield
						A__							:   Tuple					       =		self.feature_extractor
						A__							:   Dict					       =		False



