


"""simple docstring"""



import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


__snake_case             =   logging.getLogger(__name__)







class _lowerCAmelCase    :







   def __init__(     self						)     ->		Union[str, Any]:

       '''simple docstring'''




       snake_case   :				Union[str, Any]				 =   False







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						,     UpperCamelCase__						,     UpperCamelCase__						,     UpperCamelCase__						)     ->		Tuple:

       '''simple docstring'''




       if not self.initialized:
           snake_case   :				Tuple				 =   RagRetriever(
               _SCREAMING_SNAKE_CASE						,     question_encoder_tokenizer=_SCREAMING_SNAKE_CASE						,     generator_tokenizer=_SCREAMING_SNAKE_CASE						,     index=_SCREAMING_SNAKE_CASE						,     init_retrieval=_SCREAMING_SNAKE_CASE						,     )
           snake_case   :				Any				 =   True







   def 			lowerCamelCase		(     self						)     ->		Union[str, Any]:

       '''simple docstring'''




       self.retriever.index.init_index()







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						,     UpperCamelCase__						)     ->		int:

       '''simple docstring'''




       snake_case ,snake_case   :				Optional[Any]				 =   self.retriever._main_retrieve(_SCREAMING_SNAKE_CASE						,     _SCREAMING_SNAKE_CASE						)
       return doc_ids, retrieved_doc_embeds







class _lowerCAmelCase    (				lowerCAmelCase__     ):







   def __init__(     self						,     UpperCamelCase__						,     UpperCamelCase__						,     UpperCamelCase__						,     UpperCamelCase__						,     UpperCamelCase__=None						)     ->		Dict:

       '''simple docstring'''




       if index is not None and index.is_initialized() and len(_SCREAMING_SNAKE_CASE						) > 0:
           raise ValueError(
               "When using Ray for distributed fine-tuning, "
               "you'll need to provide the paths instead, "
               "as the dataset and the index are loaded "
               "separately. More info in examples/rag/use_own_knowledge_dataset.py "						)
       super().__init__(
           _SCREAMING_SNAKE_CASE						,     question_encoder_tokenizer=_SCREAMING_SNAKE_CASE						,     generator_tokenizer=_SCREAMING_SNAKE_CASE						,     index=_SCREAMING_SNAKE_CASE						,     init_retrieval=_SCREAMING_SNAKE_CASE						,     )
       snake_case   :				int				 =   retrieval_workers
       if len(self.retrieval_workers						) > 0:
           ray.get(
               [
                   worker.create_rag_retriever.remote(_SCREAMING_SNAKE_CASE						,     _SCREAMING_SNAKE_CASE						,     _SCREAMING_SNAKE_CASE						,     _SCREAMING_SNAKE_CASE						)
                   for worker in self.retrieval_workers
               ]						)







   def 			lowerCamelCase		(     self						)     ->		Optional[Any]:

       '''simple docstring'''




       logger.info("initializing retrieval"						)

       if len(self.retrieval_workers						) > 0:
           ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers]						)
       else:
           # Non-distributed training. Load index into this same process.
           self.index.init_index()







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						,     UpperCamelCase__						)     ->		Any:

       '''simple docstring'''




       if len(self.retrieval_workers						) > 0:
           # Select a random retrieval actor.
           snake_case   :				Any				 =   self.retrieval_workers[random.randint(0						,     len(self.retrieval_workers						) - 1						)]
           snake_case ,snake_case   :				str				 =   ray.get(random_worker.retrieve.remote(_SCREAMING_SNAKE_CASE						,     _SCREAMING_SNAKE_CASE						)						)
       else:
           snake_case ,snake_case   :				Optional[Any]				 =   self._main_retrieve(_SCREAMING_SNAKE_CASE						,     _SCREAMING_SNAKE_CASE						)
       return retrieved_doc_embeds, doc_ids, self.index.get_doc_dicts(_SCREAMING_SNAKE_CASE						)







   @classmethod
   def 			lowerCamelCase		(     cls						,     UpperCamelCase__						,     UpperCamelCase__=None						,     **UpperCamelCase__						)     ->		str:

       '''simple docstring'''




       return super(_SCREAMING_SNAKE_CASE						,     cls						).get_tokenizers(_SCREAMING_SNAKE_CASE						,     _SCREAMING_SNAKE_CASE						,     **_SCREAMING_SNAKE_CASE						)







   @classmethod
   def 			lowerCamelCase		(     cls						,     UpperCamelCase__						,     UpperCamelCase__						,     UpperCamelCase__=None						,     **UpperCamelCase__						)     ->		Union[str, Any]:

       '''simple docstring'''




       snake_case   :				Dict				 =   kwargs.pop("config"						,     _SCREAMING_SNAKE_CASE						) or RagConfig.from_pretrained(_SCREAMING_SNAKE_CASE						,     **_SCREAMING_SNAKE_CASE						)
       snake_case   :				List[str]				 =   RagTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE						,     config=_SCREAMING_SNAKE_CASE						)
       snake_case   :				List[str]				 =   rag_tokenizer.question_encoder
       snake_case   :				str				 =   rag_tokenizer.generator
       if indexed_dataset is not None:
           snake_case   :				Any				 =   "custom"
           snake_case   :				Union[str, Any]				 =   CustomHFIndex(config.retrieval_vector_size						,     _SCREAMING_SNAKE_CASE						)
       else:
           snake_case   :				str				 =   cls._build_index(_SCREAMING_SNAKE_CASE						)
       return cls(
           _SCREAMING_SNAKE_CASE						,     question_encoder_tokenizer=_SCREAMING_SNAKE_CASE						,     generator_tokenizer=_SCREAMING_SNAKE_CASE						,     retrieval_workers=_SCREAMING_SNAKE_CASE						,     index=_SCREAMING_SNAKE_CASE						,     )









"""simple docstring"""



import doctest
import logging
import os
import unittest
from pathlib import Path
from typing import List, Union

import transformers
from transformers.testing_utils import require_tf, require_torch, slow


__snake_case             =   logging.getLogger()







@unittest.skip('''Temporarily disable the doc tests.'''     )
@require_torch
@require_tf
@slow
class _lowerCAmelCase    (				unittest.TestCase     ):







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						,     UpperCamelCase__ = None						,     UpperCamelCase__ = None						,     UpperCamelCase__ = None						,     UpperCamelCase__ = True						,     )     ->		Tuple:

       '''simple docstring'''




       snake_case   :				List[Any]				 =   [file for file in os.listdir(UpperCamelCase__						) if os.path.isfile(os.path.join(UpperCamelCase__						,     UpperCamelCase__						)						)]

       if identifier is not None:
           snake_case   :				Optional[Any]				 =   [file for file in files if identifier in file]

       if n_identifier is not None:
           if isinstance(UpperCamelCase__						,     UpperCamelCase__						):
               for n_ in n_identifier:
                   snake_case   :				str				 =   [file for file in files if n_ not in file]
           else:
               snake_case   :				str				 =   [file for file in files if n_identifier not in file]

       snake_case   :				Tuple				 =   ignore_files or []
       ignore_files.append("__init__.py"						)
       snake_case   :				Optional[Any]				 =   [file for file in files if file not in ignore_files]

       for file in files:
           # Open all files
           print("Testing"						,     UpperCamelCase__						)

           if only_modules:
               snake_case   :				str				 =   file.split("."						)[0]
               try:
                   snake_case   :				Optional[int]				 =   getattr(UpperCamelCase__						,     UpperCamelCase__						)
                   snake_case   :				str				 =   doctest.DocTestSuite(UpperCamelCase__						)
                   snake_case   :				Optional[Any]				 =   unittest.TextTestRunner().run(UpperCamelCase__						)
                   self.assertIs(len(result.failures						)						,     0						)
               except AttributeError:
                   logger.info(F'{module_identifier} is not a module.'						)
           else:
               snake_case   :				Tuple				 =   doctest.testfile(str(".." / directory / file						)						,     optionflags=doctest.ELLIPSIS						)
               self.assertIs(result.failed						,     0						)







   def 			lowerCamelCase		(     self						)     ->		str:

       '''simple docstring'''




       snake_case   :				Tuple				 =   Path("src/transformers"						)
       snake_case   :				List[Any]				 =   "modeling"
       snake_case   :				Dict				 =   [
           "modeling_ctrl.py",
           "modeling_tf_ctrl.py",
       ]
       self.analyze_directory(UpperCamelCase__						,     identifier=UpperCamelCase__						,     ignore_files=UpperCamelCase__						)







   def 			lowerCamelCase		(     self						)     ->		Dict:

       '''simple docstring'''




       snake_case   :				Optional[Any]				 =   Path("src/transformers"						)
       snake_case   :				Optional[Any]				 =   "tokenization"
       self.analyze_directory(UpperCamelCase__						,     identifier=UpperCamelCase__						)







   def 			lowerCamelCase		(     self						)     ->		List[str]:

       '''simple docstring'''




       snake_case   :				Union[str, Any]				 =   Path("src/transformers"						)
       snake_case   :				Optional[Any]				 =   "configuration"
       self.analyze_directory(UpperCamelCase__						,     identifier=UpperCamelCase__						)







   def 			lowerCamelCase		(     self						)     ->		Optional[Any]:

       '''simple docstring'''




       snake_case   :				List[str]				 =   Path("src/transformers"						)
       snake_case   :				List[str]				 =   ["configuration", "modeling", "tokenization"]
       self.analyze_directory(UpperCamelCase__						,     n_identifier=UpperCamelCase__						)







   def 			lowerCamelCase		(     self						)     ->		List[Any]:

       '''simple docstring'''




       snake_case   :				Any				 =   Path("docs/source"						)
       snake_case   :				Tuple				 =   ["favicon.ico"]
       self.analyze_directory(UpperCamelCase__						,     ignore_files=UpperCamelCase__						,     only_modules=UpperCamelCase__						)





