




"""simple docstring"""




from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


UpperCamelCase__         =      {
    """configuration_megatron_bert""": ["""MEGATRON_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """MegatronBertConfig"""],
}

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					UpperCamelCase__         =      [
					    """MEGATRON_BERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
					    """MegatronBertForCausalLM""",
					    """MegatronBertForMaskedLM""",
					    """MegatronBertForMultipleChoice""",
					    """MegatronBertForNextSentencePrediction""",
					    """MegatronBertForPreTraining""",
					    """MegatronBertForQuestionAnswering""",
					    """MegatronBertForSequenceClassification""",
					    """MegatronBertForTokenClassification""",
					    """MegatronBertModel""",
					    """MegatronBertPreTrainedModel""",
					]

if TYPE_CHECKING:
					from .configuration_megatron_bert import MEGATRON_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, MegatronBertConfig

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_megatron_bert import (
										    MEGATRON_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
										    MegatronBertForCausalLM,
										    MegatronBertForMaskedLM,
										    MegatronBertForMultipleChoice,
										    MegatronBertForNextSentencePrediction,
										    MegatronBertForPreTraining,
										    MegatronBertForQuestionAnswering,
										    MegatronBertForSequenceClassification,
										    MegatronBertForTokenClassification,
										    MegatronBertModel,
										    MegatronBertPreTrainedModel,
										)

else:
					import sys

					UpperCamelCase__         =      _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)










import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


UpperCamelCase__         =      logging.getLogger(__name__)
class       a__   :






						def __init__(       self      ):




											"""simple docstring"""



											__lowerCAmelCase           =	False






						def     __SCREAMING_SNAKE_CASE(       self		,      _A		,      _A		,      _A		,      _A      ):




											"""simple docstring"""



											if not self.initialized:
																__lowerCAmelCase           =	RagRetriever(
																    _A		,      question_encoder_tokenizer=_A		,      generator_tokenizer=_A		,      index=_A		,      init_retrieval=_A		,      )
																__lowerCAmelCase           =	True






						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											self.retriever.index.init_index()






						def     __SCREAMING_SNAKE_CASE(       self		,      _A		,      _A      ):




											"""simple docstring"""



											__lowerCAmelCase       ,							__lowerCAmelCase           =	self.retriever._main_retrieve(_A		,      _A      )
											return doc_ids, retrieved_doc_embeds






class       a__   ( snake_case__				):






						def __init__(       self		,      _A		,      _A		,      _A		,      _A		,      _A=None      ):




											"""simple docstring"""



											if index is not None and index.is_initialized() and len(_A      ) > 0:
																raise ValueError(
																    "When using Ray for distributed fine-tuning, "
																    "you'll need to provide the paths instead, "
																    "as the dataset and the index are loaded "
																    "separately. More info in examples/rag/use_own_knowledge_dataset.py "      )
											super().__init__(
											    _A		,      question_encoder_tokenizer=_A		,      generator_tokenizer=_A		,      index=_A		,      init_retrieval=_A		,      )
											__lowerCAmelCase           =	retrieval_workers
											if len(self.retrieval_workers      ) > 0:
																ray.get(
																    [
																        worker.create_rag_retriever.remote(_A		,      _A		,      _A		,      _A      )
																        for worker in self.retrieval_workers
																    ]      )






						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											logger.info("initializing retrieval"      )

											if len(self.retrieval_workers      ) > 0:
																ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers]      )
											else:
																# Non-distributed training. Load index into this same process.
																self.index.init_index()






						def     __SCREAMING_SNAKE_CASE(       self		,      _A		,      _A      ):




											"""simple docstring"""



											if len(self.retrieval_workers      ) > 0:
																# Select a random retrieval actor.
																__lowerCAmelCase           =	self.retrieval_workers[random.randint(0		,      len(self.retrieval_workers      ) - 1      )]
																__lowerCAmelCase       ,							__lowerCAmelCase           =	ray.get(random_worker.retrieve.remote(_A		,      _A      )      )
											else:
																__lowerCAmelCase       ,							__lowerCAmelCase           =	self._main_retrieve(_A		,      _A      )
											return retrieved_doc_embeds, doc_ids, self.index.get_doc_dicts(_A      )






						@classmethod
						def     __SCREAMING_SNAKE_CASE(       cls		,      _A		,      _A=None		,      **_A      ):




											"""simple docstring"""



											return super(_A		,      cls      ).get_tokenizers(_A		,      _A		,      **_A      )






						@classmethod
						def     __SCREAMING_SNAKE_CASE(       cls		,      _A		,      _A		,      _A=None		,      **_A      ):




											"""simple docstring"""



											__lowerCAmelCase           =	kwargs.pop("config"		,      _A      ) or RagConfig.from_pretrained(_A		,      **_A      )
											__lowerCAmelCase           =	RagTokenizer.from_pretrained(_A		,      config=_A      )
											__lowerCAmelCase           =	rag_tokenizer.question_encoder
											__lowerCAmelCase           =	rag_tokenizer.generator
											if indexed_dataset is not None:
																__lowerCAmelCase           =	"custom"
																__lowerCAmelCase           =	CustomHFIndex(config.retrieval_vector_size		,      _A      )
											else:
																__lowerCAmelCase           =	cls._build_index(_A      )
											return cls(
											    _A		,      question_encoder_tokenizer=_A		,      generator_tokenizer=_A		,      retrieval_workers=_A		,      index=_A		,      )




