






import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer



class 							A   (	snake_case_							):
					'''simple docstring'''






					def   lowerCamelCase__					(self   :						Dict		)						->       str:




							"""simple docstring"""


							lowercase__					      =		tempfile.mkdtemp()
							lowercase__					      =		5

							# Realm tok
							lowercase__					      =		[
							    """[UNK]""",
							    """[CLS]""",
							    """[SEP]""",
							    """[PAD]""",
							    """[MASK]""",
							    """test""",
							    """question""",
							    """this""",
							    """is""",
							    """the""",
							    """first""",
							    """second""",
							    """third""",
							    """fourth""",
							    """fifth""",
							    """record""",
							    """want""",
							    """##want""",
							    """##ed""",
							    """wa""",
							    """un""",
							    """runn""",
							    """##ing""",
							    """,""",
							    """low""",
							    """lowest""",
							]
							lowercase__					      =		os.path.join(self.tmpdirname				, """realm_tokenizer"""		)
							os.makedirs(_UpperCAmelCase				, exist_ok=_UpperCAmelCase		)
							lowercase__					      =		os.path.join(_UpperCAmelCase				, VOCAB_FILES_NAMES["""vocab_file"""]		)
							with open(self.vocab_file				, """w"""				, encoding="""utf-8"""		) as vocab_writer:
									vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]		)		)

							lowercase__					      =		os.path.join(self.tmpdirname				, """realm_block_records"""		)
							os.makedirs(_UpperCAmelCase				, exist_ok=_UpperCAmelCase		)

					def   lowerCamelCase__					(self   :						Any		)						->       RealmTokenizer:




							"""simple docstring"""


							return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname				, """realm_tokenizer"""		)		)

					def   lowerCamelCase__					(self   :						Optional[Any]		)						->       Union[str, Any]:




							"""simple docstring"""


							shutil.rmtree(self.tmpdirname		)

					def   lowerCamelCase__					(self   :						List[Any]		)						->       Dict:




							"""simple docstring"""


							lowercase__					      =		RealmConfig(num_block_records=self.num_block_records		)
							return config

					def   lowerCamelCase__					(self   :						str		)						->       Optional[int]:




							"""simple docstring"""


							lowercase__					      =		Dataset.from_dict(
							    {
							        """id""": ["""0""", """1"""],
							        """question""": ["""foo""", """bar"""],
							        """answers""": [["""Foo""", """Bar"""], ["""Bar"""]],
							    }		)
							return dataset

					def   lowerCamelCase__					(self   :						Any		)						->       Any:




							"""simple docstring"""


							lowercase__					      =		np.array(
							    [
							        b"""This is the first record""",
							        b"""This is the second record""",
							        b"""This is the third record""",
							        b"""This is the fourth record""",
							        b"""This is the fifth record""",
							        b"""This is a longer longer longer record""",
							    ]				, dtype=_UpperCAmelCase				, )
							return block_records

					def   lowerCamelCase__					(self   :						Optional[int]		)						->       List[str]:




							"""simple docstring"""


							lowercase__					      =		RealmRetriever(
							    block_records=self.get_dummy_block_records()				, tokenizer=self.get_tokenizer()				, )
							return retriever

					def   lowerCamelCase__					(self   :						Dict		)						->       Dict:




							"""simple docstring"""


							lowercase__					      =		self.get_config()
							lowercase__					      =		self.get_dummy_retriever()
							lowercase__					      =		retriever.tokenizer

							lowercase__					      =		np.array([0, 3]				, dtype="""long"""		)
							lowercase__					      =		tokenizer(["""Test question"""]		).input_ids
							lowercase__					      =		tokenizer(
							    ["""the fourth"""]				, add_special_tokens=_UpperCAmelCase				, return_token_type_ids=_UpperCAmelCase				, return_attention_mask=_UpperCAmelCase				, ).input_ids
							lowercase__					      =		config.reader_seq_len

							lowercase__      ,     lowercase__      ,     lowercase__      ,     lowercase__					      =		retriever(
							    _UpperCAmelCase				, _UpperCAmelCase				, answer_ids=_UpperCAmelCase				, max_length=_UpperCAmelCase				, return_tensors="""np"""		)

							self.assertEqual(len(_UpperCAmelCase		)				, 2		)
							self.assertEqual(len(_UpperCAmelCase		)				, 2		)
							self.assertEqual(len(_UpperCAmelCase		)				, 2		)
							self.assertEqual(concat_inputs.input_ids.shape				, (2, 10)		)
							self.assertEqual(concat_inputs.attention_mask.shape				, (2, 10)		)
							self.assertEqual(concat_inputs.token_type_ids.shape				, (2, 10)		)
							self.assertEqual(concat_inputs.special_tokens_mask.shape				, (2, 10)		)
							self.assertEqual(
							    tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0]		)				, ["""[CLS]""", """test""", """question""", """[SEP]""", """this""", """is""", """the""", """first""", """record""", """[SEP]"""]				, )
							self.assertEqual(
							    tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1]		)				, ["""[CLS]""", """test""", """question""", """[SEP]""", """this""", """is""", """the""", """fourth""", """record""", """[SEP]"""]				, )

					def   lowerCamelCase__					(self   :						str		)						->       List[str]:




							"""simple docstring"""


							lowercase__					      =		self.get_config()
							lowercase__					      =		self.get_dummy_retriever()
							lowercase__					      =		retriever.tokenizer

							lowercase__					      =		np.array([0, 3, 5]				, dtype="""long"""		)
							lowercase__					      =		tokenizer(["""Test question"""]		).input_ids
							lowercase__					      =		tokenizer(
							    ["""the fourth""", """longer longer"""]				, add_special_tokens=_UpperCAmelCase				, return_token_type_ids=_UpperCAmelCase				, return_attention_mask=_UpperCAmelCase				, ).input_ids
							lowercase__					      =		config.reader_seq_len

							lowercase__      ,     lowercase__      ,     lowercase__      ,     lowercase__					      =		retriever(
							    _UpperCAmelCase				, _UpperCAmelCase				, answer_ids=_UpperCAmelCase				, max_length=_UpperCAmelCase				, return_tensors="""np"""		)

							self.assertEqual([False, True, True]				, _UpperCAmelCase		)
							self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]				, _UpperCAmelCase		)
							self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]				, _UpperCAmelCase		)


					def   lowerCamelCase__					(self   :						List[Any]		)						->       Union[str, Any]:




							"""simple docstring"""


							lowercase__					      =		self.get_dummy_retriever()
							retriever.save_pretrained(os.path.join(self.tmpdirname				, """realm_block_records"""		)		)

							# Test local path
							lowercase__					      =		retriever.from_pretrained(os.path.join(self.tmpdirname				, """realm_block_records"""		)		)
							self.assertEqual(retriever.block_records[0]				, b"""This is the first record"""		)

							# Test mocked remote path
							with patch("""transformers.models.realm.retrieval_realm.hf_hub_download"""		) as mock_hf_hub_download:
									lowercase__					      =		os.path.join(
									    os.path.join(self.tmpdirname				, """realm_block_records"""		)				, _REALM_BLOCK_RECORDS_FILENAME		)
									lowercase__					      =		RealmRetriever.from_pretrained("""google/realm-cc-news-pretrained-openqa"""		)

							self.assertEqual(retriever.block_records[0]				, b"""This is the first record"""		)





A	:		Union[str, Any]    =       [
    (1_0_0_0, 'M'),
    (9_0_0, 'CM'),
    (5_0_0, 'D'),
    (4_0_0, 'CD'),
    (1_0_0, 'C'),
    (9_0, 'XC'),
    (5_0, 'L'),
    (4_0, 'XL'),
    (1_0, 'X'),
    (9, 'IX'),
    (5, 'V'),
    (4, 'IV'),
    (1, 'I'),
]






def 			UpperCamelCase (     __magic_name__					:							str					)       ->  int:
		"""simple docstring"""




		lowercase__					      =		{"""I""": 1, """V""": 5, """X""": 10, """L""": 50, """C""": 100, """D""": 500, """M""": 1000}
		lowercase__					      =		0
		lowercase__					      =		0
		while place < len(__magic_name__					):
				if (place + 1 < len(__magic_name__					)) and (vals[roman[place]] < vals[roman[place + 1]]):
						total += vals[roman[place + 1]] - vals[roman[place]]
						place += 2
				else:
						total += vals[roman[place]]
						place += 1
		return total






def 			UpperCamelCase (     __magic_name__					:							int					)       ->  str:
		"""simple docstring"""




		lowercase__					      =		[]
		for arabic, roman in ROMAN:
				((lowercase__)      ,     (lowercase__))					      =		divmod(__magic_name__   ,   __magic_name__					)
				result.append(roman * factor					)
				if number == 0:
						break
		return "".join(__magic_name__					)


if __name__ == "__main__":
	import doctest

	doctest.testmod()

