






"""simple docstring"""







import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
			from test_module.custom_tokenization_fast import CustomTokenizerFast


class    __UpperCAmelCase(       unittest.TestCase							):




						"""simple docstring"""




						def 		UpperCAmelCase_	(       self      ):
												'''simple docstring'''
												# A mock response for an HTTP head request to emulate server down
												lowercase__    :				Dict=	mock.Mock()
												lowercase__    :				List[str]=	500
												lowercase__    :				List[str]=	{}
												lowercase__    :				str=	HTTPError
												lowercase__    :				Tuple=	{}

												# Download this model to make sure it's in the cache.
												lowercase__    :				Optional[Any]=	BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"      )

												# Under the mock environment we get a 500 error when trying to reach the tokenizer.
												with mock.patch("requests.Session.request"	,  return_value=snake_case__      ) as mock_head:
																		lowercase__    :				List[str]=	BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"      )
																		# This check we did call the fake head request
																		mock_head.assert_called()




						@require_tokenizers
						def 		UpperCAmelCase_	(       self      ):
												'''simple docstring'''
												# A mock response for an HTTP head request to emulate server down
												lowercase__    :				Optional[Any]=	mock.Mock()
												lowercase__    :				Dict=	500
												lowercase__    :				List[str]=	{}
												lowercase__    :				Any=	HTTPError
												lowercase__    :				Dict=	{}

												# Download this model to make sure it's in the cache.
												lowercase__    :				Optional[int]=	GPTaTokenizerFast.from_pretrained("gpt2"      )

												# Under the mock environment we get a 500 error when trying to reach the tokenizer.
												with mock.patch("requests.Session.request"	,  return_value=snake_case__      ) as mock_head:
																		lowercase__    :				Any=	GPTaTokenizerFast.from_pretrained("gpt2"      )
																		# This check we did call the fake head request
																		mock_head.assert_called()




						def 		UpperCAmelCase_	(       self      ):
												'''simple docstring'''
												# This test is for deprecated behavior and can be removed in v5
												try:
																		lowercase__    :				Optional[int]=	tempfile.mktemp()
																		with open(snake_case__	,  "wb"      ) as f:
																								http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"	,  snake_case__      )

																		lowercase__    :				Optional[Any]=	AlbertTokenizer.from_pretrained(snake_case__      )
												finally:
																		os.remove(snake_case__      )

												# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
												# the current folder and have the right name.
												if os.path.isfile("tokenizer.json"      ):
																		# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
																		return
												try:
																		with open("tokenizer.json"	,  "wb"      ) as f:
																								http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"	,  snake_case__      )
																		lowercase__    :				str=	AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"      )
																		# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
																		self.assertEqual(tokenizer.vocab_size	,  1000      )
																		# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

												finally:
																		os.remove("tokenizer.json"      )




						def 		UpperCAmelCase_	(       self      ):
												'''simple docstring'''
												# This test is for deprecated behavior and can be removed in v5
												lowercase__    :				Union[str, Any]=	AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"      )


@is_staging_test
class    __UpperCAmelCase(       unittest.TestCase							):




						"""simple docstring"""




						__lowerCamelCase    	= ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]




						@classmethod
						def 		UpperCAmelCase_	(       cls      ):
												'''simple docstring'''
												lowercase__    :				Union[str, Any]=	TOKEN
												HfFolder.save_token(snake_case__      )




						@classmethod
						def 		UpperCAmelCase_	(       cls      ):
												'''simple docstring'''
												try:
																		delete_repo(token=cls._token	,  repo_id="test-tokenizer"      )
												except HTTPError:
																		pass

												try:
																		delete_repo(token=cls._token	,  repo_id="valid_org/test-tokenizer-org"      )
												except HTTPError:
																		pass

												try:
																		delete_repo(token=cls._token	,  repo_id="test-dynamic-tokenizer"      )
												except HTTPError:
																		pass




						def 		UpperCAmelCase_	(       self      ):
												'''simple docstring'''
												with tempfile.TemporaryDirectory() as tmp_dir:
																		lowercase__    :				Union[str, Any]=	os.path.join(snake_case__	,  "vocab.txt"      )
																		with open(snake_case__	,  "w"	,  encoding="utf-8"      ) as vocab_writer:
																								vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]      )      )
																		lowercase__    :				Union[str, Any]=	BertTokenizer(snake_case__      )

												tokenizer.push_to_hub("test-tokenizer"	,  use_auth_token=self._token      )
												lowercase__    :				Optional[Any]=	BertTokenizer.from_pretrained(F'''{USER}/test-tokenizer'''      )
												self.assertDictEqual(new_tokenizer.vocab	,  tokenizer.vocab      )

												# Reset repo
												delete_repo(token=self._token	,  repo_id="test-tokenizer"      )

												# Push to hub via save_pretrained
												with tempfile.TemporaryDirectory() as tmp_dir:
																		tokenizer.save_pretrained(snake_case__	,  repo_id="test-tokenizer"	,  push_to_hub=snake_case__	,  use_auth_token=self._token      )

												lowercase__    :				Any=	BertTokenizer.from_pretrained(F'''{USER}/test-tokenizer'''      )
												self.assertDictEqual(new_tokenizer.vocab	,  tokenizer.vocab      )




						def 		UpperCAmelCase_	(       self      ):
												'''simple docstring'''
												with tempfile.TemporaryDirectory() as tmp_dir:
																		lowercase__    :				Dict=	os.path.join(snake_case__	,  "vocab.txt"      )
																		with open(snake_case__	,  "w"	,  encoding="utf-8"      ) as vocab_writer:
																								vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]      )      )
																		lowercase__    :				Optional[int]=	BertTokenizer(snake_case__      )

												tokenizer.push_to_hub("valid_org/test-tokenizer-org"	,  use_auth_token=self._token      )
												lowercase__    :				Optional[Any]=	BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"      )
												self.assertDictEqual(new_tokenizer.vocab	,  tokenizer.vocab      )

												# Reset repo
												delete_repo(token=self._token	,  repo_id="valid_org/test-tokenizer-org"      )

												# Push to hub via save_pretrained
												with tempfile.TemporaryDirectory() as tmp_dir:
																		tokenizer.save_pretrained(
																		    snake_case__	,  repo_id="valid_org/test-tokenizer-org"	,  push_to_hub=snake_case__	,  use_auth_token=self._token      )

												lowercase__    :				Optional[Any]=	BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"      )
												self.assertDictEqual(new_tokenizer.vocab	,  tokenizer.vocab      )




						@require_tokenizers
						def 		UpperCAmelCase_	(       self      ):
												'''simple docstring'''
												CustomTokenizer.register_for_auto_class()
												with tempfile.TemporaryDirectory() as tmp_dir:
																		lowercase__    :				int=	os.path.join(snake_case__	,  "vocab.txt"      )
																		with open(snake_case__	,  "w"	,  encoding="utf-8"      ) as vocab_writer:
																								vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]      )      )
																		lowercase__    :				str=	CustomTokenizer(snake_case__      )

												# No fast custom tokenizer
												tokenizer.push_to_hub("test-dynamic-tokenizer"	,  use_auth_token=self._token      )

												lowercase__    :				List[str]=	AutoTokenizer.from_pretrained(F'''{USER}/test-dynamic-tokenizer'''	,  trust_remote_code=snake_case__      )
												# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
												self.assertEqual(tokenizer.__class__.__name__	,  "CustomTokenizer"      )

												# Fast and slow custom tokenizer
												CustomTokenizerFast.register_for_auto_class()
												with tempfile.TemporaryDirectory() as tmp_dir:
																		lowercase__    :				str=	os.path.join(snake_case__	,  "vocab.txt"      )
																		with open(snake_case__	,  "w"	,  encoding="utf-8"      ) as vocab_writer:
																								vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]      )      )

																		lowercase__    :				Tuple=	BertTokenizerFast.from_pretrained(snake_case__      )
																		bert_tokenizer.save_pretrained(snake_case__      )
																		lowercase__    :				List[Any]=	CustomTokenizerFast.from_pretrained(snake_case__      )

												tokenizer.push_to_hub("test-dynamic-tokenizer"	,  use_auth_token=self._token      )

												lowercase__    :				List[str]=	AutoTokenizer.from_pretrained(F'''{USER}/test-dynamic-tokenizer'''	,  trust_remote_code=snake_case__      )
												# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
												self.assertEqual(tokenizer.__class__.__name__	,  "CustomTokenizerFast"      )
												lowercase__    :				Dict=	AutoTokenizer.from_pretrained(
												    F'''{USER}/test-dynamic-tokenizer'''	,  use_fast=snake_case__	,  trust_remote_code=snake_case__      )
												# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
												self.assertEqual(tokenizer.__class__.__name__	,  "CustomTokenizer"      )


class    __UpperCAmelCase(       unittest.TestCase							):




						"""simple docstring"""




						def 		UpperCAmelCase_	(       self      ):
												'''simple docstring'''
												lowercase__    :				Any=	Trie()
												trie.add("Hello 友達"      )
												self.assertEqual(trie.data	,  {"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}      )
												trie.add("Hello"      )
												trie.data
												self.assertEqual(trie.data	,  {"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}      )




						def 		UpperCAmelCase_	(       self      ):
												'''simple docstring'''
												lowercase__    :				List[str]=	Trie()
												self.assertEqual(trie.split("[CLS] This is a extra_id_100"      )	,  ["[CLS] This is a extra_id_100"]      )
												trie.add("[CLS]"      )
												trie.add("extra_id_1"      )
												trie.add("extra_id_100"      )
												self.assertEqual(trie.split("[CLS] This is a extra_id_100"      )	,  ["[CLS]", " This is a ", "extra_id_100"]      )




						def 		UpperCAmelCase_	(       self      ):
												'''simple docstring'''
												lowercase__    :				Tuple=	Trie()
												trie.add("A"      )
												self.assertEqual(trie.split("ABC"      )	,  ["A", "BC"]      )
												self.assertEqual(trie.split("BCA"      )	,  ["BC", "A"]      )




						def 		UpperCAmelCase_	(       self      ):
												'''simple docstring'''
												lowercase__    :				Optional[int]=	Trie()
												trie.add("TOKEN]"      )
												trie.add("[SPECIAL_TOKEN]"      )
												self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"      )	,  ["This is something ", "[SPECIAL_TOKEN]"]      )




						def 		UpperCAmelCase_	(       self      ):
												'''simple docstring'''
												lowercase__    :				Dict=	Trie()
												trie.add("A"      )
												trie.add("P"      )
												trie.add("[SPECIAL_TOKEN]"      )
												self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"      )	,  ["This is something ", "[SPECIAL_TOKEN]"]      )




						def 		UpperCAmelCase_	(       self      ):
												'''simple docstring'''
												lowercase__    :				str=	Trie()
												trie.add("AB"      )
												trie.add("B"      )
												trie.add("C"      )
												self.assertEqual(trie.split("ABC"      )	,  ["AB", "C"]      )




						def 		UpperCAmelCase_	(       self      ):
												'''simple docstring'''
												lowercase__    :				List[Any]=	Trie()
												trie.add("ABC"      )
												trie.add("B"      )
												trie.add("CD"      )
												self.assertEqual(trie.split("ABCD"      )	,  ["ABC", "D"]      )




						def 		UpperCAmelCase_	(       self      ):
												'''simple docstring'''
												# Even if the offsets are wrong, we necessarily output correct string
												# parts.
												lowercase__    :				List[Any]=	Trie()
												lowercase__    :				List[Any]=	trie.cut_text("ABC"	,  [0, 0, 2, 1, 2, 3]      )
												self.assertEqual(snake_case__	,  ["AB", "C"]      )














"""simple docstring"""







import argparse
from collections import OrderedDict
from pathlib import Path

import torch

from transformers import (
    VisualBertConfig,
    VisualBertForMultipleChoice,
    VisualBertForPreTraining,
    VisualBertForQuestionAnswering,
    VisualBertForVisualReasoning,
)
from transformers.utils import logging


logging.set_verbosity_info()
a			:    List[str]				 =		logging.get_logger(__name__)

a			:    List[Any]				 =		[
    ("""bert.bert""", """visual_bert"""),
    ("""bert.cls""", """cls"""),
    ("""bert.classifier""", """cls"""),
    ("""token_type_embeddings_visual""", """visual_token_type_embeddings"""),
    ("""position_embeddings_visual""", """visual_position_embeddings"""),
    ("""projection""", """visual_projection"""),
]

a			:    Dict				 =		[
    """nlvr2_coco_pre_trained.th""",
    """nlvr2_fine_tuned.th""",
    """nlvr2_pre_trained.th""",
    """vcr_coco_pre_train.th""",
    """vcr_fine_tune.th""",
    """vcr_pre_train.th""",
    """vqa_coco_pre_trained.th""",
    """vqa_fine_tuned.th""",
    """vqa_pre_trained.th""",
]





def 						lowercase__(A						)    ->Optional[int]:




						"""simple docstring"""



						lowercase__    :				Any=	torch.load(A ,		map_location="cpu"						)
						return sd





def 						lowercase__(A ,		A ,		A=rename_keys_prefix						)    ->List[str]:




						"""simple docstring"""



						lowercase__    :				int=	OrderedDict()
						lowercase__    :				Optional[Any]=	torch.arange(config.max_position_embeddings						).expand((1, -1)						)
						# detector_d = OrderedDict()
						for key in d:
												if "detector" in key:
																		# detector_d[key.replace('detector.','')] = d[key]
																		continue
												lowercase__    :				Union[str, Any]=	key
												for name_pair in rename_keys_prefix:
																		lowercase__    :				str=	new_key.replace(name_pair[0] ,		name_pair[1]						)
												lowercase__    :				Union[str, Any]=	d[key]
												if key == "bert.cls.predictions.decoder.weight":
																		# Old bert code didn't have `decoder.bias`, but was added separately
																		lowercase__    :				Optional[int]=	new_d["cls.predictions.bias"]
						return new_d





@torch.no_grad()
def 						lowercase__(A ,		A						)    ->str:




						"""simple docstring"""



						assert (
						    checkpoint_path.split("/"						)[-1] in ACCEPTABLE_CHECKPOINTS
						), f'''The checkpoint provided must be in {ACCEPTABLE_CHECKPOINTS}.'''

						# Get Config
						if "pre" in checkpoint_path:
												lowercase__    :				Union[str, Any]=	"pretraining"
												if "vcr" in checkpoint_path:
																		lowercase__    :				str=	{"visual_embedding_dim": 512}
												elif "vqa_advanced" in checkpoint_path:
																		lowercase__    :				Optional[Any]=	{"visual_embedding_dim": 2_048}
												elif "vqa" in checkpoint_path:
																		lowercase__    :				int=	{"visual_embedding_dim": 2_048}
												elif "nlvr" in checkpoint_path:
																		lowercase__    :				Tuple=	{"visual_embedding_dim": 1_024}
												else:
																		raise NotImplementedError(f'''No implementation found for `{checkpoint_path}`.'''						)
						else:
												if "vcr" in checkpoint_path:
																		lowercase__    :				int=	{"visual_embedding_dim": 512}
																		lowercase__    :				int=	"multichoice"
												elif "vqa_advanced" in checkpoint_path:
																		lowercase__    :				Dict=	{"visual_embedding_dim": 2_048}
																		lowercase__    :				Optional[Any]=	"vqa_advanced"
												elif "vqa" in checkpoint_path:
																		lowercase__    :				Optional[int]=	{"visual_embedding_dim": 2_048, "num_labels": 3_129}
																		lowercase__    :				List[str]=	"vqa"
												elif "nlvr" in checkpoint_path:
																		lowercase__    :				Dict=	{
																		    "visual_embedding_dim": 1_024,
																		    "num_labels": 2,
																		}
																		lowercase__    :				Any=	"nlvr"

						lowercase__    :				List[Any]=	VisualBertConfig(**A						)

						# Load State Dict
						lowercase__    :				Union[str, Any]=	load_state_dict(A						)

						lowercase__    :				List[str]=	get_new_dict(A ,		A						)

						if model_type == "pretraining":
												lowercase__    :				Optional[Any]=	VisualBertForPreTraining(A						)
						elif model_type == "vqa":
												lowercase__    :				Any=	VisualBertForQuestionAnswering(A						)
						elif model_type == "nlvr":
												lowercase__    :				Union[str, Any]=	VisualBertForVisualReasoning(A						)
						elif model_type == "multichoice":
												lowercase__    :				str=	VisualBertForMultipleChoice(A						)

						model.load_state_dict(A						)
						# Save Checkpoints
						Path(A						).mkdir(exist_ok=A						)
						model.save_pretrained(A						)


if __name__ == "__main__":
			a			:    int				 =		argparse.ArgumentParser()
			# Required parameters
			parser.add_argument("""orig_checkpoint_path""", type=str, help="""A path to .th on local filesystem.""")
			parser.add_argument("""pytorch_dump_folder_path""", type=str, help="""Path to the output PyTorch model.""")
			a			:    Dict				 =		parser.parse_args()
			convert_visual_bert_checkpoint(args.orig_checkpoint_path, args.pytorch_dump_folder_path)






