




import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
lowercase__          =data_utils.TransfoXLTokenizer
lowercase__          =data_utils.TransfoXLCorpus
lowercase__          =data_utils
lowercase__          =data_utils




def        __UpperCamelCase					(    lowerCAmelCase__							:    Optional[Any]  ,       lowerCAmelCase__							:    List[str]  ,       lowerCAmelCase__							:    Dict  ,       lowerCAmelCase__							:    Tuple  ):
 if transfo_xl_dataset_file:
  # Convert a pre-processed corpus (see original TensorFlow repo)
  with open(lowerCAmelCase__  ,       '''rb'''  ) as fp:
   __a			:  Dict		    =  pickle.load(lowerCAmelCase__  ,       encoding='''latin1'''  )
  # Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
  __a			:  Optional[int]		    =  pytorch_dump_folder_path + '''/''' + VOCAB_FILES_NAMES['''pretrained_vocab_file''']
  print(f"Save vocabulary to {pytorch_vocab_dump_path}"  )
  __a			:  Any		    =  corpus.vocab.__dict__
  torch.save(lowerCAmelCase__  ,       lowerCAmelCase__  )

  __a			:  Dict		    =  corpus.__dict__
  corpus_dict_no_vocab.pop('''vocab'''  ,       lowerCAmelCase__  )
  __a			:  Tuple		    =  pytorch_dump_folder_path + '''/''' + CORPUS_NAME
  print(f"Save dataset to {pytorch_dataset_dump_path}"  )
  torch.save(lowerCAmelCase__  ,       lowerCAmelCase__  )

 if tf_checkpoint_path:
  # Convert a pre-trained TensorFlow model
  __a			:  Optional[Any]		    =  os.path.abspath(lowerCAmelCase__  )
  __a			:  Optional[Any]		    =  os.path.abspath(lowerCAmelCase__  )

  print(f"Converting Transformer XL checkpoint from {tf_path} with config at {config_path}."  )
  # Initialise PyTorch model
  if transfo_xl_config_file == "":
   __a			:  Any		    =  TransfoXLConfig()
  else:
   __a			:  Dict		    =  TransfoXLConfig.from_json_file(lowerCAmelCase__  )
  print(f"Building PyTorch model from configuration: {config}"  )
  __a			:  int		    =  TransfoXLLMHeadModel(lowerCAmelCase__  )

  __a			:  str		    =  load_tf_weights_in_transfo_xl(lowerCAmelCase__  ,       lowerCAmelCase__  ,       lowerCAmelCase__  )
  # Save pytorch-model
  __a			:  List[Any]		    =  os.path.join(lowerCAmelCase__  ,       lowerCAmelCase__  )
  __a			:  List[Any]		    =  os.path.join(lowerCAmelCase__  ,       lowerCAmelCase__  )
  print(f"Save PyTorch model to {os.path.abspath(lowerCAmelCase__  )}"  )
  torch.save(model.state_dict()  ,       lowerCAmelCase__  )
  print(f"Save configuration file to {os.path.abspath(lowerCAmelCase__  )}"  )
  with open(lowerCAmelCase__  ,       '''w'''  ,       encoding='''utf-8'''  ) as f:
   f.write(config.to_json_string()  )


if __name__ == "__main__":
      lowercase__          =argparse.ArgumentParser()
      parser.add_argument(
          '--pytorch_dump_folder_path',
          default=None,
          type=str,
          required=True,
          help='Path to the folder to store the PyTorch model or dataset/vocab.',
      )
      parser.add_argument(
          '--tf_checkpoint_path',
          default='',
          type=str,
          help='An optional path to a TensorFlow checkpoint path to be converted.',
      )
      parser.add_argument(
          '--transfo_xl_config_file',
          default='',
          type=str,
          help=(
              'An optional config json file corresponding to the pre-trained BERT model. \n'
              'This specifies the model architecture.'
          ),
      )
      parser.add_argument(
          '--transfo_xl_dataset_file',
          default='',
          type=str,
          help='An optional dataset file to be converted in a vocabulary.',
      )
      lowercase__          =parser.parse_args()
      convert_transfo_xl_checkpoint_to_pytorch(
          args.tf_checkpoint_path,
          args.transfo_xl_config_file,
          args.pytorch_dump_folder_path,
          args.transfo_xl_dataset_file,
      )


import os
import sys
import unittest


lowercase__          =os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, 'utils'))

import get_test_info  # noqa: E402
from get_test_info import (  # noqa: E402
    get_model_to_test_mapping,
    get_model_to_tester_mapping,
    get_test_to_tester_mapping,
)


lowercase__          =os.path.join('tests', 'models', 'bert', 'test_modeling_bert.py')
lowercase__          =os.path.join('tests', 'models', 'blip', 'test_modeling_blip.py')



class 		UpperCamelCase__  (	unittest.TestCase   ):
   def 				lowerCAmelCase      (self			: List[Any]       ):
    __a			:  str		    =  get_test_to_tester_mapping(snake_case_       )
    __a			:  Tuple		    =  get_test_to_tester_mapping(snake_case_       )

    __a			:  Union[str, Any]		    =  {'''BertModelTest''': '''BertModelTester'''}

    __a			:  Tuple		    =  {
        '''BlipModelTest''': '''BlipModelTester''',
        '''BlipTextImageModelTest''': '''BlipTextImageModelsModelTester''',
        '''BlipTextModelTest''': '''BlipTextModelTester''',
        '''BlipTextRetrievalModelTest''': '''BlipTextRetrievalModelTester''',
        '''BlipVQAModelTest''': '''BlipVQAModelTester''',
        '''BlipVisionModelTest''': '''BlipVisionModelTester''',
    }

    self.assertEqual(get_test_info.to_json(snake_case_       ) ,    snake_case_       )
    self.assertEqual(get_test_info.to_json(snake_case_       ) ,    snake_case_       )
   def 				lowerCAmelCase      (self			: str       ):
    __a			:  Optional[int]		    =  get_model_to_test_mapping(snake_case_       )
    __a			:  Any		    =  get_model_to_test_mapping(snake_case_       )

    __a			:  List[Any]		    =  {
        '''BertForMaskedLM''': ['''BertModelTest'''],
        '''BertForMultipleChoice''': ['''BertModelTest'''],
        '''BertForNextSentencePrediction''': ['''BertModelTest'''],
        '''BertForPreTraining''': ['''BertModelTest'''],
        '''BertForQuestionAnswering''': ['''BertModelTest'''],
        '''BertForSequenceClassification''': ['''BertModelTest'''],
        '''BertForTokenClassification''': ['''BertModelTest'''],
        '''BertLMHeadModel''': ['''BertModelTest'''],
        '''BertModel''': ['''BertModelTest'''],
    }

    __a			:  Dict		    =  {
        '''BlipForConditionalGeneration''': ['''BlipTextImageModelTest'''],
        '''BlipForImageTextRetrieval''': ['''BlipTextRetrievalModelTest'''],
        '''BlipForQuestionAnswering''': ['''BlipVQAModelTest'''],
        '''BlipModel''': ['''BlipModelTest'''],
        '''BlipTextModel''': ['''BlipTextModelTest'''],
        '''BlipVisionModel''': ['''BlipVisionModelTest'''],
    }

    self.assertEqual(get_test_info.to_json(snake_case_       ) ,    snake_case_       )
    self.assertEqual(get_test_info.to_json(snake_case_       ) ,    snake_case_       )







   def 				lowerCAmelCase      (self			: int       ):
    __a			:  Any		    =  get_model_to_tester_mapping(snake_case_       )
    __a			:  List[str]		    =  get_model_to_tester_mapping(snake_case_       )

    __a			:  Any		    =  {
        '''BertForMaskedLM''': ['''BertModelTester'''],
        '''BertForMultipleChoice''': ['''BertModelTester'''],
        '''BertForNextSentencePrediction''': ['''BertModelTester'''],
        '''BertForPreTraining''': ['''BertModelTester'''],
        '''BertForQuestionAnswering''': ['''BertModelTester'''],
        '''BertForSequenceClassification''': ['''BertModelTester'''],
        '''BertForTokenClassification''': ['''BertModelTester'''],
        '''BertLMHeadModel''': ['''BertModelTester'''],
        '''BertModel''': ['''BertModelTester'''],
    }

    __a			:  int		    =  {
        '''BlipForConditionalGeneration''': ['''BlipTextImageModelsModelTester'''],
        '''BlipForImageTextRetrieval''': ['''BlipTextRetrievalModelTester'''],
        '''BlipForQuestionAnswering''': ['''BlipVQAModelTester'''],
        '''BlipModel''': ['''BlipModelTester'''],
        '''BlipTextModel''': ['''BlipTextModelTester'''],
        '''BlipVisionModel''': ['''BlipVisionModelTester'''],
    }

    self.assertEqual(get_test_info.to_json(snake_case_       ) ,    snake_case_       )
    self.assertEqual(get_test_info.to_json(snake_case_       ) ,    snake_case_       )
