



import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
lowerCAmelCase__		:					Tuple							=data_utils.TransfoXLTokenizer
lowerCAmelCase__		:					Optional[int]							=data_utils.TransfoXLCorpus
lowerCAmelCase__		:					int							=data_utils
lowerCAmelCase__		:					Optional[int]							=data_utils






def      __lowercase       (			a__							,       a__							,       a__							,       a__       )    ->			Optional[int]:
							if transfo_xl_dataset_file:
														# Convert a pre-processed corpus (see original TensorFlow repo)
														with open(snake_case_							,       'rb'       ) as fp:
																					__SCREAMING_SNAKE_CASE									= pickle.load(snake_case_							,       encoding='latin1'       )
														# Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
														__SCREAMING_SNAKE_CASE									= pytorch_dump_folder_path + '/' + VOCAB_FILES_NAMES['pretrained_vocab_file']
														print(f"""Save vocabulary to {pytorch_vocab_dump_path}"""       )
														__SCREAMING_SNAKE_CASE									= corpus.vocab.__dict__
														torch.save(snake_case_							,       snake_case_       )

														__SCREAMING_SNAKE_CASE									= corpus.__dict__
														corpus_dict_no_vocab.pop('vocab'							,       snake_case_       )
														__SCREAMING_SNAKE_CASE									= pytorch_dump_folder_path + '/' + CORPUS_NAME
														print(f"""Save dataset to {pytorch_dataset_dump_path}"""       )
														torch.save(snake_case_							,       snake_case_       )

							if tf_checkpoint_path:
														# Convert a pre-trained TensorFlow model
														__SCREAMING_SNAKE_CASE									= os.path.abspath(snake_case_       )
														__SCREAMING_SNAKE_CASE									= os.path.abspath(snake_case_       )

														print(f"""Converting Transformer XL checkpoint from {tf_path} with config at {config_path}."""       )
														# Initialise PyTorch model
														if transfo_xl_config_file == "":
																					__SCREAMING_SNAKE_CASE									= TransfoXLConfig()
														else:
																					__SCREAMING_SNAKE_CASE									= TransfoXLConfig.from_json_file(snake_case_       )
														print(f"""Building PyTorch model from configuration: {config}"""       )
														__SCREAMING_SNAKE_CASE									= TransfoXLLMHeadModel(snake_case_       )

														__SCREAMING_SNAKE_CASE									= load_tf_weights_in_transfo_xl(snake_case_							,       snake_case_							,       snake_case_       )
														# Save pytorch-model
														__SCREAMING_SNAKE_CASE									= os.path.join(snake_case_							,       snake_case_       )
														__SCREAMING_SNAKE_CASE									= os.path.join(snake_case_							,       snake_case_       )
														print(f"""Save PyTorch model to {os.path.abspath(snake_case_       )}"""       )
														torch.save(model.state_dict()							,       snake_case_       )
														print(f"""Save configuration file to {os.path.abspath(snake_case_       )}"""       )
														with open(snake_case_							,       'w'							,       encoding='utf-8'       ) as f:
																					f.write(config.to_json_string()       )


if __name__ == "__main__":
			lowerCAmelCase__		:					Optional[Any]							=argparse.ArgumentParser()
			parser.add_argument(
			    '''--pytorch_dump_folder_path''',
			    default=None,
			    type=str,
			    required=True,
			    help='''Path to the folder to store the PyTorch model or dataset/vocab.''',
			)
			parser.add_argument(
			    '''--tf_checkpoint_path''',
			    default='''''',
			    type=str,
			    help='''An optional path to a TensorFlow checkpoint path to be converted.''',
			)
			parser.add_argument(
			    '''--transfo_xl_config_file''',
			    default='''''',
			    type=str,
			    help=(
			        '''An optional config json file corresponding to the pre-trained BERT model. \n'''
			        '''This specifies the model architecture.'''
			    ),
			)
			parser.add_argument(
			    '''--transfo_xl_dataset_file''',
			    default='''''',
			    type=str,
			    help='''An optional dataset file to be converted in a vocabulary.''',
			)
			lowerCAmelCase__		:					int							=parser.parse_args()
			convert_transfo_xl_checkpoint_to_pytorch(
			    args.tf_checkpoint_path,
			    args.transfo_xl_config_file,
			    args.pytorch_dump_folder_path,
			    args.transfo_xl_dataset_file,
			)











import pytest

from datasets.parallel import ParallelBackendConfig, parallel_backend
from datasets.utils.py_utils import map_nested

from .utils import require_dill_gt_0_3_2, require_joblibspark, require_not_windows






def      __lowercase       (			a__       )    ->			List[Any]:  # picklable for multiprocessing
							return i + 1






@require_dill_gt_0_3_2
@require_joblibspark
@require_not_windows
def      __lowercase       (			)    ->			Any:
							with parallel_backend('spark'       ):
														assert ParallelBackendConfig.backend_name == "spark"

							__SCREAMING_SNAKE_CASE									= [1, 2, 3]
							with pytest.raises(a__       ):
														with parallel_backend('unsupported backend'       ):
																					map_nested(a__							,       a__							,       num_proc=2       )

							with pytest.raises(a__       ):
														with parallel_backend('unsupported backend'       ):
																					map_nested(a__							,       a__							,       num_proc=-1       )






@require_dill_gt_0_3_2
@require_joblibspark
@require_not_windows
@pytest.mark.parametrize('num_proc'							,       [2, -1]       )
def      __lowercase       (			a__       )    ->			Union[str, Any]:
							__SCREAMING_SNAKE_CASE									= [1, 2]
							__SCREAMING_SNAKE_CASE									= {'a': 1, 'b': 2}
							__SCREAMING_SNAKE_CASE									= {'a': [1, 2], 'b': [3, 4]}
							__SCREAMING_SNAKE_CASE									= {'a': {'1': 1}, 'b': 2}
							__SCREAMING_SNAKE_CASE									= {'a': 1, 'b': 2, 'c': 3, 'd': 4}
							__SCREAMING_SNAKE_CASE									= [2, 3]
							__SCREAMING_SNAKE_CASE									= {'a': 2, 'b': 3}
							__SCREAMING_SNAKE_CASE									= {'a': [2, 3], 'b': [4, 5]}
							__SCREAMING_SNAKE_CASE									= {'a': {'1': 2}, 'b': 3}
							__SCREAMING_SNAKE_CASE									= {'a': 2, 'b': 3, 'c': 4, 'd': 5}

							with parallel_backend('spark'       ):
														assert map_nested(a__							,       a__							,       num_proc=a__       ) == expected_map_nested_sa
														assert map_nested(a__							,       a__							,       num_proc=a__       ) == expected_map_nested_sa
														assert map_nested(a__							,       a__							,       num_proc=a__       ) == expected_map_nested_sa
														assert map_nested(a__							,       a__							,       num_proc=a__       ) == expected_map_nested_sa
														assert map_nested(a__							,       a__							,       num_proc=a__       ) == expected_map_nested_sa






