


import os
import unittest
from tempfile import TemporaryDirectory

import torch
import torch.nn as nn

from accelerate.utils import (
    OffloadedWeightsLoader,
    extract_submodules_state_dict,
    load_offloaded_weight,
    offload_state_dict,
    offload_weight,
)


class 	_a							(      nn.Module      ):

							def __init__(							self:     Optional[int]							)      ->       str:





									"""simple docstring"""







									super().__init__()
									lowercase__						=				nn.Linear(3   ,						4							)
									lowercase__						=				nn.BatchNormad(4							)
									lowercase__						=				nn.Linear(4   ,						5							)


							def 				lowerCamelCase_			(							self:     Tuple   ,						UpperCamelCase_:     List[str]							)      ->       str:





									"""simple docstring"""







									return self.lineara(self.batchnorm(self.lineara(UpperCamelCase_							)							)							)


class 	_a							(      unittest.TestCase      ):

							def 				lowerCamelCase_			(							self:     str							)      ->       int:





									"""simple docstring"""







									lowercase__						=				ModelForTest()
									with TemporaryDirectory() as tmp_dir:
											offload_state_dict(UpperCamelCase_   ,						model.state_dict()							)
											lowercase__						=				os.path.join(UpperCamelCase_   ,						'''index.json'''							)
											self.assertTrue(os.path.isfile(UpperCamelCase_							)							)
											# TODO: add tests on what is inside the index

											for key in ["linear1.weight", "linear1.bias", "linear2.weight", "linear2.bias"]:
													lowercase__						=				os.path.join(UpperCamelCase_   ,						f'{key}.dat'							)
													self.assertTrue(os.path.isfile(UpperCamelCase_							)							)
													# TODO: add tests on the fact weights are properly loaded

							def 				lowerCamelCase_			(							self:     Any							)      ->       List[str]:





									"""simple docstring"""







									lowercase__						=				[torch.floataa, torch.floataa, torch.bfloataa]

									for dtype in dtypes:
											lowercase__						=				torch.randn(2   ,						3   ,						dtype=UpperCamelCase_							)
											with TemporaryDirectory() as tmp_dir:
													lowercase__						=				offload_weight(UpperCamelCase_   ,						'''weight'''   ,						UpperCamelCase_   ,						{}							)
													lowercase__						=				os.path.join(UpperCamelCase_   ,						'''weight.dat'''							)
													self.assertTrue(os.path.isfile(UpperCamelCase_							)							)
													self.assertDictEqual(UpperCamelCase_   ,						{'''weight''': {'''shape''': [2, 3], '''dtype''': str(UpperCamelCase_							).split('''.'''							)[1]}}							)

													lowercase__						=				load_offloaded_weight(UpperCamelCase_   ,						index['''weight''']							)
													self.assertTrue(torch.equal(UpperCamelCase_   ,						UpperCamelCase_							)							)

							def 				lowerCamelCase_			(							self:     int							)      ->       Any:





									"""simple docstring"""







									lowercase__						=				ModelForTest()
									lowercase__						=				model.state_dict()
									lowercase__						=				{k: v for k, v in state_dict.items() if '''linear2''' not in k}
									lowercase__						=				{k: v for k, v in state_dict.items() if '''linear2''' in k}

									with TemporaryDirectory() as tmp_dir:
											offload_state_dict(UpperCamelCase_   ,						UpperCamelCase_							)
											lowercase__						=				OffloadedWeightsLoader(state_dict=UpperCamelCase_   ,						save_folder=UpperCamelCase_							)

											# Every key is there with the right value
											self.assertEqual(sorted(UpperCamelCase_							)   ,						sorted(state_dict.keys()							)							)
											for key, param in state_dict.items():
													self.assertTrue(torch.allclose(UpperCamelCase_   ,						weight_map[key]							)							)

									lowercase__						=				{k: v for k, v in state_dict.items() if '''weight''' in k}
									lowercase__						=				{k: v for k, v in state_dict.items() if '''weight''' not in k}

									with TemporaryDirectory() as tmp_dir:
											offload_state_dict(UpperCamelCase_   ,						UpperCamelCase_							)
											lowercase__						=				OffloadedWeightsLoader(state_dict=UpperCamelCase_   ,						save_folder=UpperCamelCase_							)

											# Every key is there with the right value
											self.assertEqual(sorted(UpperCamelCase_							)   ,						sorted(state_dict.keys()							)							)
											for key, param in state_dict.items():
													self.assertTrue(torch.allclose(UpperCamelCase_   ,						weight_map[key]							)							)

									with TemporaryDirectory() as tmp_dir:
											offload_state_dict(UpperCamelCase_   ,						UpperCamelCase_							)
											# Duplicates are removed
											lowercase__						=				OffloadedWeightsLoader(state_dict=UpperCamelCase_   ,						save_folder=UpperCamelCase_							)

											# Every key is there with the right value
											self.assertEqual(sorted(UpperCamelCase_							)   ,						sorted(state_dict.keys()							)							)
											for key, param in state_dict.items():
													self.assertTrue(torch.allclose(UpperCamelCase_   ,						weight_map[key]							)							)


							def 				lowerCamelCase_			(							self:     Dict							)      ->       Union[str, Any]:





									"""simple docstring"""







									lowercase__						=				{'''a.1''': 0, '''a.10''': 1, '''a.2''': 2}
									lowercase__						=				extract_submodules_state_dict(UpperCamelCase_   ,						['''a.1''', '''a.2''']							)
									self.assertDictEqual(UpperCamelCase_   ,						{'''a.1''': 0, '''a.2''': 2}							)

									lowercase__						=				{'''a.1.a''': 0, '''a.10.a''': 1, '''a.2.a''': 2}
									lowercase__						=				extract_submodules_state_dict(UpperCamelCase_   ,						['''a.1''', '''a.2''']							)
									self.assertDictEqual(UpperCamelCase_   ,						{'''a.1.a''': 0, '''a.2.a''': 2}							)






from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase	=							logging.get_logger(__name__)

lowerCAmelCase	=							{
    'unc-nlp/lxmert-base-uncased': 'https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/config.json',
}


class 	_a							(      UpperCamelCase__      ):
							_lowercase     :      List[Any]  				=			'''lxmert'''
							_lowercase     :      Any  				=			{}

							def __init__(							self:     Any   ,						UpperCamelCase_:     List[Any]=30_522   ,						UpperCamelCase_:     int=768   ,						UpperCamelCase_:     Optional[Any]=12   ,						UpperCamelCase_:     Dict=9_500   ,						UpperCamelCase_:     List[Any]=1_600   ,						UpperCamelCase_:     List[Any]=400   ,						UpperCamelCase_:     List[str]=3_072   ,						UpperCamelCase_:     int="gelu"   ,						UpperCamelCase_:     Any=0.1   ,						UpperCamelCase_:     int=0.1   ,						UpperCamelCase_:     List[str]=512   ,						UpperCamelCase_:     Optional[int]=2   ,						UpperCamelCase_:     Union[str, Any]=0.02   ,						UpperCamelCase_:     Dict=1E-1_2   ,						UpperCamelCase_:     List[Any]=9   ,						UpperCamelCase_:     List[Any]=5   ,						UpperCamelCase_:     Optional[Any]=5   ,						UpperCamelCase_:     str=2_048   ,						UpperCamelCase_:     Dict=4   ,						UpperCamelCase_:     Any=6.67   ,						UpperCamelCase_:     Dict=True   ,						UpperCamelCase_:     List[str]=True   ,						UpperCamelCase_:     Dict=True   ,						UpperCamelCase_:     Any=True   ,						UpperCamelCase_:     Tuple=True   ,						UpperCamelCase_:     Optional[Any]=True   ,						UpperCamelCase_:     Any=True   ,						**UpperCamelCase_:     Union[str, Any]   ,						)      ->       Union[str, Any]:





									"""simple docstring"""







									lowercase__						=				vocab_size
									lowercase__						=				hidden_size
									lowercase__						=				num_attention_heads
									lowercase__						=				hidden_act
									lowercase__						=				intermediate_size
									lowercase__						=				hidden_dropout_prob
									lowercase__						=				attention_probs_dropout_prob
									lowercase__						=				max_position_embeddings
									lowercase__						=				type_vocab_size
									lowercase__						=				initializer_range
									lowercase__						=				layer_norm_eps
									lowercase__						=				num_qa_labels
									lowercase__						=				num_object_labels
									lowercase__						=				num_attr_labels
									lowercase__						=				l_layers
									lowercase__						=				x_layers
									lowercase__						=				r_layers
									lowercase__						=				visual_feat_dim
									lowercase__						=				visual_pos_dim
									lowercase__						=				visual_loss_normalizer
									lowercase__						=				task_matched
									lowercase__						=				task_mask_lm
									lowercase__						=				task_obj_predict
									lowercase__						=				task_qa
									lowercase__						=				visual_obj_loss
									lowercase__						=				visual_attr_loss
									lowercase__						=				visual_feat_loss
									lowercase__						=				{'''vision''': r_layers, '''cross_encoder''': x_layers, '''language''': l_layers}
									super().__init__(**UpperCamelCase_							)


