
"""simple docstring"""





import os
import unittest
from tempfile import TemporaryDirectory

import torch
import torch.nn as nn

from accelerate.utils import (
    OffloadedWeightsLoader,
    extract_submodules_state_dict,
    load_offloaded_weight,
    offload_state_dict,
    offload_weight,
)






class    _UpperCAmelCase(   nn.Module     ):



		def __init__( self)	->	str:






							'''simple docstring'''
							super().__init__()
							_UpperCamelCase										=  nn.Linear(3    ,		4)
							_UpperCamelCase										=  nn.BatchNormad(4)
							_UpperCamelCase										=  nn.Linear(4    ,		5)




		def 						UpperCAmelCase							( self    ,		__a)	->	int:






							'''simple docstring'''
							return self.lineara(self.batchnorm(self.lineara(__a)))






class    _UpperCAmelCase(   unittest.TestCase     ):



		def 						UpperCAmelCase							( self)	->	Union[str, Any]:






							'''simple docstring'''
							_UpperCamelCase										=  ModelForTest()
							with TemporaryDirectory() as tmp_dir:
												offload_state_dict(__a    ,		model.state_dict())
												_UpperCamelCase										=  os.path.join(__a    ,		'''index.json''')
												self.assertTrue(os.path.isfile(__a))
												# TODO: add tests on what is inside the index

												for key in ["linear1.weight", "linear1.bias", "linear2.weight", "linear2.bias"]:
																	_UpperCamelCase										=  os.path.join(__a    ,		F'''{key}.dat''')
																	self.assertTrue(os.path.isfile(__a))
																	# TODO: add tests on the fact weights are properly loaded



		def 						UpperCAmelCase							( self)	->	Optional[int]:






							'''simple docstring'''
							_UpperCamelCase										=  [torch.floataa, torch.floataa, torch.bfloataa]

							for dtype in dtypes:
												_UpperCamelCase										=  torch.randn(2    ,		3    ,		dtype=__a)
												with TemporaryDirectory() as tmp_dir:
																	_UpperCamelCase										=  offload_weight(__a    ,		'''weight'''    ,		__a    ,		{})
																	_UpperCamelCase										=  os.path.join(__a    ,		'''weight.dat''')
																	self.assertTrue(os.path.isfile(__a))
																	self.assertDictEqual(__a    ,		{'''weight''': {'''shape''': [2, 3], '''dtype''': str(__a).split('''.''')[1]}})

																	_UpperCamelCase										=  load_offloaded_weight(__a    ,		index['''weight'''])
																	self.assertTrue(torch.equal(__a    ,		__a))



		def 						UpperCAmelCase							( self)	->	Optional[int]:






							'''simple docstring'''
							_UpperCamelCase										=  ModelForTest()
							_UpperCamelCase										=  model.state_dict()
							_UpperCamelCase										=  {k: v for k, v in state_dict.items() if '''linear2''' not in k}
							_UpperCamelCase										=  {k: v for k, v in state_dict.items() if '''linear2''' in k}

							with TemporaryDirectory() as tmp_dir:
												offload_state_dict(__a    ,		__a)
												_UpperCamelCase										=  OffloadedWeightsLoader(state_dict=__a    ,		save_folder=__a)

												# Every key is there with the right value
												self.assertEqual(sorted(__a)    ,		sorted(state_dict.keys()))
												for key, param in state_dict.items():
																	self.assertTrue(torch.allclose(__a    ,		weight_map[key]))

							_UpperCamelCase										=  {k: v for k, v in state_dict.items() if '''weight''' in k}
							_UpperCamelCase										=  {k: v for k, v in state_dict.items() if '''weight''' not in k}

							with TemporaryDirectory() as tmp_dir:
												offload_state_dict(__a    ,		__a)
												_UpperCamelCase										=  OffloadedWeightsLoader(state_dict=__a    ,		save_folder=__a)

												# Every key is there with the right value
												self.assertEqual(sorted(__a)    ,		sorted(state_dict.keys()))
												for key, param in state_dict.items():
																	self.assertTrue(torch.allclose(__a    ,		weight_map[key]))

							with TemporaryDirectory() as tmp_dir:
												offload_state_dict(__a    ,		__a)
												# Duplicates are removed
												_UpperCamelCase										=  OffloadedWeightsLoader(state_dict=__a    ,		save_folder=__a)

												# Every key is there with the right value
												self.assertEqual(sorted(__a)    ,		sorted(state_dict.keys()))
												for key, param in state_dict.items():
																	self.assertTrue(torch.allclose(__a    ,		weight_map[key]))




		def 						UpperCAmelCase							( self)	->	Union[str, Any]:






							'''simple docstring'''
							_UpperCamelCase										=  {'''a.1''': 0, '''a.10''': 1, '''a.2''': 2}
							_UpperCamelCase										=  extract_submodules_state_dict(__a    ,		['''a.1''', '''a.2'''])
							self.assertDictEqual(__a    ,		{'''a.1''': 0, '''a.2''': 2})

							_UpperCamelCase										=  {'''a.1.a''': 0, '''a.10.a''': 1, '''a.2.a''': 2}
							_UpperCamelCase										=  extract_submodules_state_dict(__a    ,		['''a.1''', '''a.2'''])
							self.assertDictEqual(__a    ,		{'''a.1.a''': 0, '''a.2.a''': 2})








"""simple docstring"""





from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


_a														=		logging.get_logger(__name__)

_a														=		{
    """bert-base-uncased""": """https://huggingface.co/bert-base-uncased/resolve/main/config.json""",
    """bert-large-uncased""": """https://huggingface.co/bert-large-uncased/resolve/main/config.json""",
    """bert-base-cased""": """https://huggingface.co/bert-base-cased/resolve/main/config.json""",
    """bert-large-cased""": """https://huggingface.co/bert-large-cased/resolve/main/config.json""",
    """bert-base-multilingual-uncased""": """https://huggingface.co/bert-base-multilingual-uncased/resolve/main/config.json""",
    """bert-base-multilingual-cased""": """https://huggingface.co/bert-base-multilingual-cased/resolve/main/config.json""",
    """bert-base-chinese""": """https://huggingface.co/bert-base-chinese/resolve/main/config.json""",
    """bert-base-german-cased""": """https://huggingface.co/bert-base-german-cased/resolve/main/config.json""",
    """bert-large-uncased-whole-word-masking""": (
        """https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/config.json"""
    ),
    """bert-large-cased-whole-word-masking""": (
        """https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/config.json"""
    ),
    """bert-large-uncased-whole-word-masking-finetuned-squad""": (
        """https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/config.json"""
    ),
    """bert-large-cased-whole-word-masking-finetuned-squad""": (
        """https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/config.json"""
    ),
    """bert-base-cased-finetuned-mrpc""": """https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/config.json""",
    """bert-base-german-dbmdz-cased""": """https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/config.json""",
    """bert-base-german-dbmdz-uncased""": """https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/config.json""",
    """cl-tohoku/bert-base-japanese""": """https://huggingface.co/cl-tohoku/bert-base-japanese/resolve/main/config.json""",
    """cl-tohoku/bert-base-japanese-whole-word-masking""": (
        """https://huggingface.co/cl-tohoku/bert-base-japanese-whole-word-masking/resolve/main/config.json"""
    ),
    """cl-tohoku/bert-base-japanese-char""": (
        """https://huggingface.co/cl-tohoku/bert-base-japanese-char/resolve/main/config.json"""
    ),
    """cl-tohoku/bert-base-japanese-char-whole-word-masking""": (
        """https://huggingface.co/cl-tohoku/bert-base-japanese-char-whole-word-masking/resolve/main/config.json"""
    ),
    """TurkuNLP/bert-base-finnish-cased-v1""": (
        """https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/config.json"""
    ),
    """TurkuNLP/bert-base-finnish-uncased-v1""": (
        """https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/config.json"""
    ),
    """wietsedv/bert-base-dutch-cased""": """https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/config.json""",
    # See all BERT models at https://huggingface.co/models?filter=bert
}






class    _UpperCAmelCase(   lowerCamelCase     ):
		lowercase__      =				'bert'



		def __init__( self    ,		__a=3_05_22    ,		__a=7_68    ,		__a=12    ,		__a=12    ,		__a=30_72    ,		__a="gelu"    ,		__a=0.1    ,		__a=0.1    ,		__a=5_12    ,		__a=2    ,		__a=0.02    ,		__a=1e-12    ,		__a=0    ,		__a="absolute"    ,		__a=True    ,		__a=None    ,		**__a    ,		)	->	Optional[Any]:






							'''simple docstring'''
							super().__init__(pad_token_id=__a    ,		**__a)

							_UpperCamelCase										=  vocab_size
							_UpperCamelCase										=  hidden_size
							_UpperCamelCase										=  num_hidden_layers
							_UpperCamelCase										=  num_attention_heads
							_UpperCamelCase										=  hidden_act
							_UpperCamelCase										=  intermediate_size
							_UpperCamelCase										=  hidden_dropout_prob
							_UpperCamelCase										=  attention_probs_dropout_prob
							_UpperCamelCase										=  max_position_embeddings
							_UpperCamelCase										=  type_vocab_size
							_UpperCamelCase										=  initializer_range
							_UpperCamelCase										=  layer_norm_eps
							_UpperCamelCase										=  position_embedding_type
							_UpperCamelCase										=  use_cache
							_UpperCamelCase										=  classifier_dropout






class    _UpperCAmelCase(   lowerCamelCase     ):



		@property
		def 						UpperCAmelCase							( self)	->	Mapping[str, Mapping[int, str]]:






							'''simple docstring'''
							if self.task == "multiple-choice":
												_UpperCamelCase										=  {0: '''batch''', 1: '''choice''', 2: '''sequence'''}
							else:
												_UpperCamelCase										=  {0: '''batch''', 1: '''sequence'''}
							return OrderedDict(
							    [
							        ('''input_ids''', dynamic_axis),
							        ('''attention_mask''', dynamic_axis),
							        ('''token_type_ids''', dynamic_axis),
							    ])






