





import logging
import math
from functools import partial
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union

import torch

from .tensor_utils import tensor_tree_map, tree_map





def       _UpperCamelCase       (a__				:Union[dict, list, tuple, torch.Tensor]    ):






							"""simple docstring"""





							UpperCamelCase__			       =     []
							if isinstance(a__				,     a__    ):
														for v in tree.values():
																					shapes.extend(_fetch_dims(a__    )    )
							elif isinstance(a__				,     (list, tuple)    ):
														for t in tree:
																					shapes.extend(_fetch_dims(a__    )    )
							elif isinstance(a__				,     torch.Tensor    ):
														shapes.append(tree.shape    )
							else:
														raise ValueError("""Not supported"""    )

							return shapes





@torch.jit.ignore
def       _UpperCamelCase       (a__				:int				,     a__				:Tuple[int, ...]    ):






							"""simple docstring"""





							UpperCamelCase__			       =     []
							for d in reversed(a__    ):
														idx.append(flat_idx % d    )
														UpperCamelCase__			       =     flat_idx // d

							return tuple(reversed(a__    )    )





@torch.jit.ignore
def       _UpperCamelCase       (a__				:Sequence[int]				,     a__				:Sequence[int]				,     a__				:Sequence[int]				,     a__				:Optional[Sequence[bool]] = None				,     a__				:Optional[Sequence[bool]] = None				,     ):






							"""simple docstring"""





							def reduce_edge_list(a__				:List[bool]    ) -> None:
														UpperCamelCase__			       =     True
														for i in range(len(a__    )    ):
																					UpperCamelCase__			       =     -1 * (i + 1)
																					l[reversed_idx] &= tally
																					UpperCamelCase__			       =     l[reversed_idx]

							if start_edges is None:
														UpperCamelCase__			       =     [s == 0 for s in start]
														reduce_edge_list(a__    )
							if end_edges is None:
														UpperCamelCase__			       =     [e == (d - 1) for e, d in zip(a__				,     a__    )]
														reduce_edge_list(a__    )

							# Base cases. Either start/end are empty and we're done, or the final,
							# one-dimensional tensor can be simply sliced
							if len(a__    ) == 0:
														return [()]
							elif len(a__    ) == 1:
														return [(slice(start[0]				,     end[0] + 1    ),)]

							UpperCamelCase__			       =     []
							UpperCamelCase__			       =     []

							# Dimensions common to start and end can be selected directly
							for s, e in zip(a__				,     a__    ):
														if s == e:
																					path_list.append(slice(a__				,     s + 1    )    )
														else:
																					break

							UpperCamelCase__			       =     tuple(a__    )
							UpperCamelCase__			       =     len(a__    )

							# start == end, and we're done
							if divergence_idx == len(a__    ):
														return [path]

							def upper() -> Tuple[Tuple[slice, ...], ...]:
														assert start_edges is not None
														assert end_edges is not None

														UpperCamelCase__			       =     start[divergence_idx]
														return tuple(
														    path + (slice(a__				,     sdi + 1    ),) + s
														    for s in _get_minimal_slice_set(
														        start[divergence_idx + 1 :]				,     [d - 1 for d in dims[divergence_idx + 1 :]]				,     dims[divergence_idx + 1 :]				,     start_edges=start_edges[divergence_idx + 1 :]				,     end_edges=[True for _ in end_edges[divergence_idx + 1 :]]				,     )    )

							def lower() -> Tuple[Tuple[slice, ...], ...]:
														assert start_edges is not None
														assert end_edges is not None

														UpperCamelCase__			       =     end[divergence_idx]
														return tuple(
														    path + (slice(a__				,     edi + 1    ),) + s
														    for s in _get_minimal_slice_set(
														        [0 for _ in start[divergence_idx + 1 :]]				,     end[divergence_idx + 1 :]				,     dims[divergence_idx + 1 :]				,     start_edges=[True for _ in start_edges[divergence_idx + 1 :]]				,     end_edges=end_edges[divergence_idx + 1 :]				,     )    )

							# If both start and end are at the edges of the subtree rooted at
							# divergence_idx, we can just select the whole subtree at once
							if start_edges[divergence_idx] and end_edges[divergence_idx]:
														slices.append(path + (slice(start[divergence_idx]				,     end[divergence_idx] + 1    ),)    )
							# If just start is at the edge, we can grab almost all of the subtree,
							# treating only the ragged bottom edge as an edge case
							elif start_edges[divergence_idx]:
														slices.append(path + (slice(start[divergence_idx]				,     end[divergence_idx]    ),)    )
														slices.extend(lower()    )
							# Analogous to the previous case, but the top is ragged this time
							elif end_edges[divergence_idx]:
														slices.extend(upper()    )
														slices.append(path + (slice(start[divergence_idx] + 1				,     end[divergence_idx] + 1    ),)    )
							# If both sides of the range are ragged, we need to handle both sides
							# separately. If there's contiguous meat in between them, we can index it
							# in one big chunk
							else:
														slices.extend(upper()    )
														UpperCamelCase__			       =     end[divergence_idx] - start[divergence_idx]
														if middle_ground > 1:
																					slices.append(path + (slice(start[divergence_idx] + 1				,     end[divergence_idx]    ),)    )
														slices.extend(lower()    )

							return slices





@torch.jit.ignore
def       _UpperCamelCase       (a__				:torch.Tensor				,     a__				:int				,     a__				:int				,     a__				:int    ):






							"""simple docstring"""





							UpperCamelCase__			       =     t.shape[:no_batch_dims]
							UpperCamelCase__			       =     list(_flat_idx_to_idx(a__				,     a__    )    )
							# _get_minimal_slice_set is inclusive
							UpperCamelCase__			       =     list(_flat_idx_to_idx(flat_end - 1				,     a__    )    )

							# Get an ordered list of slices to perform
							UpperCamelCase__			       =     _get_minimal_slice_set(
							    a__				,     a__				,     a__				,     )

							UpperCamelCase__			       =     [t[s] for s in slices]

							return torch.cat([s.view((-1,) + t.shape[no_batch_dims:]    ) for s in sliced_tensors]    )





def       _UpperCamelCase       (a__				:Callable				,     a__				:Dict[str, Any]				,     a__				:int				,     a__				:int				,     a__				:bool = False				,     a__				:Any = None				,     a__				:bool = False				,     ):






							"""simple docstring"""





							if not (len(a__    ) > 0):
														raise ValueError("""Must provide at least one input"""    )

							UpperCamelCase__			       =     [shape[:no_batch_dims] for shape in _fetch_dims(a__    )]
							UpperCamelCase__			       =     tuple([max(a__    ) for s in zip(*a__    )]    )

							def _prep_inputs(a__				:torch.Tensor    ) -> torch.Tensor:
														if not low_mem:
																					if not sum(t.shape[:no_batch_dims]    ) == no_batch_dims:
																												UpperCamelCase__			       =     t.expand(orig_batch_dims + t.shape[no_batch_dims:]    )
																					UpperCamelCase__			       =     t.reshape(-1				,     *t.shape[no_batch_dims:]    )
														else:
																					UpperCamelCase__			       =     t.expand(orig_batch_dims + t.shape[no_batch_dims:]    )
														return t

							UpperCamelCase__			       =     tensor_tree_map(_prep_inputs				,     a__    )
							UpperCamelCase__			       =     None
							if _out is not None:
														UpperCamelCase__			       =     tensor_tree_map(lambda a__    : t.view([-1] + list(t.shape[no_batch_dims:]    )    )				,     _out    )

							UpperCamelCase__			       =     1
							for d in orig_batch_dims:
														flat_batch_dim *= d

							UpperCamelCase__			       =     flat_batch_dim // chunk_size + (flat_batch_dim % chunk_size != 0)

							def _select_chunk(a__				:torch.Tensor    ) -> torch.Tensor:
														return t[i : i + chunk_size] if t.shape[0] != 1 else t

							UpperCamelCase__			       =     0
							UpperCamelCase__			       =     prepped_outputs
							for _ in range(a__    ):
														# Chunk the input
														if not low_mem:
																					UpperCamelCase__			       =     _select_chunk
														else:
																					UpperCamelCase__			       =     partial(
																					    _chunk_slice				,     flat_start=a__				,     flat_end=min(a__				,     i + chunk_size    )				,     no_batch_dims=len(a__    )				,     )

														UpperCamelCase__			       =     tensor_tree_map(a__				,     a__    )

														# Run the layer on the chunk
														UpperCamelCase__			       =     layer(**a__    )

														# Allocate space for the output
														if out is None:
																					UpperCamelCase__			       =     tensor_tree_map(lambda a__    : t.new_zeros((flat_batch_dim,) + t.shape[1:]    )				,     a__    )

														# Put the chunk in its pre-allocated space
														if isinstance(a__				,     a__    ):

																					def assign(a__				:dict				,     a__				:dict    ) -> None:
																												for k, v in da.items():
																																			if isinstance(a__				,     a__    ):
																																										assign(a__				,     da[k]    )
																																			else:
																																										if _add_into_out:
																																																	v[i : i + chunk_size] += da[k]
																																										else:
																																																	UpperCamelCase__			       =     da[k]

																					assign(a__				,     a__    )
														elif isinstance(a__				,     a__    ):
																					for xa, xa in zip(a__				,     a__    ):
																												if _add_into_out:
																																			xa[i : i + chunk_size] += xa
																												else:
																																			UpperCamelCase__			       =     xa
														elif isinstance(a__				,     torch.Tensor    ):
																					if _add_into_out:
																												out[i : i + chunk_size] += output_chunk
																					else:
																												UpperCamelCase__			       =     output_chunk
														else:
																					raise ValueError("""Not supported"""    )

														i += chunk_size

							UpperCamelCase__			       =     tensor_tree_map(lambda a__    : t.view(orig_batch_dims + t.shape[1:]    )				,     a__    )

							return out

class     __SCREAMING_SNAKE_CASE   :






							def __init__(  self					,	__lowerCAmelCase = 512					,	):
														UpperCamelCase__			       =     max_chunk_size
														UpperCamelCase__			       =     None
														UpperCamelCase__			       =     None






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase					,	__lowerCAmelCase					):
														logging.info("""Tuning chunk size..."""					)

														if min_chunk_size >= self.max_chunk_size:
																					return min_chunk_size

														UpperCamelCase__			       =     [2**l for l in range(int(math.log(self.max_chunk_size					,	2					)					) + 1					)]
														UpperCamelCase__			       =     [c for c in candidates if c > min_chunk_size]
														UpperCamelCase__			       =     [min_chunk_size] + candidates
														candidates[-1] += 4

														def test_chunk_size(__lowerCAmelCase					) -> bool:
																					try:
																												with torch.no_grad():
																																			fn(*__lowerCamelCase					,	chunk_size=__lowerCamelCase					)
																												return True
																					except RuntimeError:
																												return False

														UpperCamelCase__			       =     0
														UpperCamelCase__			       =     len(__lowerCamelCase					) - 1
														while i > min_viable_chunk_size_index:
																					UpperCamelCase__			       =     test_chunk_size(candidates[i]					)
																					if not viable:
																												UpperCamelCase__			       =     (min_viable_chunk_size_index + i) // 2
																					else:
																												UpperCamelCase__			       =     i
																												UpperCamelCase__			       =     (i + len(__lowerCamelCase					) - 1) // 2

														return candidates[min_viable_chunk_size_index]






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase					):
														UpperCamelCase__			       =     True
														for aa, aa in zip(__lowerCamelCase					,	__lowerCamelCase					):
																					assert type(__lowerCamelCase					) == type(__lowerCamelCase					)
																					if isinstance(__lowerCamelCase					,	(list, tuple)					):
																												consistent &= self._compare_arg_caches(__lowerCamelCase					,	__lowerCamelCase					)
																					elif isinstance(__lowerCamelCase					,	__lowerCamelCase					):
																												UpperCamelCase__			       =     [v for _, v in sorted(aa.items()					,	key=lambda __lowerCAmelCase					: x[0]					)]
																												UpperCamelCase__			       =     [v for _, v in sorted(aa.items()					,	key=lambda __lowerCAmelCase					: x[0]					)]
																												consistent &= self._compare_arg_caches(__lowerCamelCase					,	__lowerCamelCase					)
																					else:
																												consistent &= aa == aa

														return consistent







							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase					,	__lowerCAmelCase					,	):
														UpperCamelCase__			       =     True
														UpperCamelCase__			       =     tree_map(lambda __lowerCAmelCase					: a.shape if isinstance(__lowerCamelCase					,	torch.Tensor					) else a					,	__lowerCamelCase					,	__lowerCamelCase					)
														if self.cached_arg_data is not None:
																					# If args have changed shape/value, we need to re-tune
																					assert len(self.cached_arg_data					) == len(__lowerCamelCase					)
																					UpperCamelCase__			       =     self._compare_arg_caches(self.cached_arg_data					,	__lowerCamelCase					)
														else:
																					# Otherwise, we can reuse the precomputed value
																					UpperCamelCase__			       =     False

														if not consistent:
																					UpperCamelCase__			       =     self._determine_favorable_chunk_size(
																					    __lowerCamelCase					,	__lowerCamelCase					,	__lowerCamelCase					,	)
																					UpperCamelCase__			       =     arg_data

														assert self.cached_chunk_size is not None

														return self.cached_chunk_size











import logging

from transformers.configuration_utils import PretrainedConfig


UpperCamelCase__     =	logging.getLogger(__name__)

class     __SCREAMING_SNAKE_CASE   (       _a    ):
							snake_case       : Optional[Any]         =    """masked_bert"""






							def __init__(  self					,	__lowerCAmelCase=30522					,	__lowerCAmelCase=768					,	__lowerCAmelCase=12					,	__lowerCAmelCase=12					,	__lowerCAmelCase=3072					,	__lowerCAmelCase="gelu"					,	__lowerCAmelCase=0.1					,	__lowerCAmelCase=0.1					,	__lowerCAmelCase=512					,	__lowerCAmelCase=2					,	__lowerCAmelCase=0.02					,	__lowerCAmelCase=1E-12					,	__lowerCAmelCase=0					,	__lowerCAmelCase="topK"					,	__lowerCAmelCase="constant"					,	__lowerCAmelCase=0.0					,	**__lowerCAmelCase					,	):
														super().__init__(pad_token_id=__lowerCAmelCase					,	**__lowerCAmelCase					)

														UpperCamelCase__			       =     vocab_size
														UpperCamelCase__			       =     hidden_size
														UpperCamelCase__			       =     num_hidden_layers
														UpperCamelCase__			       =     num_attention_heads
														UpperCamelCase__			       =     hidden_act
														UpperCamelCase__			       =     intermediate_size
														UpperCamelCase__			       =     hidden_dropout_prob
														UpperCamelCase__			       =     attention_probs_dropout_prob
														UpperCamelCase__			       =     max_position_embeddings
														UpperCamelCase__			       =     type_vocab_size
														UpperCamelCase__			       =     initializer_range
														UpperCamelCase__			       =     layer_norm_eps
														UpperCamelCase__			       =     pruning_method
														UpperCamelCase__			       =     mask_init
														UpperCamelCase__			       =     mask_scale




