





import logging
import math
from functools import partial
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union

import torch

from .tensor_utils import tensor_tree_map, tree_map


def 							SCREAMING_SNAKE_CASE__  (      __a				):
	snake_case_						:     Optional[int]     =		[]
	if isinstance(__UpperCAmelCase				,	__UpperCAmelCase				):
		for v in tree.values():
			shapes.extend(_fetch_dims(__UpperCAmelCase				)				)
	elif isinstance(__UpperCAmelCase				,	(list, tuple)				):
		for t in tree:
			shapes.extend(_fetch_dims(__UpperCAmelCase				)				)
	elif isinstance(__UpperCAmelCase				,	torch.Tensor				):
		shapes.append(tree.shape				)
	else:
		raise ValueError('Not supported'				)

	return shapes


@torch.jit.ignore
def 							SCREAMING_SNAKE_CASE__  (      __a				,	__a				):
	snake_case_						:     str     =		[]
	for d in reversed(__UpperCAmelCase				):
		idx.append(flat_idx % d				)
		snake_case_						:     Optional[Any]     =		flat_idx // d

	return tuple(reversed(__UpperCAmelCase				)				)


@torch.jit.ignore
def 							SCREAMING_SNAKE_CASE__  (      __a				,	__a				,	__a				,	__a = None				,	__a = None				,	):

	# start_edges and end_edges both indicate whether, starting from any given
	# dimension, the start/end index is at the top/bottom edge of the
	# corresponding tensor, modeled as a tree
	def reduce_edge_list(__a				) -> None:
		snake_case_						:     Tuple     =		True
		for i in range(len(__UpperCAmelCase				)				):
			snake_case_						:     Optional[Any]     =		-1 * (i + 1)
			l[reversed_idx] &= tally
			snake_case_						:     Dict     =		l[reversed_idx]

	if start_edges is None:
		snake_case_						:     Optional[int]     =		[s == 0 for s in start]
		reduce_edge_list(__UpperCAmelCase				)
	if end_edges is None:
		snake_case_						:     Optional[int]     =		[e == (d - 1) for e, d in zip(__UpperCAmelCase				,	__UpperCAmelCase				)]
		reduce_edge_list(__UpperCAmelCase				)

	# Base cases. Either start/end are empty and we're done, or the final,
	# one-dimensional tensor can be simply sliced
	if len(__UpperCAmelCase				) == 0:
		return [()]
	elif len(__UpperCAmelCase				) == 1:
		return [(slice(start[0]				,	end[0] + 1				),)]

	snake_case_						:     List[Tuple[slice, ...]]     =		[]
	snake_case_						:     List[slice]     =		[]

	# Dimensions common to start and end can be selected directly
	for s, e in zip(__UpperCAmelCase				,	__UpperCAmelCase				):
		if s == e:
			path_list.append(slice(__UpperCAmelCase				,	s + 1				)				)
		else:
			break

	snake_case_						:     Tuple[slice, ...]     =		tuple(__UpperCAmelCase				)
	snake_case_						:     int     =		len(__UpperCAmelCase				)

	# start == end, and we're done
	if divergence_idx == len(__UpperCAmelCase				):
		return [path]

	def upper() -> Tuple[Tuple[slice, ...], ...]:
		assert start_edges is not None
		assert end_edges is not None

		snake_case_						:     int     =		start[divergence_idx]
		return tuple(
		    path + (slice(__UpperCAmelCase				,	sdi + 1				),) + s
		    for s in _get_minimal_slice_set(
		        start[divergence_idx + 1 :]				,	[d - 1 for d in dims[divergence_idx + 1 :]]				,	dims[divergence_idx + 1 :]				,	start_edges=start_edges[divergence_idx + 1 :]				,	end_edges=[True for _ in end_edges[divergence_idx + 1 :]]				,	)				)

	def lower() -> Tuple[Tuple[slice, ...], ...]:
		assert start_edges is not None
		assert end_edges is not None

		snake_case_						:     Dict     =		end[divergence_idx]
		return tuple(
		    path + (slice(__UpperCAmelCase				,	edi + 1				),) + s
		    for s in _get_minimal_slice_set(
		        [0 for _ in start[divergence_idx + 1 :]]				,	end[divergence_idx + 1 :]				,	dims[divergence_idx + 1 :]				,	start_edges=[True for _ in start_edges[divergence_idx + 1 :]]				,	end_edges=end_edges[divergence_idx + 1 :]				,	)				)

	# If both start and end are at the edges of the subtree rooted at
	# divergence_idx, we can just select the whole subtree at once
	if start_edges[divergence_idx] and end_edges[divergence_idx]:
		slices.append(path + (slice(start[divergence_idx]				,	end[divergence_idx] + 1				),)				)
	# If just start is at the edge, we can grab almost all of the subtree,
	# treating only the ragged bottom edge as an edge case
	elif start_edges[divergence_idx]:
		slices.append(path + (slice(start[divergence_idx]				,	end[divergence_idx]				),)				)
		slices.extend(lower()				)
	# Analogous to the previous case, but the top is ragged this time
	elif end_edges[divergence_idx]:
		slices.extend(upper()				)
		slices.append(path + (slice(start[divergence_idx] + 1				,	end[divergence_idx] + 1				),)				)
	# If both sides of the range are ragged, we need to handle both sides
	# separately. If there's contiguous meat in between them, we can index it
	# in one big chunk
	else:
		slices.extend(upper()				)
		snake_case_						:     Optional[int]     =		end[divergence_idx] - start[divergence_idx]
		if middle_ground > 1:
			slices.append(path + (slice(start[divergence_idx] + 1				,	end[divergence_idx]				),)				)
		slices.extend(lower()				)

	return slices


@torch.jit.ignore
def 							SCREAMING_SNAKE_CASE__  (      __a				,	__a				,	__a				,	__a				):
	snake_case_						:     List[Any]     =		t.shape[:no_batch_dims]
	snake_case_						:     Optional[Any]     =		list(_flat_idx_to_idx(__UpperCAmelCase				,	__UpperCAmelCase				)				)
	# _get_minimal_slice_set is inclusive
	snake_case_						:     Dict     =		list(_flat_idx_to_idx(flat_end - 1				,	__UpperCAmelCase				)				)

	# Get an ordered list of slices to perform
	snake_case_						:     int     =		_get_minimal_slice_set(
	    __UpperCAmelCase				,	__UpperCAmelCase				,	__UpperCAmelCase				,	)

	snake_case_						:     Tuple     =		[t[s] for s in slices]

	return torch.cat([s.view((-1,) + t.shape[no_batch_dims:]				) for s in sliced_tensors]				)


def 							SCREAMING_SNAKE_CASE__  (      __a				,	__a				,	__a				,	__a				,	__a = False				,	__a = None				,	__a = False				,	):
	if not (len(__UpperCAmelCase				) > 0):
		raise ValueError('Must provide at least one input'				)

	snake_case_						:     List[Any]     =		[shape[:no_batch_dims] for shape in _fetch_dims(__UpperCAmelCase				)]
	snake_case_						:     int     =		tuple([max(__UpperCAmelCase				) for s in zip(*__UpperCAmelCase				)]				)

	def _prep_inputs(__a				) -> torch.Tensor:
		if not low_mem:
			if not sum(t.shape[:no_batch_dims]				) == no_batch_dims:
				snake_case_						:     Any     =		t.expand(orig_batch_dims + t.shape[no_batch_dims:]				)
			snake_case_						:     List[str]     =		t.reshape(-1				,	*t.shape[no_batch_dims:]				)
		else:
			snake_case_						:     Union[str, Any]     =		t.expand(orig_batch_dims + t.shape[no_batch_dims:]				)
		return t

	snake_case_						:     Dict[str, Any]     =		tensor_tree_map(_prep_inputs				,	__UpperCAmelCase				)
	snake_case_						:     int     =		None
	if _out is not None:
		snake_case_						:     str     =		tensor_tree_map(lambda __a				: t.view([-1] + list(t.shape[no_batch_dims:]				)				)				,	_out				)

	snake_case_						:     Optional[Any]     =		1
	for d in orig_batch_dims:
		flat_batch_dim *= d

	snake_case_						:     Any     =		flat_batch_dim // chunk_size + (flat_batch_dim % chunk_size != 0)

	def _select_chunk(__a				) -> torch.Tensor:
		return t[i : i + chunk_size] if t.shape[0] != 1 else t

	snake_case_						:     Optional[Any]     =		0
	snake_case_						:     int     =		prepped_outputs
	for _ in range(__UpperCAmelCase				):
		# Chunk the input
		if not low_mem:
			snake_case_						:     int     =		_select_chunk
		else:
			snake_case_						:     Optional[Any]     =		partial(
			    _chunk_slice				,	flat_start=__UpperCAmelCase				,	flat_end=min(__UpperCAmelCase				,	i + chunk_size				)				,	no_batch_dims=len(__UpperCAmelCase				)				,	)

		snake_case_						:     Dict[str, Any]     =		tensor_tree_map(__UpperCAmelCase				,	__UpperCAmelCase				)

		# Run the layer on the chunk
		snake_case_						:     Optional[Any]     =		layer(**__UpperCAmelCase				)

		# Allocate space for the output
		if out is None:
			snake_case_						:     Any     =		tensor_tree_map(lambda __a				: t.new_zeros((flat_batch_dim,) + t.shape[1:]				)				,	__UpperCAmelCase				)

		# Put the chunk in its pre-allocated space
		if isinstance(__UpperCAmelCase				,	__UpperCAmelCase				):

			def assign(__a				,	__a				) -> None:
				for k, v in da.items():
					if isinstance(__UpperCAmelCase				,	__UpperCAmelCase				):
						assign(__UpperCAmelCase				,	da[k]				)
					else:
						if _add_into_out:
							v[i : i + chunk_size] += da[k]
						else:
							snake_case_						:     Optional[int]     =		da[k]

			assign(__UpperCAmelCase				,	__UpperCAmelCase				)
		elif isinstance(__UpperCAmelCase				,	__UpperCAmelCase				):
			for xa, xa in zip(__UpperCAmelCase				,	__UpperCAmelCase				):
				if _add_into_out:
					xa[i : i + chunk_size] += xa
				else:
					snake_case_						:     Optional[int]     =		xa
		elif isinstance(__UpperCAmelCase				,	torch.Tensor				):
			if _add_into_out:
				out[i : i + chunk_size] += output_chunk
			else:
				snake_case_						:     Union[str, Any]     =		output_chunk
		else:
			raise ValueError('Not supported'				)

		i += chunk_size

	snake_case_						:     Any     =		tensor_tree_map(lambda __a				: t.view(orig_batch_dims + t.shape[1:]				)				,	__UpperCAmelCase				)

	return out


class SCREAMING_SNAKE_CASE_  :






						def __init__( self : Any						,				_A : List[str] = 512						,				)							->      Optional[Any]:







							"""simple docstring"""


							snake_case_						:     int     =		max_chunk_size
							snake_case_						:     Optional[int]     =		None
							snake_case_						:     Optional[tuple]     =		None






						def UpperCAmelCase_			( self : Union[str, Any]						,				_A : Optional[Any]						,				_A : List[str]						,				_A : Optional[Any]		)							->      Any:







							"""simple docstring"""


							logging.info('Tuning chunk size...'		)

							if min_chunk_size >= self.max_chunk_size:
								return min_chunk_size

							snake_case_						:     List[int]     =		[2**l for l in range(int(math.log(self.max_chunk_size						,				2		)		) + 1		)]
							snake_case_						:     List[Any]     =		[c for c in candidates if c > min_chunk_size]
							snake_case_						:     str     =		[min_chunk_size] + candidates
							candidates[-1] += 4

							def test_chunk_size(_A : Any		) -> bool:
								try:
									with torch.no_grad():
										fn(*_UpperCAmelCase						,				chunk_size=_UpperCAmelCase		)
									return True
								except RuntimeError:
									return False

							snake_case_						:     Any     =		0
							snake_case_						:     Any     =		len(_UpperCAmelCase		) - 1
							while i > min_viable_chunk_size_index:
								snake_case_						:     List[str]     =		test_chunk_size(candidates[i]		)
								if not viable:
									snake_case_						:     Optional[int]     =		(min_viable_chunk_size_index + i) // 2
								else:
									snake_case_						:     Dict     =		i
									snake_case_						:     Union[str, Any]     =		(i + len(_UpperCAmelCase		) - 1) // 2

							return candidates[min_viable_chunk_size_index]






						def UpperCAmelCase_			( self : Any						,				_A : List[str]						,				_A : List[str]		)							->      Optional[int]:







							"""simple docstring"""


							snake_case_						:     Tuple     =		True
							for aa, aa in zip(_UpperCAmelCase						,				_UpperCAmelCase		):
								assert type(_UpperCAmelCase		) == type(_UpperCAmelCase		)
								if isinstance(_UpperCAmelCase						,				(list, tuple)		):
									consistent &= self._compare_arg_caches(_UpperCAmelCase						,				_UpperCAmelCase		)
								elif isinstance(_UpperCAmelCase						,				_UpperCAmelCase		):
									snake_case_						:     Union[str, Any]     =		[v for _, v in sorted(aa.items()						,				key=lambda _A		: x[0]		)]
									snake_case_						:     Union[str, Any]     =		[v for _, v in sorted(aa.items()						,				key=lambda _A		: x[0]		)]
									consistent &= self._compare_arg_caches(_UpperCAmelCase						,				_UpperCAmelCase		)
								else:
									consistent &= aa == aa

							return consistent






						def UpperCAmelCase_			( self : Union[str, Any]						,				_A : Dict						,				_A : Any						,				_A : Optional[Any]						,				)							->      Any:







							"""simple docstring"""


							snake_case_						:     str     =		True
							snake_case_						:     tuple     =		tree_map(lambda _A		: a.shape if isinstance(_UpperCAmelCase						,				torch.Tensor		) else a						,				_UpperCAmelCase						,				_UpperCAmelCase		)
							if self.cached_arg_data is not None:
								# If args have changed shape/value, we need to re-tune
								assert len(self.cached_arg_data		) == len(_UpperCAmelCase		)
								snake_case_						:     str     =		self._compare_arg_caches(self.cached_arg_data						,				_UpperCAmelCase		)
							else:
								# Otherwise, we can reuse the precomputed value
								snake_case_						:     Optional[int]     =		False

							if not consistent:
								snake_case_						:     List[Any]     =		self._determine_favorable_chunk_size(
								    _UpperCAmelCase						,				_UpperCAmelCase						,				_UpperCAmelCase						,				)
								snake_case_						:     str     =		arg_data

							assert self.cached_chunk_size is not None

							return self.cached_chunk_size








from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


_SCREAMING_SNAKE_CASE 		=						{
    """configuration_xlm""": ["""XLM_PRETRAINED_CONFIG_ARCHIVE_MAP""", """XLMConfig""", """XLMOnnxConfig"""],
    """tokenization_xlm""": ["""XLMTokenizer"""],
}

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						_SCREAMING_SNAKE_CASE 		=						[
						    """XLM_PRETRAINED_MODEL_ARCHIVE_LIST""",
						    """XLMForMultipleChoice""",
						    """XLMForQuestionAnswering""",
						    """XLMForQuestionAnsweringSimple""",
						    """XLMForSequenceClassification""",
						    """XLMForTokenClassification""",
						    """XLMModel""",
						    """XLMPreTrainedModel""",
						    """XLMWithLMHeadModel""",
						]

try:
						if not is_tf_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						_SCREAMING_SNAKE_CASE 		=						[
						    """TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST""",
						    """TFXLMForMultipleChoice""",
						    """TFXLMForQuestionAnsweringSimple""",
						    """TFXLMForSequenceClassification""",
						    """TFXLMForTokenClassification""",
						    """TFXLMMainLayer""",
						    """TFXLMModel""",
						    """TFXLMPreTrainedModel""",
						    """TFXLMWithLMHeadModel""",
						]


if TYPE_CHECKING:
						from .configuration_xlm import XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMConfig, XLMOnnxConfig
						from .tokenization_xlm import XLMTokenizer

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_xlm import (
												    XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
												    XLMForMultipleChoice,
												    XLMForQuestionAnswering,
												    XLMForQuestionAnsweringSimple,
												    XLMForSequenceClassification,
												    XLMForTokenClassification,
												    XLMModel,
												    XLMPreTrainedModel,
												    XLMWithLMHeadModel,
												)

						try:
												if not is_tf_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_tf_xlm import (
												    TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
												    TFXLMForMultipleChoice,
												    TFXLMForQuestionAnsweringSimple,
												    TFXLMForSequenceClassification,
												    TFXLMForTokenClassification,
												    TFXLMMainLayer,
												    TFXLMModel,
												    TFXLMPreTrainedModel,
												    TFXLMWithLMHeadModel,
												)

else:
						import sys

						_SCREAMING_SNAKE_CASE 		=						_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)

