



import logging
import math
from functools import partial
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union

import torch

from .tensor_utils import tensor_tree_map, tree_map


def  __lowerCamelCase (       lowerCamelCase__						):

				"""simple docstring"""


				lowercase__					:   Any													=					[]
				if isinstance(lowerCamelCase__					,			lowerCamelCase__						):
								for v in tree.values():
												shapes.extend(_fetch_dims(lowerCamelCase__						)						)
				elif isinstance(lowerCamelCase__					,			(list, tuple)						):
								for t in tree:
												shapes.extend(_fetch_dims(lowerCamelCase__						)						)
				elif isinstance(lowerCamelCase__					,			torch.Tensor						):
								shapes.append(tree.shape						)
				else:
								raise ValueError("Not supported"						)

				return shapes


@torch.jit.ignore
def  __lowerCamelCase (       lowerCamelCase__					,			lowerCamelCase__						):

				"""simple docstring"""


				lowercase__					:   Union[str, Any]													=					[]
				for d in reversed(lowerCamelCase__						):
								idx.append(flat_idx % d						)
								lowercase__					:   List[Any]													=					flat_idx // d

				return tuple(reversed(lowerCamelCase__						)						)


@torch.jit.ignore
def  __lowerCamelCase (       lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__ = None					,			lowerCamelCase__ = None					,			):

				"""simple docstring"""


				def reduce_edge_list(lowerCamelCase__						) -> None:
								lowercase__					:   str													=					True
								for i in range(len(lowerCamelCase__						)						):
												lowercase__					:   str													=					-1 * (i + 1)
												l[reversed_idx] &= tally
												lowercase__					:   str													=					l[reversed_idx]

				if start_edges is None:
								lowercase__					:   Union[str, Any]													=					[s == 0 for s in start]
								reduce_edge_list(lowerCamelCase__						)
				if end_edges is None:
								lowercase__					:   List[Any]													=					[e == (d - 1) for e, d in zip(lowerCamelCase__					,			lowerCamelCase__						)]
								reduce_edge_list(lowerCamelCase__						)

				# Base cases. Either start/end are empty and we're done, or the final,
				# one-dimensional tensor can be simply sliced
				if len(lowerCamelCase__						) == 0:
								return [()]
				elif len(lowerCamelCase__						) == 1:
								return [(slice(start[0]					,			end[0] + 1						),)]

				lowercase__					:   List[Tuple[slice, ...]]													=					[]
				lowercase__					:   List[slice]													=					[]

				# Dimensions common to start and end can be selected directly
				for s, e in zip(lowerCamelCase__					,			lowerCamelCase__						):
								if s == e:
												path_list.append(slice(lowerCamelCase__					,			s + 1						)						)
								else:
												break

				lowercase__					:   Tuple[slice, ...]													=					tuple(lowerCamelCase__						)
				lowercase__					:   Optional[int]													=					len(lowerCamelCase__						)

				# start == end, and we're done
				if divergence_idx == len(lowerCamelCase__						):
								return [path]

				def upper() -> Tuple[Tuple[slice, ...], ...]:
								assert start_edges is not None
								assert end_edges is not None

								lowercase__					:   str													=					start[divergence_idx]
								return tuple(
								    path + (slice(lowerCamelCase__					,			sdi + 1						),) + s
								    for s in _get_minimal_slice_set(
								        start[divergence_idx + 1 :]					,			[d - 1 for d in dims[divergence_idx + 1 :]]					,			dims[divergence_idx + 1 :]					,			start_edges=start_edges[divergence_idx + 1 :]					,			end_edges=[True for _ in end_edges[divergence_idx + 1 :]]					,			)						)

				def lower() -> Tuple[Tuple[slice, ...], ...]:
								assert start_edges is not None
								assert end_edges is not None

								lowercase__					:   List[str]													=					end[divergence_idx]
								return tuple(
								    path + (slice(lowerCamelCase__					,			edi + 1						),) + s
								    for s in _get_minimal_slice_set(
								        [0 for _ in start[divergence_idx + 1 :]]					,			end[divergence_idx + 1 :]					,			dims[divergence_idx + 1 :]					,			start_edges=[True for _ in start_edges[divergence_idx + 1 :]]					,			end_edges=end_edges[divergence_idx + 1 :]					,			)						)

				# If both start and end are at the edges of the subtree rooted at
				# divergence_idx, we can just select the whole subtree at once
				if start_edges[divergence_idx] and end_edges[divergence_idx]:
								slices.append(path + (slice(start[divergence_idx]					,			end[divergence_idx] + 1						),)						)
				# If just start is at the edge, we can grab almost all of the subtree,
				# treating only the ragged bottom edge as an edge case
				elif start_edges[divergence_idx]:
								slices.append(path + (slice(start[divergence_idx]					,			end[divergence_idx]						),)						)
								slices.extend(lower()						)
				# Analogous to the previous case, but the top is ragged this time
				elif end_edges[divergence_idx]:
								slices.extend(upper()						)
								slices.append(path + (slice(start[divergence_idx] + 1					,			end[divergence_idx] + 1						),)						)
				# If both sides of the range are ragged, we need to handle both sides
				# separately. If there's contiguous meat in between them, we can index it
				# in one big chunk
				else:
								slices.extend(upper()						)
								lowercase__					:   Optional[int]													=					end[divergence_idx] - start[divergence_idx]
								if middle_ground > 1:
												slices.append(path + (slice(start[divergence_idx] + 1					,			end[divergence_idx]						),)						)
								slices.extend(lower()						)

				return slices


@torch.jit.ignore
def  __lowerCamelCase (       lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__						):

				"""simple docstring"""


				lowercase__					:   Tuple													=					t.shape[:no_batch_dims]
				lowercase__					:   List[str]													=					list(_flat_idx_to_idx(lowerCamelCase__					,			lowerCamelCase__						)						)
				# _get_minimal_slice_set is inclusive
				lowercase__					:   Optional[Any]													=					list(_flat_idx_to_idx(flat_end - 1					,			lowerCamelCase__						)						)

				# Get an ordered list of slices to perform
				lowercase__					:   Optional[Any]													=					_get_minimal_slice_set(
				    lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			)

				lowercase__					:   Tuple													=					[t[s] for s in slices]

				return torch.cat([s.view((-1,) + t.shape[no_batch_dims:]						) for s in sliced_tensors]						)


def  __lowerCamelCase (       lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__ = False					,			lowerCamelCase__ = None					,			lowerCamelCase__ = False					,			):

				"""simple docstring"""


				if not (len(lowerCamelCase__						) > 0):
								raise ValueError("Must provide at least one input"						)

				lowercase__					:   str													=					[shape[:no_batch_dims] for shape in _fetch_dims(lowerCamelCase__						)]
				lowercase__					:   Union[str, Any]													=					tuple([max(lowerCamelCase__						) for s in zip(*lowerCamelCase__						)]						)

				def _prep_inputs(lowerCamelCase__						) -> torch.Tensor:
								if not low_mem:
												if not sum(t.shape[:no_batch_dims]						) == no_batch_dims:
																lowercase__					:   int													=					t.expand(orig_batch_dims + t.shape[no_batch_dims:]						)
												lowercase__					:   str													=					t.reshape(-1					,			*t.shape[no_batch_dims:]						)
								else:
												lowercase__					:   int													=					t.expand(orig_batch_dims + t.shape[no_batch_dims:]						)
								return t

				lowercase__					:   Dict[str, Any]													=					tensor_tree_map(_prep_inputs					,			lowerCamelCase__						)
				lowercase__					:   List[str]													=					None
				if _out is not None:
								lowercase__					:   int													=					tensor_tree_map(lambda lowerCamelCase__						: t.view([-1] + list(t.shape[no_batch_dims:]						)						)					,			_out						)

				lowercase__					:   List[Any]													=					1
				for d in orig_batch_dims:
								flat_batch_dim *= d

				lowercase__					:   List[str]													=					flat_batch_dim // chunk_size + (flat_batch_dim % chunk_size != 0)

				def _select_chunk(lowerCamelCase__						) -> torch.Tensor:
								return t[i : i + chunk_size] if t.shape[0] != 1 else t

				lowercase__					:   List[Any]													=					0
				lowercase__					:   Any													=					prepped_outputs
				for _ in range(lowerCamelCase__						):
								# Chunk the input
								if not low_mem:
												lowercase__					:   Any													=					_select_chunk
								else:
												lowercase__					:   Optional[Any]													=					partial(
												    _chunk_slice					,			flat_start=lowerCamelCase__					,			flat_end=min(lowerCamelCase__					,			i + chunk_size						)					,			no_batch_dims=len(lowerCamelCase__						)					,			)

								lowercase__					:   Dict[str, Any]													=					tensor_tree_map(lowerCamelCase__					,			lowerCamelCase__						)

								# Run the layer on the chunk
								lowercase__					:   List[str]													=					layer(**lowerCamelCase__						)

								# Allocate space for the output
								if out is None:
												lowercase__					:   List[Any]													=					tensor_tree_map(lambda lowerCamelCase__						: t.new_zeros((flat_batch_dim,) + t.shape[1:]						)					,			lowerCamelCase__						)

								# Put the chunk in its pre-allocated space
								if isinstance(lowerCamelCase__					,			lowerCamelCase__						):

												def assign(lowerCamelCase__					,			lowerCamelCase__						) -> None:
																for k, v in da.items():
																				if isinstance(lowerCamelCase__					,			lowerCamelCase__						):
																								assign(lowerCamelCase__					,			da[k]						)
																				else:
																								if _add_into_out:
																												v[i : i + chunk_size] += da[k]
																								else:
																												lowercase__					:   str													=					da[k]

												assign(lowerCamelCase__					,			lowerCamelCase__						)
								elif isinstance(lowerCamelCase__					,			lowerCamelCase__						):
												for xa, xa in zip(lowerCamelCase__					,			lowerCamelCase__						):
																if _add_into_out:
																				xa[i : i + chunk_size] += xa
																else:
																				lowercase__					:   Any													=					xa
								elif isinstance(lowerCamelCase__					,			torch.Tensor						):
												if _add_into_out:
																out[i : i + chunk_size] += output_chunk
												else:
																lowercase__					:   Optional[int]													=					output_chunk
								else:
												raise ValueError("Not supported"						)

								i += chunk_size

				lowercase__					:   Union[str, Any]													=					tensor_tree_map(lambda lowerCamelCase__						: t.view(orig_batch_dims + t.shape[1:]						)					,			lowerCamelCase__						)

				return out

class 				snake_case__:

					"""simple docstring"""






					def __init__(							self :     Optional[Any]				,       SCREAMING_SNAKE_CASE :     int = 512				,       ):
									lowercase__					:   Union[str, Any]													=					max_chunk_size
									lowercase__					:   Optional[int]													=					None
									lowercase__					:   Optional[tuple]													=					None







					def 			snake_case					(							self :     Dict				,       SCREAMING_SNAKE_CASE :     Callable				,       SCREAMING_SNAKE_CASE :     tuple				,       SCREAMING_SNAKE_CASE :     int					):
									logging.info("Tuning chunk size..."					)

									if min_chunk_size >= self.max_chunk_size:
													return min_chunk_size

									lowercase__					:   List[int]													=					[2**l for l in range(int(math.log(self.max_chunk_size				,       2					)					) + 1					)]
									lowercase__					:   List[Any]													=					[c for c in candidates if c > min_chunk_size]
									lowercase__					:   List[str]													=					[min_chunk_size] + candidates
									candidates[-1] += 4

									def test_chunk_size(SCREAMING_SNAKE_CASE :     int					) -> bool:
													try:
																	with torch.no_grad():
																					fn(*SCREAMING_SNAKE_CASE				,       chunk_size=SCREAMING_SNAKE_CASE					)
																	return True
													except RuntimeError:
																	return False

									lowercase__					:   Union[str, Any]													=					0
									lowercase__					:   List[str]													=					len(SCREAMING_SNAKE_CASE					) - 1
									while i > min_viable_chunk_size_index:
													lowercase__					:   int													=					test_chunk_size(candidates[i]					)
													if not viable:
																	lowercase__					:   Optional[Any]													=					(min_viable_chunk_size_index + i) // 2
													else:
																	lowercase__					:   List[str]													=					i
																	lowercase__					:   Optional[Any]													=					(i + len(SCREAMING_SNAKE_CASE					) - 1) // 2

									return candidates[min_viable_chunk_size_index]







					def 			snake_case					(							self :     str				,       SCREAMING_SNAKE_CASE :     Iterable				,       SCREAMING_SNAKE_CASE :     Iterable					):
									lowercase__					:   List[Any]													=					True
									for aa, aa in zip(SCREAMING_SNAKE_CASE				,       SCREAMING_SNAKE_CASE					):
													assert type(SCREAMING_SNAKE_CASE					) == type(SCREAMING_SNAKE_CASE					)
													if isinstance(SCREAMING_SNAKE_CASE				,       (list, tuple)					):
																	consistent &= self._compare_arg_caches(SCREAMING_SNAKE_CASE				,       SCREAMING_SNAKE_CASE					)
													elif isinstance(SCREAMING_SNAKE_CASE				,       SCREAMING_SNAKE_CASE					):
																	lowercase__					:   List[str]													=					[v for _, v in sorted(aa.items()				,       key=lambda SCREAMING_SNAKE_CASE					: x[0]					)]
																	lowercase__					:   Any													=					[v for _, v in sorted(aa.items()				,       key=lambda SCREAMING_SNAKE_CASE					: x[0]					)]
																	consistent &= self._compare_arg_caches(SCREAMING_SNAKE_CASE				,       SCREAMING_SNAKE_CASE					)
													else:
																	consistent &= aa == aa

									return consistent







					def 			snake_case					(							self :     Optional[Any]				,       SCREAMING_SNAKE_CASE :     Callable				,       SCREAMING_SNAKE_CASE :     tuple				,       SCREAMING_SNAKE_CASE :     int				,       ):
									lowercase__					:   List[str]													=					True
									lowercase__					:   tuple													=					tree_map(lambda SCREAMING_SNAKE_CASE					: a.shape if isinstance(SCREAMING_SNAKE_CASE				,       torch.Tensor					) else a				,       SCREAMING_SNAKE_CASE				,       SCREAMING_SNAKE_CASE					)
									if self.cached_arg_data is not None:
													# If args have changed shape/value, we need to re-tune
													assert len(self.cached_arg_data					) == len(SCREAMING_SNAKE_CASE					)
													lowercase__					:   int													=					self._compare_arg_caches(self.cached_arg_data				,       SCREAMING_SNAKE_CASE					)
									else:
													# Otherwise, we can reuse the precomputed value
													lowercase__					:   str													=					False

									if not consistent:
													lowercase__					:   List[str]													=					self._determine_favorable_chunk_size(
													    SCREAMING_SNAKE_CASE				,       SCREAMING_SNAKE_CASE				,       SCREAMING_SNAKE_CASE				,       )
													lowercase__					:   Optional[int]													=					arg_data

									assert self.cached_chunk_size is not None

									return self.cached_chunk_size









import unittest

from transformers import is_flax_available
from transformers.testing_utils import require_flax, require_sentencepiece, require_tokenizers, require_torch, slow


if is_flax_available():
				import optax
				from flax.training.common_utils import onehot

				from transformers import AutoTokenizer, FlaxMTaForConditionalGeneration
				from transformers.models.ta.modeling_flax_ta import shift_tokens_right

@require_torch
@require_sentencepiece
@require_tokenizers
@require_flax
class 				snake_case__(unittest.TestCase      ):

					"""simple docstring"""






					@slow
					def 			snake_case					(							self :     Optional[int]					):
									lowercase__					:   Dict													=					FlaxMTaForConditionalGeneration.from_pretrained("google/mt5-small"					)
									lowercase__					:   Dict													=					AutoTokenizer.from_pretrained("google/mt5-small"					)

									lowercase__					:   Optional[Any]													=					tokenizer("Hello there"				,       return_tensors="np"					).input_ids
									lowercase__					:   Optional[Any]													=					tokenizer("Hi I am"				,       return_tensors="np"					).input_ids

									lowercase__					:   int													=					shift_tokens_right(SCREAMING_SNAKE_CASE				,       model.config.pad_token_id				,       model.config.decoder_start_token_id					)

									lowercase__					:   Tuple													=					model(SCREAMING_SNAKE_CASE				,       decoder_input_ids=SCREAMING_SNAKE_CASE					).logits
									lowercase__					:   Dict													=					optax.softmax_cross_entropy(SCREAMING_SNAKE_CASE				,       onehot(SCREAMING_SNAKE_CASE				,       logits.shape[-1]					)					).mean()

									lowercase__					:   Union[str, Any]													=					-(labels.shape[-1] * loss.item())

									lowercase__					:   Union[str, Any]													=					-84.9_127
									self.assertTrue(abs(mtf_score - EXPECTED_SCORE					) < 1E-4					)




