
from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings







@maybe_allow_in_graph
class     lowercase		( nn.Module		):




							def __init__(    self       ,A__       ,A__       ,A__       ,A__=0.0       ,A__ = None       ,A__ = "geglu"       ,A__ = None       ,A__ = False       ,A__ = False       ,A__ = False       ,A__ = False       ,A__ = True       ,A__ = "layer_norm"       ,A__ = False       ,):
												super().__init__()
												lowercase						     =						only_cross_attention

												lowercase						     =						(num_embeds_ada_norm is not None) and norm_type == '''ada_norm_zero'''
												lowercase						     =						(num_embeds_ada_norm is not None) and norm_type == '''ada_norm'''

												if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
																	raise ValueError(
																	    f'`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to'
																	    f' define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}.')

												# Define 3 blocks. Each block has its own normalization layer.
												# 1. Self-Attn
												if self.use_ada_layer_norm:
																	lowercase						     =						AdaLayerNorm(A__       ,A__)
												elif self.use_ada_layer_norm_zero:
																	lowercase						     =						AdaLayerNormZero(A__       ,A__)
												else:
																	lowercase						     =						nn.LayerNorm(A__       ,elementwise_affine=A__)
												lowercase						     =						Attention(
												    query_dim=A__       ,heads=A__       ,dim_head=A__       ,dropout=A__       ,bias=A__       ,cross_attention_dim=cross_attention_dim if only_cross_attention else None       ,upcast_attention=A__       ,)

												# 2. Cross-Attn
												if cross_attention_dim is not None or double_self_attention:
																	# We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
																	# I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
																	# the second cross attention block.
																	lowercase						     =						(
																	    AdaLayerNorm(A__       ,A__)
																	    if self.use_ada_layer_norm
																	    else nn.LayerNorm(A__       ,elementwise_affine=A__)
																	)
																	lowercase						     =						Attention(
																	    query_dim=A__       ,cross_attention_dim=cross_attention_dim if not double_self_attention else None       ,heads=A__       ,dim_head=A__       ,dropout=A__       ,bias=A__       ,upcast_attention=A__       ,)  # is self-attn if encoder_hidden_states is none
												else:
																	lowercase						     =						None
																	lowercase						     =						None

												# 3. Feed-forward
												lowercase						     =						nn.LayerNorm(A__       ,elementwise_affine=A__)
												lowercase						     =						FeedForward(A__       ,dropout=A__       ,activation_fn=A__       ,final_dropout=A__)

												# let chunk size default to None
												lowercase						     =						None
												lowercase						     =						0




							def 			A__				(    self       ,A__       ,A__):
												# Sets chunk feed-forward
												lowercase						     =						chunk_size
												lowercase						     =						dim





							def 			A__				(    self       ,A__       ,A__ = None       ,A__ = None       ,A__ = None       ,A__ = None       ,A__ = None       ,A__ = None       ,):
												# Notice that normalization is always applied before the real computation in the following blocks.
												# 1. Self-Attention
												if self.use_ada_layer_norm:
																	lowercase						     =						self.norma(A__       ,A__)
												elif self.use_ada_layer_norm_zero:
																	lowercase     ,						lowercase     ,						lowercase     ,						lowercase     ,						lowercase						     =						self.norma(
																	    A__       ,A__       ,A__       ,hidden_dtype=hidden_states.dtype)
												else:
																	lowercase						     =						self.norma(A__)

												lowercase						     =						cross_attention_kwargs if cross_attention_kwargs is not None else {}

												lowercase						     =						self.attna(
												    A__       ,encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None       ,attention_mask=A__       ,**A__       ,)
												if self.use_ada_layer_norm_zero:
																	lowercase						     =						gate_msa.unsqueeze(1) * attn_output
												lowercase						     =						attn_output + hidden_states

												# 2. Cross-Attention
												if self.attna is not None:
																	lowercase						     =						(
																	    self.norma(A__       ,A__) if self.use_ada_layer_norm else self.norma(A__)
																	)

																	lowercase						     =						self.attna(
																	    A__       ,encoder_hidden_states=A__       ,attention_mask=A__       ,**A__       ,)
																	lowercase						     =						attn_output + hidden_states

												# 3. Feed-forward
												lowercase						     =						self.norma(A__)

												if self.use_ada_layer_norm_zero:
																	lowercase						     =						norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

												if self._chunk_size is not None:
																	# "feed_forward_chunk_size" can be used to save memory
																	if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
																						raise ValueError(
																						    f'`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`.')

																	lowercase						     =						norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
																	lowercase						     =						torch.cat(
																	    [self.ff(A__) for hid_slice in norm_hidden_states.chunk(A__       ,dim=self._chunk_dim)]       ,dim=self._chunk_dim       ,)
												else:
																	lowercase						     =						self.ff(A__)

												if self.use_ada_layer_norm_zero:
																	lowercase						     =						gate_mlp.unsqueeze(1) * ff_output

												lowercase						     =						ff_output + hidden_states

												return hidden_states







class     lowercase		( nn.Module		):




							def __init__(    self       ,A__       ,A__ = None       ,A__ = 4       ,A__ = 0.0       ,A__ = "geglu"       ,A__ = False       ,):
												super().__init__()
												lowercase						     =						int(dim * mult)
												lowercase						     =						dim_out if dim_out is not None else dim

												if activation_fn == "gelu":
																	lowercase						     =						GELU(A__       ,A__)
												if activation_fn == "gelu-approximate":
																	lowercase						     =						GELU(A__       ,A__       ,approximate='''tanh''')
												elif activation_fn == "geglu":
																	lowercase						     =						GEGLU(A__       ,A__)
												elif activation_fn == "geglu-approximate":
																	lowercase						     =						ApproximateGELU(A__       ,A__)

												lowercase						     =						nn.ModuleList([])
												# project in
												self.net.append(A__)
												# project dropout
												self.net.append(nn.Dropout(A__))
												# project out
												self.net.append(nn.Linear(A__       ,A__))
												# FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
												if final_dropout:
																	self.net.append(nn.Dropout(A__))





							def 			A__				(    self       ,A__):
												for module in self.net:
																	lowercase						     =						module(A__)
												return hidden_states







class     lowercase		( nn.Module		):




							def __init__(    self       ,A__       ,A__       ,A__ = "none"):
												super().__init__()
												lowercase						     =						nn.Linear(A__       ,A__)
												lowercase						     =						approximate




							def 			A__				(    self       ,A__):
												if gate.device.type != "mps":
																	return F.gelu(A__       ,approximate=self.approximate)
												# mps: gelu is not implemented for float16
												return F.gelu(gate.to(dtype=torch.floataa)       ,approximate=self.approximate).to(dtype=gate.dtype)





							def 			A__				(    self       ,A__):
												lowercase						     =						self.proj(A__)
												lowercase						     =						self.gelu(A__)
												return hidden_states







class     lowercase		( nn.Module		):




							def __init__(    self       ,A__       ,A__):
												super().__init__()
												lowercase						     =						nn.Linear(A__       ,dim_out * 2)




							def 			A__				(    self       ,A__):
												if gate.device.type != "mps":
																	return F.gelu(A__)
												# mps: gelu is not implemented for float16
												return F.gelu(gate.to(dtype=torch.floataa)).to(dtype=gate.dtype)





							def 			A__				(    self       ,A__):
												lowercase     ,						lowercase						     =						self.proj(A__).chunk(2       ,dim=-1)
												return hidden_states * self.gelu(A__)







class     lowercase		( nn.Module		):




							def __init__(    self       ,A__       ,A__):
												super().__init__()
												lowercase						     =						nn.Linear(A__       ,A__)





							def 			A__				(    self       ,A__):
												lowercase						     =						self.proj(A__)
												return x * torch.sigmoid(1.702 * x)







class     lowercase		( nn.Module		):




							def __init__(    self       ,A__       ,A__):
												super().__init__()
												lowercase						     =						nn.Embedding(A__       ,A__)
												lowercase						     =						nn.SiLU()
												lowercase						     =						nn.Linear(A__       ,embedding_dim * 2)
												lowercase						     =						nn.LayerNorm(A__       ,elementwise_affine=A__)





							def 			A__				(    self       ,A__       ,A__):
												lowercase						     =						self.linear(self.silu(self.emb(A__)))
												lowercase     ,						lowercase						     =						torch.chunk(A__       ,2)
												lowercase						     =						self.norm(A__) * (1 + scale) + shift
												return x







class     lowercase		( nn.Module		):




							def __init__(    self       ,A__       ,A__):
												super().__init__()

												lowercase						     =						CombinedTimestepLabelEmbeddings(A__       ,A__)

												lowercase						     =						nn.SiLU()
												lowercase						     =						nn.Linear(A__       ,6 * embedding_dim       ,bias=A__)
												lowercase						     =						nn.LayerNorm(A__       ,elementwise_affine=A__       ,eps=1E-6)





							def 			A__				(    self       ,A__       ,A__       ,A__       ,A__=None):
												lowercase						     =						self.linear(self.silu(self.emb(A__       ,A__       ,hidden_dtype=A__)))
												lowercase     ,						lowercase     ,						lowercase     ,						lowercase     ,						lowercase     ,						lowercase						     =						emb.chunk(6       ,dim=1)
												lowercase						     =						self.norm(A__) * (1 + scale_msa[:, None]) + shift_msa[:, None]
												return x, gate_msa, shift_mlp, scale_mlp, gate_mlp







class     lowercase		( nn.Module		):




							def __init__(    self       ,A__       ,A__       ,A__       ,A__ = None       ,A__ = 1E-5):
												super().__init__()
												lowercase						     =						num_groups
												lowercase						     =						eps

												if act_fn is None:
																	lowercase						     =						None
												else:
																	lowercase						     =						get_activation(A__)

												lowercase						     =						nn.Linear(A__       ,out_dim * 2)





							def 			A__				(    self       ,A__       ,A__):
												if self.act:
																	lowercase						     =						self.act(A__)
												lowercase						     =						self.linear(A__)
												lowercase						     =						emb[:, :, None, None]
												lowercase     ,						lowercase						     =						emb.chunk(2       ,dim=1)

												lowercase						     =						F.group_norm(A__       ,self.num_groups       ,eps=self.eps)
												lowercase						     =						x * (1 + scale) + shift
												return x






import unittest

from transformers import TrOCRConfig
from transformers.testing_utils import is_torch_available, require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers.models.trocr.modeling_trocr import TrOCRDecoder, TrOCRForCausalLM







@require_torch
class     lowercase		:




							def __init__(    self       ,A__       ,A__=9_9       ,A__=1_3       ,A__=1_6       ,A__=7       ,A__=True       ,A__=True       ,A__=True       ,A__=False       ,A__=True       ,A__=2       ,A__=3_2       ,A__=4       ,A__=4       ,A__=3_0       ,A__=0       ,A__=1       ,A__=2       ,A__=None       ,):
												lowercase						     =						parent
												lowercase						     =						batch_size
												lowercase						     =						decoder_seq_length
												# For common tests
												lowercase						     =						self.decoder_seq_length
												lowercase						     =						is_training
												lowercase						     =						use_attention_mask
												lowercase						     =						use_labels

												lowercase						     =						vocab_size
												lowercase						     =						d_model
												lowercase						     =						d_model
												lowercase						     =						decoder_layers
												lowercase						     =						decoder_layers
												lowercase						     =						decoder_ffn_dim
												lowercase						     =						decoder_attention_heads
												lowercase						     =						decoder_attention_heads
												lowercase						     =						eos_token_id
												lowercase						     =						bos_token_id
												lowercase						     =						pad_token_id
												lowercase						     =						decoder_start_token_id
												lowercase						     =						use_cache
												lowercase						     =						max_position_embeddings

												lowercase						     =						None
												lowercase						     =						decoder_seq_length
												lowercase						     =						2
												lowercase						     =						1




							def 			A__				(    self):
												lowercase						     =						ids_tensor([self.batch_size, self.decoder_seq_length]       ,self.vocab_size)

												lowercase						     =						None
												if self.use_attention_mask:
																	lowercase						     =						ids_tensor([self.batch_size, self.decoder_seq_length]       ,vocab_size=2)

												lowercase						     =						None
												if self.use_labels:
																	lowercase						     =						ids_tensor([self.batch_size, self.decoder_seq_length]       ,self.vocab_size)

												lowercase						     =						TrOCRConfig(
												    vocab_size=self.vocab_size       ,d_model=self.d_model       ,decoder_layers=self.decoder_layers       ,decoder_ffn_dim=self.decoder_ffn_dim       ,decoder_attention_heads=self.decoder_attention_heads       ,eos_token_id=self.eos_token_id       ,bos_token_id=self.bos_token_id       ,use_cache=self.use_cache       ,pad_token_id=self.pad_token_id       ,decoder_start_token_id=self.decoder_start_token_id       ,max_position_embeddings=self.max_position_embeddings       ,)

												return (config, input_ids, attention_mask, lm_labels)




							def 			A__				(    self       ,A__       ,A__       ,A__       ,A__       ,):
												lowercase						     =						True
												lowercase						     =						TrOCRDecoder(config=A__).to(A__).eval()
												lowercase						     =						input_ids[:2]

												input_ids[input_ids == 0] += 1
												# first forward pass
												lowercase						     =						model(A__       ,use_cache=A__)
												lowercase						     =						model(A__)
												lowercase						     =						model(A__       ,use_cache=A__)

												self.parent.assertTrue(len(A__) == len(A__))
												self.parent.assertTrue(len(A__) == len(A__) + 1)

												lowercase						     =						outputs['''past_key_values''']

												# create hypothetical next token and extent to next_input_ids
												lowercase						     =						ids_tensor((2, 1)       ,config.vocab_size - 1) + 1

												# append to next input_ids and
												lowercase						     =						torch.cat([input_ids, next_tokens]       ,dim=-1)

												lowercase						     =						model(A__)['''last_hidden_state''']
												lowercase						     =						model(A__       ,past_key_values=A__)['''last_hidden_state''']

												# select random slice
												lowercase						     =						ids_tensor((1,)       ,output_from_past.shape[-1]).item()
												lowercase						     =						output_from_no_past[:, next_input_ids.shape[-1] - 1, random_slice_idx].detach()
												lowercase						     =						output_from_past[:, 0, random_slice_idx].detach()

												# test that outputs are equal for slice
												assert torch.allclose(A__       ,A__       ,atol=1E-3)





							def 			A__				(    self):
												lowercase						     =						self.prepare_config_and_inputs()
												lowercase     ,						lowercase     ,						lowercase     ,						lowercase						     =						config_and_inputs

												lowercase						     =						{'''input_ids''': input_ids, '''attention_mask''': attention_mask}
												return config, inputs_dict







@require_torch
class     lowercase		( SCREAMING_SNAKE_CASE__	,  SCREAMING_SNAKE_CASE__	,  SCREAMING_SNAKE_CASE__	,  unittest.TestCase		):
							lowercase_	:	Any	 =(TrOCRDecoder, TrOCRForCausalLM) if is_torch_available() else ()
							lowercase_	:	Dict	 =(TrOCRForCausalLM,) if is_torch_available() else ()
							lowercase_	:	int	 ={'''text-generation''': TrOCRForCausalLM} if is_torch_available() else {}
							lowercase_	:	List[Any]	 =True
							lowercase_	:	int	 =False




							def 			A__				(    self):
												lowercase						     =						TrOCRStandaloneDecoderModelTester(self       ,is_training=A__)
												lowercase						     =						ConfigTester(self       ,config_class=A__)




							def 			A__				(    self):
												pass




							def 			A__				(    self):
												pass




							def 			A__				(    self):
												pass




							def 			A__				(    self):
												self.config_tester.run_common_tests()




							def 			A__				(    self):
												lowercase						     =						self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_decoder_model_past(*A__)




							def 			A__				(    self):
												return





							@unittest.skip('''The model doesn\'t support left padding''')  # and it's not used enough to be worth fixing :)
							def 			A__				(    self):
												pass




