


'''simple docstring'''






from collections import OrderedDict
from typing import TYPE_CHECKING, Any, List, Mapping, Optional

from packaging import version


if TYPE_CHECKING:
  from ... import PreTrainedTokenizer, TensorType

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import is_torch_available, logging


__A :  List[Any]    							=				logging.get_logger(__name__)

__A :  int    							=				{
    'bigscience/bloom': 'https://huggingface.co/bigscience/bloom/resolve/main/config.json',
    'bigscience/bloom-560m': 'https://huggingface.co/bigscience/bloom-560m/blob/main/config.json',
    'bigscience/bloom-1b1': 'https://huggingface.co/bigscience/bloom-1b1/blob/main/config.json',
    'bigscience/bloom-1b7': 'https://huggingface.co/bigscience/bloom-1b7/blob/main/config.json',
    'bigscience/bloom-3b': 'https://huggingface.co/bigscience/bloom-3b/blob/main/config.json',
    'bigscience/bloom-7b1': 'https://huggingface.co/bigscience/bloom-7b1/blob/main/config.json',
}






class        __snake_case	(       _lowerCAmelCase):







    """simple docstring"""



    lowercase       			= "bloom"
    lowercase       			= ["past_key_values"]
    lowercase       			= {
        "num_hidden_layers": "n_layer",
        "num_attention_heads": "n_head",
    }

    def __init__(       self   :  Union[str, Any]   ,						lowerCamelCase   :  Dict=25_08_80   ,						lowerCamelCase   :  str=64   ,						lowerCamelCase   :  int=2   ,						lowerCamelCase   :  Union[str, Any]=8   ,						lowerCamelCase   :  Optional[Any]=1E-5   ,						lowerCamelCase   :  Dict=0.02   ,						lowerCamelCase   :  Optional[int]=True   ,						lowerCamelCase   :  Any=1   ,						lowerCamelCase   :  Dict=2   ,						lowerCamelCase   :  Optional[Any]=False   ,						lowerCamelCase   :  Union[str, Any]=0.0   ,						lowerCamelCase   :  str=0.0   ,						lowerCamelCase   :  str=1   ,						lowerCamelCase   :  int=False   ,						**lowerCamelCase   :  List[str]   ,						)       ->  Tuple:
      lowerCAmelCase_      :				Union[str, Any]      =       vocab_size
      # Backward compatibility with n_embed kwarg
      lowerCAmelCase_      :				List[str]      =       kwargs.pop("""n_embed"""   ,						_lowercase		)
      lowerCAmelCase_      :				Optional[int]      =       hidden_size if n_embed is None else n_embed
      lowerCAmelCase_      :				Union[str, Any]      =       n_layer
      lowerCAmelCase_      :				List[Any]      =       n_head
      lowerCAmelCase_      :				int      =       layer_norm_epsilon
      lowerCAmelCase_      :				Dict      =       initializer_range
      lowerCAmelCase_      :				Optional[int]      =       use_cache
      lowerCAmelCase_      :				Any      =       pretraining_tp
      lowerCAmelCase_      :				str      =       apply_residual_connection_post_layernorm
      lowerCAmelCase_      :				List[str]      =       hidden_dropout
      lowerCAmelCase_      :				Any      =       attention_dropout

      lowerCAmelCase_      :				Dict      =       bos_token_id
      lowerCAmelCase_      :				Optional[Any]      =       eos_token_id
      lowerCAmelCase_      :				str      =       slow_but_exact

      super().__init__(bos_token_id=_lowercase   ,						eos_token_id=_lowercase   ,						**_lowercase		)







class        __snake_case	(       _lowerCAmelCase):







    """simple docstring"""



    lowercase       			= version.parse('1.12')

    def __init__(       self   :  Optional[int]   ,						lowerCamelCase   :  PretrainedConfig   ,						lowerCamelCase   :  str = "default"   ,						lowerCamelCase   :  List[PatchingSpec] = None   ,						lowerCamelCase   :  bool = False   ,						)       ->  Dict:
      super().__init__(_lowercase   ,						task=_lowercase   ,						patching_specs=_lowercase   ,						use_past=_lowercase		)
      if not getattr(self._config   ,						"""pad_token_id"""   ,						_lowercase		):
        # TODO: how to do that better?
        lowerCAmelCase_      :				Optional[int]      =       0

    @property
    def __lowercase       (       self   :  Optional[int]		)       ->  int:
      lowerCAmelCase_      :				int      =       OrderedDict({"""input_ids""": {0: """batch""", 1: """sequence"""}}		)
      if self.use_past:
        # BLOOM stores values on dynamic axis 2. For more details see: https://github.com/huggingface/transformers/pull/18344
        self.fill_with_past_key_values_(_lowercase   ,						direction="""inputs"""   ,						inverted_values_shape=_lowercase		)
        lowerCAmelCase_      :				List[Any]      =       {0: """batch""", 1: """past_sequence + sequence"""}
      else:
        lowerCAmelCase_      :				Tuple      =       {0: """batch""", 1: """sequence"""}

      return common_inputs

    @property
    def __lowercase       (       self   :  Any		)       ->  Optional[int]:
      return self._config.n_layer

    @property
    def __lowercase       (       self   :  Tuple		)       ->  Any:
      return self._config.n_head

    @property
    def __lowercase       (       self   :  Dict		)       ->  List[Any]:
      return 1E-3

    def __lowercase       (       self   :  List[str]   ,						lowerCamelCase   :  "PreTrainedTokenizer"   ,						lowerCamelCase   :  int = -1   ,						lowerCamelCase   :  int = -1   ,						lowerCamelCase   :  bool = False   ,						lowerCamelCase   :  Optional["TensorType"] = None   ,						)       ->  Union[str, Any]:
      lowerCAmelCase_      :				List[str]      =       super(_lowercase   ,						self		).generate_dummy_inputs(
          _lowercase   ,						batch_size=_lowercase   ,						seq_length=_lowercase   ,						is_pair=_lowercase   ,						framework=_lowercase		)

      # We need to order the input in the way they appears in the forward()
      lowerCAmelCase_      :				List[str]      =       OrderedDict({"""input_ids""": common_inputs["""input_ids"""]}		)

      # Need to add the past_keys
      if self.use_past:
        if not is_torch_available():
          raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""		)
        else:
          import torch

          lowerCAmelCase_,							lowerCAmelCase_      :				Any      =       common_inputs["""input_ids"""].shape
          # Not using the same length for past_key_values
          lowerCAmelCase_      :				Optional[Any]      =       seqlen + 2
          lowerCAmelCase_      :				Optional[int]      =       self._config.hidden_size // self.num_attention_heads
          lowerCAmelCase_      :				Optional[int]      =       (
              batch * self.num_attention_heads,
              head_dim,
              past_key_values_length,
          )
          lowerCAmelCase_      :				Tuple      =       (
              batch * self.num_attention_heads,
              past_key_values_length,
              head_dim,
          )
          lowerCAmelCase_      :				Union[str, Any]      =       [
              (torch.zeros(_lowercase		), torch.zeros(_lowercase		)) for _ in range(self.num_layers		)
          ]

      lowerCAmelCase_      :				Union[str, Any]      =       common_inputs["""attention_mask"""]
      if self.use_past:
        lowerCAmelCase_      :				List[Any]      =       ordered_inputs["""attention_mask"""].dtype
        lowerCAmelCase_      :				Dict      =       torch.cat(
            [ordered_inputs["""attention_mask"""], torch.ones(_lowercase   ,						_lowercase   ,						dtype=_lowercase		)]   ,						dim=1		)

      return ordered_inputs





    @property
    def __lowercase       (       self   :  Any		)       ->  List[str]:
      return 13







'''simple docstring'''






from dataclasses import dataclass
from typing import Optional, Tuple, Union

import torch
import torch.nn as nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput
from .embeddings import GaussianFourierProjection, TimestepEmbedding, Timesteps
from .modeling_utils import ModelMixin
from .unet_ad_blocks import get_down_block, get_mid_block, get_out_block, get_up_block






@dataclass
class        __snake_case	(       _SCREAMING_SNAKE_CASE):







    """simple docstring"""



    lowercase       			= 42







class        __snake_case	(       _SCREAMING_SNAKE_CASE     ,_SCREAMING_SNAKE_CASE):







    """simple docstring"""



    @register_to_config
    def __init__(       self   :  Optional[int]   ,						lowerCamelCase   :  int = 6_55_36   ,						lowerCamelCase   :  Optional[int] = None   ,						lowerCamelCase   :  int = 2   ,						lowerCamelCase   :  int = 2   ,						lowerCamelCase   :  int = 0   ,						lowerCamelCase   :  str = "fourier"   ,						lowerCamelCase   :  bool = True   ,						lowerCamelCase   :  bool = False   ,						lowerCamelCase   :  float = 0.0   ,						lowerCamelCase   :  Tuple[str] = ("DownBlock1DNoSkip", "DownBlock1D", "AttnDownBlock1D")   ,						lowerCamelCase   :  Tuple[str] = ("AttnUpBlock1D", "UpBlock1D", "UpBlock1DNoSkip")   ,						lowerCamelCase   :  Tuple[str] = "UNetMidBlock1D"   ,						lowerCamelCase   :  str = None   ,						lowerCamelCase   :  Tuple[int] = (32, 32, 64)   ,						lowerCamelCase   :  str = None   ,						lowerCamelCase   :  int = 8   ,						lowerCamelCase   :  int = 1   ,						lowerCamelCase   :  bool = False   ,						)       ->  List[Any]:
      super().__init__()
      lowerCAmelCase_      :				Optional[Any]      =       sample_size

      # time
      if time_embedding_type == "fourier":
        lowerCAmelCase_      :				int      =       GaussianFourierProjection(
            embedding_size=8   ,						set_W_to_weight=lowerCamelCase   ,						log=lowerCamelCase   ,						flip_sin_to_cos=lowerCamelCase		)
        lowerCAmelCase_      :				List[Any]      =       2 * block_out_channels[0]
      elif time_embedding_type == "positional":
        lowerCAmelCase_      :				Optional[int]      =       Timesteps(
            block_out_channels[0]   ,						flip_sin_to_cos=lowerCamelCase   ,						downscale_freq_shift=lowerCamelCase		)
        lowerCAmelCase_      :				Union[str, Any]      =       block_out_channels[0]

      if use_timestep_embedding:
        lowerCAmelCase_      :				List[str]      =       block_out_channels[0] * 4
        lowerCAmelCase_      :				List[str]      =       TimestepEmbedding(
            in_channels=lowerCamelCase   ,						time_embed_dim=lowerCamelCase   ,						act_fn=lowerCamelCase   ,						out_dim=block_out_channels[0]   ,						)

      lowerCAmelCase_      :				str      =       nn.ModuleList([]		)
      lowerCAmelCase_      :				Tuple      =       None
      lowerCAmelCase_      :				str      =       nn.ModuleList([]		)
      lowerCAmelCase_      :				List[str]      =       None

      # down
      lowerCAmelCase_      :				List[str]      =       in_channels
      for i, down_block_type in enumerate(lowerCamelCase		):
        lowerCAmelCase_      :				Optional[Any]      =       output_channel
        lowerCAmelCase_      :				str      =       block_out_channels[i]

        if i == 0:
          input_channel += extra_in_channels

        lowerCAmelCase_      :				int      =       i == len(lowerCamelCase		) - 1

        lowerCAmelCase_      :				List[str]      =       get_down_block(
            lowerCamelCase   ,						num_layers=lowerCamelCase   ,						in_channels=lowerCamelCase   ,						out_channels=lowerCamelCase   ,						temb_channels=block_out_channels[0]   ,						add_downsample=not is_final_block or downsample_each_block   ,						)
        self.down_blocks.append(lowerCamelCase		)

      # mid
      lowerCAmelCase_      :				Dict      =       get_mid_block(
          lowerCamelCase   ,						in_channels=block_out_channels[-1]   ,						mid_channels=block_out_channels[-1]   ,						out_channels=block_out_channels[-1]   ,						embed_dim=block_out_channels[0]   ,						num_layers=lowerCamelCase   ,						add_downsample=lowerCamelCase   ,						)

      # up
      lowerCAmelCase_      :				List[str]      =       list(reversed(lowerCamelCase		)		)
      lowerCAmelCase_      :				str      =       reversed_block_out_channels[0]
      if out_block_type is None:
        lowerCAmelCase_      :				Optional[int]      =       out_channels
      else:
        lowerCAmelCase_      :				Optional[Any]      =       block_out_channels[0]

      for i, up_block_type in enumerate(lowerCamelCase		):
        lowerCAmelCase_      :				Any      =       output_channel
        lowerCAmelCase_      :				str      =       (
            reversed_block_out_channels[i + 1] if i < len(lowerCamelCase		) - 1 else final_upsample_channels
        )

        lowerCAmelCase_      :				int      =       i == len(lowerCamelCase		) - 1

        lowerCAmelCase_      :				Any      =       get_up_block(
            lowerCamelCase   ,						num_layers=lowerCamelCase   ,						in_channels=lowerCamelCase   ,						out_channels=lowerCamelCase   ,						temb_channels=block_out_channels[0]   ,						add_upsample=not is_final_block   ,						)
        self.up_blocks.append(lowerCamelCase		)
        lowerCAmelCase_      :				Optional[int]      =       output_channel

      # out
      lowerCAmelCase_      :				str      =       norm_num_groups if norm_num_groups is not None else min(block_out_channels[0] // 4   ,						32		)
      lowerCAmelCase_      :				Union[str, Any]      =       get_out_block(
          out_block_type=lowerCamelCase   ,						num_groups_out=lowerCamelCase   ,						embed_dim=block_out_channels[0]   ,						out_channels=lowerCamelCase   ,						act_fn=lowerCamelCase   ,						fc_dim=block_out_channels[-1] // 4   ,						)





    def __lowercase       (       self   :  List[Any]   ,						lowerCamelCase   :  torch.FloatTensor   ,						lowerCamelCase   :  Union[torch.Tensor, float, int]   ,						lowerCamelCase   :  bool = True   ,						)       ->  Union[UNetaDOutput, Tuple]:
      lowerCAmelCase_      :				Optional[Any]      =       timestep
      if not torch.is_tensor(lowerCamelCase		):
        lowerCAmelCase_      :				Any      =       torch.tensor([timesteps]   ,						dtype=torch.long   ,						device=sample.device		)
      elif torch.is_tensor(lowerCamelCase		) and len(timesteps.shape		) == 0:
        lowerCAmelCase_      :				Dict      =       timesteps[None].to(sample.device		)

      lowerCAmelCase_      :				int      =       self.time_proj(lowerCamelCase		)
      if self.config.use_timestep_embedding:
        lowerCAmelCase_      :				int      =       self.time_mlp(lowerCamelCase		)
      else:
        lowerCAmelCase_      :				Tuple      =       timestep_embed[..., None]
        lowerCAmelCase_      :				Dict      =       timestep_embed.repeat([1, 1, sample.shape[2]]		).to(sample.dtype		)
        lowerCAmelCase_      :				Union[str, Any]      =       timestep_embed.broadcast_to((sample.shape[:1] + timestep_embed.shape[1:])		)

      # 2. down
      lowerCAmelCase_      :				Optional[Any]      =       ()
      for downsample_block in self.down_blocks:
        lowerCAmelCase_,							lowerCAmelCase_      :				Any      =       downsample_block(hidden_states=lowerCamelCase   ,						temb=lowerCamelCase		)
        down_block_res_samples += res_samples

      # 3. mid
      if self.mid_block:
        lowerCAmelCase_      :				int      =       self.mid_block(lowerCamelCase   ,						lowerCamelCase		)

      # 4. up
      for i, upsample_block in enumerate(self.up_blocks		):
        lowerCAmelCase_      :				str      =       down_block_res_samples[-1:]
        lowerCAmelCase_      :				Union[str, Any]      =       down_block_res_samples[:-1]
        lowerCAmelCase_      :				List[str]      =       upsample_block(lowerCamelCase   ,						res_hidden_states_tuple=lowerCamelCase   ,						temb=lowerCamelCase		)

      # 5. post-process
      if self.out_block:
        lowerCAmelCase_      :				Union[str, Any]      =       self.out_block(lowerCamelCase   ,						lowerCamelCase		)

      if not return_dict:
        return (sample,)

      return UNetaDOutput(sample=lowerCamelCase		)



