



from typing import Optional

import torch
import torch.utils.checkpoint
from torch import Tensor, nn
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss

from ...activations import ACTaFN
from ...modeling_outputs import (
    BackboneOutput,
    BaseModelOutputWithNoAttention,
    BaseModelOutputWithPoolingAndNoAttention,
    ImageClassifierOutputWithNoAttention,
)
from ...modeling_utils import PreTrainedModel
from ...utils import (
    add_code_sample_docstrings,
    add_start_docstrings,
    add_start_docstrings_to_model_forward,
    logging,
    replace_return_docstrings,
)
from ...utils.backbone_utils import BackboneMixin
from .configuration_resnet import ResNetConfig


__snake_case						:Dict						=					logging.get_logger(__name__)

# General docstring
__snake_case						:List[str]						=					'''ResNetConfig'''

# Base docstring
__snake_case						:Dict						=					'''microsoft/resnet-50'''
__snake_case						:Optional[Any]						=					[1, 2048, 7, 7]

# Image classification docstring
__snake_case						:List[Any]						=					'''microsoft/resnet-50'''
__snake_case						:List[Any]						=					'''tiger cat'''

__snake_case						:Tuple						=					[
    '''microsoft/resnet-50''',
    # See all resnet models at https://huggingface.co/models?filter=resnet
]

class    _A	(     nn.Module   ):



       def __init__(		self							:					Dict					,		__SCREAMING_SNAKE_CASE							:					int					,		__SCREAMING_SNAKE_CASE							:					int					,		__SCREAMING_SNAKE_CASE							:					int = 3					,		__SCREAMING_SNAKE_CASE							:					int = 1					,		__SCREAMING_SNAKE_CASE							:					str = "relu"):


           '''simple docstring'''

           super().__init__()
           __a							=    nn.Convad(
               __SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE					,		kernel_size=__SCREAMING_SNAKE_CASE					,		stride=__SCREAMING_SNAKE_CASE					,		padding=kernel_size // 2					,		bias=__SCREAMING_SNAKE_CASE)
           __a							=    nn.BatchNormad(__SCREAMING_SNAKE_CASE)
           __a							=    ACTaFN[activation] if activation is not None else nn.Identity()





       def 		_lowerCamelCase      (		self							:					List[Any]					,		__SCREAMING_SNAKE_CASE							:					Tensor):


           '''simple docstring'''

           __a							=    self.convolution(__SCREAMING_SNAKE_CASE)
           __a							=    self.normalization(__SCREAMING_SNAKE_CASE)
           __a							=    self.activation(__SCREAMING_SNAKE_CASE)
           return hidden_state


class    _A	(     nn.Module   ):



       def __init__(		self							:					List[str]					,		__SCREAMING_SNAKE_CASE							:					ResNetConfig):


           '''simple docstring'''

           super().__init__()
           __a							=    ResNetConvLayer(
               config.num_channels					,		config.embedding_size					,		kernel_size=7					,		stride=2					,		activation=config.hidden_act)
           __a							=    nn.MaxPoolad(kernel_size=3					,		stride=2					,		padding=1)
           __a							=    config.num_channels





       def 		_lowerCamelCase      (		self							:					List[Any]					,		__SCREAMING_SNAKE_CASE							:					Tensor):


           '''simple docstring'''

           __a							=    pixel_values.shape[1]
           if num_channels != self.num_channels:
               raise ValueError(
                   '''Make sure that the channel dimension of the pixel values match with the one set in the configuration.''')
           __a							=    self.embedder(__SCREAMING_SNAKE_CASE)
           __a							=    self.pooler(__SCREAMING_SNAKE_CASE)
           return embedding


class    _A	(     nn.Module   ):



       def __init__(		self							:					List[str]					,		__SCREAMING_SNAKE_CASE							:					int					,		__SCREAMING_SNAKE_CASE							:					int					,		__SCREAMING_SNAKE_CASE							:					int = 2):


           '''simple docstring'''

           super().__init__()
           __a							=    nn.Convad(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE					,		kernel_size=1					,		stride=__SCREAMING_SNAKE_CASE					,		bias=__SCREAMING_SNAKE_CASE)
           __a							=    nn.BatchNormad(__SCREAMING_SNAKE_CASE)





       def 		_lowerCamelCase      (		self							:					Tuple					,		__SCREAMING_SNAKE_CASE							:					Tensor):


           '''simple docstring'''

           __a							=    self.convolution(__SCREAMING_SNAKE_CASE)
           __a							=    self.normalization(__SCREAMING_SNAKE_CASE)
           return hidden_state


class    _A	(     nn.Module   ):



       def __init__(		self							:					Any					,		__SCREAMING_SNAKE_CASE							:					int					,		__SCREAMING_SNAKE_CASE							:					int					,		__SCREAMING_SNAKE_CASE							:					int = 1					,		__SCREAMING_SNAKE_CASE							:					str = "relu"):


           '''simple docstring'''

           super().__init__()
           __a							=    in_channels != out_channels or stride != 1
           __a							=    (
               ResNetShortCut(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE					,		stride=__SCREAMING_SNAKE_CASE) if should_apply_shortcut else nn.Identity()
           )
           __a							=    nn.Sequential(
               ResNetConvLayer(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE					,		stride=__SCREAMING_SNAKE_CASE)					,		ResNetConvLayer(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE					,		activation=__SCREAMING_SNAKE_CASE)					,		)
           __a							=    ACTaFN[activation]





       def 		_lowerCamelCase      (		self							:					Dict					,		__SCREAMING_SNAKE_CASE							:					str):


           '''simple docstring'''

           __a							=    hidden_state
           __a							=    self.layer(__SCREAMING_SNAKE_CASE)
           __a							=    self.shortcut(__SCREAMING_SNAKE_CASE)
           hidden_state += residual
           __a							=    self.activation(__SCREAMING_SNAKE_CASE)
           return hidden_state


class    _A	(     nn.Module   ):



       def __init__(		self							:					Dict					,		__SCREAMING_SNAKE_CASE							:					int					,		__SCREAMING_SNAKE_CASE							:					int					,		__SCREAMING_SNAKE_CASE							:					int = 1					,		__SCREAMING_SNAKE_CASE							:					str = "relu"					,		__SCREAMING_SNAKE_CASE							:					int = 4):


           '''simple docstring'''

           super().__init__()
           __a							=    in_channels != out_channels or stride != 1
           __a							=    out_channels // reduction
           __a							=    (
               ResNetShortCut(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE					,		stride=__SCREAMING_SNAKE_CASE) if should_apply_shortcut else nn.Identity()
           )
           __a							=    nn.Sequential(
               ResNetConvLayer(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE					,		kernel_size=1)					,		ResNetConvLayer(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE					,		stride=__SCREAMING_SNAKE_CASE)					,		ResNetConvLayer(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE					,		kernel_size=1					,		activation=__SCREAMING_SNAKE_CASE)					,		)
           __a							=    ACTaFN[activation]





       def 		_lowerCamelCase      (		self							:					Dict					,		__SCREAMING_SNAKE_CASE							:					Dict):


           '''simple docstring'''

           __a							=    hidden_state
           __a							=    self.layer(__SCREAMING_SNAKE_CASE)
           __a							=    self.shortcut(__SCREAMING_SNAKE_CASE)
           hidden_state += residual
           __a							=    self.activation(__SCREAMING_SNAKE_CASE)
           return hidden_state


class    _A	(     nn.Module   ):



       def __init__(		self							:					str					,		__SCREAMING_SNAKE_CASE							:					ResNetConfig					,		__SCREAMING_SNAKE_CASE							:					int					,		__SCREAMING_SNAKE_CASE							:					int					,		__SCREAMING_SNAKE_CASE							:					int = 2					,		__SCREAMING_SNAKE_CASE							:					int = 2					,		):


           '''simple docstring'''

           super().__init__()

           __a							=    ResNetBottleNeckLayer if config.layer_type == '''bottleneck''' else ResNetBasicLayer

           __a							=    nn.Sequential(
               # downsampling is done in the first layer with stride of 2
               layer(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE					,		stride=__SCREAMING_SNAKE_CASE					,		activation=config.hidden_act)					,		*[layer(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE					,		activation=config.hidden_act) for _ in range(depth - 1)]					,		)





       def 		_lowerCamelCase      (		self							:					Optional[Any]					,		__SCREAMING_SNAKE_CASE							:					Tensor):


           '''simple docstring'''

           __a							=    input
           for layer in self.layers:
               __a							=    layer(__SCREAMING_SNAKE_CASE)
           return hidden_state


class    _A	(     nn.Module   ):



       def __init__(		self							:					Optional[int]					,		__SCREAMING_SNAKE_CASE							:					ResNetConfig):


           '''simple docstring'''

           super().__init__()
           __a							=    nn.ModuleList([])
           # based on `downsample_in_first_stage` the first layer of the first stage may or may not downsample the input
           self.stages.append(
               ResNetStage(
                   __SCREAMING_SNAKE_CASE					,		config.embedding_size					,		config.hidden_sizes[0]					,		stride=2 if config.downsample_in_first_stage else 1					,		depth=config.depths[0]					,		))
           __a							=    zip(config.hidden_sizes					,		config.hidden_sizes[1:])
           for (in_channels, out_channels), depth in zip(__SCREAMING_SNAKE_CASE					,		config.depths[1:]):
               self.stages.append(ResNetStage(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE					,		depth=__SCREAMING_SNAKE_CASE))





       def 		_lowerCamelCase      (		self							:					Union[str, Any]					,		__SCREAMING_SNAKE_CASE							:					Tensor					,		__SCREAMING_SNAKE_CASE							:					bool = False					,		__SCREAMING_SNAKE_CASE							:					bool = True):


           '''simple docstring'''

           __a							=    () if output_hidden_states else None

           for stage_module in self.stages:
               if output_hidden_states:
                   __a							=    hidden_states + (hidden_state,)

               __a							=    stage_module(__SCREAMING_SNAKE_CASE)

           if output_hidden_states:
               __a							=    hidden_states + (hidden_state,)

           if not return_dict:
               return tuple(v for v in [hidden_state, hidden_states] if v is not None)

           return BaseModelOutputWithNoAttention(
               last_hidden_state=__SCREAMING_SNAKE_CASE					,		hidden_states=__SCREAMING_SNAKE_CASE					,		)


class    _A	(     __UpperCAmelCase   ):
       UpperCamelCase__  :   Any						 =						ResNetConfig
       UpperCamelCase__  :   List[Any]						 =						'''resnet'''
       UpperCamelCase__  :   List[Any]						 =						'''pixel_values'''
       UpperCamelCase__  :   str						 =						True



       def 		_lowerCamelCase      (		self							:					Any					,		__SCREAMING_SNAKE_CASE							:					List[Any]):


           '''simple docstring'''

           if isinstance(__SCREAMING_SNAKE_CASE					,		nn.Convad):
               nn.init.kaiming_normal_(module.weight					,		mode='''fan_out'''					,		nonlinearity='''relu''')
           elif isinstance(__SCREAMING_SNAKE_CASE					,		(nn.BatchNormad, nn.GroupNorm)):
               nn.init.constant_(module.weight					,		1)
               nn.init.constant_(module.bias					,		0)





       def 		_lowerCamelCase      (		self							:					int					,		__SCREAMING_SNAKE_CASE							:					int					,		__SCREAMING_SNAKE_CASE							:					Optional[Any]=False):


           '''simple docstring'''

           if isinstance(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE):
               __a							=    value


__snake_case						:Union[str, Any]						=					r'''
    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it
    as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`ResNetConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
'''

__snake_case						:List[str]						=					r'''
    Args:
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
            [`ConvNextImageProcessor.__call__`] for details.

        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
'''

@add_start_docstrings(
    '''The bare ResNet model outputting raw features without any specific head on top.'''				,__UpperCAmelCase				,)
class    _A	(     __UpperCAmelCase   ):



       def __init__(		self							:					Union[str, Any]					,		__SCREAMING_SNAKE_CASE							:					List[str]):


           '''simple docstring'''

           super().__init__(__SCREAMING_SNAKE_CASE)
           __a							=    config
           __a							=    ResNetEmbeddings(__SCREAMING_SNAKE_CASE)
           __a							=    ResNetEncoder(__SCREAMING_SNAKE_CASE)
           __a							=    nn.AdaptiveAvgPoolad((1, 1))
           # Initialize weights and apply final processing
           self.post_init()





       @add_start_docstrings_to_model_forward(__SCREAMING_SNAKE_CASE)
       @add_code_sample_docstrings(
           checkpoint=_CHECKPOINT_FOR_DOC					,		output_type=__SCREAMING_SNAKE_CASE					,		config_class=_CONFIG_FOR_DOC					,		modality='''vision'''					,		expected_output=_EXPECTED_OUTPUT_SHAPE					,		)
       def 		_lowerCamelCase      (		self							:					int					,		__SCREAMING_SNAKE_CASE							:					Tensor					,		__SCREAMING_SNAKE_CASE							:					Optional[bool] = None					,		__SCREAMING_SNAKE_CASE							:					Optional[bool] = None):


           '''simple docstring'''

           __a							=    (
               output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
           )
           __a							=    return_dict if return_dict is not None else self.config.use_return_dict

           __a							=    self.embedder(__SCREAMING_SNAKE_CASE)

           __a							=    self.encoder(
               __SCREAMING_SNAKE_CASE					,		output_hidden_states=__SCREAMING_SNAKE_CASE					,		return_dict=__SCREAMING_SNAKE_CASE)

           __a							=    encoder_outputs[0]

           __a							=    self.pooler(__SCREAMING_SNAKE_CASE)

           if not return_dict:
               return (last_hidden_state, pooled_output) + encoder_outputs[1:]

           return BaseModelOutputWithPoolingAndNoAttention(
               last_hidden_state=__SCREAMING_SNAKE_CASE					,		pooler_output=__SCREAMING_SNAKE_CASE					,		hidden_states=encoder_outputs.hidden_states					,		)


@add_start_docstrings(
    '''
    ResNet Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for
    ImageNet.
    '''				,__UpperCAmelCase				,)
class    _A	(     __UpperCAmelCase   ):



       def __init__(		self							:					str					,		__SCREAMING_SNAKE_CASE							:					List[Any]):


           '''simple docstring'''

           super().__init__(__SCREAMING_SNAKE_CASE)
           __a							=    config.num_labels
           __a							=    ResNetModel(__SCREAMING_SNAKE_CASE)
           # classification head
           __a							=    nn.Sequential(
               nn.Flatten()					,		nn.Linear(config.hidden_sizes[-1]					,		config.num_labels) if config.num_labels > 0 else nn.Identity()					,		)
           # initialize weights and apply final processing
           self.post_init()





       @add_start_docstrings_to_model_forward(__SCREAMING_SNAKE_CASE)
       @add_code_sample_docstrings(
           checkpoint=_IMAGE_CLASS_CHECKPOINT					,		output_type=__SCREAMING_SNAKE_CASE					,		config_class=_CONFIG_FOR_DOC					,		expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT					,		)
       def 		_lowerCamelCase      (		self							:					Union[str, Any]					,		__SCREAMING_SNAKE_CASE							:					Optional[torch.FloatTensor] = None					,		__SCREAMING_SNAKE_CASE							:					Optional[torch.LongTensor] = None					,		__SCREAMING_SNAKE_CASE							:					Optional[bool] = None					,		__SCREAMING_SNAKE_CASE							:					Optional[bool] = None					,		):


           '''simple docstring'''

           __a							=    return_dict if return_dict is not None else self.config.use_return_dict

           __a							=    self.resnet(__SCREAMING_SNAKE_CASE					,		output_hidden_states=__SCREAMING_SNAKE_CASE					,		return_dict=__SCREAMING_SNAKE_CASE)

           __a							=    outputs.pooler_output if return_dict else outputs[1]

           __a							=    self.classifier(__SCREAMING_SNAKE_CASE)

           __a							=    None

           if labels is not None:
               if self.config.problem_type is None:
                   if self.num_labels == 1:
                       __a							=    '''regression'''
                   elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
                       __a							=    '''single_label_classification'''
                   else:
                       __a							=    '''multi_label_classification'''
               if self.config.problem_type == "regression":
                   __a							=    MSELoss()
                   if self.num_labels == 1:
                       __a							=    loss_fct(logits.squeeze()					,		labels.squeeze())
                   else:
                       __a							=    loss_fct(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE)
               elif self.config.problem_type == "single_label_classification":
                   __a							=    CrossEntropyLoss()
                   __a							=    loss_fct(logits.view(-1					,		self.num_labels)					,		labels.view(-1))
               elif self.config.problem_type == "multi_label_classification":
                   __a							=    BCEWithLogitsLoss()
                   __a							=    loss_fct(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE)

           if not return_dict:
               __a							=    (logits,) + outputs[2:]
               return (loss,) + output if loss is not None else output

           return ImageClassifierOutputWithNoAttention(loss=__SCREAMING_SNAKE_CASE					,		logits=__SCREAMING_SNAKE_CASE					,		hidden_states=outputs.hidden_states)


@add_start_docstrings(
    '''
    ResNet backbone, to be used with frameworks like DETR and MaskFormer.
    '''				,__UpperCAmelCase				,)
class    _A	(     __UpperCAmelCase				,__UpperCAmelCase   ):



       def __init__(		self							:					List[Any]					,		__SCREAMING_SNAKE_CASE							:					Optional[Any]):


           '''simple docstring'''

           super().__init__(__SCREAMING_SNAKE_CASE)
           super()._init_backbone(__SCREAMING_SNAKE_CASE)

           __a							=    [config.embedding_size] + config.hidden_sizes
           __a							=    ResNetEmbeddings(__SCREAMING_SNAKE_CASE)
           __a							=    ResNetEncoder(__SCREAMING_SNAKE_CASE)

           # initialize weights and apply final processing
           self.post_init()





       @add_start_docstrings_to_model_forward(__SCREAMING_SNAKE_CASE)
       @replace_return_docstrings(output_type=__SCREAMING_SNAKE_CASE					,		config_class=_CONFIG_FOR_DOC)
       def 		_lowerCamelCase      (		self							:					Optional[Any]					,		__SCREAMING_SNAKE_CASE							:					Tensor					,		__SCREAMING_SNAKE_CASE							:					Optional[bool] = None					,		__SCREAMING_SNAKE_CASE							:					Optional[bool] = None):


           '''simple docstring'''

           __a							=    return_dict if return_dict is not None else self.config.use_return_dict
           __a							=    (
               output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
           )

           __a							=    self.embedder(__SCREAMING_SNAKE_CASE)

           __a							=    self.encoder(__SCREAMING_SNAKE_CASE					,		output_hidden_states=__SCREAMING_SNAKE_CASE					,		return_dict=__SCREAMING_SNAKE_CASE)

           __a							=    outputs.hidden_states

           __a							=    ()
           for idx, stage in enumerate(self.stage_names):
               if stage in self.out_features:
                   feature_maps += (hidden_states[idx],)

           if not return_dict:
               __a							=    (feature_maps,)
               if output_hidden_states:
                   output += (outputs.hidden_states,)
               return output

           return BackboneOutput(
               feature_maps=__SCREAMING_SNAKE_CASE					,		hidden_states=outputs.hidden_states if output_hidden_states else None					,		attentions=__SCREAMING_SNAKE_CASE					,		)










from ...configuration_utils import PretrainedConfig
from ...utils import logging


__snake_case						:Dict						=					logging.get_logger(__name__)

__snake_case						:List[Any]						=					{
    '''tanreinama/GPTSAN-2.8B-spout_is_uniform''': (
        '''https://huggingface.co/tanreinama/GPTSAN-2.8B-spout_is_uniform/resolve/main/config.json'''
    ),
}

class    _A	(     __UpperCAmelCase   ):
       UpperCamelCase__  :   List[str]						 =						'''gptsan-japanese'''
       UpperCamelCase__  :   Dict						 =						[
           '''past_key_values''',
       ]
       UpperCamelCase__  :   Dict						 =						{
           '''hidden_size''': '''d_model''',
           '''num_attention_heads''': '''num_heads''',
           '''num_hidden_layers''': '''num_layers''',
       }



       def __init__(		self							:					List[str]					,		__SCREAMING_SNAKE_CASE							:					Optional[Any]=36_000					,		__SCREAMING_SNAKE_CASE							:					Tuple=1_280					,		__SCREAMING_SNAKE_CASE							:					List[Any]=1_024					,		__SCREAMING_SNAKE_CASE							:					List[Any]=8_192					,		__SCREAMING_SNAKE_CASE							:					str=4_096					,		__SCREAMING_SNAKE_CASE							:					Any=128					,		__SCREAMING_SNAKE_CASE							:					int=10					,		__SCREAMING_SNAKE_CASE							:					Optional[int]=0					,		__SCREAMING_SNAKE_CASE							:					Optional[Any]=16					,		__SCREAMING_SNAKE_CASE							:					List[Any]=16					,		__SCREAMING_SNAKE_CASE							:					Optional[Any]=128					,		__SCREAMING_SNAKE_CASE							:					Tuple=0.0					,		__SCREAMING_SNAKE_CASE							:					List[Any]=1E-5					,		__SCREAMING_SNAKE_CASE							:					str=False					,		__SCREAMING_SNAKE_CASE							:					Optional[int]=0.0					,		__SCREAMING_SNAKE_CASE							:					List[str]="float32"					,		__SCREAMING_SNAKE_CASE							:					List[str]=False					,		__SCREAMING_SNAKE_CASE							:					Any=False					,		__SCREAMING_SNAKE_CASE							:					Any=False					,		__SCREAMING_SNAKE_CASE							:					int=0.0_02					,		__SCREAMING_SNAKE_CASE							:					Any=False					,		__SCREAMING_SNAKE_CASE							:					Tuple=True					,		__SCREAMING_SNAKE_CASE							:					int=35_998					,		__SCREAMING_SNAKE_CASE							:					Optional[int]=35_995					,		__SCREAMING_SNAKE_CASE							:					List[str]=35_999					,		**__SCREAMING_SNAKE_CASE							:					List[str]					,		):


           '''simple docstring'''

           __a							=    vocab_size
           __a							=    max_position_embeddings
           __a							=    d_model
           __a							=    d_ff
           __a							=    d_ext
           __a							=    d_spout
           __a							=    num_switch_layers
           __a							=    num_ext_layers
           __a							=    num_switch_layers + num_ext_layers
           __a							=    num_heads
           __a							=    num_experts
           __a							=    expert_capacity
           __a							=    dropout_rate
           __a							=    layer_norm_epsilon
           __a							=    router_bias
           __a							=    router_jitter_noise
           __a							=    router_dtype
           __a							=    router_ignore_padding_tokens
           __a							=    output_hidden_states
           __a							=    output_attentions
           __a							=    initializer_factor
           __a							=    output_router_logits
           __a							=    use_cache

           super().__init__(
               separator_token_id=__SCREAMING_SNAKE_CASE					,		pad_token_id=__SCREAMING_SNAKE_CASE					,		eos_token_id=__SCREAMING_SNAKE_CASE					,		**__SCREAMING_SNAKE_CASE					,		)





