





import torch
from transformers import PreTrainedModel, XLMRobertaConfig, XLMRobertaModel

class     __SCREAMING_SNAKE_CASE   (       _a    ):
							snake_case       : str         =    """M-CLIP"""






							def __init__(  self					,	__lowerCAmelCase=1024					,	__lowerCAmelCase=768					,	**__lowerCAmelCase					):
														UpperCamelCase__			       =     transformerDimSize
														UpperCamelCase__			       =     imageDimSize
														super().__init__(**__lowerCAmelCase					)






class     __SCREAMING_SNAKE_CASE   (       _a    ):
							snake_case       : int         =    MCLIPConfig






							def __init__(  self					,	__lowerCAmelCase					,	*__lowerCAmelCase					,	**__lowerCAmelCase					):
														super().__init__(__lowerCAmelCase					,	*__lowerCAmelCase					,	**__lowerCAmelCase					)
														UpperCamelCase__			       =     XLMRobertaModel(__lowerCAmelCase					)
														UpperCamelCase__			       =     torch.nn.Linear(
														    in_features=config.transformerDimensions					,	out_features=config.numDims					)







							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase					):
														UpperCamelCase__			       =     self.transformer(input_ids=__lowerCAmelCase					,	attention_mask=__lowerCAmelCase					)[0]
														UpperCamelCase__			       =     (embs * attention_mask.unsqueeze(2					)).sum(dim=1					) / attention_mask.sum(dim=1					)[:, None]
														return self.LinearTransformation(__lowerCAmelCase					), embs











from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


UpperCamelCase__     =	{
    "configuration_swiftformer": [
        "SWIFTFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "SwiftFormerConfig",
        "SwiftFormerOnnxConfig",
    ]
}

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					UpperCamelCase__     =	[
					    "SWIFTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST",
					    "SwiftFormerForImageClassification",
					    "SwiftFormerModel",
					    "SwiftFormerPreTrainedModel",
					]

if TYPE_CHECKING:
					from .configuration_swiftformer import (
					    SWIFTFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP,
					    SwiftFormerConfig,
					    SwiftFormerOnnxConfig,
					)

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_swiftformer import (
										    SWIFTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
										    SwiftFormerForImageClassification,
										    SwiftFormerModel,
										    SwiftFormerPreTrainedModel,
										)

else:
					import sys

					UpperCamelCase__     =	_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)




