


'''simple docstring'''



import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowercase      :      Any            =					logging.get_logger(__name__)

_lowercase      :      Dict            =					{
    """asapp/sew-d-tiny-100k""": """https://huggingface.co/asapp/sew-d-tiny-100k/resolve/main/config.json""",
    # See all SEW-D models at https://huggingface.co/models?filter=sew-d
}






class 	UpperCamelCase__(       lowerCAmelCase	):
				__magic_name__    :				Dict  		=	"sew-d"




				def __init__(		self      :			Optional[Any]				,   lowerCAmelCase      :			str=32				,   lowerCAmelCase      :			int=768				,   lowerCAmelCase      :			Optional[Any]=12				,   lowerCAmelCase      :			Optional[int]=12				,   lowerCAmelCase      :			Tuple=3072				,   lowerCAmelCase      :			str=2				,   lowerCAmelCase      :			Optional[int]=512				,   lowerCAmelCase      :			Optional[Any]=256				,   lowerCAmelCase      :			Optional[Any]=True				,   lowerCAmelCase      :			Tuple=True				,   lowerCAmelCase      :			Union[str, Any]=("p2c", "c2p")				,   lowerCAmelCase      :			Tuple="layer_norm"				,   lowerCAmelCase      :			Tuple="gelu_python"				,   lowerCAmelCase      :			Optional[int]=0.1				,   lowerCAmelCase      :			Dict=0.1				,   lowerCAmelCase      :			Optional[int]=0.1				,   lowerCAmelCase      :			Any=0.0				,   lowerCAmelCase      :			Union[str, Any]=0.1				,   lowerCAmelCase      :			int=0.02				,   lowerCAmelCase      :			List[str]=1E-7				,   lowerCAmelCase      :			Dict=1E-5				,   lowerCAmelCase      :			Any="group"				,   lowerCAmelCase      :			int="gelu"				,   lowerCAmelCase      :			Optional[Any]=(64, 128, 128, 128, 128, 256, 256, 256, 256, 512, 512, 512, 512)				,   lowerCAmelCase      :			str=(5, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1)				,   lowerCAmelCase      :			Optional[Any]=(10, 3, 1, 3, 1, 3, 1, 3, 1, 2, 1, 2, 1)				,   lowerCAmelCase      :			Tuple=False				,   lowerCAmelCase      :			str=128				,   lowerCAmelCase      :			Optional[Any]=16				,   lowerCAmelCase      :			List[str]=True				,   lowerCAmelCase      :			str=0.05				,   lowerCAmelCase      :			Optional[Any]=10				,   lowerCAmelCase      :			Tuple=2				,   lowerCAmelCase      :			Union[str, Any]=0.0				,   lowerCAmelCase      :			List[str]=10				,   lowerCAmelCase      :			Optional[int]=0				,   lowerCAmelCase      :			Union[str, Any]="mean"				,   lowerCAmelCase      :			Tuple=False				,   lowerCAmelCase      :			int=False				,   lowerCAmelCase      :			Any=256				,   lowerCAmelCase      :			int=0				,   lowerCAmelCase      :			int=1				,   lowerCAmelCase      :			List[str]=2				,   **lowerCAmelCase      :			List[str]				,   )->					int:

										"""simple docstring"""




										super().__init__(**lowerCAmelCase				,   pad_token_id=lowerCAmelCase				,   bos_token_id=lowerCAmelCase				,   eos_token_id=lowerCAmelCase				)
										UpperCAmelCase					    =     hidden_size
										UpperCAmelCase					    =     feat_extract_norm
										UpperCAmelCase					    =     feat_extract_activation
										UpperCAmelCase					    =     list(lowerCAmelCase				)
										UpperCAmelCase					    =     list(lowerCAmelCase				)
										UpperCAmelCase					    =     list(lowerCAmelCase				)
										UpperCAmelCase					    =     conv_bias
										UpperCAmelCase					    =     num_conv_pos_embeddings
										UpperCAmelCase					    =     num_conv_pos_embedding_groups
										UpperCAmelCase					    =     len(self.conv_dim				)
										UpperCAmelCase					    =     num_hidden_layers
										UpperCAmelCase					    =     intermediate_size
										UpperCAmelCase					    =     squeeze_factor
										UpperCAmelCase					    =     max_position_embeddings
										UpperCAmelCase					    =     position_buckets
										UpperCAmelCase					    =     share_att_key
										UpperCAmelCase					    =     relative_attention
										UpperCAmelCase					    =     norm_rel_ebd
										UpperCAmelCase					    =     list(lowerCAmelCase				)
										UpperCAmelCase					    =     hidden_act
										UpperCAmelCase					    =     num_attention_heads
										UpperCAmelCase					    =     hidden_dropout
										UpperCAmelCase					    =     attention_dropout
										UpperCAmelCase					    =     activation_dropout
										UpperCAmelCase					    =     feat_proj_dropout
										UpperCAmelCase					    =     final_dropout
										UpperCAmelCase					    =     layer_norm_eps
										UpperCAmelCase					    =     feature_layer_norm_eps
										UpperCAmelCase					    =     initializer_range
										UpperCAmelCase					    =     vocab_size

										if (
										    (len(self.conv_stride				) != self.num_feat_extract_layers)
										    or (len(self.conv_kernel				) != self.num_feat_extract_layers)
										    or (len(self.conv_dim				) != self.num_feat_extract_layers)
										):
																raise ValueError(
																    '''Configuration for convolutional layers is incorrect.'''
																    '''It is required that `len(config.conv_dim)` == `len(config.conv_stride)` == `len(config.conv_kernel)`,'''
																    F"""but is `len(config.conv_dim) = {len(self.conv_dim				)}`, `len(config.conv_stride)"""
																    F"""= {len(self.conv_stride				)}`, `len(config.conv_kernel) = {len(self.conv_kernel				)}`."""				)

										# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
										UpperCAmelCase					    =     apply_spec_augment
										UpperCAmelCase					    =     mask_time_prob
										UpperCAmelCase					    =     mask_time_length
										UpperCAmelCase					    =     mask_time_min_masks
										UpperCAmelCase					    =     mask_feature_prob
										UpperCAmelCase					    =     mask_feature_length
										UpperCAmelCase					    =     mask_feature_min_masks

										# ctc loss
										UpperCAmelCase					    =     ctc_loss_reduction
										UpperCAmelCase					    =     ctc_zero_infinity

										# sequence classification
										UpperCAmelCase					    =     use_weighted_layer_sum
										UpperCAmelCase					    =     classifier_proj_size





				@property
				def 					a__(		self      :			str				)->					List[str]:

										"""simple docstring"""




										return functools.reduce(operator.mul				,   self.conv_stride				,   1				)









'''simple docstring'''



import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
	from transformers.pipelines.conversational import Conversation


_lowercase      :      Optional[Any]            =					logging.get_logger(__name__)

_lowercase      :      List[str]            =					{"""tokenizer_file""": """tokenizer.json"""}

_lowercase      :      Optional[Any]            =					{
    """tokenizer_file""": {
        """bigscience/tokenizer""": """https://huggingface.co/bigscience/tokenizer/blob/main/tokenizer.json""",
        """bigscience/bloom-560m""": """https://huggingface.co/bigscience/bloom-560m/blob/main/tokenizer.json""",
        """bigscience/bloom-1b1""": """https://huggingface.co/bigscience/bloom-1b1/blob/main/tokenizer.json""",
        """bigscience/bloom-1b7""": """https://huggingface.co/bigscience/bloom-1b7/blob/main/tokenizer.json""",
        """bigscience/bloom-3b""": """https://huggingface.co/bigscience/bloom-3b/blob/main/tokenizer.json""",
        """bigscience/bloom-7b1""": """https://huggingface.co/bigscience/bloom-7b1/blob/main/tokenizer.json""",
        """bigscience/bloom""": """https://huggingface.co/bigscience/bloom/blob/main/tokenizer.json""",
    },
}






class 	UpperCamelCase__(       lowerCAmelCase	):
				__magic_name__    :				List[str]  		=	VOCAB_FILES_NAMES
				__magic_name__    :				Optional[int]  		=	PRETRAINED_VOCAB_FILES_MAP
				__magic_name__    :				Any  		=	["input_ids", "attention_mask"]
				__magic_name__    :				Optional[int]  		=	None




				def __init__(		self      :			List[Any]				,   lowerCAmelCase      :			str=None				,   lowerCAmelCase      :			Tuple=None				,   lowerCAmelCase      :			str=None				,   lowerCAmelCase      :			Optional[int]="<unk>"				,   lowerCAmelCase      :			int="<s>"				,   lowerCAmelCase      :			Dict="</s>"				,   lowerCAmelCase      :			Union[str, Any]="<pad>"				,   lowerCAmelCase      :			int=False				,   lowerCAmelCase      :			Optional[int]=False				,   **lowerCAmelCase      :			Optional[int]				,   )->					Union[str, Any]:

										"""simple docstring"""




										super().__init__(
										    lowerCAmelCase				,   lowerCAmelCase				,   tokenizer_file=lowerCAmelCase				,   unk_token=lowerCAmelCase				,   bos_token=lowerCAmelCase				,   eos_token=lowerCAmelCase				,   pad_token=lowerCAmelCase				,   add_prefix_space=lowerCAmelCase				,   clean_up_tokenization_spaces=lowerCAmelCase				,   **lowerCAmelCase				,   )
										UpperCAmelCase					    =     json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()				)
										if pre_tok_state.get('''add_prefix_space'''				,   lowerCAmelCase				) != add_prefix_space:
																UpperCAmelCase					    =     getattr(lowerCAmelCase				,   pre_tok_state.pop('''type'''				)				)
																UpperCAmelCase					    =     add_prefix_space
																UpperCAmelCase					    =     pre_tok_class(**lowerCAmelCase				)

										UpperCAmelCase					    =     add_prefix_space




				def 					a__(		self      :			str				,   *lowerCAmelCase      :			Dict				,   **lowerCAmelCase      :			List[str]				)->					BatchEncoding:

										"""simple docstring"""




										UpperCAmelCase					    =     kwargs.get('''is_split_into_words'''				,   lowerCAmelCase				)
										if not (self.add_prefix_space or not is_split_into_words):
																raise Exception(
																    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with"""
																    ''' pretokenized inputs.'''				)

										return super()._batch_encode_plus(*lowerCAmelCase				,   **lowerCAmelCase				)




				def 					a__(		self      :			Union[str, Any]				,   *lowerCAmelCase      :			List[str]				,   **lowerCAmelCase      :			Union[str, Any]				)->					BatchEncoding:

										"""simple docstring"""




										UpperCAmelCase					    =     kwargs.get('''is_split_into_words'''				,   lowerCAmelCase				)

										if not (self.add_prefix_space or not is_split_into_words):
																raise Exception(
																    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with"""
																    ''' pretokenized inputs.'''				)

										return super()._encode_plus(*lowerCAmelCase				,   **lowerCAmelCase				)




				def 					a__(		self      :			Union[str, Any]				,   lowerCAmelCase      :			str				,   lowerCAmelCase      :			Optional[str] = None				)->					Tuple[str]:

										"""simple docstring"""




										UpperCAmelCase					    =     self._tokenizer.model.save(lowerCAmelCase				,   name=lowerCAmelCase				)
										return tuple(lowerCAmelCase				)





				def 					a__(		self      :			List[Any]				,   lowerCAmelCase      :			"Conversation"				)->					List[int]:

										"""simple docstring"""




										UpperCAmelCase					    =     []
										for is_user, text in conversation.iter_texts():
																input_ids.extend(self.encode(lowerCAmelCase				,   add_special_tokens=lowerCAmelCase				) + [self.eos_token_id]				)

										if len(lowerCAmelCase				) > self.model_max_length:
																UpperCAmelCase					    =     input_ids[-self.model_max_length :]
										return input_ids





