





import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
					from transformers.pipelines.conversational import Conversation

UpperCamelCase__     =	logging.get_logger(__name__)


UpperCamelCase__     =	{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
    "tokenizer_config_file": "tokenizer_config.json",
}

UpperCamelCase__     =	{
    "vocab_file": {"facebook/blenderbot-3B": "https://huggingface.co/facebook/blenderbot-3B/resolve/main/vocab.json"},
    "merges_file": {"facebook/blenderbot-3B": "https://huggingface.co/facebook/blenderbot-3B/resolve/main/merges.txt"},
    "tokenizer_config_file": {
        "facebook/blenderbot-3B": "https://huggingface.co/facebook/blenderbot-3B/resolve/main/tokenizer_config.json"
    },
}

UpperCamelCase__     =	{"facebook/blenderbot-3B": 128}





@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def       _UpperCamelCase       ():






							"""simple docstring"""





							UpperCamelCase__			       =     (
							    list(range(ord("""!"""    )				,     ord("""~"""    ) + 1    )    ) + list(range(ord("""¡"""    )				,     ord("""¬"""    ) + 1    )    ) + list(range(ord("""®"""    )				,     ord("""ÿ"""    ) + 1    )    )
							)
							UpperCamelCase__			       =     bs[:]
							UpperCamelCase__			       =     0
							for b in range(2**8    ):
														if b not in bs:
																					bs.append(__lowerCamelCase    )
																					cs.append(2**8 + n    )
																					n += 1
							UpperCamelCase__			       =     [chr(__lowerCamelCase    ) for n in cs]
							return dict(zip(__lowerCamelCase				,     __lowerCamelCase    )    )





def       _UpperCamelCase       (a__				:Optional[int]    ):






							"""simple docstring"""





							UpperCamelCase__			       =     set()
							UpperCamelCase__			       =     word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)    )
														UpperCamelCase__			       =     char
							return pairs

class     __SCREAMING_SNAKE_CASE   (       A_    ):
							snake_case       : List[str]         =    VOCAB_FILES_NAMES
							snake_case       : str         =    PRETRAINED_VOCAB_FILES_MAP
							snake_case       : int         =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							snake_case       : Tuple         =    ["""input_ids""", """attention_mask"""]






							def __init__(  self					,	__lowerCAmelCase					,	__lowerCAmelCase					,	__lowerCAmelCase="replace"					,	__lowerCAmelCase="<s>"					,	__lowerCAmelCase="</s>"					,	__lowerCAmelCase="</s>"					,	__lowerCAmelCase="<s>"					,	__lowerCAmelCase="<unk>"					,	__lowerCAmelCase="<pad>"					,	__lowerCAmelCase="<mask>"					,	__lowerCAmelCase=False					,	**__lowerCAmelCase					,	):
														UpperCamelCase__			       =     AddedToken(_lowerCamelCase					,	lstrip=_lowerCamelCase					,	rstrip=_lowerCamelCase					) if isinstance(_lowerCamelCase					,	_lowerCamelCase					) else bos_token
														UpperCamelCase__			       =     AddedToken(_lowerCamelCase					,	lstrip=_lowerCamelCase					,	rstrip=_lowerCamelCase					) if isinstance(_lowerCamelCase					,	_lowerCamelCase					) else eos_token
														UpperCamelCase__			       =     AddedToken(_lowerCamelCase					,	lstrip=_lowerCamelCase					,	rstrip=_lowerCamelCase					) if isinstance(_lowerCamelCase					,	_lowerCamelCase					) else sep_token
														UpperCamelCase__			       =     AddedToken(_lowerCamelCase					,	lstrip=_lowerCamelCase					,	rstrip=_lowerCamelCase					) if isinstance(_lowerCamelCase					,	_lowerCamelCase					) else cls_token
														UpperCamelCase__			       =     AddedToken(_lowerCamelCase					,	lstrip=_lowerCamelCase					,	rstrip=_lowerCamelCase					) if isinstance(_lowerCamelCase					,	_lowerCamelCase					) else unk_token
														UpperCamelCase__			       =     AddedToken(_lowerCamelCase					,	lstrip=_lowerCamelCase					,	rstrip=_lowerCamelCase					) if isinstance(_lowerCamelCase					,	_lowerCamelCase					) else pad_token

														# Mask token behave like a normal word, i.e. include the space before it
														UpperCamelCase__			       =     AddedToken(_lowerCamelCase					,	lstrip=_lowerCamelCase					,	rstrip=_lowerCamelCase					) if isinstance(_lowerCamelCase					,	_lowerCamelCase					) else mask_token

														super().__init__(
														    errors=_lowerCamelCase					,	bos_token=_lowerCamelCase					,	eos_token=_lowerCamelCase					,	unk_token=_lowerCamelCase					,	sep_token=_lowerCamelCase					,	cls_token=_lowerCamelCase					,	pad_token=_lowerCamelCase					,	mask_token=_lowerCamelCase					,	add_prefix_space=_lowerCamelCase					,	**_lowerCamelCase					,	)

														with open(_lowerCamelCase					,	encoding="""utf-8"""					) as vocab_handle:
																					UpperCamelCase__			       =     json.load(_lowerCamelCase					)
														UpperCamelCase__			       =     {v: k for k, v in self.encoder.items()}
														UpperCamelCase__			       =     errors  # how to handle errors in decoding
														UpperCamelCase__			       =     bytes_to_unicode()
														UpperCamelCase__			       =     {v: k for k, v in self.byte_encoder.items()}
														with open(_lowerCamelCase					,	encoding="""utf-8"""					) as merges_handle:
																					UpperCamelCase__			       =     merges_handle.read().split("""\n"""					)[1:-1]
														UpperCamelCase__			       =     [tuple(merge.split()					) for merge in bpe_merges]
														UpperCamelCase__			       =     dict(zip(_lowerCamelCase					,	range(len(_lowerCamelCase					)					)					)					)
														UpperCamelCase__			       =     {}
														UpperCamelCase__			       =     add_prefix_space

														# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
														UpperCamelCase__			       =     re.compile(r"""\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""					)






							@property
							# Copied from transformers.models.roberta.tokenization_roberta.RobertaTokenizer.vocab_size with Roberta->Blenderbot, RoBERTa->Blenderbot
							def 			_lowerCamelCase						(  self					):
														return len(self.encoder					)






							def 			_lowerCamelCase						(  self					):
														return dict(self.encoder					,	**self.added_tokens_encoder					)






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					):
														if token in self.cache:
																					return self.cache[token]
														UpperCamelCase__			       =     tuple(_lowerCamelCase					)
														UpperCamelCase__			       =     get_pairs(_lowerCamelCase					)

														if not pairs:
																					return token

														while True:
																					UpperCamelCase__			       =     min(_lowerCamelCase					,	key=lambda __lowerCAmelCase					: self.bpe_ranks.get(_lowerCamelCase					,	float("""inf"""					)					)					)
																					if bigram not in self.bpe_ranks:
																												break
																					UpperCamelCase__       ,       UpperCamelCase__			       =     bigram
																					UpperCamelCase__			       =     []
																					UpperCamelCase__			       =     0
																					while i < len(_lowerCamelCase					):
																												try:
																																			UpperCamelCase__			       =     word.index(_lowerCamelCase					,	_lowerCamelCase					)
																												except ValueError:
																																			new_word.extend(word[i:]					)
																																			break
																												else:
																																			new_word.extend(word[i:j]					)
																																			UpperCamelCase__			       =     j

																												if word[i] == first and i < len(_lowerCamelCase					) - 1 and word[i + 1] == second:
																																			new_word.append(first + second					)
																																			i += 2
																												else:
																																			new_word.append(word[i]					)
																																			i += 1
																					UpperCamelCase__			       =     tuple(_lowerCamelCase					)
																					UpperCamelCase__			       =     new_word
																					if len(_lowerCamelCase					) == 1:
																												break
																					else:
																												UpperCamelCase__			       =     get_pairs(_lowerCamelCase					)
														UpperCamelCase__			       =     """ """.join(_lowerCamelCase					)
														UpperCamelCase__			       =     word
														return word






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					):
														UpperCamelCase__			       =     []
														for token in re.findall(self.pat					,	_lowerCamelCase					):
																					UpperCamelCase__			       =     """""".join(
																					    self.byte_encoder[b] for b in token.encode("""utf-8"""					)					)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																					bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_lowerCamelCase					).split(""" """					)					)
														return bpe_tokens






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					):
														return self.encoder.get(_lowerCamelCase					,	self.encoder.get(self.unk_token					)					)






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					):
														return self.decoder.get(_lowerCamelCase					)






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					):
														UpperCamelCase__			       =     """""".join(_lowerCamelCase					)
														UpperCamelCase__			       =     bytearray([self.byte_decoder[c] for c in text]					).decode("""utf-8"""					,	errors=self.errors					)
														return text






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase = None					):
														if not os.path.isdir(_lowerCamelCase					):
																					logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""					)
																					return
														UpperCamelCase__			       =     os.path.join(
														    _lowerCamelCase					,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]					)
														UpperCamelCase__			       =     os.path.join(
														    _lowerCamelCase					,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]					)

														with open(_lowerCamelCase					,	"""w"""					,	encoding="""utf-8"""					) as f:
																					f.write(json.dumps(self.encoder					,	indent=2					,	sort_keys=_lowerCamelCase					,	ensure_ascii=_lowerCamelCase					) + """\n"""					)

														UpperCamelCase__			       =     0
														with open(_lowerCamelCase					,	"""w"""					,	encoding="""utf-8"""					) as writer:
																					writer.write("""#version: 0.2\n"""					)
																					for bpe_tokens, token_index in sorted(self.bpe_ranks.items()					,	key=lambda __lowerCAmelCase					: kv[1]					):
																												if index != token_index:
																																			logger.warning(
																																			    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																																			    """ Please check that the tokenizer is not corrupted!"""					)
																																			UpperCamelCase__			       =     token_index
																												writer.write(""" """.join(_lowerCamelCase					) + """\n"""					)
																												index += 1

														return vocab_file, merge_file






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase = None					,	__lowerCAmelCase = False					):
														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=_lowerCamelCase					,	token_ids_a=_lowerCamelCase					,	already_has_special_tokens=_lowerCamelCase					)

														if token_ids_a is None:
																					return [1] + ([0] * len(_lowerCamelCase					)) + [1]
														return [1] + ([0] * len(_lowerCamelCase					)) + [1, 1] + ([0] * len(_lowerCamelCase					)) + [1]






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase = None					):
														UpperCamelCase__			       =     [self.sep_token_id]
														UpperCamelCase__			       =     [self.cls_token_id]

														if token_ids_a is None:
																					return len(cls + token_ids_a + sep					) * [0]
														return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase=False					,	**__lowerCAmelCase					):
														UpperCamelCase__			       =     kwargs.pop("""add_prefix_space"""					,	self.add_prefix_space					)
														if (is_split_into_words or add_prefix_space) and (len(_lowerCamelCase					) > 0 and not text[0].isspace()):
																					UpperCamelCase__			       =     """ """ + text
														return (text, kwargs)






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase = None					):
														return token_ids_a + [self.eos_token_id]







							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					):
														UpperCamelCase__			       =     []
														for is_user, text in conversation.iter_texts():
																					if is_user:
																												# We need to space prefix as it's being done within blenderbot
																												inputs.append(""" """ + text					)
																					else:
																												# Generated responses should contain them already.
																												inputs.append(_lowerCamelCase					)

														UpperCamelCase__			       =     """  """.join(_lowerCamelCase					)
														UpperCamelCase__			       =     self.encode(_lowerCamelCase					)
														if len(_lowerCamelCase					) > self.model_max_length:
																					UpperCamelCase__			       =     input_ids[-self.model_max_length :]
																					logger.warning(f"""Trimmed input from conversation as it was longer than {self.model_max_length} tokens."""					)
														return input_ids











import logging

from transformers.configuration_utils import PretrainedConfig


UpperCamelCase__     =	logging.getLogger(__name__)

class     __SCREAMING_SNAKE_CASE   (       _a    ):
							snake_case       : Optional[Any]         =    """masked_bert"""






							def __init__(  self					,	__lowerCAmelCase=30522					,	__lowerCAmelCase=768					,	__lowerCAmelCase=12					,	__lowerCAmelCase=12					,	__lowerCAmelCase=3072					,	__lowerCAmelCase="gelu"					,	__lowerCAmelCase=0.1					,	__lowerCAmelCase=0.1					,	__lowerCAmelCase=512					,	__lowerCAmelCase=2					,	__lowerCAmelCase=0.02					,	__lowerCAmelCase=1E-12					,	__lowerCAmelCase=0					,	__lowerCAmelCase="topK"					,	__lowerCAmelCase="constant"					,	__lowerCAmelCase=0.0					,	**__lowerCAmelCase					,	):
														super().__init__(pad_token_id=__lowerCAmelCase					,	**__lowerCAmelCase					)

														UpperCamelCase__			       =     vocab_size
														UpperCamelCase__			       =     hidden_size
														UpperCamelCase__			       =     num_hidden_layers
														UpperCamelCase__			       =     num_attention_heads
														UpperCamelCase__			       =     hidden_act
														UpperCamelCase__			       =     intermediate_size
														UpperCamelCase__			       =     hidden_dropout_prob
														UpperCamelCase__			       =     attention_probs_dropout_prob
														UpperCamelCase__			       =     max_position_embeddings
														UpperCamelCase__			       =     type_vocab_size
														UpperCamelCase__			       =     initializer_range
														UpperCamelCase__			       =     layer_norm_eps
														UpperCamelCase__			       =     pruning_method
														UpperCamelCase__			       =     mask_init
														UpperCamelCase__			       =     mask_scale




