

import warnings

from transformers import AutoTokenizer
from transformers.utils import is_torch_available
from transformers.utils.generic import ExplicitEnum

from ...processing_utils import ProcessorMixin


if is_torch_available():
						import torch



class   __lowercase    (			UpperCAmelCase_     ):




			"""simple docstring"""







			_UpperCAmelCase       :							Dict      							=					'''char'''
			_UpperCAmelCase       :							List[str]      							=					'''bpe'''
			_UpperCAmelCase       :							List[Any]      							=					'''wp'''







lowerCAmelCase							:  Any            =						(DecodeType.CHARACTER, DecodeType.BPE, DecodeType.WORDPIECE)



class   __lowercase    (			UpperCAmelCase_     ):




			"""simple docstring"""







			_UpperCAmelCase       :							List[str]      							=					['''image_processor''', '''char_tokenizer''']
			_UpperCAmelCase       :							Tuple      							=					'''ViTImageProcessor'''
			_UpperCAmelCase       :							Tuple      							=					'''MgpstrTokenizer'''



			def __init__(			self  :					int       ,						lowerCAmelCase__  :					int=None       ,						lowerCAmelCase__  :					Optional[Any]=None       ,						**lowerCAmelCase__  :					List[Any]):
									SCREAMING_SNAKE_CASE_:			List[str]  		=	None
									if "feature_extractor" in kwargs:
															warnings.warn(
															    "The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"
															    " instead."       ,						lowerCAmelCase__       ,						)
															SCREAMING_SNAKE_CASE_:			Dict  		=	kwargs.pop("feature_extractor")

									SCREAMING_SNAKE_CASE_:			Any  		=	image_processor if image_processor is not None else feature_extractor
									if image_processor is None:
															raise ValueError("You need to specify an `image_processor`.")
									if tokenizer is None:
															raise ValueError("You need to specify a `tokenizer`.")

									SCREAMING_SNAKE_CASE_:			Dict  		=	tokenizer
									SCREAMING_SNAKE_CASE_:			Any  		=	AutoTokenizer.from_pretrained("gpt2")
									SCREAMING_SNAKE_CASE_:			List[Any]  		=	AutoTokenizer.from_pretrained("bert-base-uncased")

									super().__init__(lowerCAmelCase__       ,						lowerCAmelCase__)



			def __call__(			self  :					int       ,						lowerCAmelCase__  :					Dict=None       ,						lowerCAmelCase__  :					Optional[int]=None       ,						lowerCAmelCase__  :					Union[str, Any]=None       ,						**lowerCAmelCase__  :					str):
									if images is None and text is None:
															raise ValueError("You need to specify either an `images` or `text` input to process.")

									if images is not None:
															SCREAMING_SNAKE_CASE_:			Any  		=	self.image_processor(lowerCAmelCase__       ,						return_tensors=lowerCAmelCase__       ,						**lowerCAmelCase__)
									if text is not None:
															SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	self.char_tokenizer(lowerCAmelCase__       ,						return_tensors=lowerCAmelCase__       ,						**lowerCAmelCase__)

									if text is None:
															return inputs
									elif images is None:
															return encodings
									else:
															SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	encodings["input_ids"]
															return inputs



			def 	_SCREAMING_SNAKE_CASE      (			self  :					List[Any]       ,						lowerCAmelCase__  :					Dict):
									SCREAMING_SNAKE_CASE_		,   SCREAMING_SNAKE_CASE_		,   SCREAMING_SNAKE_CASE_:			Dict  		=	sequences
									SCREAMING_SNAKE_CASE_:			Tuple  		=	char_preds.size(0)

									SCREAMING_SNAKE_CASE_		,   SCREAMING_SNAKE_CASE_:			Optional[int]  		=	self._decode_helper(lowerCAmelCase__       ,						"char")
									SCREAMING_SNAKE_CASE_		,   SCREAMING_SNAKE_CASE_:			List[str]  		=	self._decode_helper(lowerCAmelCase__       ,						"bpe")
									SCREAMING_SNAKE_CASE_		,   SCREAMING_SNAKE_CASE_:			str  		=	self._decode_helper(lowerCAmelCase__       ,						"wp")

									SCREAMING_SNAKE_CASE_:			Dict  		=	[]
									SCREAMING_SNAKE_CASE_:			Tuple  		=	[]
									for i in range(lowerCAmelCase__):
															SCREAMING_SNAKE_CASE_:			Optional[int]  		=	[char_scores[i], bpe_scores[i], wp_scores[i]]
															SCREAMING_SNAKE_CASE_:			Any  		=	[char_strs[i], bpe_strs[i], wp_strs[i]]
															SCREAMING_SNAKE_CASE_:			Optional[int]  		=	scores.index(max(lowerCAmelCase__))
															final_strs.append(strs[max_score_index])
															final_scores.append(scores[max_score_index])

									SCREAMING_SNAKE_CASE_:			int  		=	{}
									SCREAMING_SNAKE_CASE_:			Dict  		=	final_strs
									SCREAMING_SNAKE_CASE_:			Tuple  		=	final_scores
									SCREAMING_SNAKE_CASE_:			Tuple  		=	char_strs
									SCREAMING_SNAKE_CASE_:			Tuple  		=	bpe_strs
									SCREAMING_SNAKE_CASE_:			Any  		=	wp_strs
									return out



			def 	_SCREAMING_SNAKE_CASE      (			self  :					List[Any]       ,						lowerCAmelCase__  :					List[Any]       ,						lowerCAmelCase__  :					Any):
									if format == DecodeType.CHARACTER:
															SCREAMING_SNAKE_CASE_:			Optional[int]  		=	self.char_decode
															SCREAMING_SNAKE_CASE_:			str  		=	1
															SCREAMING_SNAKE_CASE_:			Tuple  		=	"[s]"
									elif format == DecodeType.BPE:
															SCREAMING_SNAKE_CASE_:			str  		=	self.bpe_decode
															SCREAMING_SNAKE_CASE_:			Optional[int]  		=	2
															SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	"#"
									elif format == DecodeType.WORDPIECE:
															SCREAMING_SNAKE_CASE_:			Dict  		=	self.wp_decode
															SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	102
															SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	"[SEP]"
									else:
															raise ValueError(F"Format {format} is not supported.")

									SCREAMING_SNAKE_CASE_		,   SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	[], []
									SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	pred_logits.size(0)
									SCREAMING_SNAKE_CASE_:			int  		=	pred_logits.size(1)
									SCREAMING_SNAKE_CASE_		,   SCREAMING_SNAKE_CASE_:			Optional[int]  		=	pred_logits.topk(1       ,						dim=-1       ,						largest=lowerCAmelCase__       ,						sorted=lowerCAmelCase__)
									SCREAMING_SNAKE_CASE_:			int  		=	preds_index.view(-1       ,						lowerCAmelCase__)[:, 1:]
									SCREAMING_SNAKE_CASE_:			Dict  		=	decoder(lowerCAmelCase__)
									SCREAMING_SNAKE_CASE_		,   SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	torch.nn.functional.softmax(lowerCAmelCase__       ,						dim=2).max(dim=2)
									SCREAMING_SNAKE_CASE_:			int  		=	preds_max_prob[:, 1:]

									for index in range(lowerCAmelCase__):
															SCREAMING_SNAKE_CASE_:			str  		=	preds_str[index].find(lowerCAmelCase__)
															SCREAMING_SNAKE_CASE_:			int  		=	preds_str[index][:pred_eos]
															SCREAMING_SNAKE_CASE_:			List[str]  		=	preds_index[index].cpu().tolist()
															SCREAMING_SNAKE_CASE_:			Any  		=	pred_index.index(lowerCAmelCase__) if eos_token in pred_index else -1
															SCREAMING_SNAKE_CASE_:			Any  		=	preds_max_prob[index][: pred_eos_index + 1]
															SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	pred_max_prob.cumprod(dim=0)[-1] if pred_max_prob.nelement() != 0 else 0.0
															dec_strs.append(lowerCAmelCase__)
															conf_scores.append(lowerCAmelCase__)

									return dec_strs, conf_scores



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Optional[int]       ,						lowerCAmelCase__  :					List[str]):
									SCREAMING_SNAKE_CASE_:			int  		=	[seq.replace(" "       ,						"") for seq in self.char_tokenizer.batch_decode(lowerCAmelCase__)]
									return decode_strs



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Any       ,						lowerCAmelCase__  :					Tuple):
									return self.bpe_tokenizer.batch_decode(lowerCAmelCase__)




			def 	_SCREAMING_SNAKE_CASE      (			self  :					str       ,						lowerCAmelCase__  :					Optional[int]):
									SCREAMING_SNAKE_CASE_:			Tuple  		=	[seq.replace(" "       ,						"") for seq in self.wp_tokenizer.batch_decode(lowerCAmelCase__)]
									return decode_strs









from typing import Dict, List, Optional

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase							:  List[str]            =						logging.get_logger(__name__)


lowerCAmelCase							:  Tuple            =						{
    """nielsr/canine-s""": 2048,
}

# Unicode defines 1,114,112 total “codepoints”
lowerCAmelCase							:  Any            =						1114112

# Below: Constants defining canonical codepoints for special, pseudo-characters.
# Copied from https://github.com/google-research/language/blob/master/language/canine/special_codepoints.py
lowerCAmelCase							:  Dict            =						0

lowerCAmelCase							:  Dict            =						0xe000
lowerCAmelCase							:  str            =						0xe001
lowerCAmelCase							:  str            =						0xe002
lowerCAmelCase							:  Optional[int]            =						0xe003
lowerCAmelCase							:  List[Any]            =						0xe004

# Maps special codepoints to human-readable names.
lowerCAmelCase							:  Dict[int, str]            =						{
    # Special symbols are represented using codepoints values that are valid,
    # but designated as "Private Use", meaning that they will never be assigned
    # characters by the Unicode Consortium, and are thus safe for use here.
    #
    # NOTE: Do *NOT* add any sort of [UNK_CHAR] here. They are explicitly
    # excluded and should fail with a hard error.
    CLS: "[CLS]",
    SEP: "[SEP]",
    BOS: "[BOS]",
    MASK: "[MASK]",
    PAD: "[PAD]",
    RESERVED: "[RESERVED]",
}

# Maps special codepoint human-readable names to their codepoint values.
lowerCAmelCase							:  Dict[str, int]            =						{name: codepoint for codepoint, name in SPECIAL_CODEPOINTS.items()}



class   __lowercase    (			UpperCAmelCase_     ):




			"""simple docstring"""







			_UpperCAmelCase       :							List[str]      							=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES



			def __init__(			self  :					Optional[Any]       ,						lowerCAmelCase__  :					Tuple=chr(lowerCAmelCase__)       ,						lowerCAmelCase__  :					Tuple=chr(lowerCAmelCase__)       ,						lowerCAmelCase__  :					str=chr(lowerCAmelCase__)       ,						lowerCAmelCase__  :					List[Any]=chr(lowerCAmelCase__)       ,						lowerCAmelCase__  :					Optional[int]=chr(lowerCAmelCase__)       ,						lowerCAmelCase__  :					Union[str, Any]=chr(lowerCAmelCase__)       ,						lowerCAmelCase__  :					Union[str, Any]=False       ,						lowerCAmelCase__  :					int=2048       ,						**lowerCAmelCase__  :					Tuple       ,						):
									SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	AddedToken(lowerCAmelCase__       ,						lstrip=lowerCAmelCase__       ,						rstrip=lowerCAmelCase__) if isinstance(lowerCAmelCase__       ,						lowerCAmelCase__) else bos_token
									SCREAMING_SNAKE_CASE_:			List[Any]  		=	AddedToken(lowerCAmelCase__       ,						lstrip=lowerCAmelCase__       ,						rstrip=lowerCAmelCase__) if isinstance(lowerCAmelCase__       ,						lowerCAmelCase__) else eos_token
									SCREAMING_SNAKE_CASE_:			List[Any]  		=	AddedToken(lowerCAmelCase__       ,						lstrip=lowerCAmelCase__       ,						rstrip=lowerCAmelCase__) if isinstance(lowerCAmelCase__       ,						lowerCAmelCase__) else sep_token
									SCREAMING_SNAKE_CASE_:			Any  		=	AddedToken(lowerCAmelCase__       ,						lstrip=lowerCAmelCase__       ,						rstrip=lowerCAmelCase__) if isinstance(lowerCAmelCase__       ,						lowerCAmelCase__) else cls_token
									SCREAMING_SNAKE_CASE_:			str  		=	AddedToken(lowerCAmelCase__       ,						lstrip=lowerCAmelCase__       ,						rstrip=lowerCAmelCase__) if isinstance(lowerCAmelCase__       ,						lowerCAmelCase__) else pad_token

									# Mask token behave like a normal word, i.e. include the space before it
									SCREAMING_SNAKE_CASE_:			Dict  		=	AddedToken(lowerCAmelCase__       ,						lstrip=lowerCAmelCase__       ,						rstrip=lowerCAmelCase__) if isinstance(lowerCAmelCase__       ,						lowerCAmelCase__) else mask_token

									super().__init__(
									    bos_token=lowerCAmelCase__       ,						eos_token=lowerCAmelCase__       ,						sep_token=lowerCAmelCase__       ,						cls_token=lowerCAmelCase__       ,						pad_token=lowerCAmelCase__       ,						mask_token=lowerCAmelCase__       ,						add_prefix_space=lowerCAmelCase__       ,						model_max_length=lowerCAmelCase__       ,						**lowerCAmelCase__       ,						)

									# Creates a mapping for looking up the IDs of special symbols.
									SCREAMING_SNAKE_CASE_:			Dict[str, int]  		=	{}
									for codepoint, name in SPECIAL_CODEPOINTS.items():
															SCREAMING_SNAKE_CASE_:			List[Any]  		=	codepoint

									# Creates a mapping for looking up the string forms of special symbol IDs.
									SCREAMING_SNAKE_CASE_:			Dict[int, str]  		=	{
									    codepoint: name for name, codepoint in self._special_codepoints.items()
									}

									SCREAMING_SNAKE_CASE_:			List[str]  		=	UNICODE_VOCAB_SIZE
									SCREAMING_SNAKE_CASE_:			Tuple  		=	len(self._special_codepoints)



			@property
			def 	_SCREAMING_SNAKE_CASE      (			self  :					Optional[int]):
									return self._unicode_vocab_size



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Dict       ,						lowerCAmelCase__  :					str):
									return list(lowerCAmelCase__)



			def 	_SCREAMING_SNAKE_CASE      (			self  :					List[str]       ,						lowerCAmelCase__  :					str):
									try:
															return ord(lowerCAmelCase__)
									except TypeError:
															raise ValueError(F"invalid token: '{token}'")



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Optional[int]       ,						lowerCAmelCase__  :					int):
									try:
															if index in SPECIAL_CODEPOINTS:
																					return SPECIAL_CODEPOINTS[index]
															return chr(lowerCAmelCase__)
									except TypeError:
															raise ValueError(F"invalid id: {index}")



			def 	_SCREAMING_SNAKE_CASE      (			self  :					List[str]       ,						lowerCAmelCase__  :					Dict):
									return "".join(lowerCAmelCase__)



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Optional[int]       ,						lowerCAmelCase__  :					List[int]       ,						lowerCAmelCase__  :					Optional[List[int]] = None):
									SCREAMING_SNAKE_CASE_:			List[str]  		=	[self.sep_token_id]
									SCREAMING_SNAKE_CASE_:			Tuple  		=	[self.cls_token_id]

									SCREAMING_SNAKE_CASE_:			int  		=	cls + token_ids_a + sep
									if token_ids_a is not None:
															result += token_ids_a + sep
									return result



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Union[str, Any]       ,						lowerCAmelCase__  :					List[int]       ,						lowerCAmelCase__  :					Optional[List[int]] = None       ,						lowerCAmelCase__  :					bool = False):
									if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=lowerCAmelCase__       ,						token_ids_a=lowerCAmelCase__       ,						already_has_special_tokens=lowerCAmelCase__)

									SCREAMING_SNAKE_CASE_:			Any  		=	[1] + ([0] * len(lowerCAmelCase__)) + [1]
									if token_ids_a is not None:
															result += ([0] * len(lowerCAmelCase__)) + [1]
									return result



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Dict       ,						lowerCAmelCase__  :					List[int]       ,						lowerCAmelCase__  :					Optional[List[int]] = None):
									SCREAMING_SNAKE_CASE_:			Dict  		=	[self.sep_token_id]
									SCREAMING_SNAKE_CASE_:			int  		=	[self.cls_token_id]

									SCREAMING_SNAKE_CASE_:			List[str]  		=	len(cls + token_ids_a + sep) * [0]
									if token_ids_a is not None:
															result += len(token_ids_a + sep) * [1]
									return result




			def 	_SCREAMING_SNAKE_CASE      (			self  :					str       ,						lowerCAmelCase__  :					str       ,						lowerCAmelCase__  :					Optional[str] = None):
									return ()






