"""simple docstring"""







import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_a					:					Tuple=			logging.get_logger(__name__)
_a					:					str=			{"vocab_file": "spiece.model"}

_a					:					Union[str, Any]=			{
    "vocab_file": {
        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/spiece.model",
        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/spiece.model",
        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model",
        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model",
        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/spiece.model",
        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/spiece.model",
        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model",
        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model",
    }
}

_a					:					List[str]=			{
    "albert-base-v1": 512,
    "albert-large-v1": 512,
    "albert-xlarge-v1": 512,
    "albert-xxlarge-v1": 512,
    "albert-base-v2": 512,
    "albert-large-v2": 512,
    "albert-xlarge-v2": 512,
    "albert-xxlarge-v2": 512,
}

_a					:					Tuple=			"▁"

class 		UpperCamelCase      (   lowercase						):
				UpperCAmelCase    :				Any   =		VOCAB_FILES_NAMES
				UpperCAmelCase    :				Optional[Any]   =		PRETRAINED_VOCAB_FILES_MAP
				UpperCAmelCase    :				List[Any]   =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES



				def __init__(self    :				Dict     ,  _A    :				Dict     ,  _A    :				Optional[Any]=True     ,  _A    :				Union[str, Any]=True     ,  _A    :				int=False     ,  _A    :				Tuple="[CLS]"     ,  _A    :				Optional[int]="[SEP]"     ,  _A    :				List[Any]="<unk>"     ,  _A    :				List[str]="[SEP]"     ,  _A    :				Tuple="<pad>"     ,  _A    :				Tuple="[CLS]"     ,  _A    :				int="[MASK]"     ,  _A    :				Optional[Dict[str, Any]] = None     ,  **_A    :				Union[str, Any]     ,  )   ->     None:
					# Mask token behave like a normal word, i.e. include the space before it and
					# is included in the raw text, there should be a match in a non-normalized sentence.
					__snake_case      :							Dict						=			(
					    AddedToken(_A     ,  lstrip=_A     ,  rstrip=_A     ,  normalized=_A)
					    if isinstance(_A     ,  _A)
					    else mask_token
					)

					__snake_case      :							List[str]						=			{} if sp_model_kwargs is None else sp_model_kwargs

					super().__init__(
					    do_lower_case=_A     ,  remove_space=_A     ,  keep_accents=_A     ,  bos_token=_A     ,  eos_token=_A     ,  unk_token=_A     ,  sep_token=_A     ,  pad_token=_A     ,  cls_token=_A     ,  mask_token=_A     ,  sp_model_kwargs=self.sp_model_kwargs     ,  **_A     ,  )

					__snake_case      :							Optional[Any]						=			do_lower_case
					__snake_case      :							Tuple						=			remove_space
					__snake_case      :							str						=			keep_accents
					__snake_case      :							List[Any]						=			vocab_file

					__snake_case      :							Optional[Any]						=			spm.SentencePieceProcessor(**self.sp_model_kwargs)
					self.sp_model.Load(_A)



				@property
				def  _lowercase					(self    :				Any)   ->     Optional[Any]:
					return len(self.sp_model)



				def  _lowercase					(self    :				Dict)   ->     Any:
					__snake_case      :							str						=			{self.convert_ids_to_tokens(_A): i for i in range(self.vocab_size)}
					vocab.update(self.added_tokens_encoder)
					return vocab



				def __getstate__(self    :				Any)   ->     List[str]:
					__snake_case      :							Union[str, Any]						=			self.__dict__.copy()
					__snake_case      :							str						=			None
					return state



				def __setstate__(self    :				Optional[int]     ,  _A    :				Optional[int])   ->     Union[str, Any]:
					__snake_case      :							Any						=			d

					# for backward compatibility
					if not hasattr(self     ,  'sp_model_kwargs'):
						__snake_case      :							str						=			{}

					__snake_case      :							Any						=			spm.SentencePieceProcessor(**self.sp_model_kwargs)
					self.sp_model.Load(self.vocab_file)



				def  _lowercase					(self    :				Union[str, Any]     ,  _A    :				List[str])   ->     str:
					if self.remove_space:
						__snake_case      :							Union[str, Any]						=			' '.join(inputs.strip().split())
					else:
						__snake_case      :							List[str]						=			inputs
					__snake_case      :							List[str]						=			outputs.replace('``'     ,  '"').replace('\'\''     ,  '"')

					if not self.keep_accents:
						__snake_case      :							Dict						=			unicodedata.normalize('NFKD'     ,  _A)
						__snake_case      :							Dict						=			''.join([c for c in outputs if not unicodedata.combining(_A)])
					if self.do_lower_case:
						__snake_case      :							Any						=			outputs.lower()

					return outputs



				def  _lowercase					(self    :				int     ,  _A    :				str)   ->     List[str]:
					__snake_case      :							Optional[Any]						=			self.preprocess_text(_A)
					__snake_case      :							Optional[int]						=			self.sp_model.encode(_A     ,  out_type=_A)
					__snake_case      :							Union[str, Any]						=			[]
					for piece in pieces:
						if len(_A) > 1 and piece[-1] == str(',') and piece[-2].isdigit():
							__snake_case      :							Tuple						=			self.sp_model.EncodeAsPieces(piece[:-1].replace(_A     ,  ''))
							if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
								if len(cur_pieces[0]) == 1:
									__snake_case      :							str						=			cur_pieces[1:]
								else:
									__snake_case      :							Union[str, Any]						=			cur_pieces[0][1:]
							cur_pieces.append(piece[-1])
							new_pieces.extend(_A)
						else:
							new_pieces.append(_A)

					return new_pieces



				def  _lowercase					(self    :				Optional[int]     ,  _A    :				Union[str, Any])   ->     Union[str, Any]:
					return self.sp_model.PieceToId(_A)



				def  _lowercase					(self    :				List[str]     ,  _A    :				Tuple)   ->     Tuple:
					return self.sp_model.IdToPiece(_A)



				def  _lowercase					(self    :				List[Any]     ,  _A    :				str)   ->     str:
					__snake_case      :							str						=			[]
					__snake_case      :							Optional[int]						=			''
					__snake_case      :							Dict						=			False
					for token in tokens:
						# make sure that special tokens are not decoded using sentencepiece model
						if token in self.all_special_tokens:
							if not prev_is_special:
								out_string += " "
							out_string += self.sp_model.decode(_A) + token
							__snake_case      :							Dict						=			True
							__snake_case      :							Dict						=			[]
						else:
							current_sub_tokens.append(_A)
							__snake_case      :							Union[str, Any]						=			False
					out_string += self.sp_model.decode(_A)
					return out_string.strip()



				def  _lowercase					(self    :				Union[str, Any]     ,  _A    :				List[int]     ,  _A    :				Optional[List[int]] = None)   ->     List[int]:
					__snake_case      :							int						=			[self.sep_token_id]
					__snake_case      :							Tuple						=			[self.cls_token_id]
					if token_ids_a is None:
						return cls + token_ids_a + sep
					return cls + token_ids_a + sep + token_ids_a + sep



				def  _lowercase					(self    :				List[Any]     ,  _A    :				List[int]     ,  _A    :				Optional[List[int]] = None     ,  _A    :				bool = False)   ->     List[int]:

					if already_has_special_tokens:
						return super().get_special_tokens_mask(
						    token_ids_a=_A     ,  token_ids_a=_A     ,  already_has_special_tokens=_A)

					if token_ids_a is not None:
						return [1] + ([0] * len(_A)) + [1] + ([0] * len(_A)) + [1]
					return [1] + ([0] * len(_A)) + [1]



				def  _lowercase					(self    :				Union[str, Any]     ,  _A    :				List[int]     ,  _A    :				Optional[List[int]] = None)   ->     List[int]:
					__snake_case      :							List[Any]						=			[self.sep_token_id]
					__snake_case      :							Tuple						=			[self.cls_token_id]

					if token_ids_a is None:
						return len(cls + token_ids_a + sep) * [0]
					return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]






				def  _lowercase					(self    :				List[str]     ,  _A    :				str     ,  _A    :				Optional[str] = None)   ->     Tuple[str]:
					if not os.path.isdir(_A):
						logger.error(f"Vocabulary path ({save_directory}) should be a directory")
						return
					__snake_case      :							Tuple						=			os.path.join(
					    _A     ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])

					if os.path.abspath(self.vocab_file) != os.path.abspath(_A) and os.path.isfile(self.vocab_file):
						copyfile(self.vocab_file     ,  _A)
					elif not os.path.isfile(self.vocab_file):
						with open(_A     ,  'wb') as fi:
							__snake_case      :							int						=			self.sp_model.serialized_model_proto()
							fi.write(_A)

					return (out_vocab_file,)






"""simple docstring"""







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


_a					:					Any=			logging.get_logger(__name__)

_a					:					str=			{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_a					:					Optional[Any]=			[
    "small",
    "small-base",
    "medium",
    "medium-base",
    "intermediate",
    "intermediate-base",
    "large",
    "large-base",
    "xlarge",
    "xlarge-base",
]

_a					:					List[Any]=			{
    "vocab_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt",
        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt",
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt",
        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt",
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json",
        "funnel-transformer/small-base": (
            "https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json",
        "funnel-transformer/large-base": (
            "https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"
        ),
    },
}
_a					:					str=			{f'''funnel-transformer/{name}''': 512 for name in _model_names}
_a					:					List[Any]=			{f'''funnel-transformer/{name}''': {"do_lower_case": True} for name in _model_names}

class 		UpperCamelCase      (   lowercase						):
				UpperCAmelCase    :				List[str]   =		VOCAB_FILES_NAMES
				UpperCAmelCase    :				Tuple   =		PRETRAINED_VOCAB_FILES_MAP
				UpperCAmelCase    :				int   =		PRETRAINED_INIT_CONFIGURATION
				UpperCAmelCase    :				Tuple   =		FunnelTokenizer
				UpperCAmelCase    :				Tuple   =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				UpperCAmelCase    :				int   =		2



				def __init__(self    :				int     ,  _A    :				Any=None     ,  _A    :				Union[str, Any]=None     ,  _A    :				Union[str, Any]=True     ,  _A    :				List[str]="<unk>"     ,  _A    :				Any="<sep>"     ,  _A    :				Dict="<pad>"     ,  _A    :				Tuple="<cls>"     ,  _A    :				Dict="<mask>"     ,  _A    :				Optional[Any]="<s>"     ,  _A    :				List[Any]="</s>"     ,  _A    :				Optional[int]=True     ,  _A    :				Dict=True     ,  _A    :				Tuple=None     ,  _A    :				int="##"     ,  **_A    :				Any     ,  )   ->     str:
					super().__init__(
					    _A     ,  tokenizer_file=_A     ,  do_lower_case=_A     ,  unk_token=_A     ,  sep_token=_A     ,  pad_token=_A     ,  cls_token=_A     ,  mask_token=_A     ,  bos_token=_A     ,  eos_token=_A     ,  clean_text=_A     ,  tokenize_chinese_chars=_A     ,  strip_accents=_A     ,  wordpieces_prefix=_A     ,  **_A     ,  )

					__snake_case      :							Optional[int]						=			json.loads(self.backend_tokenizer.normalizer.__getstate__())
					if (
					    normalizer_state.get('lowercase'     ,  _A) != do_lower_case
					    or normalizer_state.get('strip_accents'     ,  _A) != strip_accents
					    or normalizer_state.get('handle_chinese_chars'     ,  _A) != tokenize_chinese_chars
					):
						__snake_case      :							List[str]						=			getattr(_A     ,  normalizer_state.pop('type'))
						__snake_case      :							int						=			do_lower_case
						__snake_case      :							Optional[int]						=			strip_accents
						__snake_case      :							str						=			tokenize_chinese_chars
						__snake_case      :							Optional[int]						=			normalizer_class(**_A)

					__snake_case      :							str						=			do_lower_case



				def  _lowercase					(self    :				Optional[Any]     ,  _A    :				Dict     ,  _A    :				Tuple=None)   ->     Any:
					__snake_case      :							List[Any]						=			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

					if token_ids_a:
						output += token_ids_a + [self.sep_token_id]

					return output



				def  _lowercase					(self    :				str     ,  _A    :				List[int]     ,  _A    :				Optional[List[int]] = None)   ->     List[int]:
					__snake_case      :							Union[str, Any]						=			[self.sep_token_id]
					__snake_case      :							Union[str, Any]						=			[self.cls_token_id]
					if token_ids_a is None:
						return len(cls) * [self.cls_token_type_id] + len(token_ids_a + sep) * [0]
					return len(cls) * [self.cls_token_type_id] + len(token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]






				def  _lowercase					(self    :				Tuple     ,  _A    :				str     ,  _A    :				Optional[str] = None)   ->     Tuple[str]:
					__snake_case      :							int						=			self._tokenizer.model.save(_A     ,  name=_A)
					return tuple(_A)





