"""simple docstring"""







import json
import os
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_a					:					Union[str, Any]=			logging.get_logger(__name__)


_a					:					List[str]=			{
    "vocab_file": "vocab.json",
    "tokenizer_config_file": "tokenizer_config.json",
    "merges_file": "merges.txt",
}

_a					:					List[str]=			{
    "vocab_file": {
        "facebook/s2t-wav2vec2-large-en-de": (
            "https://huggingface.co/facebook/s2t-wav2vec2-large-en-de/resolve/main/vocab.json"
        ),
    },
    "tokenizer_config_file": {
        "facebook/s2t-wav2vec2-large-en-de": (
            "https://huggingface.co/facebook/s2t-wav2vec2-large-en-de/resolve/main/tokenizer_config.json"
        ),
    },
    "merges_file": {
        "facebook/s2t-wav2vec2-large-en-de": (
            "https://huggingface.co/facebook/s2t-wav2vec2-large-en-de/resolve/main/merges.txt"
        ),
    },
}

_a					:					List[str]=			"</w>"
_a					:					Dict=			"@@ "
def   __UpperCAmelCase    (   UpperCAmelCase_   :    int  )							->      Union[str, Any]:

	'''simple docstring'''





	__snake_case      :							Optional[Any]						=			set()
	__snake_case      :							Optional[Any]						=			word[0]
	for char in word[1:]:
		pairs.add((prev_char, char)  )
		__snake_case      :							List[str]						=			char
	return pairs


# Speech2Text2 has no max input length
_a					:					Union[str, Any]=			{"facebook/s2t-wav2vec2-large-en-de": 1_024}

class 		UpperCamelCase      (   lowercase						):
				UpperCAmelCase    :				List[str]   =		VOCAB_FILES_NAMES
				UpperCAmelCase    :				List[str]   =		PRETRAINED_VOCAB_FILES_MAP
				UpperCAmelCase    :				List[str]   =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				UpperCAmelCase    :				int   =		["""input_ids""", """attention_mask"""]



				def __init__(self    :				str     ,  _A    :				Any     ,  _A    :				Union[str, Any]="<s>"     ,  _A    :				Optional[int]="<pad>"     ,  _A    :				Tuple="</s>"     ,  _A    :				List[Any]="<unk>"     ,  _A    :				List[str]=False     ,  _A    :				List[Any]=None     ,  **_A    :				int     ,  )   ->     List[str]:
					super().__init__(
					    unk_token=_A     ,  bos_token=_A     ,  eos_token=_A     ,  pad_token=_A     ,  do_lower_case=_A     ,  **_A     ,  )

					__snake_case      :							List[str]						=			do_lower_case

					with open(_A     ,  encoding='utf-8') as vocab_handle:
						__snake_case      :							Optional[Any]						=			json.load(_A)
					__snake_case      :							Dict						=			{v: k for k, v in self.encoder.items()}

					if merges_file is None:
						logger.info(f"No merges files provided. {self.__class__.__name__} can only be used for decoding.")

						__snake_case      :							List[Any]						=			None
						__snake_case      :							List[str]						=			None
					else:
						with open(_A     ,  encoding='utf-8') as merges_handle:
							__snake_case      :							Tuple						=			merges_handle.read().split('\n')[:-1]

						__snake_case      :							List[Any]						=			[tuple(merge.split()[:2]) for merge in merges]
						__snake_case      :							List[str]						=			dict(zip(_A     ,  range(len(_A))))
						__snake_case      :							Tuple						=			{}



				@property
				def  _lowercase					(self    :				Dict)   ->     int:
					return len(self.decoder)



				def  _lowercase					(self    :				Tuple)   ->     Dict:
					return dict(self.encoder     ,  **self.added_tokens_encoder)



				def  _lowercase					(self    :				Optional[int]     ,  _A    :				Dict)   ->     Any:
					__snake_case      :							Tuple						=			tuple(token[:-1]) + (token[-1] + BPE_TOKEN_MERGES,)
					if token in self.cache:
						return self.cache[token]
					__snake_case      :							List[Any]						=			get_pairs(_A)

					if not pairs:
						return token

					while True:
						__snake_case      :							Dict						=			min(_A     ,  key=lambda _A: self.bpe_ranks.get(_A     ,  float('inf')))
						if bigram not in self.bpe_ranks:
							break
						__snake_case      :							Dict						=			bigram
						__snake_case      :							Union[str, Any]						=			[]
						__snake_case      :							List[Any]						=			0
						while i < len(_A):
							try:
								__snake_case      :							Any						=			word.index(_A     ,  _A)
							except ValueError:
								new_word.extend(word[i:])
								break
							else:
								new_word.extend(word[i:j])
								__snake_case      :							List[Any]						=			j

							if word[i] == first and i < len(_A) - 1 and word[i + 1] == second:
								new_word.append(first + second)
								i += 2
							else:
								new_word.append(word[i])
								i += 1
						__snake_case      :							Tuple						=			tuple(_A)
						__snake_case      :							List[str]						=			new_word
						if len(_A) == 1:
							break
						else:
							__snake_case      :							int						=			get_pairs(_A)
					__snake_case      :							Tuple						=			' '.join(_A)
					if word == "\n  " + BPE_TOKEN_MERGES:
						__snake_case      :							Tuple						=			'\n' + BPE_TOKEN_MERGES

					if word.endswith(_A):
						__snake_case      :							List[str]						=			word.replace(_A     ,  '')

					__snake_case      :							Tuple						=			word.replace(' '     ,  _A)
					__snake_case      :							Optional[int]						=			word
					return word



				def  _lowercase					(self    :				str     ,  _A    :				List[str])   ->     str:

					if self.bpe_ranks is None:
						raise ValueError(
						    'This tokenizer was instantiated without a `merges.txt` file, so'
						    ' that it can only be used for decoding, not for encoding.'
						    'Make sure to provide `merges.txt` file at instantiation to enable '
						    'encoding.')

					if self.do_lower_case:
						__snake_case      :							Dict						=			text.lower()

					__snake_case      :							str						=			text.split()

					__snake_case      :							Optional[int]						=			[]
					for token in text:
						if token:
							split_tokens.extend(list(self.bpe(_A).split(' ')))

					return split_tokens



				def  _lowercase					(self    :				str     ,  _A    :				str)   ->     int:
					return self.encoder.get(_A     ,  self.encoder.get(self.unk_token))



				def  _lowercase					(self    :				int     ,  _A    :				int)   ->     str:
					__snake_case      :							str						=			self.decoder.get(_A     ,  self.unk_token)
					return result



				def  _lowercase					(self    :				Optional[int]     ,  _A    :				List[str])   ->     str:
					__snake_case      :							Union[str, Any]						=			' '.join(_A)

					# make sure @@ tokens are concatenated
					__snake_case      :							Tuple						=			''.join(string.split(_A))

					return string






				def  _lowercase					(self    :				str     ,  _A    :				str     ,  _A    :				Optional[str] = None)   ->     Tuple[str]:
					if not os.path.isdir(_A):
						logger.error(f"Vocabulary path ({save_directory}) should be a directory")
						return
					__snake_case      :							Any						=			os.path.join(
					    _A     ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])
					__snake_case      :							List[Any]						=			os.path.join(
					    _A     ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file'])

					with open(_A     ,  'w'     ,  encoding='utf-8') as f:
						f.write(json.dumps(self.encoder     ,  indent=2     ,  sort_keys=_A     ,  ensure_ascii=_A) + '\n')

					__snake_case      :							int						=			0
					if self.bpe_ranks is None:
						return (vocab_file,)

					with open(_A     ,  'w'     ,  encoding='utf-8') as writer:
						for bpe_tokens, token_index in sorted(self.bpe_ranks.items()     ,  key=lambda _A: kv[1]):
							if index != token_index:
								logger.warning(
								    f"Saving vocabulary to {merges_file}: BPE merge indices are not consecutive."
								    ' Please check that the tokenizer is not corrupted!')
								__snake_case      :							str						=			token_index
							writer.write(' '.join(_A) + '\n')
							index += 1

					return (vocab_file, merges_file)





"""simple docstring"""







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


_a					:					Any=			logging.get_logger(__name__)

_a					:					str=			{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_a					:					Optional[Any]=			[
    "small",
    "small-base",
    "medium",
    "medium-base",
    "intermediate",
    "intermediate-base",
    "large",
    "large-base",
    "xlarge",
    "xlarge-base",
]

_a					:					List[Any]=			{
    "vocab_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt",
        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt",
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt",
        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt",
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json",
        "funnel-transformer/small-base": (
            "https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json",
        "funnel-transformer/large-base": (
            "https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"
        ),
    },
}
_a					:					str=			{f'''funnel-transformer/{name}''': 512 for name in _model_names}
_a					:					List[Any]=			{f'''funnel-transformer/{name}''': {"do_lower_case": True} for name in _model_names}

class 		UpperCamelCase      (   lowercase						):
				UpperCAmelCase    :				List[str]   =		VOCAB_FILES_NAMES
				UpperCAmelCase    :				Tuple   =		PRETRAINED_VOCAB_FILES_MAP
				UpperCAmelCase    :				int   =		PRETRAINED_INIT_CONFIGURATION
				UpperCAmelCase    :				Tuple   =		FunnelTokenizer
				UpperCAmelCase    :				Tuple   =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				UpperCAmelCase    :				int   =		2



				def __init__(self    :				int     ,  _A    :				Any=None     ,  _A    :				Union[str, Any]=None     ,  _A    :				Union[str, Any]=True     ,  _A    :				List[str]="<unk>"     ,  _A    :				Any="<sep>"     ,  _A    :				Dict="<pad>"     ,  _A    :				Tuple="<cls>"     ,  _A    :				Dict="<mask>"     ,  _A    :				Optional[Any]="<s>"     ,  _A    :				List[Any]="</s>"     ,  _A    :				Optional[int]=True     ,  _A    :				Dict=True     ,  _A    :				Tuple=None     ,  _A    :				int="##"     ,  **_A    :				Any     ,  )   ->     str:
					super().__init__(
					    _A     ,  tokenizer_file=_A     ,  do_lower_case=_A     ,  unk_token=_A     ,  sep_token=_A     ,  pad_token=_A     ,  cls_token=_A     ,  mask_token=_A     ,  bos_token=_A     ,  eos_token=_A     ,  clean_text=_A     ,  tokenize_chinese_chars=_A     ,  strip_accents=_A     ,  wordpieces_prefix=_A     ,  **_A     ,  )

					__snake_case      :							Optional[int]						=			json.loads(self.backend_tokenizer.normalizer.__getstate__())
					if (
					    normalizer_state.get('lowercase'     ,  _A) != do_lower_case
					    or normalizer_state.get('strip_accents'     ,  _A) != strip_accents
					    or normalizer_state.get('handle_chinese_chars'     ,  _A) != tokenize_chinese_chars
					):
						__snake_case      :							List[str]						=			getattr(_A     ,  normalizer_state.pop('type'))
						__snake_case      :							int						=			do_lower_case
						__snake_case      :							Optional[int]						=			strip_accents
						__snake_case      :							str						=			tokenize_chinese_chars
						__snake_case      :							Optional[int]						=			normalizer_class(**_A)

					__snake_case      :							str						=			do_lower_case



				def  _lowercase					(self    :				Optional[Any]     ,  _A    :				Dict     ,  _A    :				Tuple=None)   ->     Any:
					__snake_case      :							List[Any]						=			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

					if token_ids_a:
						output += token_ids_a + [self.sep_token_id]

					return output



				def  _lowercase					(self    :				str     ,  _A    :				List[int]     ,  _A    :				Optional[List[int]] = None)   ->     List[int]:
					__snake_case      :							Union[str, Any]						=			[self.sep_token_id]
					__snake_case      :							Union[str, Any]						=			[self.cls_token_id]
					if token_ids_a is None:
						return len(cls) * [self.cls_token_type_id] + len(token_ids_a + sep) * [0]
					return len(cls) * [self.cls_token_type_id] + len(token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]






				def  _lowercase					(self    :				Tuple     ,  _A    :				str     ,  _A    :				Optional[str] = None)   ->     Tuple[str]:
					__snake_case      :							int						=			self._tokenizer.model.save(_A     ,  name=_A)
					return tuple(_A)





