

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


a		=logging.get_logger(__name__)

a		={"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

a		=[
    """small""",
    """small-base""",
    """medium""",
    """medium-base""",
    """intermediate""",
    """intermediate-base""",
    """large""",
    """large-base""",
    """xlarge""",
    """xlarge-base""",
]

a		={
    """vocab_file""": {
        """funnel-transformer/small""": """https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt""",
        """funnel-transformer/small-base""": """https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt""",
        """funnel-transformer/medium""": """https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt""",
        """funnel-transformer/medium-base""": (
            """https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/intermediate""": (
            """https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/intermediate-base""": (
            """https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/large""": """https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt""",
        """funnel-transformer/large-base""": """https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt""",
        """funnel-transformer/xlarge""": """https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt""",
        """funnel-transformer/xlarge-base""": (
            """https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """funnel-transformer/small""": """https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json""",
        """funnel-transformer/small-base""": (
            """https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/medium""": """https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json""",
        """funnel-transformer/medium-base""": (
            """https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/intermediate""": (
            """https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/intermediate-base""": (
            """https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/large""": """https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json""",
        """funnel-transformer/large-base""": (
            """https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/xlarge""": """https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json""",
        """funnel-transformer/xlarge-base""": (
            """https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"""
        ),
    },
}
a		={F"""funnel-transformer/{name}""": 512 for name in _model_names}
a		={F"""funnel-transformer/{name}""": {"""do_lower_case""": True} for name in _model_names}






class 	A_		(	SCREAMING_SNAKE_CASE    ):
							_UpperCAmelCase   :  str										=				VOCAB_FILES_NAMES
							_UpperCAmelCase   :  int										=				PRETRAINED_VOCAB_FILES_MAP
							_UpperCAmelCase   :  List[str]										=				PRETRAINED_INIT_CONFIGURATION
							_UpperCAmelCase   :  str										=				FunnelTokenizer
							_UpperCAmelCase   :  Dict										=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							_UpperCAmelCase   :  int										=				2
							def __init__(		self		:      List[Any]       ,SCREAMING_SNAKE_CASE__		:      str=None       ,SCREAMING_SNAKE_CASE__		:      str=None       ,SCREAMING_SNAKE_CASE__		:      List[str]=True       ,SCREAMING_SNAKE_CASE__		:      Tuple="<unk>"       ,SCREAMING_SNAKE_CASE__		:      Union[str, Any]="<sep>"       ,SCREAMING_SNAKE_CASE__		:      Dict="<pad>"       ,SCREAMING_SNAKE_CASE__		:      Union[str, Any]="<cls>"       ,SCREAMING_SNAKE_CASE__		:      Union[str, Any]="<mask>"       ,SCREAMING_SNAKE_CASE__		:      List[Any]="<s>"       ,SCREAMING_SNAKE_CASE__		:      Optional[Any]="</s>"       ,SCREAMING_SNAKE_CASE__		:      Any=True       ,SCREAMING_SNAKE_CASE__		:      Tuple=True       ,SCREAMING_SNAKE_CASE__		:      Optional[Any]=None       ,SCREAMING_SNAKE_CASE__		:      int="##"       ,**SCREAMING_SNAKE_CASE__		:      Tuple       ,):
								super().__init__(
								    SCREAMING_SNAKE_CASE__       ,tokenizer_file=SCREAMING_SNAKE_CASE__       ,do_lower_case=SCREAMING_SNAKE_CASE__       ,unk_token=SCREAMING_SNAKE_CASE__       ,sep_token=SCREAMING_SNAKE_CASE__       ,pad_token=SCREAMING_SNAKE_CASE__       ,cls_token=SCREAMING_SNAKE_CASE__       ,mask_token=SCREAMING_SNAKE_CASE__       ,bos_token=SCREAMING_SNAKE_CASE__       ,eos_token=SCREAMING_SNAKE_CASE__       ,clean_text=SCREAMING_SNAKE_CASE__       ,tokenize_chinese_chars=SCREAMING_SNAKE_CASE__       ,strip_accents=SCREAMING_SNAKE_CASE__       ,wordpieces_prefix=SCREAMING_SNAKE_CASE__       ,**SCREAMING_SNAKE_CASE__       ,)

								__lowerCamelCase    :    Optional[int] 	=			json.loads(self.backend_tokenizer.normalizer.__getstate__())
								if (
								    normalizer_state.get('lowercase'       ,SCREAMING_SNAKE_CASE__) != do_lower_case
								    or normalizer_state.get('strip_accents'       ,SCREAMING_SNAKE_CASE__) != strip_accents
								    or normalizer_state.get('handle_chinese_chars'       ,SCREAMING_SNAKE_CASE__) != tokenize_chinese_chars
								):
									__lowerCamelCase    :    str 	=			getattr(SCREAMING_SNAKE_CASE__       ,normalizer_state.pop('type'))
									__lowerCamelCase    :    str 	=			do_lower_case
									__lowerCamelCase    :    Dict 	=			strip_accents
									__lowerCamelCase    :    Optional[Any] 	=			tokenize_chinese_chars
									__lowerCamelCase    :    Dict 	=			normalizer_class(**SCREAMING_SNAKE_CASE__)

								__lowerCamelCase    :    int 	=			do_lower_case
							def    lowerCAmelCase					(		self		:      List[str]       ,SCREAMING_SNAKE_CASE__		:      Optional[Any]       ,SCREAMING_SNAKE_CASE__		:      List[str]=None):
								__lowerCamelCase    :    Union[str, Any] 	=			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

								return output
							def    lowerCAmelCase					(		self		:      Dict       ,SCREAMING_SNAKE_CASE__		:      List[int]       ,SCREAMING_SNAKE_CASE__		:      Optional[List[int]] = None):
								__lowerCamelCase    :    Tuple 	=			[self.sep_token_id]
								__lowerCamelCase    :    Tuple 	=			[self.cls_token_id]
								if token_ids_a is None:
									return len(cls) * [self.cls_token_type_id] + len(token_ids_a + sep) * [0]
								return len(cls) * [self.cls_token_type_id] + len(token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]


							def    lowerCAmelCase					(		self		:      str       ,SCREAMING_SNAKE_CASE__		:      str       ,SCREAMING_SNAKE_CASE__		:      Optional[str] = None):
								__lowerCamelCase    :    str 	=			self._tokenizer.model.save(SCREAMING_SNAKE_CASE__       ,name=SCREAMING_SNAKE_CASE__)
								return tuple(SCREAMING_SNAKE_CASE__)







# This is the module that test_patching.py uses to test patch_submodule()

import os  # noqa: this is just for tests
import os as renamed_os  # noqa: this is just for tests
from os import path  # noqa: this is just for tests
from os import path as renamed_path  # noqa: this is just for tests
from os.path import join  # noqa: this is just for tests
from os.path import join as renamed_join  # noqa: this is just for tests


a		=open  # noqa: we just need to have a builtin inside this module to test it properly




