"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


_a				:		Any	   =       logging.get_logger(__name__)

_a				:		List[str]	   =       {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

_a				:		Optional[int]	   =       [
    'small',
    'small-base',
    'medium',
    'medium-base',
    'intermediate',
    'intermediate-base',
    'large',
    'large-base',
    'xlarge',
    'xlarge-base',
]

_a				:		List[str]	   =       {
    'vocab_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt',
        'funnel-transformer/small-base': 'https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt',
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt',
        'funnel-transformer/large-base': 'https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt',
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json',
        'funnel-transformer/small-base': (
            'https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json',
        'funnel-transformer/large-base': (
            'https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'
        ),
    },
}
_a				:		int	   =       {F"""funnel-transformer/{name}""": 512 for name in _model_names}
_a				:		Optional[int]	   =       {F"""funnel-transformer/{name}""": {'do_lower_case': True} for name in _model_names}
class 						__A							(			__SCREAMING_SNAKE_CASE		):
     _UpperCamelCase							:      int		 =		VOCAB_FILES_NAMES
     _UpperCamelCase							:      Optional[Any]		 =		PRETRAINED_VOCAB_FILES_MAP
     _UpperCamelCase							:      Dict		 =		PRETRAINED_INIT_CONFIGURATION
     _UpperCamelCase							:      Optional[int]		 =		FunnelTokenizer
     _UpperCamelCase							:      List[Any]		 =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _UpperCamelCase							:      int		 =		2






     def __init__(		self    ,					a__=None    ,					a__=None    ,					a__=True    ,					a__="<unk>"    ,					a__="<sep>"    ,					a__="<pad>"    ,					a__="<cls>"    ,					a__="<mask>"    ,					a__="<s>"    ,					a__="</s>"    ,					a__=True    ,					a__=True    ,					a__=None    ,					a__="##"    ,					**a__    ,					):
       super().__init__(
           __UpperCAmelCase    ,					tokenizer_file=__UpperCAmelCase    ,					do_lower_case=__UpperCAmelCase    ,					unk_token=__UpperCAmelCase    ,					sep_token=__UpperCAmelCase    ,					pad_token=__UpperCAmelCase    ,					cls_token=__UpperCAmelCase    ,					mask_token=__UpperCAmelCase    ,					bos_token=__UpperCAmelCase    ,					eos_token=__UpperCAmelCase    ,					clean_text=__UpperCAmelCase    ,					tokenize_chinese_chars=__UpperCAmelCase    ,					strip_accents=__UpperCAmelCase    ,					wordpieces_prefix=__UpperCAmelCase    ,					**__UpperCAmelCase    ,					)

       _lowerCAmelCase						:	Optional[Any]        =					json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
       if (
           normalizer_state.get("""lowercase"""    ,					__UpperCAmelCase  ) != do_lower_case
           or normalizer_state.get("""strip_accents"""    ,					__UpperCAmelCase  ) != strip_accents
           or normalizer_state.get("""handle_chinese_chars"""    ,					__UpperCAmelCase  ) != tokenize_chinese_chars
       ):
         _lowerCAmelCase						:	Optional[int]        =					getattr(__UpperCAmelCase    ,					normalizer_state.pop("""type"""  )  )
         _lowerCAmelCase						:	List[Any]        =					do_lower_case
         _lowerCAmelCase						:	List[str]        =					strip_accents
         _lowerCAmelCase						:	Optional[Any]        =					tokenize_chinese_chars
         _lowerCAmelCase						:	List[Any]        =					normalizer_class(**__UpperCAmelCase  )

       _lowerCAmelCase						:	Union[str, Any]        =					do_lower_case






     def    __A  (		self    ,					a__    ,					a__=None  ):
       _lowerCAmelCase						:	Tuple        =					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

       if token_ids_a:
         output += token_ids_a + [self.sep_token_id]

       return output






     def    __A  (		self    ,					a__    ,					a__ = None  ):
       _lowerCAmelCase						:	Optional[Any]        =					[self.sep_token_id]
       _lowerCAmelCase						:	Optional[int]        =					[self.cls_token_id]
       if token_ids_a is None:
         return len(cls  ) * [self.cls_token_type_id] + len(token_ids_a + sep  ) * [0]
       return len(cls  ) * [self.cls_token_type_id] + len(token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]






     def    __A  (		self    ,					a__    ,					a__ = None  ):
       _lowerCAmelCase						:	List[str]        =					self._tokenizer.model.save(__UpperCAmelCase    ,					name=__UpperCAmelCase  )
       return tuple(__UpperCAmelCase  )






"""simple docstring"""




def 							SCREAMING_SNAKE_CASE					(      _lowerCamelCase						:	str      ,_lowerCamelCase						:	bool = False     )	->				str:

  if not isinstance(_lowerCamelCase      ,_lowerCamelCase     ):
    _lowerCAmelCase						:	Union[str, Any]        =					f"Expected string as input, found {type(_lowerCamelCase     )}"
    raise ValueError(_lowerCamelCase     )
  if not isinstance(_lowerCamelCase      ,_lowerCamelCase     ):
    _lowerCAmelCase						:	Union[str, Any]        =					f"Expected boolean as use_pascal parameter, found {type(_lowerCamelCase     )}"
    raise ValueError(_lowerCamelCase     )

  _lowerCAmelCase						:	str        =					input_str.split("""_"""     )

  _lowerCAmelCase						:	Optional[int]        =					0 if use_pascal else 1

  _lowerCAmelCase						:	List[Any]        =					words[start_index:]

  _lowerCAmelCase						:	Union[str, Any]        =					[word[0].upper() + word[1:] for word in words_to_capitalize]

  _lowerCAmelCase						:	List[Any]        =					"""""" if use_pascal else words[0]

  return "".join([initial_word, *capitalized_words]     )


if __name__ == "__main__":
     from doctest import testmod

     testmod()




