import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


__lowercase										= logging.get_logger(__name__)

__lowercase										= {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__lowercase										= [
    '''small''',
    '''small-base''',
    '''medium''',
    '''medium-base''',
    '''intermediate''',
    '''intermediate-base''',
    '''large''',
    '''large-base''',
    '''xlarge''',
    '''xlarge-base''',
]

__lowercase										= {
    '''vocab_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt''',
        '''funnel-transformer/small-base''': '''https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt''',
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt''',
        '''funnel-transformer/large-base''': '''https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json''',
        '''funnel-transformer/small-base''': (
            '''https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json''',
        '''funnel-transformer/large-base''': (
            '''https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'''
        ),
    },
}
__lowercase										= {F'funnel-transformer/{name}': 512 for name in _model_names}
__lowercase										= {F'funnel-transformer/{name}': {'''do_lower_case''': True} for name in _model_names}


class      lowerCamelCase_					(       UpperCAmelCase_			):






    '''simple docstring'''






    a__     :						List[str]      =				VOCAB_FILES_NAMES
    a__     :						List[str]      =				PRETRAINED_VOCAB_FILES_MAP
    a__     :						Any      =				PRETRAINED_INIT_CONFIGURATION
    a__     :						List[Any]      =				FunnelTokenizer
    a__     :						Optional[Any]      =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    a__     :						int      =				2







    def __init__(     self			,			__lowercase=None			,			__lowercase=None			,			__lowercase=True			,			__lowercase="<unk>"			,			__lowercase="<sep>"			,			__lowercase="<pad>"			,			__lowercase="<cls>"			,			__lowercase="<mask>"			,			__lowercase="<s>"			,			__lowercase="</s>"			,			__lowercase=True			,			__lowercase=True			,			__lowercase=None			,			__lowercase="##"			,			**__lowercase			,			)	->     Tuple:
        super().__init__(
            __lowercase			,			tokenizer_file=__lowercase			,			do_lower_case=__lowercase			,			unk_token=__lowercase			,			sep_token=__lowercase			,			pad_token=__lowercase			,			cls_token=__lowercase			,			mask_token=__lowercase			,			bos_token=__lowercase			,			eos_token=__lowercase			,			clean_text=__lowercase			,			tokenize_chinese_chars=__lowercase			,			strip_accents=__lowercase			,			wordpieces_prefix=__lowercase			,			**__lowercase			,			)

        __UpperCamelCase					:int        =		json.loads(self.backend_tokenizer.normalizer.__getstate__())
        if (
            normalizer_state.get('''lowercase'''			,			__lowercase) != do_lower_case
            or normalizer_state.get('''strip_accents'''			,			__lowercase) != strip_accents
            or normalizer_state.get('''handle_chinese_chars'''			,			__lowercase) != tokenize_chinese_chars
        ):
            __UpperCamelCase					:Optional[int]        =		getattr(__lowercase			,			normalizer_state.pop('''type'''))
            __UpperCamelCase					:str        =		do_lower_case
            __UpperCamelCase					:Dict        =		strip_accents
            __UpperCamelCase					:Optional[int]        =		tokenize_chinese_chars
            __UpperCamelCase					:Union[str, Any]        =		normalizer_class(**__lowercase)

        __UpperCamelCase					:int        =		do_lower_case







    def       UpperCamelCase__       (     self			,			__lowercase			,			__lowercase=None)	->     Any:
        __UpperCamelCase					:List[str]        =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
            output += token_ids_a + [self.sep_token_id]

        return output







    def       UpperCamelCase__       (     self			,			__lowercase			,			__lowercase = None)	->     List[int]:
        __UpperCamelCase					:Optional[int]        =		[self.sep_token_id]
        __UpperCamelCase					:List[Any]        =		[self.cls_token_id]
        if token_ids_a is None:
            return len(cls) * [self.cls_token_type_id] + len(token_ids_a + sep) * [0]
        return len(cls) * [self.cls_token_type_id] + len(token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]







    def       UpperCamelCase__       (     self			,			__lowercase			,			__lowercase = None)	->     Tuple[str]:
        __UpperCamelCase					:Any        =		self._tokenizer.model.save(__lowercase			,			name=__lowercase)
        return tuple(__lowercase)






from ...utils import is_note_seq_available, is_transformers_available, is_torch_available
from ...utils import OptionalDependencyNotAvailable


try:
    if not (is_transformers_available() and is_torch_available()):
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    from ...utils.dummy_torch_and_transformers_objects import *  # noqa F403
else:
    from .notes_encoder import SpectrogramNotesEncoder
    from .continous_encoder import SpectrogramContEncoder
    from .pipeline_spectrogram_diffusion import (
        SpectrogramContEncoder,
        SpectrogramDiffusionPipeline,
        TaFilmDecoder,
    )

try:
    if not (is_transformers_available() and is_torch_available() and is_note_seq_available()):
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    from ...utils.dummy_transformers_and_torch_and_note_seq_objects import *  # noqa F403
else:
    from .midi_utils import MidiProcessor





