






"""simple docstring"""







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


a			:    Dict				 =		logging.get_logger(__name__)

a			:    Optional[Any]				 =		{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

a			:    Optional[int]				 =		[
    """small""",
    """small-base""",
    """medium""",
    """medium-base""",
    """intermediate""",
    """intermediate-base""",
    """large""",
    """large-base""",
    """xlarge""",
    """xlarge-base""",
]

a			:    int				 =		{
    """vocab_file""": {
        """funnel-transformer/small""": """https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt""",
        """funnel-transformer/small-base""": """https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt""",
        """funnel-transformer/medium""": """https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt""",
        """funnel-transformer/medium-base""": (
            """https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/intermediate""": (
            """https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/intermediate-base""": (
            """https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/large""": """https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt""",
        """funnel-transformer/large-base""": """https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt""",
        """funnel-transformer/xlarge""": """https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt""",
        """funnel-transformer/xlarge-base""": (
            """https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """funnel-transformer/small""": """https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json""",
        """funnel-transformer/small-base""": (
            """https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/medium""": """https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json""",
        """funnel-transformer/medium-base""": (
            """https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/intermediate""": (
            """https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/intermediate-base""": (
            """https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/large""": """https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json""",
        """funnel-transformer/large-base""": (
            """https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/xlarge""": """https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json""",
        """funnel-transformer/xlarge-base""": (
            """https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"""
        ),
    },
}
a			:    List[Any]				 =		{F"""funnel-transformer/{name}""": 512 for name in _model_names}
a			:    Optional[int]				 =		{F"""funnel-transformer/{name}""": {"""do_lower_case""": True} for name in _model_names}


class    __UpperCAmelCase(       SCREAMING_SNAKE_CASE__							):




						"""simple docstring"""




						__lowerCamelCase    	= VOCAB_FILES_NAMES
						__lowerCamelCase    	= PRETRAINED_VOCAB_FILES_MAP
						__lowerCamelCase    	= PRETRAINED_INIT_CONFIGURATION
						__lowerCamelCase    	= FunnelTokenizer
						__lowerCamelCase    	= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						__lowerCamelCase    	= 2




						def __init__(       self	,  snake_case__=None	,  snake_case__=None	,  snake_case__=True	,  snake_case__="<unk>"	,  snake_case__="<sep>"	,  snake_case__="<pad>"	,  snake_case__="<cls>"	,  snake_case__="<mask>"	,  snake_case__="<s>"	,  snake_case__="</s>"	,  snake_case__=True	,  snake_case__=True	,  snake_case__=None	,  snake_case__="##"	,  **snake_case__	,  ):
												'''simple docstring'''
												super().__init__(
												    snake_case__	,  tokenizer_file=snake_case__	,  do_lower_case=snake_case__	,  unk_token=snake_case__	,  sep_token=snake_case__	,  pad_token=snake_case__	,  cls_token=snake_case__	,  mask_token=snake_case__	,  bos_token=snake_case__	,  eos_token=snake_case__	,  clean_text=snake_case__	,  tokenize_chinese_chars=snake_case__	,  strip_accents=snake_case__	,  wordpieces_prefix=snake_case__	,  **snake_case__	,  )

												lowercase__    :				Optional[int]=	json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
												if (
												    normalizer_state.get("lowercase"	,  snake_case__      ) != do_lower_case
												    or normalizer_state.get("strip_accents"	,  snake_case__      ) != strip_accents
												    or normalizer_state.get("handle_chinese_chars"	,  snake_case__      ) != tokenize_chinese_chars
												):
																		lowercase__    :				Optional[Any]=	getattr(snake_case__	,  normalizer_state.pop("type"      )      )
																		lowercase__    :				Any=	do_lower_case
																		lowercase__    :				int=	strip_accents
																		lowercase__    :				int=	tokenize_chinese_chars
																		lowercase__    :				Dict=	normalizer_class(**snake_case__      )

												lowercase__    :				Tuple=	do_lower_case




						def 		UpperCAmelCase_	(       self	,  snake_case__	,  snake_case__=None      ):
												'''simple docstring'''
												lowercase__    :				Optional[Any]=	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

												if token_ids_a:
																		output += token_ids_a + [self.sep_token_id]

												return output




						def 		UpperCAmelCase_	(       self	,  snake_case__	,  snake_case__ = None      ):
												'''simple docstring'''
												lowercase__    :				Union[str, Any]=	[self.sep_token_id]
												lowercase__    :				Union[str, Any]=	[self.cls_token_id]
												if token_ids_a is None:
																		return len(cls      ) * [self.cls_token_type_id] + len(token_ids_a + sep      ) * [0]
												return len(cls      ) * [self.cls_token_type_id] + len(token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]




						def 		UpperCAmelCase_	(       self	,  snake_case__	,  snake_case__ = None      ):
												'''simple docstring'''
												lowercase__    :				int=	self._tokenizer.model.save(snake_case__	,  name=snake_case__      )
												return tuple(snake_case__      )














"""simple docstring"""







import os
from pathlib import Path





def 						lowercase__()    ->List[Any]:




						"""simple docstring"""



						from torch.utils.cpp_extension import load

						lowercase__    :				Any=	Path(A						).resolve().parent.parent.parent / "kernels" / "deformable_detr"
						lowercase__    :				Any=	[
						    root / filename
						    for filename in [
						        "vision.cpp",
						        os.path.join("cpu" ,		"ms_deform_attn_cpu.cpp"						),
						        os.path.join("cuda" ,		"ms_deform_attn_cuda.cu"						),
						    ]
						]

						load(
						    "MultiScaleDeformableAttention" ,		A ,		with_cuda=A ,		extra_include_paths=[str(A						)] ,		extra_cflags=["-DWITH_CUDA=1"] ,		extra_cuda_cflags=[
						        "-DCUDA_HAS_FP16=1",
						        "-D__CUDA_NO_HALF_OPERATORS__",
						        "-D__CUDA_NO_HALF_CONVERSIONS__",
						        "-D__CUDA_NO_HALF2_OPERATORS__",
						    ] ,		)

						import MultiScaleDeformableAttention as MSDA

						return MSDA






