"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


_a				:		Optional[Any]	   =       logging.get_logger(__name__)

_a				:		Dict	   =       {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

_a				:		Tuple	   =       {
    'vocab_file': {
        'google/realm-cc-news-pretrained-embedder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-encoder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-scorer': (
            'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-openqa': (
            'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt'
        ),
        'google/realm-orqa-nq-openqa': 'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt',
        'google/realm-orqa-nq-reader': 'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt',
        'google/realm-orqa-wq-openqa': 'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt',
        'google/realm-orqa-wq-reader': 'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt',
    },
    'tokenizer_file': {
        'google/realm-cc-news-pretrained-embedder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont'
        ),
        'google/realm-cc-news-pretrained-encoder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json'
        ),
        'google/realm-cc-news-pretrained-scorer': (
            'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json'
        ),
        'google/realm-cc-news-pretrained-openqa': (
            'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json'
        ),
        'google/realm-orqa-nq-openqa': (
            'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-nq-reader': (
            'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-wq-openqa': (
            'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-wq-reader': (
            'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json'
        ),
    },
}

_a				:		List[Any]	   =       {
    'google/realm-cc-news-pretrained-embedder': 512,
    'google/realm-cc-news-pretrained-encoder': 512,
    'google/realm-cc-news-pretrained-scorer': 512,
    'google/realm-cc-news-pretrained-openqa': 512,
    'google/realm-orqa-nq-openqa': 512,
    'google/realm-orqa-nq-reader': 512,
    'google/realm-orqa-wq-openqa': 512,
    'google/realm-orqa-wq-reader': 512,
}

_a				:		Any	   =       {
    'google/realm-cc-news-pretrained-embedder': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-encoder': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-scorer': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-openqa': {'do_lower_case': True},
    'google/realm-orqa-nq-openqa': {'do_lower_case': True},
    'google/realm-orqa-nq-reader': {'do_lower_case': True},
    'google/realm-orqa-wq-openqa': {'do_lower_case': True},
    'google/realm-orqa-wq-reader': {'do_lower_case': True},
}
class 						__A							(			_UpperCamelCase		):
     _UpperCamelCase							:      List[Any]		 =		VOCAB_FILES_NAMES
     _UpperCamelCase							:      int		 =		PRETRAINED_VOCAB_FILES_MAP
     _UpperCamelCase							:      List[Any]		 =		PRETRAINED_INIT_CONFIGURATION
     _UpperCamelCase							:      str		 =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _UpperCamelCase							:      str		 =		RealmTokenizer






     def __init__(		self    ,					a__=None    ,					a__=None    ,					a__=True    ,					a__="[UNK]"    ,					a__="[SEP]"    ,					a__="[PAD]"    ,					a__="[CLS]"    ,					a__="[MASK]"    ,					a__=True    ,					a__=None    ,					**a__    ,					):
       super().__init__(
           _SCREAMING_SNAKE_CASE    ,					tokenizer_file=_SCREAMING_SNAKE_CASE    ,					do_lower_case=_SCREAMING_SNAKE_CASE    ,					unk_token=_SCREAMING_SNAKE_CASE    ,					sep_token=_SCREAMING_SNAKE_CASE    ,					pad_token=_SCREAMING_SNAKE_CASE    ,					cls_token=_SCREAMING_SNAKE_CASE    ,					mask_token=_SCREAMING_SNAKE_CASE    ,					tokenize_chinese_chars=_SCREAMING_SNAKE_CASE    ,					strip_accents=_SCREAMING_SNAKE_CASE    ,					**_SCREAMING_SNAKE_CASE    ,					)

       _lowerCAmelCase						:	Dict        =					json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
       if (
           normalizer_state.get("""lowercase"""    ,					_SCREAMING_SNAKE_CASE  ) != do_lower_case
           or normalizer_state.get("""strip_accents"""    ,					_SCREAMING_SNAKE_CASE  ) != strip_accents
           or normalizer_state.get("""handle_chinese_chars"""    ,					_SCREAMING_SNAKE_CASE  ) != tokenize_chinese_chars
       ):
         _lowerCAmelCase						:	Optional[Any]        =					getattr(_SCREAMING_SNAKE_CASE    ,					normalizer_state.pop("""type"""  )  )
         _lowerCAmelCase						:	Dict        =					do_lower_case
         _lowerCAmelCase						:	Union[str, Any]        =					strip_accents
         _lowerCAmelCase						:	Optional[Any]        =					tokenize_chinese_chars
         _lowerCAmelCase						:	Optional[Any]        =					normalizer_class(**_SCREAMING_SNAKE_CASE  )

       _lowerCAmelCase						:	List[str]        =					do_lower_case






     def    __A  (		self    ,					a__    ,					**a__  ):
       _lowerCAmelCase						:	Dict        =					PaddingStrategy.MAX_LENGTH

       _lowerCAmelCase						:	List[str]        =					text
       _lowerCAmelCase						:	Tuple        =					kwargs.pop("""text_pair"""    ,					_SCREAMING_SNAKE_CASE  )
       _lowerCAmelCase						:	Any        =					kwargs.pop("""return_tensors"""    ,					_SCREAMING_SNAKE_CASE  )

       _lowerCAmelCase						:	List[str]        =					{
           """input_ids""": [],
           """attention_mask""": [],
           """token_type_ids""": [],
       }

       for idx, candidate_text in enumerate(_SCREAMING_SNAKE_CASE  ):
         if batch_text_pair is not None:
           _lowerCAmelCase						:	Dict        =					batch_text_pair[idx]
         else:
           _lowerCAmelCase						:	Optional[int]        =					None

         _lowerCAmelCase						:	Union[str, Any]        =					super().__call__(_SCREAMING_SNAKE_CASE    ,					_SCREAMING_SNAKE_CASE    ,					return_tensors=_SCREAMING_SNAKE_CASE    ,					**_SCREAMING_SNAKE_CASE  )

         _lowerCAmelCase						:	List[str]        =					encoded_candidates.get("""input_ids"""  )
         _lowerCAmelCase						:	Tuple        =					encoded_candidates.get("""attention_mask"""  )
         _lowerCAmelCase						:	Optional[Any]        =					encoded_candidates.get("""token_type_ids"""  )

         if encoded_input_ids is not None:
           output_data["input_ids"].append(_SCREAMING_SNAKE_CASE  )
         if encoded_attention_mask is not None:
           output_data["attention_mask"].append(_SCREAMING_SNAKE_CASE  )
         if encoded_token_type_ids is not None:
           output_data["token_type_ids"].append(_SCREAMING_SNAKE_CASE  )

       _lowerCAmelCase						:	Optional[int]        =					{key: item for key, item in output_data.items() if len(_SCREAMING_SNAKE_CASE  ) != 0}

       return BatchEncoding(_SCREAMING_SNAKE_CASE    ,					tensor_type=_SCREAMING_SNAKE_CASE  )






     def    __A  (		self    ,					a__    ,					a__=None  ):
       _lowerCAmelCase						:	str        =					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

       if token_ids_a:
         output += token_ids_a + [self.sep_token_id]

       return output






     def    __A  (		self    ,					a__    ,					a__ = None  ):
       _lowerCAmelCase						:	List[Any]        =					[self.sep_token_id]
       _lowerCAmelCase						:	Optional[int]        =					[self.cls_token_id]
       if token_ids_a is None:
         return len(cls + token_ids_a + sep  ) * [0]
       return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]






     def    __A  (		self    ,					a__    ,					a__ = None  ):
       _lowerCAmelCase						:	Dict        =					self._tokenizer.model.save(_SCREAMING_SNAKE_CASE    ,					name=_SCREAMING_SNAKE_CASE  )
       return tuple(_SCREAMING_SNAKE_CASE  )





"""simple docstring"""




_a				:		Any	   =       '\n# Transformers installation\n! pip install transformers datasets\n# To install from source instead of the last release, comment the command above and uncomment the following one.\n# ! pip install git+https://github.com/huggingface/transformers.git\n'

_a				:		List[Any]	   =       [{'type': 'code', 'content': INSTALL_CONTENT}]
_a				:		Union[str, Any]	   =       {
    '{processor_class}': 'FakeProcessorClass',
    '{model_class}': 'FakeModelClass',
    '{object_class}': 'FakeObjectClass',
}




