



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


__UpperCAmelCase	       =	logging.get_logger(__name__)

__UpperCAmelCase	       =	{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__UpperCAmelCase	       =	{
    '''vocab_file''': {
        '''google/realm-cc-news-pretrained-embedder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-encoder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-scorer''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-openqa''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt'''
        ),
        '''google/realm-orqa-nq-openqa''': '''https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt''',
        '''google/realm-orqa-nq-reader''': '''https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt''',
        '''google/realm-orqa-wq-openqa''': '''https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt''',
        '''google/realm-orqa-wq-reader''': '''https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt''',
    },
    '''tokenizer_file''': {
        '''google/realm-cc-news-pretrained-embedder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont'''
        ),
        '''google/realm-cc-news-pretrained-encoder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json'''
        ),
        '''google/realm-cc-news-pretrained-scorer''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json'''
        ),
        '''google/realm-cc-news-pretrained-openqa''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-nq-openqa''': (
            '''https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-nq-reader''': (
            '''https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-wq-openqa''': (
            '''https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-wq-reader''': (
            '''https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json'''
        ),
    },
}

__UpperCAmelCase	       =	{
    '''google/realm-cc-news-pretrained-embedder''': 512,
    '''google/realm-cc-news-pretrained-encoder''': 512,
    '''google/realm-cc-news-pretrained-scorer''': 512,
    '''google/realm-cc-news-pretrained-openqa''': 512,
    '''google/realm-orqa-nq-openqa''': 512,
    '''google/realm-orqa-nq-reader''': 512,
    '''google/realm-orqa-wq-openqa''': 512,
    '''google/realm-orqa-wq-reader''': 512,
}

__UpperCAmelCase	       =	{
    '''google/realm-cc-news-pretrained-embedder''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-encoder''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-scorer''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-nq-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-nq-reader''': {'''do_lower_case''': True},
    '''google/realm-orqa-wq-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-wq-reader''': {'''do_lower_case''': True},
}



class     lowerCAmelCase_		(  a__							):
       UpperCAmelCase__    : Union[str, Any]       					=  VOCAB_FILES_NAMES
       UpperCAmelCase__    : Tuple       					=  PRETRAINED_VOCAB_FILES_MAP
       UpperCAmelCase__    : List[str]       					=  PRETRAINED_INIT_CONFIGURATION
       UpperCAmelCase__    : Any       					=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       UpperCAmelCase__    : Optional[Any]       					=  RealmTokenizer
       def __init__(      self,      SCREAMING_SNAKE_CASE_=None,      SCREAMING_SNAKE_CASE_=None,      SCREAMING_SNAKE_CASE_=True,      SCREAMING_SNAKE_CASE_="[UNK]",      SCREAMING_SNAKE_CASE_="[SEP]",      SCREAMING_SNAKE_CASE_="[PAD]",      SCREAMING_SNAKE_CASE_="[CLS]",      SCREAMING_SNAKE_CASE_="[MASK]",      SCREAMING_SNAKE_CASE_=True,      SCREAMING_SNAKE_CASE_=None,      **SCREAMING_SNAKE_CASE_,      )					->		Tuple:
          super().__init__(
              SCREAMING_SNAKE_CASE_,      tokenizer_file=SCREAMING_SNAKE_CASE_,      do_lower_case=SCREAMING_SNAKE_CASE_,      unk_token=SCREAMING_SNAKE_CASE_,      sep_token=SCREAMING_SNAKE_CASE_,      pad_token=SCREAMING_SNAKE_CASE_,      cls_token=SCREAMING_SNAKE_CASE_,      mask_token=SCREAMING_SNAKE_CASE_,      tokenize_chinese_chars=SCREAMING_SNAKE_CASE_,      strip_accents=SCREAMING_SNAKE_CASE_,      **SCREAMING_SNAKE_CASE_,      )

          UpperCamelCase   :  List[str]						     =     json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
          if (
              normalizer_state.get('lowercase',      SCREAMING_SNAKE_CASE_	) != do_lower_case
              or normalizer_state.get('strip_accents',      SCREAMING_SNAKE_CASE_	) != strip_accents
              or normalizer_state.get('handle_chinese_chars',      SCREAMING_SNAKE_CASE_	) != tokenize_chinese_chars
          ):
             UpperCamelCase   :  Optional[Any]						     =     getattr(SCREAMING_SNAKE_CASE_,      normalizer_state.pop('type'	)	)
             UpperCamelCase   :  Tuple						     =     do_lower_case
             UpperCamelCase   :  List[Any]						     =     strip_accents
             UpperCamelCase   :  Dict						     =     tokenize_chinese_chars
             UpperCamelCase   :  List[Any]						     =     normalizer_class(**SCREAMING_SNAKE_CASE_	)

          UpperCamelCase   :  Any						     =     do_lower_case
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_,      **SCREAMING_SNAKE_CASE_	)					->		Tuple:
          UpperCamelCase   :  List[str]						     =     PaddingStrategy.MAX_LENGTH

          UpperCamelCase   :  Optional[Any]						     =     text
          UpperCamelCase   :  Optional[Any]						     =     kwargs.pop('text_pair',      SCREAMING_SNAKE_CASE_	)
          UpperCamelCase   :  int						     =     kwargs.pop('return_tensors',      SCREAMING_SNAKE_CASE_	)

          UpperCamelCase   :  Any						     =     {
              'input_ids': [],
              'attention_mask': [],
              'token_type_ids': [],
          }

          for idx, candidate_text in enumerate(SCREAMING_SNAKE_CASE_	):
             if batch_text_pair is not None:
                UpperCamelCase   :  Tuple						     =     batch_text_pair[idx]
             else:
                UpperCamelCase   :  Optional[Any]						     =     None

             UpperCamelCase   :  Any						     =     super().__call__(SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      return_tensors=SCREAMING_SNAKE_CASE_,      **SCREAMING_SNAKE_CASE_	)

             UpperCamelCase   :  List[str]						     =     encoded_candidates.get('input_ids'	)
             UpperCamelCase   :  Optional[Any]						     =     encoded_candidates.get('attention_mask'	)
             UpperCamelCase   :  str						     =     encoded_candidates.get('token_type_ids'	)

             if encoded_input_ids is not None:
                output_data["input_ids"].append(SCREAMING_SNAKE_CASE_	)
             if encoded_attention_mask is not None:
                output_data["attention_mask"].append(SCREAMING_SNAKE_CASE_	)
             if encoded_token_type_ids is not None:
                output_data["token_type_ids"].append(SCREAMING_SNAKE_CASE_	)

          UpperCamelCase   :  List[str]						     =     {key: item for key, item in output_data.items() if len(SCREAMING_SNAKE_CASE_	) != 0}

          return BatchEncoding(SCREAMING_SNAKE_CASE_,      tensor_type=SCREAMING_SNAKE_CASE_	)
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_=None	)					->		List[Any]:
          UpperCamelCase   :  Any						     =     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
             output += token_ids_a + [self.sep_token_id]

          return output
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_ = None	)					->		List[int]:
          UpperCamelCase   :  Dict						     =     [self.sep_token_id]
          UpperCamelCase   :  Optional[Any]						     =     [self.cls_token_id]
          if token_ids_a is None:
             return len(cls + token_ids_a + sep	) * [0]
          return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]






       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_ = None	)					->		Tuple[str]:
          UpperCamelCase   :  List[str]						     =     self._tokenizer.model.save(SCREAMING_SNAKE_CASE_,      name=SCREAMING_SNAKE_CASE_	)
          return tuple(SCREAMING_SNAKE_CASE_	)








import re
def 		UpperCamelCase     (       snake_case__	:   str     )     ->					str:

   if len(re.findall('[ATCG]'	,					snake_case__     )     ) != len(snake_case__     ):
      raise ValueError('Invalid Strand'     )

   return dna.translate(dna.maketrans('ATCG'	,					'TAGC'     )     )


if __name__ == "__main__":
    import doctest

    doctest.testmod()



