

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


__UpperCAmelCase								=			logging.get_logger(__name__)

__UpperCAmelCase								=			{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

__UpperCAmelCase								=			{
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

__UpperCAmelCase								=			{
    """google/realm-cc-news-pretrained-embedder""": 512,
    """google/realm-cc-news-pretrained-encoder""": 512,
    """google/realm-cc-news-pretrained-scorer""": 512,
    """google/realm-cc-news-pretrained-openqa""": 512,
    """google/realm-orqa-nq-openqa""": 512,
    """google/realm-orqa-nq-reader""": 512,
    """google/realm-orqa-wq-openqa""": 512,
    """google/realm-orqa-wq-reader""": 512,
}

__UpperCAmelCase								=			{
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}



class SCREAMING_SNAKE_CASE     (  a_			):
  """simple docstring"""






  lowerCamelCase   :      str										=VOCAB_FILES_NAMES
  lowerCamelCase   :      str										=PRETRAINED_VOCAB_FILES_MAP
  lowerCamelCase   :      List[Any]										=PRETRAINED_INIT_CONFIGURATION
  lowerCamelCase   :      Dict										=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  lowerCamelCase   :      List[str]										=RealmTokenizer







  def __init__(      self			:   str	,							lowerCAmelCase			:   List[str]=None	,							lowerCAmelCase			:   List[Any]=None	,							lowerCAmelCase			:   List[str]=True	,							lowerCAmelCase			:   Dict="[UNK]"	,							lowerCAmelCase			:   Tuple="[SEP]"	,							lowerCAmelCase			:   Tuple="[PAD]"	,							lowerCAmelCase			:   Optional[int]="[CLS]"	,							lowerCAmelCase			:   Tuple="[MASK]"	,							lowerCAmelCase			:   Tuple=True	,							lowerCAmelCase			:   Tuple=None	,							**lowerCAmelCase			:   str	,							)							->     List[Any]:






     """simple docstring"""







     super().__init__(
         lowerCAmelCase	,							tokenizer_file=lowerCAmelCase	,							do_lower_case=lowerCAmelCase	,							unk_token=lowerCAmelCase	,							sep_token=lowerCAmelCase	,							pad_token=lowerCAmelCase	,							cls_token=lowerCAmelCase	,							mask_token=lowerCAmelCase	,							tokenize_chinese_chars=lowerCAmelCase	,							strip_accents=lowerCAmelCase	,							**lowerCAmelCase	,							)

     __lowerCAmelCase  :      Optional[int]    =						json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
     if (
         normalizer_state.get("""lowercase"""	,							lowerCAmelCase   ) != do_lower_case
         or normalizer_state.get("""strip_accents"""	,							lowerCAmelCase   ) != strip_accents
         or normalizer_state.get("""handle_chinese_chars"""	,							lowerCAmelCase   ) != tokenize_chinese_chars
     ):
        __lowerCAmelCase  :      Tuple    =						getattr(lowerCAmelCase	,							normalizer_state.pop("""type"""   )   )
        __lowerCAmelCase  :      Union[str, Any]    =						do_lower_case
        __lowerCAmelCase  :      Any    =						strip_accents
        __lowerCAmelCase  :      List[str]    =						tokenize_chinese_chars
        __lowerCAmelCase  :      Any    =						normalizer_class(**lowerCAmelCase   )

     __lowerCAmelCase  :      Tuple    =						do_lower_case







  def 	SCREAMING_SNAKE_CASE   (      self			:   List[Any]	,							lowerCAmelCase			:   Dict	,							**lowerCAmelCase			:   str   )							->     Optional[Any]:






     """simple docstring"""







     __lowerCAmelCase  :      Optional[Any]    =						PaddingStrategy.MAX_LENGTH

     __lowerCAmelCase  :      Tuple    =						text
     __lowerCAmelCase  :      Tuple    =						kwargs.pop("""text_pair"""	,							lowerCAmelCase   )
     __lowerCAmelCase  :      List[Any]    =						kwargs.pop("""return_tensors"""	,							lowerCAmelCase   )

     __lowerCAmelCase  :      Optional[int]    =						{
         """input_ids""": [],
         """attention_mask""": [],
         """token_type_ids""": [],
     }

     for idx, candidate_text in enumerate(lowerCAmelCase   ):
        if batch_text_pair is not None:
           __lowerCAmelCase  :      Union[str, Any]    =						batch_text_pair[idx]
        else:
           __lowerCAmelCase  :      List[Any]    =						None

        __lowerCAmelCase  :      Any    =						super().__call__(lowerCAmelCase	,							lowerCAmelCase	,							return_tensors=lowerCAmelCase	,							**lowerCAmelCase   )

        __lowerCAmelCase  :      Dict    =						encoded_candidates.get("""input_ids"""   )
        __lowerCAmelCase  :      int    =						encoded_candidates.get("""attention_mask"""   )
        __lowerCAmelCase  :      List[str]    =						encoded_candidates.get("""token_type_ids"""   )

        if encoded_input_ids is not None:
           output_data["input_ids"].append(lowerCAmelCase   )
        if encoded_attention_mask is not None:
           output_data["attention_mask"].append(lowerCAmelCase   )
        if encoded_token_type_ids is not None:
           output_data["token_type_ids"].append(lowerCAmelCase   )

     __lowerCAmelCase  :      List[Any]    =						{key: item for key, item in output_data.items() if len(lowerCAmelCase   ) != 0}

     return BatchEncoding(lowerCAmelCase	,							tensor_type=lowerCAmelCase   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   str	,							lowerCAmelCase			:   str	,							lowerCAmelCase			:   str=None   )							->     Any:






     """simple docstring"""







     __lowerCAmelCase  :      Any    =						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

     if token_ids_a:
        output += token_ids_a + [self.sep_token_id]

     return output







  def 	SCREAMING_SNAKE_CASE   (      self			:   Optional[int]	,							lowerCAmelCase			:   List[int]	,							lowerCAmelCase			:   Optional[List[int]] = None   )							->     List[int]:






     """simple docstring"""







     __lowerCAmelCase  :      int    =						[self.sep_token_id]
     __lowerCAmelCase  :      Any    =						[self.cls_token_id]
     if token_ids_a is None:
        return len(cls + token_ids_a + sep   ) * [0]
     return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]







  def 	SCREAMING_SNAKE_CASE   (      self			:   Optional[int]	,							lowerCAmelCase			:   str	,							lowerCAmelCase			:   Optional[str] = None   )							->     Tuple[str]:






     """simple docstring"""







     __lowerCAmelCase  :      List[Any]    =						self._tokenizer.model.save(lowerCAmelCase	,							name=lowerCAmelCase   )
     return tuple(lowerCAmelCase   )





import unittest

import numpy as np

from transformers import BertConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_flax_available():
   from transformers.models.bert.modeling_flax_bert import (
       FlaxBertForMaskedLM,
       FlaxBertForMultipleChoice,
       FlaxBertForNextSentencePrediction,
       FlaxBertForPreTraining,
       FlaxBertForQuestionAnswering,
       FlaxBertForSequenceClassification,
       FlaxBertForTokenClassification,
       FlaxBertModel,
   )



class SCREAMING_SNAKE_CASE     (  unittest.TestCase			):
  """simple docstring"""






  def __init__(      self			:   Tuple	,							lowerCAmelCase			:   Tuple	,							lowerCAmelCase			:   str=13	,							lowerCAmelCase			:   Optional[Any]=7	,							lowerCAmelCase			:   Optional[Any]=True	,							lowerCAmelCase			:   Any=True	,							lowerCAmelCase			:   str=True	,							lowerCAmelCase			:   Optional[Any]=True	,							lowerCAmelCase			:   Dict=99	,							lowerCAmelCase			:   Any=32	,							lowerCAmelCase			:   int=5	,							lowerCAmelCase			:   Tuple=4	,							lowerCAmelCase			:   Optional[Any]=37	,							lowerCAmelCase			:   str="gelu"	,							lowerCAmelCase			:   str=0.1	,							lowerCAmelCase			:   str=0.1	,							lowerCAmelCase			:   Any=5_12	,							lowerCAmelCase			:   Optional[Any]=16	,							lowerCAmelCase			:   Dict=2	,							lowerCAmelCase			:   Tuple=0.02	,							lowerCAmelCase			:   Optional[int]=4	,							)							->     List[Any]:






     """simple docstring"""







     __lowerCAmelCase  :      int    =						parent
     __lowerCAmelCase  :      Dict    =						batch_size
     __lowerCAmelCase  :      Dict    =						seq_length
     __lowerCAmelCase  :      Union[str, Any]    =						is_training
     __lowerCAmelCase  :      List[Any]    =						use_attention_mask
     __lowerCAmelCase  :      List[Any]    =						use_token_type_ids
     __lowerCAmelCase  :      Optional[int]    =						use_labels
     __lowerCAmelCase  :      str    =						vocab_size
     __lowerCAmelCase  :      Any    =						hidden_size
     __lowerCAmelCase  :      Optional[int]    =						num_hidden_layers
     __lowerCAmelCase  :      Optional[int]    =						num_attention_heads
     __lowerCAmelCase  :      Dict    =						intermediate_size
     __lowerCAmelCase  :      Tuple    =						hidden_act
     __lowerCAmelCase  :      Dict    =						hidden_dropout_prob
     __lowerCAmelCase  :      Any    =						attention_probs_dropout_prob
     __lowerCAmelCase  :      Union[str, Any]    =						max_position_embeddings
     __lowerCAmelCase  :      int    =						type_vocab_size
     __lowerCAmelCase  :      Tuple    =						type_sequence_label_size
     __lowerCAmelCase  :      int    =						initializer_range
     __lowerCAmelCase  :      Optional[int]    =						num_choices







  def 	SCREAMING_SNAKE_CASE   (      self			:   Optional[int]   )							->     Dict:






     """simple docstring"""







     __lowerCAmelCase  :      Optional[Any]    =						ids_tensor([self.batch_size, self.seq_length]	,							self.vocab_size   )

     __lowerCAmelCase  :      Dict    =						None
     if self.use_attention_mask:
        __lowerCAmelCase  :      Optional[int]    =						random_attention_mask([self.batch_size, self.seq_length]   )

     __lowerCAmelCase  :      Union[str, Any]    =						None
     if self.use_token_type_ids:
        __lowerCAmelCase  :      str    =						ids_tensor([self.batch_size, self.seq_length]	,							self.type_vocab_size   )

     __lowerCAmelCase  :      List[str]    =						BertConfig(
         vocab_size=self.vocab_size	,							hidden_size=self.hidden_size	,							num_hidden_layers=self.num_hidden_layers	,							num_attention_heads=self.num_attention_heads	,							intermediate_size=self.intermediate_size	,							hidden_act=self.hidden_act	,							hidden_dropout_prob=self.hidden_dropout_prob	,							attention_probs_dropout_prob=self.attention_probs_dropout_prob	,							max_position_embeddings=self.max_position_embeddings	,							type_vocab_size=self.type_vocab_size	,							is_decoder=lowerCAmelCase	,							initializer_range=self.initializer_range	,							)

     return config, input_ids, token_type_ids, attention_mask







  def 	SCREAMING_SNAKE_CASE   (      self			:   List[Any]   )							->     Optional[Any]:






     """simple docstring"""







     __lowerCAmelCase  :      Any    =						self.prepare_config_and_inputs()
     __lowerCAmelCase   ,__lowerCAmelCase   ,__lowerCAmelCase   ,__lowerCAmelCase  :      List[str]    =						config_and_inputs
     __lowerCAmelCase  :      Any    =						{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": attention_mask}
     return config, inputs_dict







  def 	SCREAMING_SNAKE_CASE   (      self			:   Union[str, Any]   )							->     Tuple:






     """simple docstring"""







     __lowerCAmelCase  :      List[str]    =						self.prepare_config_and_inputs()
     __lowerCAmelCase   ,__lowerCAmelCase   ,__lowerCAmelCase   ,__lowerCAmelCase  :      Dict    =						config_and_inputs

     __lowerCAmelCase  :      Any    =						True
     __lowerCAmelCase  :      str    =						floats_tensor([self.batch_size, self.seq_length, self.hidden_size]   )
     __lowerCAmelCase  :      int    =						ids_tensor([self.batch_size, self.seq_length]	,							vocab_size=2   )

     return (
         config,
         input_ids,
         attention_mask,
         encoder_hidden_states,
         encoder_attention_mask,
     )






@require_flax
class SCREAMING_SNAKE_CASE     (  a_       ,							unittest.TestCase			):
  """simple docstring"""






  lowerCamelCase   :      int										=True

  lowerCamelCase   :      Any										=(
      (
          FlaxBertModel,
          FlaxBertForPreTraining,
          FlaxBertForMaskedLM,
          FlaxBertForMultipleChoice,
          FlaxBertForQuestionAnswering,
          FlaxBertForNextSentencePrediction,
          FlaxBertForSequenceClassification,
          FlaxBertForTokenClassification,
          FlaxBertForQuestionAnswering,
      )
      if is_flax_available()
      else ()
  )







  def 	SCREAMING_SNAKE_CASE   (      self			:   str   )							->     Union[str, Any]:






     """simple docstring"""







     __lowerCAmelCase  :      Union[str, Any]    =						FlaxBertModelTester(self   )







  @slow
  def 	SCREAMING_SNAKE_CASE   (      self			:   Dict   )							->     Dict:






     """simple docstring"""







     __lowerCAmelCase  :      int    =						FlaxBertModel.from_pretrained("""bert-base-cased"""   )
     __lowerCAmelCase  :      str    =						model(np.ones((1, 1)   )   )
     self.assertIsNotNone(lowerCAmelCase   )


