

'''simple docstring'''







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


__lowerCamelCase						    =      logging.get_logger(__name__)

__lowerCamelCase						    =      {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__lowerCamelCase						    =      {
    '''vocab_file''': {
        '''bert-base-uncased''': '''https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt''',
        '''bert-large-uncased''': '''https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt''',
        '''bert-base-cased''': '''https://huggingface.co/bert-base-cased/resolve/main/vocab.txt''',
        '''bert-large-cased''': '''https://huggingface.co/bert-large-cased/resolve/main/vocab.txt''',
        '''bert-base-multilingual-uncased''': (
            '''https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt'''
        ),
        '''bert-base-multilingual-cased''': '''https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt''',
        '''bert-base-chinese''': '''https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt''',
        '''bert-base-german-cased''': '''https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt''',
        '''bert-large-uncased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt'''
        ),
        '''bert-large-cased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt'''
        ),
        '''bert-large-uncased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'''
        ),
        '''bert-large-cased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'''
        ),
        '''bert-base-cased-finetuned-mrpc''': (
            '''https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt'''
        ),
        '''bert-base-german-dbmdz-cased''': '''https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt''',
        '''bert-base-german-dbmdz-uncased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt'''
        ),
        '''TurkuNLP/bert-base-finnish-cased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt'''
        ),
        '''TurkuNLP/bert-base-finnish-uncased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt'''
        ),
        '''wietsedv/bert-base-dutch-cased''': (
            '''https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''bert-base-uncased''': '''https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json''',
        '''bert-large-uncased''': '''https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json''',
        '''bert-base-cased''': '''https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json''',
        '''bert-large-cased''': '''https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json''',
        '''bert-base-multilingual-uncased''': (
            '''https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-multilingual-cased''': (
            '''https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-chinese''': '''https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json''',
        '''bert-base-german-cased''': '''https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json''',
        '''bert-large-uncased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json'''
        ),
        '''bert-large-cased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json'''
        ),
        '''bert-large-uncased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'''
        ),
        '''bert-large-cased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'''
        ),
        '''bert-base-cased-finetuned-mrpc''': (
            '''https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json'''
        ),
        '''bert-base-german-dbmdz-cased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-german-dbmdz-uncased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json'''
        ),
        '''TurkuNLP/bert-base-finnish-cased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json'''
        ),
        '''TurkuNLP/bert-base-finnish-uncased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json'''
        ),
        '''wietsedv/bert-base-dutch-cased''': (
            '''https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json'''
        ),
    },
}

__lowerCamelCase						    =      {
    '''bert-base-uncased''': 512,
    '''bert-large-uncased''': 512,
    '''bert-base-cased''': 512,
    '''bert-large-cased''': 512,
    '''bert-base-multilingual-uncased''': 512,
    '''bert-base-multilingual-cased''': 512,
    '''bert-base-chinese''': 512,
    '''bert-base-german-cased''': 512,
    '''bert-large-uncased-whole-word-masking''': 512,
    '''bert-large-cased-whole-word-masking''': 512,
    '''bert-large-uncased-whole-word-masking-finetuned-squad''': 512,
    '''bert-large-cased-whole-word-masking-finetuned-squad''': 512,
    '''bert-base-cased-finetuned-mrpc''': 512,
    '''bert-base-german-dbmdz-cased''': 512,
    '''bert-base-german-dbmdz-uncased''': 512,
    '''TurkuNLP/bert-base-finnish-cased-v1''': 512,
    '''TurkuNLP/bert-base-finnish-uncased-v1''': 512,
    '''wietsedv/bert-base-dutch-cased''': 512,
}

__lowerCamelCase						    =      {
    '''bert-base-uncased''': {'''do_lower_case''': True},
    '''bert-large-uncased''': {'''do_lower_case''': True},
    '''bert-base-cased''': {'''do_lower_case''': False},
    '''bert-large-cased''': {'''do_lower_case''': False},
    '''bert-base-multilingual-uncased''': {'''do_lower_case''': True},
    '''bert-base-multilingual-cased''': {'''do_lower_case''': False},
    '''bert-base-chinese''': {'''do_lower_case''': False},
    '''bert-base-german-cased''': {'''do_lower_case''': False},
    '''bert-large-uncased-whole-word-masking''': {'''do_lower_case''': True},
    '''bert-large-cased-whole-word-masking''': {'''do_lower_case''': False},
    '''bert-large-uncased-whole-word-masking-finetuned-squad''': {'''do_lower_case''': True},
    '''bert-large-cased-whole-word-masking-finetuned-squad''': {'''do_lower_case''': False},
    '''bert-base-cased-finetuned-mrpc''': {'''do_lower_case''': False},
    '''bert-base-german-dbmdz-cased''': {'''do_lower_case''': False},
    '''bert-base-german-dbmdz-uncased''': {'''do_lower_case''': True},
    '''TurkuNLP/bert-base-finnish-cased-v1''': {'''do_lower_case''': False},
    '''TurkuNLP/bert-base-finnish-uncased-v1''': {'''do_lower_case''': True},
    '''wietsedv/bert-base-dutch-cased''': {'''do_lower_case''': False},
}


class    A__ ( _snake_case     ):
    lowercase		   =						VOCAB_FILES_NAMES
    lowercase		   =						PRETRAINED_VOCAB_FILES_MAP
    lowercase		   =						PRETRAINED_INIT_CONFIGURATION
    lowercase		   =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowercase		   =						BertTokenizer



    def __init__(   self						,			UpperCamelCase__=None						,			UpperCamelCase__=None						,			UpperCamelCase__=True						,			UpperCamelCase__="[UNK]"						,			UpperCamelCase__="[SEP]"						,			UpperCamelCase__="[PAD]"						,			UpperCamelCase__="[CLS]"						,			UpperCamelCase__="[MASK]"						,			UpperCamelCase__=True						,			UpperCamelCase__=None						,			**UpperCamelCase__						,			)  -> Union[str, Any]:





           '''simple docstring'''
           super().__init__(
               UpperCamelCase__						,			tokenizer_file=UpperCamelCase__						,			do_lower_case=UpperCamelCase__						,			unk_token=UpperCamelCase__						,			sep_token=UpperCamelCase__						,			pad_token=UpperCamelCase__						,			cls_token=UpperCamelCase__						,			mask_token=UpperCamelCase__						,			tokenize_chinese_chars=UpperCamelCase__						,			strip_accents=UpperCamelCase__						,			**UpperCamelCase__						,			)

           A_		   = json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
           if (
               normalizer_state.get("""lowercase"""						,			UpperCamelCase__    ) != do_lower_case
               or normalizer_state.get("""strip_accents"""						,			UpperCamelCase__    ) != strip_accents
               or normalizer_state.get("""handle_chinese_chars"""						,			UpperCamelCase__    ) != tokenize_chinese_chars
           ):
                  A_		   = getattr(UpperCamelCase__						,			normalizer_state.pop("""type"""    )    )
                  A_		   = do_lower_case
                  A_		   = strip_accents
                  A_		   = tokenize_chinese_chars
                  A_		   = normalizer_class(**UpperCamelCase__    )

           A_		   = do_lower_case



    def    snake_case_     (   self						,			UpperCamelCase__						,			UpperCamelCase__=None    )  -> str:





           '''simple docstring'''
           A_		   = [self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
                  output += token_ids_a + [self.sep_token_id]

           return output



    def    snake_case_     (   self						,			UpperCamelCase__						,			UpperCamelCase__ = None    )  -> List[int]:





           '''simple docstring'''
           A_		   = [self.sep_token_id]
           A_		   = [self.cls_token_id]
           if token_ids_a is None:
                  return len(cls + token_ids_a + sep    ) * [0]
           return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]







    def    snake_case_     (   self						,			UpperCamelCase__						,			UpperCamelCase__ = None    )  -> Tuple[str]:





           '''simple docstring'''
           A_		   = self._tokenizer.model.save(UpperCamelCase__						,			name=UpperCamelCase__    )
           return tuple(UpperCamelCase__    )



'''simple docstring'''







from tempfile import TemporaryDirectory
from unittest import TestCase
from unittest.mock import MagicMock, patch

from transformers import AutoModel, TFAutoModel
from transformers.onnx import FeaturesManager
from transformers.testing_utils import SMALL_MODEL_IDENTIFIER, require_tf, require_torch


@require_torch
@require_tf
class    A__ ( _snake_case     ):



    def    snake_case_     (   self    )  -> Dict:





           '''simple docstring'''
           A_		   = SMALL_MODEL_IDENTIFIER
           A_		   = """pt"""
           A_		   = """tf"""



    def    snake_case_     (   self						,			UpperCamelCase__    )  -> Tuple:





           '''simple docstring'''
           A_		   = AutoModel.from_pretrained(self.test_model    )
           model_pt.save_pretrained(UpperCamelCase__    )



    def    snake_case_     (   self						,			UpperCamelCase__    )  -> List[str]:





           '''simple docstring'''
           A_		   = TFAutoModel.from_pretrained(self.test_model						,			from_pt=UpperCamelCase__    )
           model_tf.save_pretrained(UpperCamelCase__    )



    def    snake_case_     (   self    )  -> Dict:





           '''simple docstring'''
           A_		   = """mock_framework"""

           # Framework provided - return whatever the user provides
           A_		   = FeaturesManager.determine_framework(self.test_model						,			UpperCamelCase__    )
           self.assertEqual(UpperCamelCase__						,			UpperCamelCase__    )

           # Local checkpoint and framework provided - return provided framework
           # PyTorch checkpoint
           with TemporaryDirectory() as local_pt_ckpt:
                  self._setup_pt_ckpt(UpperCamelCase__    )
                  A_		   = FeaturesManager.determine_framework(UpperCamelCase__						,			UpperCamelCase__    )
                  self.assertEqual(UpperCamelCase__						,			UpperCamelCase__    )

           # TensorFlow checkpoint
           with TemporaryDirectory() as local_tf_ckpt:
                  self._setup_tf_ckpt(UpperCamelCase__    )
                  A_		   = FeaturesManager.determine_framework(UpperCamelCase__						,			UpperCamelCase__    )
                  self.assertEqual(UpperCamelCase__						,			UpperCamelCase__    )



    def    snake_case_     (   self    )  -> int:





           '''simple docstring'''
           # PyTorch checkpoint
           with TemporaryDirectory() as local_pt_ckpt:
                  self._setup_pt_ckpt(UpperCamelCase__    )
                  A_		   = FeaturesManager.determine_framework(UpperCamelCase__    )
                  self.assertEqual(UpperCamelCase__						,			self.framework_pt    )

           # TensorFlow checkpoint
           with TemporaryDirectory() as local_tf_ckpt:
                  self._setup_tf_ckpt(UpperCamelCase__    )
                  A_		   = FeaturesManager.determine_framework(UpperCamelCase__    )
                  self.assertEqual(UpperCamelCase__						,			self.framework_tf    )

           # Invalid local checkpoint
           with TemporaryDirectory() as local_invalid_ckpt:
                  with self.assertRaises(UpperCamelCase__    ):
                         A_		   = FeaturesManager.determine_framework(UpperCamelCase__    )







    def    snake_case_     (   self    )  -> List[Any]:





           '''simple docstring'''
           A_		   = MagicMock(return_value=UpperCamelCase__    )
           with patch("""transformers.onnx.features.is_tf_available"""						,			UpperCamelCase__    ):
                  A_		   = FeaturesManager.determine_framework(self.test_model    )
                  self.assertEqual(UpperCamelCase__						,			self.framework_pt    )

           # PyTorch not in environment -> use TensorFlow
           A_		   = MagicMock(return_value=UpperCamelCase__    )
           with patch("""transformers.onnx.features.is_torch_available"""						,			UpperCamelCase__    ):
                  A_		   = FeaturesManager.determine_framework(self.test_model    )
                  self.assertEqual(UpperCamelCase__						,			self.framework_tf    )

           # Both in environment -> use PyTorch
           A_		   = MagicMock(return_value=UpperCamelCase__    )
           A_		   = MagicMock(return_value=UpperCamelCase__    )
           with patch("""transformers.onnx.features.is_tf_available"""						,			UpperCamelCase__    ), patch(
               """transformers.onnx.features.is_torch_available"""						,			UpperCamelCase__    ):
                  A_		   = FeaturesManager.determine_framework(self.test_model    )
                  self.assertEqual(UpperCamelCase__						,			self.framework_pt    )

           # Both not in environment -> raise error
           A_		   = MagicMock(return_value=UpperCamelCase__    )
           A_		   = MagicMock(return_value=UpperCamelCase__    )
           with patch("""transformers.onnx.features.is_tf_available"""						,			UpperCamelCase__    ), patch(
               """transformers.onnx.features.is_torch_available"""						,			UpperCamelCase__    ):
                  with self.assertRaises(UpperCamelCase__    ):
                         A_		   = FeaturesManager.determine_framework(self.test_model    )
