


import unittest
from pathlib import Path
from tempfile import NamedTemporaryFile, TemporaryDirectory

from transformers import BertConfig, BertTokenizerFast, FeatureExtractionPipeline
from transformers.convert_graph_to_onnx import (
    convert,
    ensure_valid_input,
    generate_identified_filename,
    infer_shapes,
    quantize,
)
from transformers.testing_utils import require_tf, require_tokenizers, require_torch, slow






class    UpperCAmelCase__	:



    """simple docstring"""






    def _a (      self ,       A_ ,       A_ ,       A_   )  ->	List[str]:
         return None






class    UpperCAmelCase__	:



    """simple docstring"""






    def _a (      self ,       A_ ,       A_ ,       A_ ,       A_   )  ->	int:
         return None






class    UpperCAmelCase__	( unittest.TestCase		):



    """simple docstring"""






    UpperCAmelCase__						:		Tuple              =  [
        # (model_name, model_kwargs)
        ("""bert-base-cased""", {}),
        ("""gpt2""", {"""use_cache""": False}),  # We don't support exporting GPT2 past keys anymore
    ]






    @require_tf
    @slow
    def _a (      self   )  ->	str:

         for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
              self._test_export(A_ ,       'tf' ,       12 ,       **A_   )






    @require_torch
    @slow
    def _a (      self   )  ->	List[str]:

         for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
              self._test_export(A_ ,       'pt' ,       12 ,       **A_   )






    @require_torch
    @slow
    def _a (      self   )  ->	List[Any]:
         from transformers import BertModel

         __UpperCamelCase   							=['[UNK]', '[SEP]', '[CLS]', '[PAD]', '[MASK]', 'some', 'other', 'words']
         with NamedTemporaryFile(mode='w+t'   ) as vocab_file:
              vocab_file.write('\n'.join(A_   )   )
              vocab_file.flush()
              __UpperCamelCase   							=BertTokenizerFast(vocab_file.name   )

         with TemporaryDirectory() as bert_save_dir:
              __UpperCamelCase   							=BertModel(BertConfig(vocab_size=len(A_   )   )   )
              model.save_pretrained(A_   )
              self._test_export(A_ ,       'pt' ,       12 ,       A_   )






    @require_tf
    @slow
    def _a (      self   )  ->	Optional[Any]:

         for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
              __UpperCamelCase   							=self._test_export(A_ ,       'tf' ,       12 ,       **A_   )
              __UpperCamelCase   							=quantize(Path(A_   )   )

              # Ensure the actual quantized model is not bigger than the original one
              if quantized_path.stat().st_size >= Path(A_   ).stat().st_size:
                   self.fail('Quantized model is bigger than initial ONNX model'   )






    @require_torch
    @slow
    def _a (      self   )  ->	str:

         for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
              __UpperCamelCase   							=self._test_export(A_ ,       'pt' ,       12 ,       **A_   )
              __UpperCamelCase   							=quantize(A_   )

              # Ensure the actual quantized model is not bigger than the original one
              if quantized_path.stat().st_size >= Path(A_   ).stat().st_size:
                   self.fail('Quantized model is bigger than initial ONNX model'   )






    def _a (      self ,       A_ ,       A_ ,       A_ ,       A_=None ,       **A_   )  ->	List[str]:

         try:
              # Compute path
              with TemporaryDirectory() as tempdir:
                   __UpperCamelCase   							=Path(A_   ).joinpath('model.onnx'   )

              # Remove folder if exists
              if path.parent.exists():
                   path.parent.rmdir()

              # Export
              convert(A_ ,       A_ ,       A_ ,       A_ ,       A_ ,       **A_   )

              return path
         except Exception as e:
              self.fail(A_   )






    @require_torch
    @require_tokenizers
    @slow
    def _a (      self   )  ->	str:
         from transformers import BertModel

         __UpperCamelCase   							=BertModel(BertConfig.from_pretrained('lysandre/tiny-bert-random'   )   )
         __UpperCamelCase   							=BertTokenizerFast.from_pretrained('lysandre/tiny-bert-random'   )
         self._test_infer_dynamic_axis(A_ ,       A_ ,       'pt'   )






    @require_tf
    @require_tokenizers
    @slow
    def _a (      self   )  ->	Tuple:
         from transformers import TFBertModel

         __UpperCamelCase   							=TFBertModel(BertConfig.from_pretrained('lysandre/tiny-bert-random'   )   )
         __UpperCamelCase   							=BertTokenizerFast.from_pretrained('lysandre/tiny-bert-random'   )
         self._test_infer_dynamic_axis(A_ ,       A_ ,       'tf'   )






    def _a (      self ,       A_ ,       A_ ,       A_   )  ->	List[Any]:
         __UpperCamelCase   							=FeatureExtractionPipeline(A_ ,       A_   )

         __UpperCamelCase   							=['input_ids', 'token_type_ids', 'attention_mask', 'output_0', 'output_1']
         __UpperCamelCase   							=infer_shapes(A_ ,       A_   )

         # Assert all variables are present
         self.assertEqual(len(A_   ) ,       len(A_   )   )
         self.assertTrue(all(var_name in shapes for var_name in variable_names   )   )
         self.assertSequenceEqual(variable_names[:3] ,       A_   )
         self.assertSequenceEqual(variable_names[3:] ,       A_   )

         # Assert inputs are {0: batch, 1: sequence}
         for var_name in ["input_ids", "token_type_ids", "attention_mask"]:
              self.assertDictEqual(shapes[var_name] ,       {0: 'batch', 1: 'sequence'}   )

         # Assert outputs are {0: batch, 1: sequence} and {0: batch}
         self.assertDictEqual(shapes['output_0'] ,       {0: 'batch', 1: 'sequence'}   )
         self.assertDictEqual(shapes['output_1'] ,       {0: 'batch'}   )






    def _a (      self   )  ->	int:
         __UpperCamelCase   							=['input_ids', 'attention_mask', 'token_type_ids']
         __UpperCamelCase   							={'input_ids': [1, 2, 3, 4], 'attention_mask': [0, 0, 0, 0], 'token_type_ids': [1, 1, 1, 1]}
         __UpperCamelCase   							=ensure_valid_input(FuncContiguousArgs() ,       A_ ,       A_   )

         # Should have exactly the same number of args (all are valid)
         self.assertEqual(len(A_   ) ,       3   )

         # Should have exactly the same input names
         self.assertEqual(set(A_   ) ,       set(A_   )   )

         # Parameter should be reordered according to their respective place in the function:
         # (input_ids, token_type_ids, attention_mask)
         self.assertEqual(A_ ,       (tokens['input_ids'], tokens['token_type_ids'], tokens['attention_mask'])   )

         # Generated args are interleaved with another args (for instance parameter "past" in GPT2)
         __UpperCamelCase   							=ensure_valid_input(FuncNonContiguousArgs() ,       A_ ,       A_   )

         # Should have exactly the one arg (all before the one not provided "some_other_args")
         self.assertEqual(len(A_   ) ,       1   )
         self.assertEqual(len(A_   ) ,       1   )

         # Should have only "input_ids"
         self.assertEqual(inputs_args[0] ,       tokens['input_ids']   )
         self.assertEqual(ordered_input_names[0] ,       'input_ids'   )






    def _a (      self   )  ->	Dict:
         __UpperCamelCase   							=generate_identified_filename(Path('/home/something/my_fake_model.onnx'   ) ,       '-test'   )
         self.assertEqual('/home/something/my_fake_model-test.onnx' ,       generated.as_posix()   )









import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


_A			=						logging.get_logger(__name__)

_A			=						{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

_A			=						{
    'vocab_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/vocab.txt',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/vocab.txt',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt'
        ),
        'bert-base-multilingual-cased': 'https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt',
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt'
        ),
        'bert-base-german-dbmdz-cased': 'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt',
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json'
        ),
        'bert-base-multilingual-cased': (
            'https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-cased': (
            'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json'
        ),
    },
}

_A			=						{
    'bert-base-uncased': 512,
    'bert-large-uncased': 512,
    'bert-base-cased': 512,
    'bert-large-cased': 512,
    'bert-base-multilingual-uncased': 512,
    'bert-base-multilingual-cased': 512,
    'bert-base-chinese': 512,
    'bert-base-german-cased': 512,
    'bert-large-uncased-whole-word-masking': 512,
    'bert-large-cased-whole-word-masking': 512,
    'bert-large-uncased-whole-word-masking-finetuned-squad': 512,
    'bert-large-cased-whole-word-masking-finetuned-squad': 512,
    'bert-base-cased-finetuned-mrpc': 512,
    'bert-base-german-dbmdz-cased': 512,
    'bert-base-german-dbmdz-uncased': 512,
    'TurkuNLP/bert-base-finnish-cased-v1': 512,
    'TurkuNLP/bert-base-finnish-uncased-v1': 512,
    'wietsedv/bert-base-dutch-cased': 512,
}

_A			=						{
    'bert-base-uncased': {'do_lower_case': True},
    'bert-large-uncased': {'do_lower_case': True},
    'bert-base-cased': {'do_lower_case': False},
    'bert-large-cased': {'do_lower_case': False},
    'bert-base-multilingual-uncased': {'do_lower_case': True},
    'bert-base-multilingual-cased': {'do_lower_case': False},
    'bert-base-chinese': {'do_lower_case': False},
    'bert-base-german-cased': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking-finetuned-squad': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking-finetuned-squad': {'do_lower_case': False},
    'bert-base-cased-finetuned-mrpc': {'do_lower_case': False},
    'bert-base-german-dbmdz-cased': {'do_lower_case': False},
    'bert-base-german-dbmdz-uncased': {'do_lower_case': True},
    'TurkuNLP/bert-base-finnish-cased-v1': {'do_lower_case': False},
    'TurkuNLP/bert-base-finnish-uncased-v1': {'do_lower_case': True},
    'wietsedv/bert-base-dutch-cased': {'do_lower_case': False},
}






class    UpperCAmelCase__	( A_		):



    """simple docstring"""






    UpperCAmelCase__						:		Any              =  VOCAB_FILES_NAMES
    UpperCAmelCase__						:		List[str]              =  PRETRAINED_VOCAB_FILES_MAP
    UpperCAmelCase__						:		Optional[int]              =  PRETRAINED_INIT_CONFIGURATION
    UpperCAmelCase__						:		List[str]              =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    UpperCAmelCase__						:		Dict              =  BertTokenizer






    def __init__(      self ,       A_=None ,       A_=None ,       A_=True ,       A_="[UNK]" ,       A_="[SEP]" ,       A_="[PAD]" ,       A_="[CLS]" ,       A_="[MASK]" ,       A_=True ,       A_=None ,       **A_ ,       )  ->	Any:
         super().__init__(
             A_ ,       tokenizer_file=A_ ,       do_lower_case=A_ ,       unk_token=A_ ,       sep_token=A_ ,       pad_token=A_ ,       cls_token=A_ ,       mask_token=A_ ,       tokenize_chinese_chars=A_ ,       strip_accents=A_ ,       **A_ ,       )

         __UpperCamelCase   							=json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
         if (
             normalizer_state.get('lowercase' ,       A_   ) != do_lower_case
             or normalizer_state.get('strip_accents' ,       A_   ) != strip_accents
             or normalizer_state.get('handle_chinese_chars' ,       A_   ) != tokenize_chinese_chars
         ):
              __UpperCamelCase   							=getattr(A_ ,       normalizer_state.pop('type'   )   )
              __UpperCamelCase   							=do_lower_case
              __UpperCamelCase   							=strip_accents
              __UpperCamelCase   							=tokenize_chinese_chars
              __UpperCamelCase   							=normalizer_class(**A_   )

         __UpperCamelCase   							=do_lower_case






    def _a (      self ,       A_ ,       A_=None   )  ->	List[str]:
         __UpperCamelCase   							=[self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
              output += token_ids_a + [self.sep_token_id]

         return output






    def _a (      self ,       A_ ,       A_ = None   )  ->	List[int]:
         __UpperCamelCase   							=[self.sep_token_id]
         __UpperCamelCase   							=[self.cls_token_id]
         if token_ids_a is None:
              return len(cls + token_ids_a + sep   ) * [0]
         return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]






    def _a (      self ,       A_ ,       A_ = None   )  ->	Tuple[str]:
         __UpperCamelCase   							=self._tokenizer.model.save(A_ ,       name=A_   )
         return tuple(A_   )





