




'''simple docstring'''



import os
import tempfile
import unittest

from transformers import FlaubertConfig, is_torch_available
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import (
     FlaubertForMultipleChoice,
     FlaubertForQuestionAnswering,
     FlaubertForQuestionAnsweringSimple,
     FlaubertForSequenceClassification,
     FlaubertForTokenClassification,
     FlaubertModel,
     FlaubertWithLMHeadModel,
 )
 from transformers.models.flaubert.modeling_flaubert import FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST







class 		lowercase_   (			A       ):


   """simple docstring"""

   def __init__(   self :     Optional[int]  ,				__lowerCamelCase :     Dict  ,				__lowerCamelCase :     Dict=1_3  ,				__lowerCamelCase :     Union[str, Any]=7  ,				__lowerCamelCase :     str=True  ,				__lowerCamelCase :     Optional[int]=True  ,				__lowerCamelCase :     Optional[int]=True  ,				__lowerCamelCase :     List[str]=True  ,				__lowerCamelCase :     Union[str, Any]=True  ,				__lowerCamelCase :     Tuple=False  ,				__lowerCamelCase :     str=False  ,				__lowerCamelCase :     List[str]=False  ,				__lowerCamelCase :     Union[str, Any]=2  ,				__lowerCamelCase :     str=9_9  ,				__lowerCamelCase :     int=0  ,				__lowerCamelCase :     Tuple=3_2  ,				__lowerCamelCase :     List[Any]=5  ,				__lowerCamelCase :     Tuple=4  ,				__lowerCamelCase :     Optional[int]=0.1  ,				__lowerCamelCase :     List[Any]=0.1  ,				__lowerCamelCase :     List[Any]=5_1_2  ,				__lowerCamelCase :     List[Any]=1_2  ,				__lowerCamelCase :     Union[str, Any]=2  ,				__lowerCamelCase :     Tuple=0.0_2  ,				__lowerCamelCase :     Any=3  ,				__lowerCamelCase :     int=4  ,				__lowerCamelCase :     Optional[Any]="last"  ,				__lowerCamelCase :     Tuple=None  ,				__lowerCamelCase :     str=None  ,				):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				parent
      _SCREAMING_SNAKE_CASE	=				batch_size
      _SCREAMING_SNAKE_CASE	=				seq_length
      _SCREAMING_SNAKE_CASE	=				is_training
      _SCREAMING_SNAKE_CASE	=				use_input_lengths
      _SCREAMING_SNAKE_CASE	=				use_token_type_ids
      _SCREAMING_SNAKE_CASE	=				use_labels
      _SCREAMING_SNAKE_CASE	=				gelu_activation
      _SCREAMING_SNAKE_CASE	=				sinusoidal_embeddings
      _SCREAMING_SNAKE_CASE	=				causal
      _SCREAMING_SNAKE_CASE	=				asm
      _SCREAMING_SNAKE_CASE	=				n_langs
      _SCREAMING_SNAKE_CASE	=				vocab_size
      _SCREAMING_SNAKE_CASE	=				n_special
      _SCREAMING_SNAKE_CASE	=				hidden_size
      _SCREAMING_SNAKE_CASE	=				num_hidden_layers
      _SCREAMING_SNAKE_CASE	=				num_attention_heads
      _SCREAMING_SNAKE_CASE	=				hidden_dropout_prob
      _SCREAMING_SNAKE_CASE	=				attention_probs_dropout_prob
      _SCREAMING_SNAKE_CASE	=				max_position_embeddings
      _SCREAMING_SNAKE_CASE	=				type_vocab_size
      _SCREAMING_SNAKE_CASE	=				type_sequence_label_size
      _SCREAMING_SNAKE_CASE	=				initializer_range
      _SCREAMING_SNAKE_CASE	=				num_labels
      _SCREAMING_SNAKE_CASE	=				num_choices
      _SCREAMING_SNAKE_CASE	=				summary_type
      _SCREAMING_SNAKE_CASE	=				use_proj
      _SCREAMING_SNAKE_CASE	=				scope



   def 							lowerCAmelCase_      (   self :     Optional[int]   ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				ids_tensor([self.batch_size, self.seq_length]  ,				self.vocab_size   )
      _SCREAMING_SNAKE_CASE	=				random_attention_mask([self.batch_size, self.seq_length]   )

      _SCREAMING_SNAKE_CASE	=				None
      if self.use_input_lengths:
         _SCREAMING_SNAKE_CASE	=				(
             ids_tensor([self.batch_size]  ,				vocab_size=2   ) + self.seq_length - 2
         )  # small variation of seq_length

      _SCREAMING_SNAKE_CASE	=				None
      if self.use_token_type_ids:
         _SCREAMING_SNAKE_CASE	=				ids_tensor([self.batch_size, self.seq_length]  ,				self.n_langs   )

      _SCREAMING_SNAKE_CASE	=				None
      _SCREAMING_SNAKE_CASE	=				None
      _SCREAMING_SNAKE_CASE	=				None
      if self.use_labels:
         _SCREAMING_SNAKE_CASE	=				ids_tensor([self.batch_size]  ,				self.type_sequence_label_size   )
         _SCREAMING_SNAKE_CASE	=				ids_tensor([self.batch_size, self.seq_length]  ,				self.num_labels   )
         _SCREAMING_SNAKE_CASE	=				ids_tensor([self.batch_size]  ,				2   ).float()
         _SCREAMING_SNAKE_CASE	=				ids_tensor([self.batch_size]  ,				self.num_choices   )

      _SCREAMING_SNAKE_CASE	=				self.get_config()

      return (
          config,
          input_ids,
          token_type_ids,
          input_lengths,
          sequence_labels,
          token_labels,
          is_impossible_labels,
          choice_labels,
          input_mask,
      )



   def 							lowerCAmelCase_      (   self :     Tuple   ):

      """simple docstring"""
      return FlaubertConfig(
          vocab_size=self.vocab_size  ,				n_special=self.n_special  ,				emb_dim=self.hidden_size  ,				n_layers=self.num_hidden_layers  ,				n_heads=self.num_attention_heads  ,				dropout=self.hidden_dropout_prob  ,				attention_dropout=self.attention_probs_dropout_prob  ,				gelu_activation=self.gelu_activation  ,				sinusoidal_embeddings=self.sinusoidal_embeddings  ,				asm=self.asm  ,				causal=self.causal  ,				n_langs=self.n_langs  ,				max_position_embeddings=self.max_position_embeddings  ,				initializer_range=self.initializer_range  ,				summary_type=self.summary_type  ,				use_proj=self.use_proj  ,				)



   def 							lowerCAmelCase_      (   self :     List[Any]  ,				__lowerCamelCase :     Optional[Any]  ,				__lowerCamelCase :     Union[str, Any]  ,				__lowerCamelCase :     List[str]  ,				__lowerCamelCase :     Any  ,				__lowerCamelCase :     Optional[int]  ,				__lowerCamelCase :     Dict  ,				__lowerCamelCase :     List[str]  ,				__lowerCamelCase :     int  ,				__lowerCamelCase :     Any  ,				):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				FlaubertModel(config=__lowerCamelCase   )
      model.to(__lowerCamelCase   )
      model.eval()
      _SCREAMING_SNAKE_CASE	=				model(__lowerCamelCase  ,				lengths=__lowerCamelCase  ,				langs=__lowerCamelCase   )
      _SCREAMING_SNAKE_CASE	=				model(__lowerCamelCase  ,				langs=__lowerCamelCase   )
      _SCREAMING_SNAKE_CASE	=				model(__lowerCamelCase   )
      self.parent.assertEqual(result.last_hidden_state.shape  ,				(self.batch_size, self.seq_length, self.hidden_size)   )



   def 							lowerCAmelCase_      (   self :     List[str]  ,				__lowerCamelCase :     Optional[int]  ,				__lowerCamelCase :     List[str]  ,				__lowerCamelCase :     Dict  ,				__lowerCamelCase :     Optional[int]  ,				__lowerCamelCase :     int  ,				__lowerCamelCase :     Optional[int]  ,				__lowerCamelCase :     Optional[int]  ,				__lowerCamelCase :     str  ,				__lowerCamelCase :     Dict  ,				):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				FlaubertWithLMHeadModel(__lowerCamelCase   )
      model.to(__lowerCamelCase   )
      model.eval()

      _SCREAMING_SNAKE_CASE	=				model(__lowerCamelCase  ,				token_type_ids=__lowerCamelCase  ,				labels=__lowerCamelCase   )
      self.parent.assertEqual(result.loss.shape  ,				()   )
      self.parent.assertEqual(result.logits.shape  ,				(self.batch_size, self.seq_length, self.vocab_size)   )



   def 							lowerCAmelCase_      (   self :     Any  ,				__lowerCamelCase :     List[str]  ,				__lowerCamelCase :     Optional[Any]  ,				__lowerCamelCase :     Union[str, Any]  ,				__lowerCamelCase :     Tuple  ,				__lowerCamelCase :     Dict  ,				__lowerCamelCase :     Union[str, Any]  ,				__lowerCamelCase :     List[Any]  ,				__lowerCamelCase :     Any  ,				__lowerCamelCase :     List[str]  ,				):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				FlaubertForQuestionAnsweringSimple(__lowerCamelCase   )
      model.to(__lowerCamelCase   )
      model.eval()

      _SCREAMING_SNAKE_CASE	=				model(__lowerCamelCase   )

      _SCREAMING_SNAKE_CASE	=				model(__lowerCamelCase  ,				start_positions=__lowerCamelCase  ,				end_positions=__lowerCamelCase   )
      self.parent.assertEqual(result.start_logits.shape  ,				(self.batch_size, self.seq_length)   )
      self.parent.assertEqual(result.end_logits.shape  ,				(self.batch_size, self.seq_length)   )



   def 							lowerCAmelCase_      (   self :     Optional[int]  ,				__lowerCamelCase :     Any  ,				__lowerCamelCase :     Union[str, Any]  ,				__lowerCamelCase :     Optional[Any]  ,				__lowerCamelCase :     Dict  ,				__lowerCamelCase :     Optional[int]  ,				__lowerCamelCase :     Optional[int]  ,				__lowerCamelCase :     str  ,				__lowerCamelCase :     Optional[Any]  ,				__lowerCamelCase :     Any  ,				):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				FlaubertForQuestionAnswering(__lowerCamelCase   )
      model.to(__lowerCamelCase   )
      model.eval()

      _SCREAMING_SNAKE_CASE	=				model(__lowerCamelCase   )

      _SCREAMING_SNAKE_CASE	=				model(
          __lowerCamelCase  ,				start_positions=__lowerCamelCase  ,				end_positions=__lowerCamelCase  ,				cls_index=__lowerCamelCase  ,				is_impossible=__lowerCamelCase  ,				p_mask=__lowerCamelCase  ,				)

      _SCREAMING_SNAKE_CASE	=				model(
          __lowerCamelCase  ,				start_positions=__lowerCamelCase  ,				end_positions=__lowerCamelCase  ,				cls_index=__lowerCamelCase  ,				is_impossible=__lowerCamelCase  ,				)

      ((_SCREAMING_SNAKE_CASE),	)	=				result_with_labels.to_tuple()

      _SCREAMING_SNAKE_CASE	=				model(__lowerCamelCase  ,				start_positions=__lowerCamelCase  ,				end_positions=__lowerCamelCase   )

      ((_SCREAMING_SNAKE_CASE),	)	=				result_with_labels.to_tuple()

      self.parent.assertEqual(result_with_labels.loss.shape  ,				()   )
      self.parent.assertEqual(result.start_top_log_probs.shape  ,				(self.batch_size, model.config.start_n_top)   )
      self.parent.assertEqual(result.start_top_index.shape  ,				(self.batch_size, model.config.start_n_top)   )
      self.parent.assertEqual(
          result.end_top_log_probs.shape  ,				(self.batch_size, model.config.start_n_top * model.config.end_n_top)   )
      self.parent.assertEqual(
          result.end_top_index.shape  ,				(self.batch_size, model.config.start_n_top * model.config.end_n_top)   )
      self.parent.assertEqual(result.cls_logits.shape  ,				(self.batch_size,)   )



   def 							lowerCAmelCase_      (   self :     Dict  ,				__lowerCamelCase :     Optional[int]  ,				__lowerCamelCase :     List[str]  ,				__lowerCamelCase :     Optional[int]  ,				__lowerCamelCase :     Any  ,				__lowerCamelCase :     List[Any]  ,				__lowerCamelCase :     Union[str, Any]  ,				__lowerCamelCase :     str  ,				__lowerCamelCase :     Any  ,				__lowerCamelCase :     str  ,				):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				FlaubertForSequenceClassification(__lowerCamelCase   )
      model.to(__lowerCamelCase   )
      model.eval()

      _SCREAMING_SNAKE_CASE	=				model(__lowerCamelCase   )
      _SCREAMING_SNAKE_CASE	=				model(__lowerCamelCase  ,				labels=__lowerCamelCase   )

      self.parent.assertEqual(result.loss.shape  ,				()   )
      self.parent.assertEqual(result.logits.shape  ,				(self.batch_size, self.type_sequence_label_size)   )



   def 							lowerCAmelCase_      (   self :     Optional[int]  ,				__lowerCamelCase :     Tuple  ,				__lowerCamelCase :     Optional[Any]  ,				__lowerCamelCase :     Optional[Any]  ,				__lowerCamelCase :     int  ,				__lowerCamelCase :     Optional[Any]  ,				__lowerCamelCase :     Dict  ,				__lowerCamelCase :     int  ,				__lowerCamelCase :     Union[str, Any]  ,				__lowerCamelCase :     List[Any]  ,				):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				self.num_labels
      _SCREAMING_SNAKE_CASE	=				FlaubertForTokenClassification(__lowerCamelCase   )
      model.to(__lowerCamelCase   )
      model.eval()

      _SCREAMING_SNAKE_CASE	=				model(__lowerCamelCase  ,				attention_mask=__lowerCamelCase  ,				labels=__lowerCamelCase   )
      self.parent.assertEqual(result.logits.shape  ,				(self.batch_size, self.seq_length, self.num_labels)   )



   def 							lowerCAmelCase_      (   self :     Any  ,				__lowerCamelCase :     Union[str, Any]  ,				__lowerCamelCase :     Any  ,				__lowerCamelCase :     Optional[Any]  ,				__lowerCamelCase :     List[str]  ,				__lowerCamelCase :     List[str]  ,				__lowerCamelCase :     Any  ,				__lowerCamelCase :     List[Any]  ,				__lowerCamelCase :     List[Any]  ,				__lowerCamelCase :     Optional[Any]  ,				):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				self.num_choices
      _SCREAMING_SNAKE_CASE	=				FlaubertForMultipleChoice(config=__lowerCamelCase   )
      model.to(__lowerCamelCase   )
      model.eval()
      _SCREAMING_SNAKE_CASE	=				input_ids.unsqueeze(1   ).expand(-1  ,				self.num_choices  ,				-1   ).contiguous()
      _SCREAMING_SNAKE_CASE	=				token_type_ids.unsqueeze(1   ).expand(-1  ,				self.num_choices  ,				-1   ).contiguous()
      _SCREAMING_SNAKE_CASE	=				input_mask.unsqueeze(1   ).expand(-1  ,				self.num_choices  ,				-1   ).contiguous()
      _SCREAMING_SNAKE_CASE	=				model(
          __lowerCamelCase  ,				attention_mask=__lowerCamelCase  ,				token_type_ids=__lowerCamelCase  ,				labels=__lowerCamelCase  ,				)
      self.parent.assertEqual(result.logits.shape  ,				(self.batch_size, self.num_choices)   )



   def 							lowerCAmelCase_      (   self :     List[Any]   ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				self.prepare_config_and_inputs()
      (
          (
          _SCREAMING_SNAKE_CASE
      ),	(
          _SCREAMING_SNAKE_CASE
      ),	(
          _SCREAMING_SNAKE_CASE
      ),	(
          _SCREAMING_SNAKE_CASE
      ),	(
          _SCREAMING_SNAKE_CASE
      ),	(
          _SCREAMING_SNAKE_CASE
      ),	(
          _SCREAMING_SNAKE_CASE
      ),	(
          _SCREAMING_SNAKE_CASE
      ),	(
          _SCREAMING_SNAKE_CASE
      ),	
      )	=				config_and_inputs
      _SCREAMING_SNAKE_CASE	=				{
          "input_ids": input_ids,
          "token_type_ids": token_type_ids,
          "lengths": input_lengths,
          "attention_mask": input_mask,
      }
      return config, inputs_dict







@require_torch
class 		lowercase_   (			A					,  A					,  unittest.TestCase       ):


   """simple docstring"""

   lowerCamelCase_      =						(
       (
           FlaubertModel,
           FlaubertWithLMHeadModel,
           FlaubertForQuestionAnswering,
           FlaubertForQuestionAnsweringSimple,
           FlaubertForSequenceClassification,
           FlaubertForTokenClassification,
           FlaubertForMultipleChoice,
       )
       if is_torch_available()
       else ()
   )
   lowerCamelCase_      =						(
       {
           '''feature-extraction''': FlaubertModel,
           '''fill-mask''': FlaubertWithLMHeadModel,
           '''question-answering''': FlaubertForQuestionAnsweringSimple,
           '''text-classification''': FlaubertForSequenceClassification,
           '''token-classification''': FlaubertForTokenClassification,
           '''zero-shot''': FlaubertForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )



   def 							lowerCAmelCase_      (   self :     Any  ,				__lowerCamelCase :     List[Any]  ,				__lowerCamelCase :     Optional[int]  ,				__lowerCamelCase :     Any  ,				__lowerCamelCase :     Optional[int]  ,				__lowerCamelCase :     List[Any]   ):

      """simple docstring"""
      if (
          pipeline_test_casse_name == "QAPipelineTests"
          and tokenizer_name is not None
          and not tokenizer_name.endswith("Fast"   )
      ):
         # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
         # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
         # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
         return True

      return False



   def 							lowerCAmelCase_      (   self :     Dict  ,				__lowerCamelCase :     List[str]  ,				__lowerCamelCase :     str  ,				__lowerCamelCase :     List[str]=False   ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				super()._prepare_for_class(__lowerCamelCase  ,				__lowerCamelCase  ,				return_labels=__lowerCamelCase   )

      if return_labels:
         if model_class.__name__ == "FlaubertForQuestionAnswering":
            _SCREAMING_SNAKE_CASE	=				torch.zeros(
                self.model_tester.batch_size  ,				dtype=torch.long  ,				device=__lowerCamelCase   )
            _SCREAMING_SNAKE_CASE	=				torch.zeros(
                self.model_tester.batch_size  ,				dtype=torch.long  ,				device=__lowerCamelCase   )

      return inputs_dict



   def 							lowerCAmelCase_      (   self :     List[str]   ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				FlaubertModelTester(self   )
      _SCREAMING_SNAKE_CASE	=				ConfigTester(self  ,				config_class=__lowerCamelCase  ,				emb_dim=3_7   )



   def 							lowerCAmelCase_      (   self :     Union[str, Any]   ):

      """simple docstring"""
      self.config_tester.run_common_tests()



   def 							lowerCAmelCase_      (   self :     List[str]   ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_flaubert_model(*__lowerCamelCase   )



   def 							lowerCAmelCase_      (   self :     List[str]   ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_flaubert_lm_head(*__lowerCamelCase   )



   def 							lowerCAmelCase_      (   self :     Dict   ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_flaubert_simple_qa(*__lowerCamelCase   )



   def 							lowerCAmelCase_      (   self :     Union[str, Any]   ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_flaubert_qa(*__lowerCamelCase   )



   def 							lowerCAmelCase_      (   self :     Tuple   ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_flaubert_sequence_classif(*__lowerCamelCase   )



   def 							lowerCAmelCase_      (   self :     Dict   ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_flaubert_token_classif(*__lowerCamelCase   )



   def 							lowerCAmelCase_      (   self :     int   ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_flaubert_multiple_choice(*__lowerCamelCase   )



   @slow
   def 							lowerCAmelCase_      (   self :     str   ):

      """simple docstring"""
      for model_name in FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
         _SCREAMING_SNAKE_CASE	=				FlaubertModel.from_pretrained(__lowerCamelCase   )
         self.assertIsNotNone(__lowerCamelCase   )



   @slow
   @require_torch_gpu
   def 							lowerCAmelCase_      (   self :     Any   ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE,	_SCREAMING_SNAKE_CASE	=				self.model_tester.prepare_config_and_inputs_for_common()
      for model_class in self.all_model_classes:
         # FlauBertForMultipleChoice behaves incorrectly in JIT environments.
         if model_class == FlaubertForMultipleChoice:
            return

         _SCREAMING_SNAKE_CASE	=				True
         _SCREAMING_SNAKE_CASE	=				model_class(config=__lowerCamelCase   )

         _SCREAMING_SNAKE_CASE	=				self._prepare_for_class(__lowerCamelCase  ,				__lowerCamelCase   )
         _SCREAMING_SNAKE_CASE	=				torch.jit.trace(
             __lowerCamelCase  ,				(inputs_dict["input_ids"].to("cpu"   ), inputs_dict["attention_mask"].to("cpu"   ))   )

         with tempfile.TemporaryDirectory() as tmp:
            torch.jit.save(__lowerCamelCase  ,				os.path.join(__lowerCamelCase  ,				"traced_model.pt"   )   )
            _SCREAMING_SNAKE_CASE	=				torch.jit.load(os.path.join(__lowerCamelCase  ,				"traced_model.pt"   )  ,				map_location=__lowerCamelCase   )
            loaded(inputs_dict["input_ids"].to(__lowerCamelCase   )  ,				inputs_dict["attention_mask"].to(__lowerCamelCase   )   )







@require_torch
class 		lowercase_   (			unittest.TestCase       ):


   """simple docstring"""

   @slow
   def 							lowerCAmelCase_      (   self :     Optional[int]   ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				FlaubertModel.from_pretrained("flaubert/flaubert_base_cased"   )
      _SCREAMING_SNAKE_CASE	=				torch.tensor([[0, 3_4_5, 2_3_2, 3_2_8, 7_4_0, 1_4_0, 1_6_9_5, 6_9, 6_0_7_8, 1_5_8_8, 2]]   )
      with torch.no_grad():
         _SCREAMING_SNAKE_CASE	=				model(__lowerCamelCase   )[0]
      _SCREAMING_SNAKE_CASE	=				torch.Size((1, 1_1, 7_6_8)   )
      self.assertEqual(output.shape  ,				__lowerCamelCase   )
      _SCREAMING_SNAKE_CASE	=				torch.tensor(
          [[[-2.6_2_5_1, -1.4_2_9_8, -0.0_2_2_7], [-2.8_5_1_0, -1.6_3_8_7, 0.2_2_5_8], [-2.8_1_1_4, -1.1_8_3_2, -0.3_0_6_6]]]   )

      self.assertTrue(torch.allclose(output[:, :3, :3]  ,				__lowerCamelCase  ,				atol=1e-4   )   )







'''simple docstring'''



import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

lowerCamelCase_			=						logging.get_logger(__name__)


lowerCamelCase_			=						{name: getattr(transformers, name + 'Fast') for name in SLOW_TO_FAST_CONVERTERS}
def 			SCREAMING_SNAKE_CASE_			(							__A    :			Optional[int]	,					__A    :			Tuple	,					__A    :			List[str]	,					__A    :			List[str]  )			->			List[Any]:
   if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
      raise ValueError(f"""Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()  )}."""  )

   if tokenizer_name is None:
      _SCREAMING_SNAKE_CASE	=				TOKENIZER_CLASSES
   else:
      _SCREAMING_SNAKE_CASE	=				{tokenizer_name: getattr(__A	,					tokenizer_name + "Fast"  )}

   logger.info(f"""Loading tokenizer classes: {tokenizer_names}"""  )

   for tokenizer_name in tokenizer_names:
      _SCREAMING_SNAKE_CASE	=				TOKENIZER_CLASSES[tokenizer_name]

      _SCREAMING_SNAKE_CASE	=				True
      if checkpoint_name is None:
         _SCREAMING_SNAKE_CASE	=				list(tokenizer_class.max_model_input_sizes.keys()  )
      else:
         _SCREAMING_SNAKE_CASE	=				[checkpoint_name]

      logger.info(f"""For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}"""  )

      for checkpoint in checkpoint_names:
         logger.info(f"""Loading {tokenizer_class.__class__.__name__} {checkpoint}"""  )

         # Load tokenizer
         _SCREAMING_SNAKE_CASE	=				tokenizer_class.from_pretrained(__A	,					force_download=__A  )

         # Save fast tokenizer
         logger.info(f"""Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}"""  )

         # For organization names we create sub-directories
         if "/" in checkpoint:
            _SCREAMING_SNAKE_CASE,	_SCREAMING_SNAKE_CASE	=				checkpoint.split("/"  )
            _SCREAMING_SNAKE_CASE	=				os.path.join(__A	,					__A  )
         elif add_prefix:
            _SCREAMING_SNAKE_CASE	=				checkpoint
            _SCREAMING_SNAKE_CASE	=				dump_path
         else:
            _SCREAMING_SNAKE_CASE	=				None
            _SCREAMING_SNAKE_CASE	=				dump_path

         logger.info(f"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""  )

         if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()  )[0]:
            _SCREAMING_SNAKE_CASE	=				list(tokenizer.pretrained_vocab_files_map.values()  )[0][checkpoint]
            _SCREAMING_SNAKE_CASE	=				file_path.split(__A  )[-1][0]
            if next_char == "/":
               _SCREAMING_SNAKE_CASE	=				os.path.join(__A	,					__A  )
               _SCREAMING_SNAKE_CASE	=				None

            logger.info(f"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""  )

         _SCREAMING_SNAKE_CASE	=				tokenizer.save_pretrained(
             __A	,					legacy_format=__A	,					filename_prefix=__A  )
         logger.info(f"""=> File names {file_names}"""  )

         for file_name in file_names:
            if not file_name.endswith("tokenizer.json"  ):
               os.remove(__A  )
               logger.info(f"""=> removing {file_name}"""  )


if __name__ == "__main__":
 lowerCamelCase_			=						argparse.ArgumentParser()
 # Required parameters
 parser.add_argument(
     '--dump_path', default=None, type=str, required=True, help='Path to output generated fast tokenizer files.'
 )
 parser.add_argument(
     '--tokenizer_name',
     default=None,
     type=str,
     help=(
         f'''Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will '''
         'download and convert all the checkpoints from AWS.'
     ),
 )
 parser.add_argument(
     '--checkpoint_name',
     default=None,
     type=str,
     help='Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.',
 )
 parser.add_argument(
     '--force_download',
     action='store_true',
     help='Re-download checkpoints.',
 )
 lowerCamelCase_			=						parser.parse_args()

 convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)

