





'''simple docstring'''

import os
import tempfile
import unittest

from transformers import DistilBertConfig, is_torch_available
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      DistilBertForMaskedLM,
      DistilBertForMultipleChoice,
      DistilBertForQuestionAnswering,
      DistilBertForSequenceClassification,
      DistilBertForTokenClassification,
      DistilBertModel,
  )






class        UpperCAmelCase     (	UpperCamelCase__    ):
 def __init__(    self				:Dict						,	lowercase_				:List[Any]						,	lowercase_				:str=13						,	lowercase_				:int=7						,	lowercase_				:Optional[Any]=True						,	lowercase_				:Tuple=True						,	lowercase_				:Union[str, Any]=False						,	lowercase_				:Optional[Any]=True						,	lowercase_				:Optional[Any]=99						,	lowercase_				:Optional[int]=32						,	lowercase_				:Union[str, Any]=5						,	lowercase_				:int=4						,	lowercase_				:Any=37						,	lowercase_				:Optional[int]="gelu"						,	lowercase_				:Optional[Any]=0.1						,	lowercase_				:List[Any]=0.1						,	lowercase_				:List[Any]=5_12						,	lowercase_				:str=16						,	lowercase_				:Optional[Any]=2						,	lowercase_				:List[str]=0.0_2						,	lowercase_				:Optional[Any]=3						,	lowercase_				:Tuple=4						,	lowercase_				:Union[str, Any]=None						,	)->							Dict:
    A__    =      parent
    A__    =      batch_size
    A__    =      seq_length
    A__    =      is_training
    A__    =      use_input_mask
    A__    =      use_token_type_ids
    A__    =      use_labels
    A__    =      vocab_size
    A__    =      hidden_size
    A__    =      num_hidden_layers
    A__    =      num_attention_heads
    A__    =      intermediate_size
    A__    =      hidden_act
    A__    =      hidden_dropout_prob
    A__    =      attention_probs_dropout_prob
    A__    =      max_position_embeddings
    A__    =      type_vocab_size
    A__    =      type_sequence_label_size
    A__    =      initializer_range
    A__    =      num_labels
    A__    =      num_choices
    A__    =      scope
 def 					UpperCAmelCase_							(    self				:List[str]							)->							List[Any]:
    A__    =      ids_tensor([self.batch_size, self.seq_length]						,	self.vocab_size							)

    A__    =      None
    if self.use_input_mask:
       A__    =      random_attention_mask([self.batch_size, self.seq_length]							)

    A__    =      None
    A__    =      None
    A__    =      None
    if self.use_labels:
       A__    =      ids_tensor([self.batch_size]						,	self.type_sequence_label_size							)
       A__    =      ids_tensor([self.batch_size, self.seq_length]						,	self.num_labels							)
       A__    =      ids_tensor([self.batch_size]						,	self.num_choices							)

    A__    =      self.get_config()

    return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
 def 					UpperCAmelCase_							(    self				:Optional[Any]							)->							Optional[Any]:
    return DistilBertConfig(
        vocab_size=self.vocab_size						,	dim=self.hidden_size						,	n_layers=self.num_hidden_layers						,	n_heads=self.num_attention_heads						,	hidden_dim=self.intermediate_size						,	hidden_act=self.hidden_act						,	dropout=self.hidden_dropout_prob						,	attention_dropout=self.attention_probs_dropout_prob						,	max_position_embeddings=self.max_position_embeddings						,	initializer_range=self.initializer_range						,	)
 def 					UpperCAmelCase_							(    self				:Union[str, Any]						,	lowercase_				:Optional[int]						,	lowercase_				:int						,	lowercase_				:List[str]						,	lowercase_				:int						,	lowercase_				:int						,	lowercase_				:Union[str, Any]							)->							List[str]:
    A__    =      DistilBertModel(config=lowercase_							)
    model.to(lowercase_							)
    model.eval()
    A__    =      model(lowercase_						,	lowercase_							)
    A__    =      model(lowercase_							)
    self.parent.assertEqual(result.last_hidden_state.shape						,	(self.batch_size, self.seq_length, self.hidden_size)							)
 def 					UpperCAmelCase_							(    self				:List[str]						,	lowercase_				:List[Any]						,	lowercase_				:Dict						,	lowercase_				:List[Any]						,	lowercase_				:Dict						,	lowercase_				:Tuple						,	lowercase_				:List[Any]							)->							Optional[int]:
    A__    =      DistilBertForMaskedLM(config=lowercase_							)
    model.to(lowercase_							)
    model.eval()
    A__    =      model(lowercase_						,	attention_mask=lowercase_						,	labels=lowercase_							)
    self.parent.assertEqual(result.logits.shape						,	(self.batch_size, self.seq_length, self.vocab_size)							)
 def 					UpperCAmelCase_							(    self				:Dict						,	lowercase_				:Optional[int]						,	lowercase_				:Optional[int]						,	lowercase_				:Any						,	lowercase_				:str						,	lowercase_				:Tuple						,	lowercase_				:Union[str, Any]							)->							int:
    A__    =      DistilBertForQuestionAnswering(config=lowercase_							)
    model.to(lowercase_							)
    model.eval()
    A__    =      model(
        lowercase_						,	attention_mask=lowercase_						,	start_positions=lowercase_						,	end_positions=lowercase_							)
    self.parent.assertEqual(result.start_logits.shape						,	(self.batch_size, self.seq_length)							)
    self.parent.assertEqual(result.end_logits.shape						,	(self.batch_size, self.seq_length)							)
 def 					UpperCAmelCase_							(    self				:List[str]						,	lowercase_				:str						,	lowercase_				:int						,	lowercase_				:Tuple						,	lowercase_				:List[Any]						,	lowercase_				:Any						,	lowercase_				:Union[str, Any]							)->							Dict:
    A__    =      self.num_labels
    A__    =      DistilBertForSequenceClassification(lowercase_							)
    model.to(lowercase_							)
    model.eval()
    A__    =      model(lowercase_						,	attention_mask=lowercase_						,	labels=lowercase_							)
    self.parent.assertEqual(result.logits.shape						,	(self.batch_size, self.num_labels)							)
 def 					UpperCAmelCase_							(    self				:Dict						,	lowercase_				:str						,	lowercase_				:Optional[int]						,	lowercase_				:Tuple						,	lowercase_				:Tuple						,	lowercase_				:Union[str, Any]						,	lowercase_				:Union[str, Any]							)->							Union[str, Any]:
    A__    =      self.num_labels
    A__    =      DistilBertForTokenClassification(config=lowercase_							)
    model.to(lowercase_							)
    model.eval()

    A__    =      model(lowercase_						,	attention_mask=lowercase_						,	labels=lowercase_							)
    self.parent.assertEqual(result.logits.shape						,	(self.batch_size, self.seq_length, self.num_labels)							)
 def 					UpperCAmelCase_							(    self				:Tuple						,	lowercase_				:Tuple						,	lowercase_				:Dict						,	lowercase_				:Optional[int]						,	lowercase_				:Optional[int]						,	lowercase_				:int						,	lowercase_				:Union[str, Any]							)->							Any:
    A__    =      self.num_choices
    A__    =      DistilBertForMultipleChoice(config=lowercase_							)
    model.to(lowercase_							)
    model.eval()
    A__    =      input_ids.unsqueeze(1							).expand(-1						,	self.num_choices						,	-1							).contiguous()
    A__    =      input_mask.unsqueeze(1							).expand(-1						,	self.num_choices						,	-1							).contiguous()
    A__    =      model(
        lowercase_						,	attention_mask=lowercase_						,	labels=lowercase_						,	)
    self.parent.assertEqual(result.logits.shape						,	(self.batch_size, self.num_choices)							)



 def 					UpperCAmelCase_							(    self				:Optional[int]							)->							Tuple:
    A__    =      self.prepare_config_and_inputs()
    ((A__),     (A__),     (A__),     (A__),     (A__),     (A__))    =      config_and_inputs
    A__    =      {"input_ids": input_ids, "attention_mask": input_mask}
    return config, inputs_dict






@require_torch
class        UpperCAmelCase     (	UpperCamelCase__      ,	UpperCamelCase__      ,	unittest.TestCase    ):
 __lowercase       		=    (
     (
         DistilBertModel,
         DistilBertForMaskedLM,
         DistilBertForMultipleChoice,
         DistilBertForQuestionAnswering,
         DistilBertForSequenceClassification,
         DistilBertForTokenClassification,
     )
     if is_torch_available()
     else None
 )
 __lowercase       		=    (
     {
         """feature-extraction""": DistilBertModel,
         """fill-mask""": DistilBertForMaskedLM,
         """question-answering""": DistilBertForQuestionAnswering,
         """text-classification""": DistilBertForSequenceClassification,
         """token-classification""": DistilBertForTokenClassification,
         """zero-shot""": DistilBertForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )
 __lowercase       		=    True
 __lowercase       		=    True
 __lowercase       		=    True
 __lowercase       		=    True
 def 					UpperCAmelCase_							(    self				:List[Any]							)->							str:
    A__    =      DistilBertModelTester(self							)
    A__    =      ConfigTester(self						,	config_class=lowercase_						,	dim=37							)
 def 					UpperCAmelCase_							(    self				:Tuple							)->							Union[str, Any]:
    self.config_tester.run_common_tests()
 def 					UpperCAmelCase_							(    self				:List[str]							)->							Any:
    A__    =      self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_distilbert_model(*lowercase_							)
 def 					UpperCAmelCase_							(    self				:str							)->							Any:
    A__    =      self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_distilbert_for_masked_lm(*lowercase_							)
 def 					UpperCAmelCase_							(    self				:List[Any]							)->							Dict:
    A__    =      self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_distilbert_for_question_answering(*lowercase_							)
 def 					UpperCAmelCase_							(    self				:List[Any]							)->							List[str]:
    A__    =      self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_distilbert_for_sequence_classification(*lowercase_							)
 def 					UpperCAmelCase_							(    self				:List[str]							)->							Dict:
    A__    =      self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_distilbert_for_token_classification(*lowercase_							)
 def 					UpperCAmelCase_							(    self				:Optional[int]							)->							str:
    A__    =      self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_distilbert_for_multiple_choice(*lowercase_							)
 @slow
 def 					UpperCAmelCase_							(    self				:Tuple							)->							Optional[Any]:
    for model_name in DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
       A__    =      DistilBertModel.from_pretrained(lowercase_							)
       self.assertIsNotNone(lowercase_							)



 @slow
 @require_torch_gpu
 def 					UpperCAmelCase_							(    self				:Optional[int]							)->							int:
    A__,     A__    =      self.model_tester.prepare_config_and_inputs_for_common()
    for model_class in self.all_model_classes:
       # BertForMultipleChoice behaves incorrectly in JIT environments.
       if model_class == DistilBertForMultipleChoice:
          return

       A__    =      True
       A__    =      model_class(config=lowercase_							)

       A__    =      self._prepare_for_class(lowercase_						,	lowercase_							)
       A__    =      torch.jit.trace(
           lowercase_						,	(inputs_dict["input_ids"].to("cpu"							), inputs_dict["attention_mask"].to("cpu"							))							)

       with tempfile.TemporaryDirectory() as tmp:
          torch.jit.save(lowercase_						,	os.path.join(lowercase_						,	"traced_model.pt"							)							)
          A__    =      torch.jit.load(os.path.join(lowercase_						,	"traced_model.pt"							)						,	map_location=lowercase_							)
          loaded(inputs_dict["input_ids"].to(lowercase_							)						,	inputs_dict["attention_mask"].to(lowercase_							)							)






@require_torch
class        UpperCAmelCase     (	unittest.TestCase    ):
 @slow
 def 					UpperCAmelCase_							(    self				:Union[str, Any]							)->							int:
    A__    =      DistilBertModel.from_pretrained("distilbert-base-uncased"							)
    A__    =      torch.tensor([[0, 3_45, 2_32, 3_28, 7_40, 1_40, 16_95, 69, 60_78, 15_88, 2]]							)
    A__    =      torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]							)
    with torch.no_grad():
       A__    =      model(lowercase_						,	attention_mask=lowercase_							)[0]
    A__    =      torch.Size((1, 11, 7_68)							)
    self.assertEqual(output.shape						,	lowercase_							)
    A__    =      torch.tensor(
        [[[-0.1_6_3_9, 0.3_2_9_9, 0.1_6_4_8], [-0.1_7_4_6, 0.3_2_8_9, 0.1_7_1_0], [-0.1_8_8_4, 0.3_3_5_7, 0.1_8_1_0]]]							)

    self.assertTrue(torch.allclose(output[:, 1:4, 1:4]						,	lowercase_						,	atol=1E-4							)							)










'''simple docstring'''

import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class        UpperCAmelCase     (	UpperCamelCase__      ,	unittest.TestCase    ):
 __lowercase       		=    CLIPTokenizer
 __lowercase       		=    CLIPTokenizerFast
 __lowercase       		=    True
 __lowercase       		=    {}
 __lowercase       		=    False
 def 					UpperCAmelCase_							(    self				:Optional[int]							)->							Optional[Any]:
     super().setUp()

     # fmt: off
     A__    =      ["l", "o", "w", "e", "r", "s", "t", "i", "d", "n", "lo", "l</w>", "w</w>", "r</w>", "t</w>", "low</w>", "er</w>", "lowest</w>", "newer</w>", "wider", "<unk>", "<|startoftext|>", "<|endoftext|>"]
     # fmt: on
     A__    =      dict(zip(lowercase_						,	range(len(lowercase_							)							)							)							)
     A__    =      ["#version: 0.2", "l o", "lo w</w>", "e r</w>"]
     A__    =      {"unk_token": "<unk>"}

     A__    =      os.path.join(self.tmpdirname						,	VOCAB_FILES_NAMES["vocab_file"]							)
     A__    =      os.path.join(self.tmpdirname						,	VOCAB_FILES_NAMES["merges_file"]							)
     with open(self.vocab_file						,	"w"						,	encoding="utf-8"							) as fp:
         fp.write(json.dumps(lowercase_							) + "\n"							)
     with open(self.merges_file						,	"w"						,	encoding="utf-8"							) as fp:
         fp.write("\n".join(lowercase_							)							)
 def 					UpperCAmelCase_							(    self				:Optional[int]						,	**lowercase_				:List[str]							)->							str:
     kwargs.update(self.special_tokens_map							)
     return CLIPTokenizer.from_pretrained(self.tmpdirname						,	**lowercase_							)
 def 					UpperCAmelCase_							(    self				:Optional[int]						,	**lowercase_				:Optional[int]							)->							Tuple:
     kwargs.update(self.special_tokens_map							)
     return CLIPTokenizerFast.from_pretrained(self.tmpdirname						,	**lowercase_							)
 def 					UpperCAmelCase_							(    self				:Optional[int]						,	lowercase_				:List[Any]							)->							List[str]:
     A__    =      "lower newer"
     A__    =      "lower newer"
     return input_text, output_text
 def 					UpperCAmelCase_							(    self				:int							)->							List[str]:
     A__    =      CLIPTokenizer(self.vocab_file						,	self.merges_file						,	**self.special_tokens_map							)
     A__    =      "lower newer"
     A__    =      ["lo", "w", "er</w>", "n", "e", "w", "er</w>"]
     A__    =      tokenizer.tokenize(lowercase_							)
     self.assertListEqual(lowercase_						,	lowercase_							)

     A__    =      tokens + [tokenizer.unk_token]
     A__    =      [10, 2, 16, 9, 3, 2, 16, 20]
     self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase_							)						,	lowercase_							)
 @require_ftfy
 def 					UpperCAmelCase_							(    self				:Dict							)->							str:
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"							):
             A__    =      self.tokenizer_class.from_pretrained(lowercase_						,	**lowercase_							)
             A__    =      self.rust_tokenizer_class.from_pretrained(lowercase_						,	**lowercase_							)

             A__    =      "A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."
             A__    =      tokenizer_s.tokenize(lowercase_							)
             A__    =      tokenizer_r.tokenize(lowercase_							)

             self.assertListEqual(lowercase_						,	lowercase_							)

             # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
             # with Tilde) encoded in 2 different ways
             A__    =      "xa\u0303y" + " " + "x\xe3y"
             A__    =      tokenizer_s.tokenize(lowercase_							)
             A__    =      tokenizer_r.tokenize(lowercase_							)

             self.assertListEqual(lowercase_						,	lowercase_							)

             # Test that the tokenization is identical on unicode of space type
             A__    =      [
                 "\u0009",  # (horizontal tab, '\t')
                 "\u000B",  # (vertical tab)
                 "\u000C",  # (form feed)
                 "\u0020",  # (space, ' ')
                 "\u200E",  # (left-to-right mark):w
                 "\u200F",  # (right-to-left mark)
             ]
             for unicode_seq in spaces_unicodes:
                 A__    =      tokenizer_s.tokenize(lowercase_							)
                 A__    =      tokenizer_r.tokenize(lowercase_							)

                 self.assertListEqual(lowercase_						,	lowercase_							)

             # Test that the tokenization is identical on unicode of line break type
             A__    =      [
                 "\u000A",  # (line feed, '\n')
                 "\r\n",  # (carriage return and line feed, '\r\n')
                 "\u000D",  # (carriage return, '\r')
                 "\r",  # (carriage return, '\r')
                 "\u000D",  # (carriage return, '\r')
                 "\u2028",  # (line separator)
                 "\u2029",  # (paragraph separator)
                 # "\u0085", # (next line)
             ]

             # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
             # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
             # space (and thus into an empty list).

             for unicode_seq in line_break_unicodes:
                 A__    =      tokenizer_s.tokenize(lowercase_							)
                 A__    =      tokenizer_r.tokenize(lowercase_							)

                 self.assertListEqual(lowercase_						,	lowercase_							)
 def 					UpperCAmelCase_							(    self				:Optional[int]							)->							Optional[Any]:
     # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"							):
             A__    =      "hello"  # `hello` is a token in the vocabulary of `pretrained_name`
             A__    =      F"{text_of_1_token} {text_of_1_token}"

             A__    =      self.rust_tokenizer_class.from_pretrained(
                 lowercase_						,	use_fast=lowercase_						,	)
             A__    =      tokenizer_r(lowercase_						,	return_offsets_mapping=lowercase_						,	add_special_tokens=lowercase_							)
             self.assertEqual(encoding.offset_mapping[0]						,	(0, len(lowercase_							))							)
             self.assertEqual(
                 encoding.offset_mapping[1]						,	(len(lowercase_							) + 1, len(lowercase_							) + 1 + len(lowercase_							))						,	)

             A__    =      F" {text}"

             A__    =      self.rust_tokenizer_class.from_pretrained(
                 lowercase_						,	use_fast=lowercase_						,	)
             A__    =      tokenizer_r(lowercase_						,	return_offsets_mapping=lowercase_						,	add_special_tokens=lowercase_							)
             self.assertEqual(encoding.offset_mapping[0]						,	(1, 1 + len(lowercase_							))							)
             self.assertEqual(
                 encoding.offset_mapping[1]						,	(1 + len(lowercase_							) + 1, 1 + len(lowercase_							) + 1 + len(lowercase_							))						,	)
 def 					UpperCAmelCase_							(    self				:Dict							)->							Optional[Any]:
     # Test related to the breaking change introduced in transformers v4.17.0
     # We need to check that an error in raised when the user try to load a previous version of the tokenizer.
     with self.assertRaises(lowercase_							) as context:
         self.rust_tokenizer_class.from_pretrained("robot-test/old-clip-tokenizer"							)

     self.assertTrue(
         context.exception.args[0].startswith(
             "The `backend_tokenizer` provided does not match the expected format."							)							)
 @require_ftfy
 def 					UpperCAmelCase_							(    self				:str							)->							Any:
     super().test_tokenization_python_rust_equals()



 def 					UpperCAmelCase_							(    self				:Optional[int]							)->							Union[str, Any]:
     # CLIP always lower cases letters
     pass



