





'''simple docstring'''


from __future__ import annotations

import copy
import tempfile
import unittest

from transformers import CONFIG_MAPPING, AutoConfig, BertConfig, GPTaConfig, TaConfig, TapasConfig, is_tf_available
from transformers.testing_utils import (
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tensorflow_probability,
    require_tf,
    slow,
)

from ..bert.test_modeling_bert import BertModelTester


if is_tf_available():
 from transformers import (
     TFAutoModel,
     TFAutoModelForCausalLM,
     TFAutoModelForMaskedLM,
     TFAutoModelForPreTraining,
     TFAutoModelForQuestionAnswering,
     TFAutoModelForSeqaSeqLM,
     TFAutoModelForSequenceClassification,
     TFAutoModelForTableQuestionAnswering,
     TFAutoModelForTokenClassification,
     TFAutoModelWithLMHead,
     TFBertForMaskedLM,
     TFBertForPreTraining,
     TFBertForQuestionAnswering,
     TFBertForSequenceClassification,
     TFBertModel,
     TFFunnelBaseModel,
     TFFunnelModel,
     TFGPTaLMHeadModel,
     TFRobertaForMaskedLM,
     TFTaForConditionalGeneration,
     TFTapasForQuestionAnswering,
 )
 from transformers.models.auto.modeling_tf_auto import (
     TF_MODEL_FOR_CAUSAL_LM_MAPPING,
     TF_MODEL_FOR_MASKED_LM_MAPPING,
     TF_MODEL_FOR_PRETRAINING_MAPPING,
     TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
     TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
     TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
     TF_MODEL_MAPPING,
 )
 from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
 from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
 from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST
 from transformers.models.tapas.modeling_tf_tapas import TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST
class 				a_       (    _a							):
     __A								=      "new-model"


if is_tf_available():
 class 				a_       (    _a							):
      __A								=      NewModelConfig
@require_tf
class 				a_       (    unittest.TestCase							):





     @slow
     def 							lowercase__				(			self		: int						):





        """simple docstring"""




        lowercase_      :Dict         =			"bert-base-cased"
        lowercase_      :Any         =			AutoConfig.from_pretrained(_a						)
        self.assertIsNotNone(_a						)
        self.assertIsInstance(_a     ,				_a						)

        lowercase_      :Optional[int]         =			TFAutoModel.from_pretrained(_a						)
        self.assertIsNotNone(_a						)
        self.assertIsInstance(_a     ,				_a						)





     @slow
     def 							lowercase__				(			self		: Union[str, Any]						):





        """simple docstring"""




        lowercase_      :Optional[int]         =			"bert-base-cased"
        lowercase_      :Optional[int]         =			AutoConfig.from_pretrained(_a						)
        self.assertIsNotNone(_a						)
        self.assertIsInstance(_a     ,				_a						)

        lowercase_      :Union[str, Any]         =			TFAutoModelForPreTraining.from_pretrained(_a						)
        self.assertIsNotNone(_a						)
        self.assertIsInstance(_a     ,				_a						)





     @slow
     def 							lowercase__				(			self		: Any						):





        """simple docstring"""




        for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
           lowercase_      :Union[str, Any]         =			AutoConfig.from_pretrained(_a						)
           self.assertIsNotNone(_a						)
           self.assertIsInstance(_a     ,				_a						)

           lowercase_      :Any         =			TFAutoModelForCausalLM.from_pretrained(_a						)
           lowercase_      :List[str]         =			TFAutoModelForCausalLM.from_pretrained(_a     ,				output_loading_info=_a						)
           self.assertIsNotNone(_a						)
           self.assertIsInstance(_a     ,				_a						)





     @slow
     def 							lowercase__				(			self		: Union[str, Any]						):





        """simple docstring"""




        for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
           lowercase_      :Optional[int]         =			AutoConfig.from_pretrained(_a						)
           self.assertIsNotNone(_a						)
           self.assertIsInstance(_a     ,				_a						)

           lowercase_      :List[Any]         =			TFAutoModelWithLMHead.from_pretrained(_a						)
           self.assertIsNotNone(_a						)
           self.assertIsInstance(_a     ,				_a						)





     @slow
     def 							lowercase__				(			self		: str						):





        """simple docstring"""




        for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
           lowercase_      :str         =			AutoConfig.from_pretrained(_a						)
           self.assertIsNotNone(_a						)
           self.assertIsInstance(_a     ,				_a						)

           lowercase_      :int         =			TFAutoModelForMaskedLM.from_pretrained(_a						)
           lowercase_      :Optional[Any]         =			TFAutoModelForMaskedLM.from_pretrained(_a     ,				output_loading_info=_a						)
           self.assertIsNotNone(_a						)
           self.assertIsInstance(_a     ,				_a						)





     @slow
     def 							lowercase__				(			self		: int						):





        """simple docstring"""




        for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
           lowercase_      :Optional[int]         =			AutoConfig.from_pretrained(_a						)
           self.assertIsNotNone(_a						)
           self.assertIsInstance(_a     ,				_a						)

           lowercase_      :List[Any]         =			TFAutoModelForSeqaSeqLM.from_pretrained(_a						)
           lowercase_      :Optional[Any]         =			TFAutoModelForSeqaSeqLM.from_pretrained(_a     ,				output_loading_info=_a						)
           self.assertIsNotNone(_a						)
           self.assertIsInstance(_a     ,				_a						)





     @slow
     def 							lowercase__				(			self		: Optional[int]						):





        """simple docstring"""




        for model_name in ["bert-base-uncased"]:
           lowercase_      :int         =			AutoConfig.from_pretrained(_a						)
           self.assertIsNotNone(_a						)
           self.assertIsInstance(_a     ,				_a						)

           lowercase_      :Optional[int]         =			TFAutoModelForSequenceClassification.from_pretrained(_a						)
           self.assertIsNotNone(_a						)
           self.assertIsInstance(_a     ,				_a						)





     @slow
     def 							lowercase__				(			self		: List[str]						):





        """simple docstring"""




        for model_name in ["bert-base-uncased"]:
           lowercase_      :Dict         =			AutoConfig.from_pretrained(_a						)
           self.assertIsNotNone(_a						)
           self.assertIsInstance(_a     ,				_a						)

           lowercase_      :Union[str, Any]         =			TFAutoModelForQuestionAnswering.from_pretrained(_a						)
           self.assertIsNotNone(_a						)
           self.assertIsInstance(_a     ,				_a						)





     @slow
     @require_tensorflow_probability
     def 							lowercase__				(			self		: Union[str, Any]						):





        """simple docstring"""




        for model_name in TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST[5:6]:
           lowercase_      :int         =			AutoConfig.from_pretrained(_a						)
           self.assertIsNotNone(_a						)
           self.assertIsInstance(_a     ,				_a						)

           lowercase_      :Tuple         =			TFAutoModelForTableQuestionAnswering.from_pretrained(_a						)
           lowercase_      :Optional[Any]         =			TFAutoModelForTableQuestionAnswering.from_pretrained(
               _a     ,				output_loading_info=_a						)
           self.assertIsNotNone(_a						)
           self.assertIsInstance(_a     ,				_a						)





     def 							lowercase__				(			self		: str						):





        """simple docstring"""




        lowercase_      :int         =			TFAutoModelWithLMHead.from_pretrained(_a						)
        self.assertIsInstance(_a     ,				_a						)
        self.assertEqual(model.num_parameters()     ,				14_410						)
        self.assertEqual(model.num_parameters(only_trainable=_a						)     ,				14_410						)





     def 							lowercase__				(			self		: Union[str, Any]						):





        """simple docstring"""




        lowercase_      :List[str]         =			TFAutoModelWithLMHead.from_pretrained(_a						)
        self.assertIsInstance(_a     ,				_a						)
        self.assertEqual(model.num_parameters()     ,				14_410						)
        self.assertEqual(model.num_parameters(only_trainable=_a						)     ,				14_410						)





     def 							lowercase__				(			self		: Tuple						):





        """simple docstring"""




        lowercase_      :Union[str, Any]         =			TFAutoModel.from_pretrained("sgugger/funnel-random-tiny"						)
        self.assertIsInstance(_a     ,				_a						)

        lowercase_      :str         =			copy.deepcopy(model.config						)
        lowercase_      :Optional[int]         =			["FunnelBaseModel"]
        lowercase_      :Any         =			TFAutoModel.from_config(_a						)
        self.assertIsInstance(_a     ,				_a						)

        with tempfile.TemporaryDirectory() as tmp_dir:
           model.save_pretrained(_a						)
           lowercase_      :Any         =			TFAutoModel.from_pretrained(_a						)
           self.assertIsInstance(_a     ,				_a						)





     def 							lowercase__				(			self		: List[str]						):





        """simple docstring"""




        try:
           AutoConfig.register("new-model"     ,				_a						)

           lowercase_      :Dict         =			[
               TFAutoModel,
               TFAutoModelForCausalLM,
               TFAutoModelForMaskedLM,
               TFAutoModelForPreTraining,
               TFAutoModelForQuestionAnswering,
               TFAutoModelForSequenceClassification,
               TFAutoModelForTokenClassification,
           ]

           for auto_class in auto_classes:
              with self.subTest(auto_class.__name__						):
                 # Wrong config class will raise an error
                 with self.assertRaises(_a						):
                    auto_class.register(_a     ,				_a						)
                 auto_class.register(_a     ,				_a						)
                 # Trying to register something existing in the Transformers library will raise an error
                 with self.assertRaises(_a						):
                    auto_class.register(_a     ,				_a						)

                 # Now that the config is registered, it can be used as any other config with the auto-API
                 lowercase_      :str         =			BertModelTester(self						).get_config()
                 lowercase_      :Optional[int]         =			NewModelConfig(**tiny_config.to_dict()						)
                 lowercase_      :Optional[int]         =			auto_class.from_config(_a						)
                 self.assertIsInstance(_a     ,				_a						)

                 with tempfile.TemporaryDirectory() as tmp_dir:
                    model.save_pretrained(_a						)
                    lowercase_      :List[Any]         =			auto_class.from_pretrained(_a						)
                    self.assertIsInstance(_a     ,				_a						)

        finally:
           if "new-model" in CONFIG_MAPPING._extra_content:
              del CONFIG_MAPPING._extra_content["new-model"]
           for mapping in (
               TF_MODEL_MAPPING,
               TF_MODEL_FOR_PRETRAINING_MAPPING,
               TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
               TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
               TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
               TF_MODEL_FOR_CAUSAL_LM_MAPPING,
               TF_MODEL_FOR_MASKED_LM_MAPPING,
           ):
              if NewModelConfig in mapping._extra_content:
                 del mapping._extra_content[NewModelConfig]





     def 							lowercase__				(			self		: Dict						):





        """simple docstring"""




        with self.assertRaisesRegex(
            _a     ,				"bert-base is not a local folder and is not a valid model identifier"						):
           lowercase_      :Optional[Any]         =			TFAutoModel.from_pretrained("bert-base"						)





     def 							lowercase__				(			self		: Optional[Any]						):





        """simple docstring"""




        with self.assertRaisesRegex(
            _a     ,				R"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"						):
           lowercase_      :str         =			TFAutoModel.from_pretrained(_a     ,				revision="aaaaaa"						)





     def 							lowercase__				(			self		: Tuple						):





        """simple docstring"""




        with self.assertRaisesRegex(
            _a     ,				"hf-internal-testing/config-no-model does not appear to have a file named pytorch_model.bin"     ,				):
           lowercase_      :int         =			TFAutoModel.from_pretrained("hf-internal-testing/config-no-model"						)





     def 							lowercase__				(			self		: Optional[Any]						):





        """simple docstring"""




        with self.assertRaisesRegex(_a     ,				"Use `from_pt=True` to load this model"						):
           lowercase_      :List[str]         =			TFAutoModel.from_pretrained("hf-internal-testing/tiny-bert-pt-only"						)







     def 							lowercase__				(			self		: Union[str, Any]						):





        """simple docstring"""




        lowercase_      :Optional[Any]         =			TFAutoModel.from_pretrained("hf-internal-testing/tiny-random-bert"						)
        with RequestCounter() as counter:
           lowercase_      :List[Any]         =			TFAutoModel.from_pretrained("hf-internal-testing/tiny-random-bert"						)
           self.assertEqual(counter.get_request_count     ,				0						)
           self.assertEqual(counter.head_request_count     ,				1						)
           self.assertEqual(counter.other_request_count     ,				0						)

        # With a sharded checkpoint
        lowercase_      :Union[str, Any]         =			TFAutoModel.from_pretrained("ArthurZ/tiny-random-bert-sharded"						)
        with RequestCounter() as counter:
           lowercase_      :Tuple         =			TFAutoModel.from_pretrained("ArthurZ/tiny-random-bert-sharded"						)
           self.assertEqual(counter.get_request_count     ,				0						)
           self.assertEqual(counter.head_request_count     ,				1						)
           self.assertEqual(counter.other_request_count     ,				0						)







'''simple docstring'''


import warnings
from typing import List, Optional, Union

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType
class 				a_       (    _lowerCAmelCase							):
     __A								=      ["image_processor", "tokenizer"]
     __A								=      "LayoutLMv3ImageProcessor"
     __A								=      ("LayoutLMv3Tokenizer", "LayoutLMv3TokenizerFast")





     def __init__(			self		: int     ,				lowercase		: Optional[Any]=None     ,				lowercase		: List[str]=None     ,				**lowercase		: Optional[int]						):





        """simple docstring"""




        lowercase_      :int         =			None
        if "feature_extractor" in kwargs:
           warnings.warn(
               "The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"
               " instead."     ,				lowercase     ,				)
           lowercase_      :Optional[int]         =			kwargs.pop("feature_extractor"						)

        lowercase_      :Union[str, Any]         =			image_processor if image_processor is not None else feature_extractor
        if image_processor is None:
           raise ValueError("You need to specify an `image_processor`."						)
        if tokenizer is None:
           raise ValueError("You need to specify a `tokenizer`."						)

        super().__init__(lowercase     ,				lowercase						)





     def __call__(			self		: Optional[Any]     ,				lowercase		: List[str]     ,				lowercase		: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None     ,				lowercase		: Optional[Union[PreTokenizedInput, List[PreTokenizedInput]]] = None     ,				lowercase		: Union[List[List[int]], List[List[List[int]]]] = None     ,				lowercase		: Optional[Union[List[int], List[List[int]]]] = None     ,				lowercase		: bool = True     ,				lowercase		: Union[bool, str, PaddingStrategy] = False     ,				lowercase		: Union[bool, str, TruncationStrategy] = None     ,				lowercase		: Optional[int] = None     ,				lowercase		: int = 0     ,				lowercase		: Optional[int] = None     ,				lowercase		: Optional[bool] = None     ,				lowercase		: Optional[bool] = None     ,				lowercase		: bool = False     ,				lowercase		: bool = False     ,				lowercase		: bool = False     ,				lowercase		: bool = False     ,				lowercase		: bool = True     ,				lowercase		: Optional[Union[str, TensorType]] = None     ,				**lowercase		: List[Any]     ,				):





        """simple docstring"""




        if self.image_processor.apply_ocr and (boxes is not None):
           raise ValueError(
               "You cannot provide bounding boxes if you initialized the image processor with apply_ocr set to True."						)

        if self.image_processor.apply_ocr and (word_labels is not None):
           raise ValueError(
               "You cannot provide word labels if you initialized the image processor with apply_ocr set to True."						)

        # first, apply the image processor
        lowercase_      :Dict         =			self.image_processor(images=lowercase     ,				return_tensors=lowercase						)

        # second, apply the tokenizer
        if text is not None and self.image_processor.apply_ocr and text_pair is None:
           if isinstance(lowercase     ,				lowercase						):
              lowercase_      :str         =			[text]  # add batch dimension (as the image processor always adds a batch dimension)
           lowercase_      :Union[str, Any]         =			features["words"]

        lowercase_      :Optional[Any]         =			self.tokenizer(
            text=text if text is not None else features["words"]     ,				text_pair=text_pair if text_pair is not None else None     ,				boxes=boxes if boxes is not None else features["boxes"]     ,				word_labels=lowercase     ,				add_special_tokens=lowercase     ,				padding=lowercase     ,				truncation=lowercase     ,				max_length=lowercase     ,				stride=lowercase     ,				pad_to_multiple_of=lowercase     ,				return_token_type_ids=lowercase     ,				return_attention_mask=lowercase     ,				return_overflowing_tokens=lowercase     ,				return_special_tokens_mask=lowercase     ,				return_offsets_mapping=lowercase     ,				return_length=lowercase     ,				verbose=lowercase     ,				return_tensors=lowercase     ,				**lowercase     ,				)

        # add pixel values
        lowercase_      :Any         =			features.pop("pixel_values"						)
        if return_overflowing_tokens is True:
           lowercase_      :Any         =			self.get_overflowing_images(lowercase     ,				encoded_inputs["overflow_to_sample_mapping"]						)
        lowercase_      :Any         =			images

        return encoded_inputs





     def 							lowercase__				(			self		: List[Any]     ,				lowercase		: Any     ,				lowercase		: Optional[Any]						):





        """simple docstring"""




        lowercase_      :Union[str, Any]         =			[]
        for sample_idx in overflow_to_sample_mapping:
           images_with_overflow.append(images[sample_idx]						)

        if len(lowercase						) != len(lowercase						):
           raise ValueError(
               "Expected length of images to be the same as the length of `overflow_to_sample_mapping`, but got"
               F' {len(lowercase						)} and {len(lowercase						)}'						)

        return images_with_overflow





     def 							lowercase__				(			self		: Union[str, Any]     ,				*lowercase		: List[Any]     ,				**lowercase		: Optional[Any]						):





        """simple docstring"""




        return self.tokenizer.batch_decode(*lowercase     ,				**lowercase						)





     def 							lowercase__				(			self		: List[Any]     ,				*lowercase		: Any     ,				**lowercase		: str						):





        """simple docstring"""




        return self.tokenizer.decode(*lowercase     ,				**lowercase						)





     @property
     def 							lowercase__				(			self		: Optional[Any]						):





        """simple docstring"""




        return ["input_ids", "bbox", "attention_mask", "pixel_values"]





     @property
     def 							lowercase__				(			self		: str						):





        """simple docstring"""




        warnings.warn(
            "`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead."     ,				lowercase     ,				)
        return self.image_processor_class







     @property
     def 							lowercase__				(			self		: Tuple						):





        """simple docstring"""




        warnings.warn(
            "`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead."     ,				lowercase     ,				)
        return self.image_processor


