


"""simple docstring"""



import json
import os
import unittest

from transformers import MgpstrTokenizer
from transformers.models.mgp_str.tokenization_mgp_str import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class _lowerCAmelCase    (				snake_case_   ,							unittest.TestCase     ):
   __UpperCAmelCase			:	List[Any]							  =  MgpstrTokenizer
   __UpperCAmelCase			:	Optional[int]							  =  False
   __UpperCAmelCase			:	Optional[int]							  =  {}
   __UpperCAmelCase			:	List[str]							  =  False







   def 			lowerCamelCase		(     self						)     ->		int:

       '''simple docstring'''




       super().setUp()

       # fmt: off
       snake_case   :				List[str]				 =   ["[GO]", "[s]", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"]
       # fmt: on
       snake_case   :				Union[str, Any]				 =   dict(zip(UpperCamelCase__						,     range(len(UpperCamelCase__						)						)						)						)

       snake_case   :				int				 =   os.path.join(self.tmpdirname						,     VOCAB_FILES_NAMES["vocab_file"]						)
       with open(self.vocab_file						,     "w"						,     encoding="utf-8"						) as fp:
           fp.write(json.dumps(UpperCamelCase__						) + "\n"						)







   def 			lowerCamelCase		(     self						,     **UpperCamelCase__						)     ->		List[str]:

       '''simple docstring'''




       return MgpstrTokenizer.from_pretrained(self.tmpdirname						,     **UpperCamelCase__						)







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						)     ->		List[Any]:

       '''simple docstring'''




       snake_case   :				int				 =   "tester"
       snake_case   :				List[str]				 =   "tester"
       return input_text, output_text







   @unittest.skip("MGP-STR always lower cases letters."						)
   def 			lowerCamelCase		(     self						)     ->		str:

       '''simple docstring'''




       pass







   def 			lowerCamelCase		(     self						)     ->		Tuple:

       '''simple docstring'''




       snake_case   :				int				 =   self.get_tokenizers(do_lower_case=UpperCamelCase__						)
       for tokenizer in tokenizers:
           with self.subTest(F'{tokenizer.__class__.__name__}'						):
               snake_case   :				List[Any]				 =   "[SPECIAL_TOKEN]"

               tokenizer.add_special_tokens({"cls_token": special_token}						)
               snake_case   :				str				 =   tokenizer.encode([special_token]						,     add_special_tokens=UpperCamelCase__						)
               self.assertEqual(len(UpperCamelCase__						)						,     1						)

               snake_case   :				List[Any]				 =   tokenizer.decode(UpperCamelCase__						,     skip_special_tokens=UpperCamelCase__						)
               self.assertTrue(special_token not in decoded						)







   def 			lowerCamelCase		(     self						)     ->		List[Any]:

       '''simple docstring'''




       snake_case   :				List[str]				 =   self.get_tokenizers()
       for tokenizer in tokenizers:
           with self.subTest(F'{tokenizer.__class__.__name__}'						):
               snake_case ,snake_case   :				Union[str, Any]				 =   self.get_input_output_texts(UpperCamelCase__						)

               snake_case   :				Dict				 =   tokenizer.tokenize(UpperCamelCase__						)
               snake_case   :				Tuple				 =   tokenizer.convert_tokens_to_ids(UpperCamelCase__						)
               snake_case   :				List[Any]				 =   tokenizer.encode(UpperCamelCase__						,     add_special_tokens=UpperCamelCase__						)
               self.assertListEqual(UpperCamelCase__						,     UpperCamelCase__						)

               snake_case   :				Dict				 =   tokenizer.convert_ids_to_tokens(UpperCamelCase__						)
               self.assertNotEqual(len(UpperCamelCase__						)						,     0						)
               snake_case   :				Tuple				 =   tokenizer.decode(UpperCamelCase__						)
               self.assertIsInstance(UpperCamelCase__						,     UpperCamelCase__						)

               self.assertEqual(text_a.replace(" "						,     ""						)						,     UpperCamelCase__						)







   @unittest.skip("MGP-STR tokenizer only handles one sequence."						)
   def 			lowerCamelCase		(     self						)     ->		List[Any]:

       '''simple docstring'''




       pass







   @unittest.skip("inputs cannot be pretokenized in MgpstrTokenizer"						)
   def 			lowerCamelCase		(     self						)     ->		Tuple:

       '''simple docstring'''




       pass









"""simple docstring"""



import warnings
from typing import List, Optional, Union

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType







class _lowerCAmelCase    (				snake_case_     ):
   __UpperCAmelCase			:	List[str]							  =  ['''image_processor''', '''tokenizer''']
   __UpperCAmelCase			:	str							  =  '''LayoutLMv2ImageProcessor'''
   __UpperCAmelCase			:	Dict							  =  ('''LayoutXLMTokenizer''', '''LayoutXLMTokenizerFast''')







   def __init__(     self						,     UpperCamelCase__=None						,     UpperCamelCase__=None						,     **UpperCamelCase__						)     ->		Union[str, Any]:

       '''simple docstring'''




       if "feature_extractor" in kwargs:
           warnings.warn(
               "The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"
               " instead."						,     UpperCamelCase__						,     )
           snake_case   :				Dict				 =   kwargs.pop("feature_extractor"						)

       snake_case   :				int				 =   image_processor if image_processor is not None else feature_extractor
       if image_processor is None:
           raise ValueError("You need to specify an `image_processor`."						)
       if tokenizer is None:
           raise ValueError("You need to specify a `tokenizer`."						)

       super().__init__(UpperCamelCase__						,     UpperCamelCase__						)







   def __call__(     self						,     UpperCamelCase__						,     UpperCamelCase__ = None						,     UpperCamelCase__ = None						,     UpperCamelCase__ = None						,     UpperCamelCase__ = None						,     UpperCamelCase__ = True						,     UpperCamelCase__ = False						,     UpperCamelCase__ = None						,     UpperCamelCase__ = None						,     UpperCamelCase__ = 0						,     UpperCamelCase__ = None						,     UpperCamelCase__ = None						,     UpperCamelCase__ = None						,     UpperCamelCase__ = False						,     UpperCamelCase__ = False						,     UpperCamelCase__ = False						,     UpperCamelCase__ = False						,     UpperCamelCase__ = True						,     UpperCamelCase__ = None						,     **UpperCamelCase__						,     )     ->		BatchEncoding:

       '''simple docstring'''




       if self.image_processor.apply_ocr and (boxes is not None):
           raise ValueError(
               "You cannot provide bounding boxes "
               "if you initialized the image processor with apply_ocr set to True."						)

       if self.image_processor.apply_ocr and (word_labels is not None):
           raise ValueError(
               "You cannot provide word labels if you initialized the image processor with apply_ocr set to True."						)

       if return_overflowing_tokens is True and return_offsets_mapping is False:
           raise ValueError("You cannot return overflowing tokens without returning the offsets mapping."						)

       # first, apply the image processor
       snake_case   :				Any				 =   self.image_processor(images=UpperCamelCase__						,     return_tensors=UpperCamelCase__						)

       # second, apply the tokenizer
       if text is not None and self.image_processor.apply_ocr and text_pair is None:
           if isinstance(UpperCamelCase__						,     UpperCamelCase__						):
               snake_case   :				Optional[int]				 =   [text]  # add batch dimension (as the image processor always adds a batch dimension)
           snake_case   :				Optional[Any]				 =   features["words"]

       snake_case   :				Dict				 =   self.tokenizer(
           text=text if text is not None else features["words"]						,     text_pair=text_pair if text_pair is not None else None						,     boxes=boxes if boxes is not None else features["boxes"]						,     word_labels=UpperCamelCase__						,     add_special_tokens=UpperCamelCase__						,     padding=UpperCamelCase__						,     truncation=UpperCamelCase__						,     max_length=UpperCamelCase__						,     stride=UpperCamelCase__						,     pad_to_multiple_of=UpperCamelCase__						,     return_token_type_ids=UpperCamelCase__						,     return_attention_mask=UpperCamelCase__						,     return_overflowing_tokens=UpperCamelCase__						,     return_special_tokens_mask=UpperCamelCase__						,     return_offsets_mapping=UpperCamelCase__						,     return_length=UpperCamelCase__						,     verbose=UpperCamelCase__						,     return_tensors=UpperCamelCase__						,     **UpperCamelCase__						,     )

       # add pixel values
       snake_case   :				Dict				 =   features.pop("pixel_values"						)
       if return_overflowing_tokens is True:
           snake_case   :				Any				 =   self.get_overflowing_images(UpperCamelCase__						,     encoded_inputs["overflow_to_sample_mapping"]						)
       snake_case   :				str				 =   images

       return encoded_inputs







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						,     UpperCamelCase__						)     ->		str:

       '''simple docstring'''




       snake_case   :				Union[str, Any]				 =   []
       for sample_idx in overflow_to_sample_mapping:
           images_with_overflow.append(images[sample_idx]						)

       if len(UpperCamelCase__						) != len(UpperCamelCase__						):
           raise ValueError(
               "Expected length of images to be the same as the length of `overflow_to_sample_mapping`, but got"
               F' {len(UpperCamelCase__						)} and {len(UpperCamelCase__						)}'						)

       return images_with_overflow







   def 			lowerCamelCase		(     self						,     *UpperCamelCase__						,     **UpperCamelCase__						)     ->		Optional[Any]:

       '''simple docstring'''




       return self.tokenizer.batch_decode(*UpperCamelCase__						,     **UpperCamelCase__						)







   def 			lowerCamelCase		(     self						,     *UpperCamelCase__						,     **UpperCamelCase__						)     ->		Tuple:

       '''simple docstring'''




       return self.tokenizer.decode(*UpperCamelCase__						,     **UpperCamelCase__						)







   @property
   def 			lowerCamelCase		(     self						)     ->		Dict:

       '''simple docstring'''




       return ["input_ids", "bbox", "attention_mask", "image"]







   @property
   def 			lowerCamelCase		(     self						)     ->		Optional[Any]:

       '''simple docstring'''




       warnings.warn(
           "`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead."						,     UpperCamelCase__						,     )
       return self.image_processor_class







   @property
   def 			lowerCamelCase		(     self						)     ->		Any:

       '''simple docstring'''




       warnings.warn(
           "`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead."						,     UpperCamelCase__						,     )
       return self.image_processor





