





"""simple docstring"""




import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
						from test_module.custom_tokenization_fast import CustomTokenizerFast
class SCREAMING_SNAKE_CASE_  (   unittest.TestCase   ):






						def UpperCAmelCase_			( self : Dict		)							->      Optional[Any]:







							"""simple docstring"""


							snake_case_						:     List[str]     =		mock.Mock()
							snake_case_						:     List[Any]     =		500
							snake_case_						:     int     =		{}
							snake_case_						:     List[Any]     =		HTTPError
							snake_case_						:     List[str]     =		{}

							# Download this model to make sure it's in the cache.
							snake_case_						:     Union[str, Any]     =		BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'		)

							# Under the mock environment we get a 500 error when trying to reach the tokenizer.
							with mock.patch('requests.Session.request'						,				return_value=_snake_case		) as mock_head:
								snake_case_						:     Any     =		BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'		)
								# This check we did call the fake head request
								mock_head.assert_called()






						@require_tokenizers
						def UpperCAmelCase_			( self : str		)							->      str:







							"""simple docstring"""


							snake_case_						:     List[Any]     =		mock.Mock()
							snake_case_						:     List[str]     =		500
							snake_case_						:     int     =		{}
							snake_case_						:     List[str]     =		HTTPError
							snake_case_						:     List[Any]     =		{}

							# Download this model to make sure it's in the cache.
							snake_case_						:     Tuple     =		GPTaTokenizerFast.from_pretrained('gpt2'		)

							# Under the mock environment we get a 500 error when trying to reach the tokenizer.
							with mock.patch('requests.Session.request'						,				return_value=_snake_case		) as mock_head:
								snake_case_						:     Tuple     =		GPTaTokenizerFast.from_pretrained('gpt2'		)
								# This check we did call the fake head request
								mock_head.assert_called()






						def UpperCAmelCase_			( self : int		)							->      Optional[int]:







							"""simple docstring"""


							try:
								snake_case_						:     str     =		tempfile.mktemp()
								with open(_snake_case						,				'wb'		) as f:
									http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'						,				_snake_case		)

								snake_case_						:     Tuple     =		AlbertTokenizer.from_pretrained(_snake_case		)
							finally:
								os.remove(_snake_case		)

							# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
							# the current folder and have the right name.
							if os.path.isfile('tokenizer.json'		):
								# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
								return
							try:
								with open('tokenizer.json'						,				'wb'		) as f:
									http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'						,				_snake_case		)
								snake_case_						:     Tuple     =		AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2'		)
								# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
								self.assertEqual(tokenizer.vocab_size						,				1000		)
								# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

							finally:
								os.remove('tokenizer.json'		)






						def UpperCAmelCase_			( self : int		)							->      List[Any]:







							"""simple docstring"""


							snake_case_						:     Tuple     =		AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'		)
@is_staging_test
class SCREAMING_SNAKE_CASE_  (   unittest.TestCase   ):
						__magic_name__:     Optional[int]	 =    ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]






						@classmethod
						def UpperCAmelCase_			( cls : Dict		)							->      Dict:







							"""simple docstring"""


							snake_case_						:     List[str]     =		TOKEN
							HfFolder.save_token(_snake_case		)






						@classmethod
						def UpperCAmelCase_			( cls : Optional[Any]		)							->      Dict:







							"""simple docstring"""


							try:
								delete_repo(token=cls._token						,				repo_id='test-tokenizer'		)
							except HTTPError:
								pass

							try:
								delete_repo(token=cls._token						,				repo_id='valid_org/test-tokenizer-org'		)
							except HTTPError:
								pass

							try:
								delete_repo(token=cls._token						,				repo_id='test-dynamic-tokenizer'		)
							except HTTPError:
								pass






						def UpperCAmelCase_			( self : Optional[int]		)							->      List[Any]:







							"""simple docstring"""


							with tempfile.TemporaryDirectory() as tmp_dir:
								snake_case_						:     int     =		os.path.join(_snake_case						,				'vocab.txt'		)
								with open(_snake_case						,				'w'						,				encoding='utf-8'		) as vocab_writer:
									vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]		)		)
								snake_case_						:     Optional[int]     =		BertTokenizer(_snake_case		)

							tokenizer.push_to_hub('test-tokenizer'						,				use_auth_token=self._token		)
							snake_case_						:     Dict     =		BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""		)
							self.assertDictEqual(new_tokenizer.vocab						,				tokenizer.vocab		)

							# Reset repo
							delete_repo(token=self._token						,				repo_id='test-tokenizer'		)

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
								tokenizer.save_pretrained(_snake_case						,				repo_id='test-tokenizer'						,				push_to_hub=_snake_case						,				use_auth_token=self._token		)

							snake_case_						:     int     =		BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""		)
							self.assertDictEqual(new_tokenizer.vocab						,				tokenizer.vocab		)






						def UpperCAmelCase_			( self : Optional[int]		)							->      List[Any]:







							"""simple docstring"""


							with tempfile.TemporaryDirectory() as tmp_dir:
								snake_case_						:     Union[str, Any]     =		os.path.join(_snake_case						,				'vocab.txt'		)
								with open(_snake_case						,				'w'						,				encoding='utf-8'		) as vocab_writer:
									vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]		)		)
								snake_case_						:     Union[str, Any]     =		BertTokenizer(_snake_case		)

							tokenizer.push_to_hub('valid_org/test-tokenizer-org'						,				use_auth_token=self._token		)
							snake_case_						:     Optional[int]     =		BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'		)
							self.assertDictEqual(new_tokenizer.vocab						,				tokenizer.vocab		)

							# Reset repo
							delete_repo(token=self._token						,				repo_id='valid_org/test-tokenizer-org'		)

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
								tokenizer.save_pretrained(
								    _snake_case						,				repo_id='valid_org/test-tokenizer-org'						,				push_to_hub=_snake_case						,				use_auth_token=self._token		)

							snake_case_						:     int     =		BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'		)
							self.assertDictEqual(new_tokenizer.vocab						,				tokenizer.vocab		)






						@require_tokenizers
						def UpperCAmelCase_			( self : Any		)							->      Union[str, Any]:







							"""simple docstring"""


							CustomTokenizer.register_for_auto_class()
							with tempfile.TemporaryDirectory() as tmp_dir:
								snake_case_						:     Dict     =		os.path.join(_snake_case						,				'vocab.txt'		)
								with open(_snake_case						,				'w'						,				encoding='utf-8'		) as vocab_writer:
									vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]		)		)
								snake_case_						:     Any     =		CustomTokenizer(_snake_case		)

							# No fast custom tokenizer
							tokenizer.push_to_hub('test-dynamic-tokenizer'						,				use_auth_token=self._token		)

							snake_case_						:     str     =		AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""						,				trust_remote_code=_snake_case		)
							# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__						,				'CustomTokenizer'		)

							# Fast and slow custom tokenizer
							CustomTokenizerFast.register_for_auto_class()
							with tempfile.TemporaryDirectory() as tmp_dir:
								snake_case_						:     Any     =		os.path.join(_snake_case						,				'vocab.txt'		)
								with open(_snake_case						,				'w'						,				encoding='utf-8'		) as vocab_writer:
									vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]		)		)

								snake_case_						:     Optional[Any]     =		BertTokenizerFast.from_pretrained(_snake_case		)
								bert_tokenizer.save_pretrained(_snake_case		)
								snake_case_						:     Dict     =		CustomTokenizerFast.from_pretrained(_snake_case		)

							tokenizer.push_to_hub('test-dynamic-tokenizer'						,				use_auth_token=self._token		)

							snake_case_						:     Any     =		AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""						,				trust_remote_code=_snake_case		)
							# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__						,				'CustomTokenizerFast'		)
							snake_case_						:     Optional[Any]     =		AutoTokenizer.from_pretrained(
							    F"""{USER}/test-dynamic-tokenizer"""						,				use_fast=_snake_case						,				trust_remote_code=_snake_case		)
							# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__						,				'CustomTokenizer'		)
class SCREAMING_SNAKE_CASE_  (   unittest.TestCase   ):






						def UpperCAmelCase_			( self : List[Any]		)							->      str:







							"""simple docstring"""


							snake_case_						:     Tuple     =		Trie()
							trie.add('Hello 友達'		)
							self.assertEqual(trie.data						,				{'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}}		)
							trie.add('Hello'		)
							trie.data
							self.assertEqual(trie.data						,				{'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}}		)






						def UpperCAmelCase_			( self : int		)							->      List[Any]:







							"""simple docstring"""


							snake_case_						:     Optional[int]     =		Trie()
							self.assertEqual(trie.split('[CLS] This is a extra_id_100'		)						,				['[CLS] This is a extra_id_100']		)
							trie.add('[CLS]'		)
							trie.add('extra_id_1'		)
							trie.add('extra_id_100'		)
							self.assertEqual(trie.split('[CLS] This is a extra_id_100'		)						,				['[CLS]', ' This is a ', 'extra_id_100']		)






						def UpperCAmelCase_			( self : List[str]		)							->      Optional[int]:







							"""simple docstring"""


							snake_case_						:     Any     =		Trie()
							trie.add('A'		)
							self.assertEqual(trie.split('ABC'		)						,				['A', 'BC']		)
							self.assertEqual(trie.split('BCA'		)						,				['BC', 'A']		)






						def UpperCAmelCase_			( self : Dict		)							->      Union[str, Any]:







							"""simple docstring"""


							snake_case_						:     List[str]     =		Trie()
							trie.add('TOKEN]'		)
							trie.add('[SPECIAL_TOKEN]'		)
							self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'		)						,				['This is something ', '[SPECIAL_TOKEN]']		)






						def UpperCAmelCase_			( self : Optional[Any]		)							->      Optional[int]:







							"""simple docstring"""


							snake_case_						:     Optional[int]     =		Trie()
							trie.add('A'		)
							trie.add('P'		)
							trie.add('[SPECIAL_TOKEN]'		)
							self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'		)						,				['This is something ', '[SPECIAL_TOKEN]']		)






						def UpperCAmelCase_			( self : Any		)							->      List[Any]:







							"""simple docstring"""


							snake_case_						:     Tuple     =		Trie()
							trie.add('AB'		)
							trie.add('B'		)
							trie.add('C'		)
							self.assertEqual(trie.split('ABC'		)						,				['AB', 'C']		)






						def UpperCAmelCase_			( self : Any		)							->      Tuple:







							"""simple docstring"""


							snake_case_						:     List[str]     =		Trie()
							trie.add('ABC'		)
							trie.add('B'		)
							trie.add('CD'		)
							self.assertEqual(trie.split('ABCD'		)						,				['ABC', 'D']		)






						def UpperCAmelCase_			( self : str		)							->      Optional[int]:







							"""simple docstring"""


							snake_case_						:     Optional[Any]     =		Trie()
							snake_case_						:     Dict     =		trie.cut_text('ABC'						,				[0, 0, 2, 1, 2, 3]		)
							self.assertEqual(_snake_case						,				['AB', 'C']		)








import warnings
from typing import List, Optional, Union

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType
class SCREAMING_SNAKE_CASE_  (   snake_case_   ):
						__magic_name__:     Optional[Any]	 =    ["image_processor", "tokenizer"]
						__magic_name__:     Optional[Any]	 =    "LayoutLMv3ImageProcessor"
						__magic_name__:     str	 =    ("LayoutLMv3Tokenizer", "LayoutLMv3TokenizerFast")






						def __init__( self : int						,				_A : List[str]=None						,				_A : Dict=None						,				**_A : int		)							->      List[str]:







							"""simple docstring"""


							snake_case_						:     Any     =		None
							if "feature_extractor" in kwargs:
								warnings.warn(
								    'The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'
								    ' instead.'						,				_A						,				)
								snake_case_						:     Any     =		kwargs.pop('feature_extractor'		)

							snake_case_						:     Optional[int]     =		image_processor if image_processor is not None else feature_extractor
							if image_processor is None:
								raise ValueError('You need to specify an `image_processor`.'		)
							if tokenizer is None:
								raise ValueError('You need to specify a `tokenizer`.'		)

							super().__init__(_A						,				_A		)






						def __call__( self : List[str]						,				_A : Optional[Any]						,				_A : Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None						,				_A : Optional[Union[PreTokenizedInput, List[PreTokenizedInput]]] = None						,				_A : Union[List[List[int]], List[List[List[int]]]] = None						,				_A : Optional[Union[List[int], List[List[int]]]] = None						,				_A : bool = True						,				_A : Union[bool, str, PaddingStrategy] = False						,				_A : Union[bool, str, TruncationStrategy] = None						,				_A : Optional[int] = None						,				_A : int = 0						,				_A : Optional[int] = None						,				_A : Optional[bool] = None						,				_A : Optional[bool] = None						,				_A : bool = False						,				_A : bool = False						,				_A : bool = False						,				_A : bool = False						,				_A : bool = True						,				_A : Optional[Union[str, TensorType]] = None						,				**_A : str						,				)							->      BatchEncoding:







							"""simple docstring"""


							if self.image_processor.apply_ocr and (boxes is not None):
								raise ValueError(
								    'You cannot provide bounding boxes if you initialized the image processor with apply_ocr set to True.'		)

							if self.image_processor.apply_ocr and (word_labels is not None):
								raise ValueError(
								    'You cannot provide word labels if you initialized the image processor with apply_ocr set to True.'		)

							# first, apply the image processor
							snake_case_						:     str     =		self.image_processor(images=_A						,				return_tensors=_A		)

							# second, apply the tokenizer
							if text is not None and self.image_processor.apply_ocr and text_pair is None:
								if isinstance(_A						,				_A		):
									snake_case_						:     List[Any]     =		[text]  # add batch dimension (as the image processor always adds a batch dimension)
								snake_case_						:     str     =		features['words']

							snake_case_						:     Optional[int]     =		self.tokenizer(
							    text=text if text is not None else features['words']						,				text_pair=text_pair if text_pair is not None else None						,				boxes=boxes if boxes is not None else features['boxes']						,				word_labels=_A						,				add_special_tokens=_A						,				padding=_A						,				truncation=_A						,				max_length=_A						,				stride=_A						,				pad_to_multiple_of=_A						,				return_token_type_ids=_A						,				return_attention_mask=_A						,				return_overflowing_tokens=_A						,				return_special_tokens_mask=_A						,				return_offsets_mapping=_A						,				return_length=_A						,				verbose=_A						,				return_tensors=_A						,				**_A						,				)

							# add pixel values
							snake_case_						:     List[str]     =		features.pop('pixel_values'		)
							if return_overflowing_tokens is True:
								snake_case_						:     Dict     =		self.get_overflowing_images(_A						,				encoded_inputs['overflow_to_sample_mapping']		)
							snake_case_						:     Optional[Any]     =		images

							return encoded_inputs






						def UpperCAmelCase_			( self : Dict						,				_A : Tuple						,				_A : Dict		)							->      Union[str, Any]:







							"""simple docstring"""


							snake_case_						:     List[str]     =		[]
							for sample_idx in overflow_to_sample_mapping:
								images_with_overflow.append(images[sample_idx]		)

							if len(_A		) != len(_A		):
								raise ValueError(
								    'Expected length of images to be the same as the length of `overflow_to_sample_mapping`, but got'
								    F""" {len(_A		)} and {len(_A		)}"""		)

							return images_with_overflow






						def UpperCAmelCase_			( self : Optional[Any]						,				*_A : Optional[Any]						,				**_A : List[Any]		)							->      List[str]:







							"""simple docstring"""


							return self.tokenizer.batch_decode(*_A						,				**_A		)






						def UpperCAmelCase_			( self : Union[str, Any]						,				*_A : Dict						,				**_A : str		)							->      Any:







							"""simple docstring"""


							return self.tokenizer.decode(*_A						,				**_A		)






						@property
						def UpperCAmelCase_			( self : Optional[int]		)							->      int:







							"""simple docstring"""


							return ["input_ids", "bbox", "attention_mask", "pixel_values"]






						@property
						def UpperCAmelCase_			( self : Any		)							->      Any:







							"""simple docstring"""


							warnings.warn(
							    '`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'						,				_A						,				)
							return self.image_processor_class






						@property
						def UpperCAmelCase_			( self : List[Any]		)							->      int:







							"""simple docstring"""


							warnings.warn(
							    '`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'						,				_A						,				)
							return self.image_processor

