






import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, PerceiverTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
		__UpperCAmelCase     							=						'pt'
elif is_tf_available():
		__UpperCAmelCase     							=						'tf'
else:
		__UpperCAmelCase     							=						'jax'



class   lowerCamelCase				(_snake_case			,  unittest.TestCase      ):







		'''simple docstring'''



		_snake_case   :   str							=  PerceiverTokenizer
		_snake_case   :   List[Any]							=  False



		def  __UpperCAmelCase      (							self       )     ->							Optional[int]:
							super().setUp()
							UpperCAmelCase_   :				int						=						PerceiverTokenizer()
							tokenizer.save_pretrained(self.tmpdirname       )



		@cached_property
		def  __UpperCAmelCase      (							self       )     ->							str:
							return PerceiverTokenizer.from_pretrained('deepmind/language-perceiver'       )



		def  __UpperCAmelCase      (							self    ,		**_UpperCamelCase       )     ->							PerceiverTokenizer:
							return self.tokenizer_class.from_pretrained(self.tmpdirname    ,		**_UpperCamelCase       )



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase=False    ,		_UpperCamelCase=2_0    ,		_UpperCamelCase=5       )     ->							Tuple[str, list]:
							# XXX The default common tokenizer tests assume that every ID is decodable on its own.
							# This assumption is invalid for Perceiver because single bytes might not be
							# valid utf-8 (byte 128 for instance).
							# Here we're overriding the smallest possible method to provide
							# a clean sequence without making the same assumption.

							UpperCAmelCase_   :				List[str]						=						[]
							for i in range(len(_UpperCamelCase       )       ):
												try:
																	UpperCAmelCase_   :				Tuple						=						tokenizer.decode([i]    ,		clean_up_tokenization_spaces=_UpperCamelCase       )
												except UnicodeDecodeError:
																	pass
												toks.append((i, tok)       )

							UpperCAmelCase_   :				str						=						list(filter(lambda _UpperCamelCase       : re.match(r'^[ a-zA-Z]+$'    ,		t[1]       )    ,		_UpperCamelCase       )       )
							UpperCAmelCase_   :				Optional[Any]						=						list(filter(lambda _UpperCamelCase       : [t[0]] == tokenizer.encode(t[1]    ,		add_special_tokens=_UpperCamelCase       )    ,		_UpperCamelCase       )       )
							if max_length is not None and len(_UpperCamelCase       ) > max_length:
												UpperCAmelCase_   :				Optional[int]						=						toks[:max_length]
							if min_length is not None and len(_UpperCamelCase       ) < min_length and len(_UpperCamelCase       ) > 0:
												while len(_UpperCamelCase       ) < min_length:
																	UpperCAmelCase_   :				Dict						=						toks + toks
        # toks_str = [t[1] for t in toks]
							UpperCAmelCase_   :				List[Any]						=						[t[0] for t in toks]

							# Ensure consistency
							UpperCAmelCase_   :				List[Any]						=						tokenizer.decode(_UpperCamelCase    ,		clean_up_tokenization_spaces=_UpperCamelCase       )
							if " " not in output_txt and len(_UpperCamelCase       ) > 1:
												UpperCAmelCase_   :				List[str]						=						(
												    tokenizer.decode([toks_ids[0]]    ,		clean_up_tokenization_spaces=_UpperCamelCase       )
												    + ' '
												    + tokenizer.decode(toks_ids[1:]    ,		clean_up_tokenization_spaces=_UpperCamelCase       )
												)
							if with_prefix_space:
												UpperCAmelCase_   :				Any						=						' ' + output_txt
							UpperCAmelCase_   :				Union[str, Any]						=						tokenizer.encode(_UpperCamelCase    ,		add_special_tokens=_UpperCamelCase       )
							return output_txt, output_ids



		def  __UpperCAmelCase      (							self       )     ->							Tuple:
							UpperCAmelCase_   :				List[str]						=						self.perceiver_tokenizer
							UpperCAmelCase_   :				Union[str, Any]						=						'Unicode €.'
							UpperCAmelCase_   :				List[str]						=						tokenizer(_UpperCamelCase       )
							UpperCAmelCase_   :				Optional[int]						=						[4, 9_1, 1_1_6, 1_1_1, 1_0_5, 1_1_7, 1_0_6, 1_0_7, 3_8, 2_3_2, 1_3_6, 1_7_8, 5_2, 5]
							self.assertEqual(encoded['input_ids']    ,		_UpperCamelCase       )

							# decoding
							UpperCAmelCase_   :				Optional[Any]						=						tokenizer.decode(_UpperCamelCase       )
							self.assertEqual(_UpperCamelCase    ,		'[CLS]Unicode €.[SEP]'       )

							UpperCAmelCase_   :				int						=						tokenizer('e è é ê ë'       )
							UpperCAmelCase_   :				Tuple						=						[4, 1_0_7, 3_8, 2_0_1, 1_7_4, 3_8, 2_0_1, 1_7_5, 3_8, 2_0_1, 1_7_6, 3_8, 2_0_1, 1_7_7, 5]
							self.assertEqual(encoded['input_ids']    ,		_UpperCamelCase       )
							# decoding
							UpperCAmelCase_   :				List[str]						=						tokenizer.decode(_UpperCamelCase       )
							self.assertEqual(_UpperCamelCase    ,		'[CLS]e è é ê ë[SEP]'       )

							# encode/decode, but with `encode` instead of `__call__`
							self.assertEqual(tokenizer.decode(tokenizer.encode('e è é ê ë'       )       )    ,		'[CLS]e è é ê ë[SEP]'       )



		def  __UpperCAmelCase      (							self       )     ->							Tuple:
							UpperCAmelCase_   :				Dict						=						self.perceiver_tokenizer
							UpperCAmelCase_   :				Dict						=						['A long paragraph for summarization.', 'Another paragraph for summarization.']
							# fmt: off
							UpperCAmelCase_   :				List[str]						=						[4, 7_1, 3_8, 1_1_4, 1_1_7, 1_1_6, 1_0_9, 3_8, 1_1_8, 1_0_3, 1_2_0, 1_0_3, 1_0_9, 1_2_0, 1_0_3, 1_1_8, 1_1_0, 3_8, 1_0_8, 1_1_7, 1_2_0, 3_8, 1_2_1, 1_2_3, 1_1_5, 1_1_5, 1_0_3, 1_2_0, 1_1_1, 1_2_8, 1_0_3, 1_2_2, 1_1_1, 1_1_7, 1_1_6, 5_2, 5, 0]
							# fmt: on
							UpperCAmelCase_   :				Tuple						=						tokenizer(_UpperCamelCase    ,		padding=_UpperCamelCase    ,		return_tensors=_UpperCamelCase       )
							self.assertIsInstance(_UpperCamelCase    ,		_UpperCamelCase       )

							if FRAMEWORK != "jax":
												UpperCAmelCase_   :				Union[str, Any]						=						list(batch.input_ids.numpy()[0]       )
							else:
												UpperCAmelCase_   :				Dict						=						list(batch.input_ids.tolist()[0]       )

							self.assertListEqual(_UpperCamelCase    ,		_UpperCamelCase       )

							self.assertEqual((2, 3_8)    ,		batch.input_ids.shape       )
							self.assertEqual((2, 3_8)    ,		batch.attention_mask.shape       )



		def  __UpperCAmelCase      (							self       )     ->							Tuple:
							UpperCAmelCase_   :				Optional[int]						=						self.perceiver_tokenizer
							UpperCAmelCase_   :				Optional[Any]						=						['A long paragraph for summarization.', 'Another paragraph for summarization.']
							UpperCAmelCase_   :				str						=						tokenizer(_UpperCamelCase    ,		padding=_UpperCamelCase    ,		return_tensors=_UpperCamelCase       )
							# check if input_ids are returned and no decoder_input_ids
							self.assertIn('input_ids'    ,		_UpperCamelCase       )
							self.assertIn('attention_mask'    ,		_UpperCamelCase       )
							self.assertNotIn('decoder_input_ids'    ,		_UpperCamelCase       )
							self.assertNotIn('decoder_attention_mask'    ,		_UpperCamelCase       )



		def  __UpperCAmelCase      (							self       )     ->							Optional[Any]:
							UpperCAmelCase_   :				Any						=						self.perceiver_tokenizer
							UpperCAmelCase_   :				Tuple						=						[
							    'Summary of the text.',
							    'Another summary.',
							]
							UpperCAmelCase_   :				str						=						tokenizer(
							    text_target=_UpperCamelCase    ,		max_length=3_2    ,		padding='max_length'    ,		truncation=_UpperCamelCase    ,		return_tensors=_UpperCamelCase       )
							self.assertEqual(3_2    ,		targets['input_ids'].shape[1]       )



		def  __UpperCAmelCase      (							self       )     ->							List[str]:
							# safety check on max_len default value so we are sure the test works
							UpperCAmelCase_   :				int						=						self.get_tokenizers()
							for tokenizer in tokenizers:
												with self.subTest(f"{tokenizer.__class__.__name__}"       ):
																	self.assertNotEqual(tokenizer.model_max_length    ,		4_2       )

        # Now let's start the test
							UpperCAmelCase_   :				int						=						self.get_tokenizers()
							for tokenizer in tokenizers:
												with self.subTest(f"{tokenizer.__class__.__name__}"       ):
																	# Isolate this from the other tests because we save additional tokens/etc
																	UpperCAmelCase_   :				Dict						=						tempfile.mkdtemp()

																	UpperCAmelCase_   :				Any						=						' He is very happy, UNwant\u00E9d,running'
																	UpperCAmelCase_   :				Dict						=						tokenizer.encode(_UpperCamelCase    ,		add_special_tokens=_UpperCamelCase       )
																	tokenizer.save_pretrained(_UpperCamelCase       )

																	UpperCAmelCase_   :				Dict						=						tokenizer.__class__.from_pretrained(_UpperCamelCase       )
																	UpperCAmelCase_   :				List[str]						=						after_tokenizer.encode(_UpperCamelCase    ,		add_special_tokens=_UpperCamelCase       )
																	self.assertListEqual(_UpperCamelCase    ,		_UpperCamelCase       )

																	shutil.rmtree(_UpperCamelCase       )

							UpperCAmelCase_   :				str						=						self.get_tokenizers(model_max_length=4_2       )
							for tokenizer in tokenizers:
												with self.subTest(f"{tokenizer.__class__.__name__}"       ):
																	# Isolate this from the other tests because we save additional tokens/etc
																	UpperCAmelCase_   :				Dict						=						tempfile.mkdtemp()

																	UpperCAmelCase_   :				Any						=						' He is very happy, UNwant\u00E9d,running'
																	tokenizer.add_tokens(['bim', 'bambam']       )
																	UpperCAmelCase_   :				Tuple						=						tokenizer.additional_special_tokens
																	additional_special_tokens.append('new_additional_special_token'       )
																	tokenizer.add_special_tokens({'additional_special_tokens': additional_special_tokens}       )
																	UpperCAmelCase_   :				Optional[int]						=						tokenizer.encode(_UpperCamelCase    ,		add_special_tokens=_UpperCamelCase       )
																	tokenizer.save_pretrained(_UpperCamelCase       )

																	UpperCAmelCase_   :				str						=						tokenizer.__class__.from_pretrained(_UpperCamelCase       )
																	UpperCAmelCase_   :				List[Any]						=						after_tokenizer.encode(_UpperCamelCase    ,		add_special_tokens=_UpperCamelCase       )
																	self.assertListEqual(_UpperCamelCase    ,		_UpperCamelCase       )
																	self.assertIn('new_additional_special_token'    ,		after_tokenizer.additional_special_tokens       )
																	self.assertEqual(after_tokenizer.model_max_length    ,		4_2       )

																	UpperCAmelCase_   :				str						=						tokenizer.__class__.from_pretrained(_UpperCamelCase    ,		model_max_length=4_3       )
																	self.assertEqual(tokenizer.model_max_length    ,		4_3       )

																	shutil.rmtree(_UpperCamelCase       )



		def  __UpperCAmelCase      (							self       )     ->							int:
							UpperCAmelCase_   :				Tuple						=						[]
							if self.test_slow_tokenizer:
												tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())       )

							if self.test_rust_tokenizer:
												tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())       )

							for tokenizer_class, tokenizer_utils in tokenizer_list:
												with tempfile.TemporaryDirectory() as tmp_dir:
																	tokenizer_utils.save_pretrained(_UpperCamelCase       )

																	with open(os.path.join(_UpperCamelCase    ,		'special_tokens_map.json'       )    ,		encoding='utf-8'       ) as json_file:
																						UpperCAmelCase_   :				int						=						json.load(_UpperCamelCase       )

																	with open(os.path.join(_UpperCamelCase    ,		'tokenizer_config.json'       )    ,		encoding='utf-8'       ) as json_file:
																						UpperCAmelCase_   :				Union[str, Any]						=						json.load(_UpperCamelCase       )

																	UpperCAmelCase_   :				Union[str, Any]						=						[f"<extra_id_{i}>" for i in range(1_2_5       )]

																	UpperCAmelCase_   :				List[str]						=						added_tokens_extra_ids + [
																	    'an_additional_special_token'
																	]
																	UpperCAmelCase_   :				Optional[int]						=						added_tokens_extra_ids + [
																	    'an_additional_special_token'
																	]

																	with open(os.path.join(_UpperCamelCase    ,		'special_tokens_map.json'       )    ,		'w'    ,		encoding='utf-8'       ) as outfile:
																						json.dump(_UpperCamelCase    ,		_UpperCamelCase       )
																	with open(os.path.join(_UpperCamelCase    ,		'tokenizer_config.json'       )    ,		'w'    ,		encoding='utf-8'       ) as outfile:
																						json.dump(_UpperCamelCase    ,		_UpperCamelCase       )

																	# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
																	# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
																	# "special_tokens_map.json" files
																	UpperCAmelCase_   :				Union[str, Any]						=						tokenizer_class.from_pretrained(
																	    _UpperCamelCase    ,		)
																	self.assertIn(
																	    'an_additional_special_token'    ,		tokenizer_without_change_in_init.additional_special_tokens       )
																	self.assertEqual(
																	    ['an_additional_special_token']    ,		tokenizer_without_change_in_init.convert_ids_to_tokens(
																	        tokenizer_without_change_in_init.convert_tokens_to_ids(['an_additional_special_token']       )       )    ,		)

																	# Now we test that we can change the value of additional_special_tokens in the from_pretrained
																	UpperCAmelCase_   :				Tuple						=						added_tokens_extra_ids + [AddedToken('a_new_additional_special_token'    ,		lstrip=_UpperCamelCase       )]
																	UpperCAmelCase_   :				List[Any]						=						tokenizer_class.from_pretrained(
																	    _UpperCamelCase    ,		additional_special_tokens=_UpperCamelCase    ,		)

																	self.assertIn('a_new_additional_special_token'    ,		tokenizer.additional_special_tokens       )
																	self.assertEqual(
																	    ['a_new_additional_special_token']    ,		tokenizer.convert_ids_to_tokens(
																	        tokenizer.convert_tokens_to_ids(['a_new_additional_special_token']       )       )    ,		)



		def  __UpperCAmelCase      (							self       )     ->							Tuple:
							UpperCAmelCase_   :				Dict						=						self.perceiver_tokenizer
							self.assertEqual(tokenizer.decode([1_7_8]       )    ,		'�'       )



		def  __UpperCAmelCase      (							self       )     ->							str:
							pass



		def  __UpperCAmelCase      (							self       )     ->							Dict:
							pass



		def  __UpperCAmelCase      (							self       )     ->							Dict:
							pass



		def  __UpperCAmelCase      (							self       )     ->							Any:
							pass



		def  __UpperCAmelCase      (							self       )     ->							List[str]:
							# The default common tokenizer tests uses invalid tokens for Perceiver that can only accept one-character
							# strings and special added tokens as tokens
							UpperCAmelCase_   :				Union[str, Any]						=						self.get_tokenizers(fast=_UpperCamelCase    ,		do_lower_case=_UpperCamelCase       )
							for tokenizer in tokenizers:
												with self.subTest(f"{tokenizer.__class__.__name__}"       ):
																	UpperCAmelCase_   :				int						=						['[CLS]', 't', 'h', 'i', 's', ' ', 'i', 's', ' ', 'a', ' ', 't', 'e', 's', 't', '[SEP]']
																	UpperCAmelCase_   :				Dict						=						tokenizer.convert_tokens_to_string(_UpperCamelCase       )

																	self.assertIsInstance(_UpperCamelCase    ,		_UpperCamelCase       )













import unittest

from huggingface_hub import hf_hub_download

from transformers import MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING, VideoMAEFeatureExtractor
from transformers.pipelines import VideoClassificationPipeline, pipeline
from transformers.testing_utils import (
    is_pipeline_test,
    nested_simplify,
    require_decord,
    require_tf,
    require_torch,
    require_torch_or_tf,
    require_vision,
)

from .test_pipelines_common import ANY



@is_pipeline_test
@require_torch_or_tf
@require_vision
@require_decord
class   lowerCamelCase				(unittest.TestCase      ):







		'''simple docstring'''



		_snake_case   :   List[str]							=  MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase       )     ->							Optional[Any]:
							UpperCAmelCase_   :				List[str]						=						hf_hub_download(
							    repo_id='nateraw/video-demo'    ,		filename='archery.mp4'    ,		repo_type='dataset'       )
							UpperCAmelCase_   :				str						=						VideoClassificationPipeline(model=_UpperCamelCase    ,		image_processor=_UpperCamelCase    ,		top_k=2       )
							UpperCAmelCase_   :				List[str]						=						[
							    example_video_filepath,
							    'https://huggingface.co/datasets/nateraw/video-demo/resolve/main/archery.mp4',
							]
							return video_classifier, examples



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase       )     ->							Dict:
							for example in examples:
												UpperCAmelCase_   :				str						=						video_classifier(_UpperCamelCase       )

												self.assertEqual(
												    _UpperCamelCase    ,		[
												        {'score': ANY(_UpperCamelCase       ), 'label': ANY(_UpperCamelCase       )},
												        {'score': ANY(_UpperCamelCase       ), 'label': ANY(_UpperCamelCase       )},
												    ]    ,		)



		@require_torch
		def  __UpperCAmelCase      (							self       )     ->							List[Any]:
							UpperCAmelCase_   :				str						=						'hf-internal-testing/tiny-random-VideoMAEForVideoClassification'
							UpperCAmelCase_   :				Optional[Any]						=						VideoMAEFeatureExtractor(
							    size={'shortest_edge': 1_0}    ,		crop_size={'height': 1_0, 'width': 1_0}       )
							UpperCAmelCase_   :				str						=						pipeline(
							    'video-classification'    ,		model=_UpperCamelCase    ,		feature_extractor=_UpperCamelCase    ,		frame_sampling_rate=4       )

							UpperCAmelCase_   :				Any						=						hf_hub_download(repo_id='nateraw/video-demo'    ,		filename='archery.mp4'    ,		repo_type='dataset'       )
							UpperCAmelCase_   :				List[str]						=						video_classifier(_UpperCamelCase    ,		top_k=2       )
							self.assertEqual(
							    nested_simplify(_UpperCamelCase    ,		decimals=4       )    ,		[{'score': 0.51_99, 'label': 'LABEL_0'}, {'score': 0.48_01, 'label': 'LABEL_1'}]    ,		)

							UpperCAmelCase_   :				Tuple						=						video_classifier(
							    [
							        video_file_path,
							        video_file_path,
							    ]    ,		top_k=2    ,		)
							self.assertEqual(
							    nested_simplify(_UpperCamelCase    ,		decimals=4       )    ,		[
							        [{'score': 0.51_99, 'label': 'LABEL_0'}, {'score': 0.48_01, 'label': 'LABEL_1'}],
							        [{'score': 0.51_99, 'label': 'LABEL_0'}, {'score': 0.48_01, 'label': 'LABEL_1'}],
							    ]    ,		)



		@require_tf
		def  __UpperCAmelCase      (							self       )     ->							Dict:
							pass






