
"""simple docstring"""





import json
import os
import unittest

from transformers import BatchEncoding, LEDTokenizer, LEDTokenizerFast
from transformers.models.led.tokenization_led import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class    _UpperCAmelCase(   _a				,					unittest.TestCase     ):
		lowercase__      =				LEDTokenizer
		lowercase__      =				LEDTokenizerFast
		lowercase__      =				True



		def 						UpperCAmelCase							( self)	->	Dict:






							'''simple docstring'''
							super().setUp()
							_UpperCamelCase										=  [
							    '''l''',
							    '''o''',
							    '''w''',
							    '''e''',
							    '''r''',
							    '''s''',
							    '''t''',
							    '''i''',
							    '''d''',
							    '''n''',
							    '''\u0120''',
							    '''\u0120l''',
							    '''\u0120n''',
							    '''\u0120lo''',
							    '''\u0120low''',
							    '''er''',
							    '''\u0120lowest''',
							    '''\u0120newer''',
							    '''\u0120wider''',
							    '''<unk>''',
							]
							_UpperCamelCase										=  dict(zip(__lowerCAmelCase    ,		range(len(__lowerCAmelCase))))
							_UpperCamelCase										=  ['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
							_UpperCamelCase										=  {'''unk_token''': '''<unk>'''}

							_UpperCamelCase										=  os.path.join(self.tmpdirname    ,		VOCAB_FILES_NAMES['''vocab_file'''])
							_UpperCamelCase										=  os.path.join(self.tmpdirname    ,		VOCAB_FILES_NAMES['''merges_file'''])
							with open(self.vocab_file    ,		'''w'''    ,		encoding='''utf-8''') as fp:
												fp.write(json.dumps(__lowerCAmelCase) + '''\n''')
							with open(self.merges_file    ,		'''w'''    ,		encoding='''utf-8''') as fp:
												fp.write('''\n'''.join(__lowerCAmelCase))



		def 						UpperCAmelCase							( self    ,		**__a)	->	str:






							'''simple docstring'''
							kwargs.update(self.special_tokens_map)
							return self.tokenizer_class.from_pretrained(self.tmpdirname    ,		**__lowerCAmelCase)



		def 						UpperCAmelCase							( self    ,		**__a)	->	Union[str, Any]:






							'''simple docstring'''
							kwargs.update(self.special_tokens_map)
							return self.rust_tokenizer_class.from_pretrained(self.tmpdirname    ,		**__lowerCAmelCase)



		def 						UpperCAmelCase							( self    ,		__a)	->	Optional[int]:






							'''simple docstring'''
							return "lower newer", "lower newer"



		@cached_property
		def 						UpperCAmelCase							( self)	->	Tuple:






							'''simple docstring'''
							return LEDTokenizer.from_pretrained('''allenai/led-base-16384''')



		@cached_property
		def 						UpperCAmelCase							( self)	->	Dict:






							'''simple docstring'''
							return LEDTokenizerFast.from_pretrained('''allenai/led-base-16384''')



		@require_torch
		def 						UpperCAmelCase							( self)	->	List[str]:






							'''simple docstring'''
							_UpperCamelCase										=  ['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
							_UpperCamelCase										=  [0, 2_50, 2_51, 1_78_18, 13, 3_91_86, 19_38, 4, 2]

							for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
												_UpperCamelCase										=  tokenizer(__lowerCAmelCase    ,		max_length=len(__lowerCAmelCase)    ,		padding=__lowerCAmelCase    ,		return_tensors='''pt''')
												self.assertIsInstance(__lowerCAmelCase    ,		__lowerCAmelCase)

												self.assertEqual((2, 9)    ,		batch.input_ids.shape)
												self.assertEqual((2, 9)    ,		batch.attention_mask.shape)
												_UpperCamelCase										=  batch.input_ids.tolist()[0]
												self.assertListEqual(__lowerCAmelCase    ,		__lowerCAmelCase)



		@require_torch
		def 						UpperCAmelCase							( self)	->	Dict:






							'''simple docstring'''
							_UpperCamelCase										=  ['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
							for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
												_UpperCamelCase										=  tokenizer(__lowerCAmelCase    ,		padding=__lowerCAmelCase    ,		return_tensors='''pt''')
												self.assertIn('''input_ids'''    ,		__lowerCAmelCase)
												self.assertIn('''attention_mask'''    ,		__lowerCAmelCase)
												self.assertNotIn('''labels'''    ,		__lowerCAmelCase)
												self.assertNotIn('''decoder_attention_mask'''    ,		__lowerCAmelCase)



		@require_torch
		def 						UpperCAmelCase							( self)	->	Tuple:






							'''simple docstring'''
							_UpperCamelCase										=  [
							    '''Summary of the text.''',
							    '''Another summary.''',
							]
							for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
												_UpperCamelCase										=  tokenizer(text_target=__lowerCAmelCase    ,		max_length=32    ,		padding='''max_length'''    ,		return_tensors='''pt''')
												self.assertEqual(32    ,		targets['''input_ids'''].shape[1])



		@require_torch
		def 						UpperCAmelCase							( self)	->	Tuple:






							'''simple docstring'''
							for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
												_UpperCamelCase										=  tokenizer(
												    ['''I am a small frog''' * 10_24, '''I am a small frog''']    ,		padding=__lowerCAmelCase    ,		truncation=__lowerCAmelCase    ,		return_tensors='''pt''')
												self.assertIsInstance(__lowerCAmelCase    ,		__lowerCAmelCase)
												self.assertEqual(batch.input_ids.shape    ,		(2, 51_22))



		@require_torch
		def 						UpperCAmelCase							( self)	->	str:






							'''simple docstring'''
							_UpperCamelCase										=  ['''A long paragraph for summarization.''']
							_UpperCamelCase										=  [
							    '''Summary of the text.''',
							]
							for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
												_UpperCamelCase										=  tokenizer(__lowerCAmelCase    ,		return_tensors='''pt''')
												_UpperCamelCase										=  tokenizer(text_target=__lowerCAmelCase    ,		return_tensors='''pt''')
												_UpperCamelCase										=  inputs['''input_ids''']
												_UpperCamelCase										=  targets['''input_ids''']
												self.assertTrue((input_ids[:, 0] == tokenizer.bos_token_id).all().item())
												self.assertTrue((labels[:, 0] == tokenizer.bos_token_id).all().item())
												self.assertTrue((input_ids[:, -1] == tokenizer.eos_token_id).all().item())
												self.assertTrue((labels[:, -1] == tokenizer.eos_token_id).all().item())



		@require_torch
		def 						UpperCAmelCase							( self)	->	List[str]:






							'''simple docstring'''
							for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
												_UpperCamelCase										=  ['''Summary of the text.''', '''Another summary.''']
												_UpperCamelCase										=  [[0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, -1, -1]]

												_UpperCamelCase										=  tokenizer(__lowerCAmelCase    ,		padding=__lowerCAmelCase)
												_UpperCamelCase										=  [[0] * len(__lowerCAmelCase) for x in encoded_output['''input_ids''']]
												_UpperCamelCase										=  tokenizer.pad(__lowerCAmelCase)
												self.assertSequenceEqual(outputs['''global_attention_mask''']    ,		__lowerCAmelCase)



		def 						UpperCAmelCase							( self)	->	str:






							'''simple docstring'''
							pass




		def 						UpperCAmelCase							( self)	->	str:






							'''simple docstring'''
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''):
																	_UpperCamelCase										=  self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase    ,		**__lowerCAmelCase)
																	_UpperCamelCase										=  self.tokenizer_class.from_pretrained(__lowerCAmelCase    ,		**__lowerCAmelCase)
																	_UpperCamelCase										=  '''A, <mask> AllenNLP sentence.'''
																	_UpperCamelCase										=  tokenizer_r.encode_plus(__lowerCAmelCase    ,		add_special_tokens=__lowerCAmelCase    ,		return_token_type_ids=__lowerCAmelCase)
																	_UpperCamelCase										=  tokenizer_p.encode_plus(__lowerCAmelCase    ,		add_special_tokens=__lowerCAmelCase    ,		return_token_type_ids=__lowerCAmelCase)
																	self.assertEqual(sum(tokens_r['''token_type_ids'''])    ,		sum(tokens_p['''token_type_ids''']))
																	self.assertEqual(
																	    sum(tokens_r['''attention_mask''']) / len(tokens_r['''attention_mask'''])    ,		sum(tokens_p['''attention_mask''']) / len(tokens_p['''attention_mask'''])    ,		)

																	_UpperCamelCase										=  tokenizer_r.convert_ids_to_tokens(tokens_r['''input_ids'''])
																	_UpperCamelCase										=  tokenizer_p.convert_ids_to_tokens(tokens_p['''input_ids'''])
																	self.assertSequenceEqual(tokens_p['''input_ids''']    ,		[0, 2_50, 6, 5_02_64, 38_23, 4_87, 2_19_92, 36_45, 4, 2])
																	self.assertSequenceEqual(tokens_r['''input_ids''']    ,		[0, 2_50, 6, 5_02_64, 38_23, 4_87, 2_19_92, 36_45, 4, 2])

																	self.assertSequenceEqual(
																	    __lowerCAmelCase    ,		['''<s>''', '''A''', ''',''', '''<mask>''', '''ĠAllen''', '''N''', '''LP''', '''Ġsentence''', '''.''', '''</s>'''])
																	self.assertSequenceEqual(
																	    __lowerCAmelCase    ,		['''<s>''', '''A''', ''',''', '''<mask>''', '''ĠAllen''', '''N''', '''LP''', '''Ġsentence''', '''.''', '''</s>'''])








"""simple docstring"""





import importlib
import json
import os
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
import transformers.models.auto
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.bert.configuration_bert import BertConfig
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, get_tests_dir


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402


_a														=		get_tests_dir("""fixtures/dummy-config.json""")






class    _UpperCAmelCase(   unittest.TestCase     ):



		def 						UpperCAmelCase							( self)	->	str:






							'''simple docstring'''
							_UpperCamelCase										=  0



		def 						UpperCAmelCase							( self)	->	List[str]:






							'''simple docstring'''
							self.assertIsNotNone(transformers.models.auto.__spec__)
							self.assertIsNotNone(importlib.util.find_spec('''transformers.models.auto'''))



		def 						UpperCAmelCase							( self)	->	str:






							'''simple docstring'''
							_UpperCamelCase										=  AutoConfig.from_pretrained('''bert-base-uncased''')
							self.assertIsInstance(__a    ,		__a)



		def 						UpperCAmelCase							( self)	->	Tuple:






							'''simple docstring'''
							_UpperCamelCase										=  AutoConfig.from_pretrained(__a)
							self.assertIsInstance(__a    ,		__a)



		def 						UpperCAmelCase							( self)	->	Optional[Any]:






							'''simple docstring'''
							_UpperCamelCase										=  AutoConfig.from_pretrained(__a)
							self.assertIsInstance(__a    ,		__a)



		def 						UpperCAmelCase							( self)	->	Union[str, Any]:






							'''simple docstring'''
							_UpperCamelCase										=  AutoConfig.for_model('''roberta''')
							self.assertIsInstance(__a    ,		__a)



		def 						UpperCAmelCase							( self)	->	Dict:






							'''simple docstring'''
							with tempfile.TemporaryDirectory() as tmp_dir:
												# This model name contains bert and roberta, but roberta ends up being picked.
												_UpperCamelCase										=  os.path.join(__a    ,		'''fake-roberta''')
												os.makedirs(__a    ,		exist_ok=__a)
												with open(os.path.join(__a    ,		'''config.json''')    ,		'''w''') as f:
																	f.write(json.dumps({}))
												_UpperCamelCase										=  AutoConfig.from_pretrained(__a)
												self.assertEqual(type(__a)    ,		__a)



		def 						UpperCAmelCase							( self)	->	Tuple:






							'''simple docstring'''
							try:
												AutoConfig.register('''custom'''    ,		__a)
												# Wrong model type will raise an error
												with self.assertRaises(__a):
																	AutoConfig.register('''model'''    ,		__a)
												# Trying to register something existing in the Transformers library will raise an error
												with self.assertRaises(__a):
																	AutoConfig.register('''bert'''    ,		__a)

												# Now that the config is registered, it can be used as any other config with the auto-API
												_UpperCamelCase										=  CustomConfig()
												with tempfile.TemporaryDirectory() as tmp_dir:
																	config.save_pretrained(__a)
																	_UpperCamelCase										=  AutoConfig.from_pretrained(__a)
																	self.assertIsInstance(__a    ,		__a)

							finally:
												if "custom" in CONFIG_MAPPING._extra_content:
																	del CONFIG_MAPPING._extra_content["custom"]



		def 						UpperCAmelCase							( self)	->	List[Any]:






							'''simple docstring'''
							with self.assertRaisesRegex(
							    __a    ,		'''bert-base is not a local folder and is not a valid model identifier'''):
												_UpperCamelCase										=  AutoConfig.from_pretrained('''bert-base''')



		def 						UpperCAmelCase							( self)	->	List[Any]:






							'''simple docstring'''
							with self.assertRaisesRegex(
							    __a    ,		R'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'''):
												_UpperCamelCase										=  AutoConfig.from_pretrained(__a    ,		revision='''aaaaaa''')



		def 						UpperCAmelCase							( self)	->	Optional[Any]:






							'''simple docstring'''
							with self.assertRaisesRegex(
							    __a    ,		'''hf-internal-testing/no-config-test-repo does not appear to have a file named config.json.'''    ,		):
												_UpperCamelCase										=  AutoConfig.from_pretrained('''hf-internal-testing/no-config-test-repo''')



		def 						UpperCAmelCase							( self)	->	int:






							'''simple docstring'''
							# If remote code is not set, we will time out when asking whether to load the model.
							with self.assertRaises(__a):
												_UpperCamelCase										=  AutoConfig.from_pretrained('''hf-internal-testing/test_dynamic_model''')
							# If remote code is disabled, we can't load this config.
							with self.assertRaises(__a):
												_UpperCamelCase										=  AutoConfig.from_pretrained('''hf-internal-testing/test_dynamic_model'''    ,		trust_remote_code=__a)

							_UpperCamelCase										=  AutoConfig.from_pretrained('''hf-internal-testing/test_dynamic_model'''    ,		trust_remote_code=__a)
							self.assertEqual(config.__class__.__name__    ,		'''NewModelConfig''')

							# Test config can be reloaded.
							with tempfile.TemporaryDirectory() as tmp_dir:
												config.save_pretrained(__a)
												_UpperCamelCase										=  AutoConfig.from_pretrained(__a    ,		trust_remote_code=__a)
							self.assertEqual(reloaded_config.__class__.__name__    ,		'''NewModelConfig''')




		def 						UpperCAmelCase							( self)	->	Any:






							'''simple docstring'''






							class    _UpperCAmelCase(   lowerCamelCase     ):
									lowercase__      =				'new-model'

							try:
												AutoConfig.register('''new-model'''    ,		__a)
												# If remote code is not set, the default is to use local
												_UpperCamelCase										=  AutoConfig.from_pretrained('''hf-internal-testing/test_dynamic_model''')
												self.assertEqual(config.__class__.__name__    ,		'''NewModelConfigLocal''')

												# If remote code is disabled, we load the local one.
												_UpperCamelCase										=  AutoConfig.from_pretrained('''hf-internal-testing/test_dynamic_model'''    ,		trust_remote_code=__a)
												self.assertEqual(config.__class__.__name__    ,		'''NewModelConfigLocal''')

												# If remote is enabled, we load from the Hub
												_UpperCamelCase										=  AutoConfig.from_pretrained('''hf-internal-testing/test_dynamic_model'''    ,		trust_remote_code=__a)
												self.assertEqual(config.__class__.__name__    ,		'''NewModelConfig''')

							finally:
												if "new-model" in CONFIG_MAPPING._extra_content:
																	del CONFIG_MAPPING._extra_content["new-model"]






