

import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


a		=get_tests_dir("""fixtures/test_sentencepiece.model""")


if is_torch_available():
					from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

a		=256047
a		=256145






@require_sentencepiece
@require_tokenizers
class 	A_		(	SCREAMING_SNAKE_CASE      ,							unittest.TestCase    ):
							_UpperCAmelCase   :  Dict										=				NllbTokenizer
							_UpperCAmelCase   :  Tuple										=				NllbTokenizerFast
							_UpperCAmelCase   :  int										=				True
							_UpperCAmelCase   :  Union[str, Any]										=				True
							_UpperCAmelCase   :  Tuple										=				{}
							def    lowerCAmelCase					(		self		:      Dict):
								super().setUp()

								# We have a SentencePiece fixture for testing
								__lowerCamelCase    :    List[str] 	=			NllbTokenizer(SCREAMING_SNAKE_CASE__       ,keep_accents=SCREAMING_SNAKE_CASE__)
								tokenizer.save_pretrained(self.tmpdirname)
							def    lowerCAmelCase					(		self		:      List[str]):
								__lowerCamelCase    :    List[str] 	=			NllbTokenizer(SCREAMING_SNAKE_CASE__       ,keep_accents=SCREAMING_SNAKE_CASE__)

								__lowerCamelCase    :    List[str] 	=			tokenizer.tokenize('This is a test')
								self.assertListEqual(SCREAMING_SNAKE_CASE__       ,['▁This', '▁is', '▁a', '▁t', 'est'])

								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__)       ,[value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]       ,)

								__lowerCamelCase    :    List[Any] 	=			tokenizer.tokenize('I was born in 92000, and this is falsé.')
								self.assertListEqual(
								    SCREAMING_SNAKE_CASE__       ,[
								        SPIECE_UNDERLINE + 'I',
								        SPIECE_UNDERLINE + 'was',
								        SPIECE_UNDERLINE + 'b',
								        'or',
								        'n',
								        SPIECE_UNDERLINE + 'in',
								        SPIECE_UNDERLINE + '',
								        '9',
								        '2',
								        '0',
								        '0',
								        '0',
								        ',',
								        SPIECE_UNDERLINE + 'and',
								        SPIECE_UNDERLINE + 'this',
								        SPIECE_UNDERLINE + 'is',
								        SPIECE_UNDERLINE + 'f',
								        'al',
								        's',
								        'é',
								        '.',
								    ]       ,)
								__lowerCamelCase    :    str 	=			tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__)
								self.assertListEqual(
								    SCREAMING_SNAKE_CASE__       ,[
								        value + tokenizer.fairseq_offset
								        for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
								    ]       ,)

								__lowerCamelCase    :    List[Any] 	=			tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__)
								self.assertListEqual(
								    SCREAMING_SNAKE_CASE__       ,[
								        SPIECE_UNDERLINE + 'I',
								        SPIECE_UNDERLINE + 'was',
								        SPIECE_UNDERLINE + 'b',
								        'or',
								        'n',
								        SPIECE_UNDERLINE + 'in',
								        SPIECE_UNDERLINE + '',
								        '<unk>',
								        '2',
								        '0',
								        '0',
								        '0',
								        ',',
								        SPIECE_UNDERLINE + 'and',
								        SPIECE_UNDERLINE + 'this',
								        SPIECE_UNDERLINE + 'is',
								        SPIECE_UNDERLINE + 'f',
								        'al',
								        's',
								        '<unk>',
								        '.',
								    ]       ,)
							def    lowerCAmelCase					(		self		:      List[Any]):
								__lowerCamelCase    :    List[str] 	=			(self.rust_tokenizer_class, 'hf-internal-testing/tiny-random-nllb', {})
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"):
										__lowerCamelCase    :    List[Any] 	=			self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__       ,**SCREAMING_SNAKE_CASE__)
										__lowerCamelCase    :    Any 	=			self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__       ,**SCREAMING_SNAKE_CASE__)

										__lowerCamelCase    :    Union[str, Any] 	=			tempfile.mkdtemp()

										__lowerCamelCase    :    Tuple 	=			tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE__)
										__lowerCamelCase    :    List[str] 	=			tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE__)

										# Checks it save with the same files + the tokenizer.json file for the fast one
										self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files))
										__lowerCamelCase    :    List[Any] 	=			tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f)
										self.assertSequenceEqual(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)

										# Checks everything loads correctly in the same way
										__lowerCamelCase    :    Dict 	=			tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE__)
										__lowerCamelCase    :    Any 	=			tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE__)

										# Check special tokens are set accordingly on Rust and Python
										for key in tokenizer_pp.special_tokens_map:
											self.assertTrue(hasattr(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__))

										shutil.rmtree(SCREAMING_SNAKE_CASE__)

										# Save tokenizer rust, legacy_format=True
										__lowerCamelCase    :    Tuple 	=			tempfile.mkdtemp()

										__lowerCamelCase    :    Optional[int] 	=			tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE__       ,legacy_format=SCREAMING_SNAKE_CASE__)
										__lowerCamelCase    :    List[Any] 	=			tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE__)

										# Checks it save with the same files
										self.assertSequenceEqual(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)

										# Checks everything loads correctly in the same way
										__lowerCamelCase    :    Any 	=			tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE__)
										__lowerCamelCase    :    int 	=			tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE__)

										# Check special tokens are set accordingly on Rust and Python
										for key in tokenizer_pp.special_tokens_map:
											self.assertTrue(hasattr(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__))

										shutil.rmtree(SCREAMING_SNAKE_CASE__)

										# Save tokenizer rust, legacy_format=False
										__lowerCamelCase    :    List[Any] 	=			tempfile.mkdtemp()

										__lowerCamelCase    :    List[str] 	=			tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE__       ,legacy_format=SCREAMING_SNAKE_CASE__)
										__lowerCamelCase    :    Dict 	=			tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE__)

										# Checks it saved the tokenizer.json file
										self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files))

										# Checks everything loads correctly in the same way
										__lowerCamelCase    :    List[Any] 	=			tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE__)
										__lowerCamelCase    :    Any 	=			tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE__)

										# Check special tokens are set accordingly on Rust and Python
										for key in tokenizer_pp.special_tokens_map:
											self.assertTrue(hasattr(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__))

										shutil.rmtree(SCREAMING_SNAKE_CASE__)
							@require_torch
							def    lowerCAmelCase					(		self		:      Dict):
								if not self.test_seqaseq:
									return

								__lowerCamelCase    :    Optional[int] 	=			self.get_tokenizers()
								for tokenizer in tokenizers:
									with self.subTest(F"{tokenizer.__class__.__name__}"):
										# Longer text that will definitely require truncation.
										__lowerCamelCase    :    Dict 	=			[
										    ' UN Chief Says There Is No Military Solution in Syria',
										    ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'
										    ' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'
										    ' will only worsen the violence and misery for millions of people.',
										]
										__lowerCamelCase    :    Optional[Any] 	=			[
										    'Şeful ONU declară că nu există o soluţie militară în Siria',
										    'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'
										    ' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'
										    ' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
										]
										try:
											__lowerCamelCase    :    Union[str, Any] 	=			tokenizer.prepare_seqaseq_batch(
											    src_texts=SCREAMING_SNAKE_CASE__       ,tgt_texts=SCREAMING_SNAKE_CASE__       ,max_length=3       ,max_target_length=1_0       ,return_tensors='pt'       ,src_lang='eng_Latn'       ,tgt_lang='ron_Latn'       ,)
										except NotImplementedError:
											return
										self.assertEqual(batch.input_ids.shape[1]       ,3)
										self.assertEqual(batch.labels.shape[1]       ,1_0)
										# max_target_length will default to max_length if not specified
										__lowerCamelCase    :    Tuple 	=			tokenizer.prepare_seqaseq_batch(
										    SCREAMING_SNAKE_CASE__       ,tgt_texts=SCREAMING_SNAKE_CASE__       ,max_length=3       ,return_tensors='pt')
										self.assertEqual(batch.input_ids.shape[1]       ,3)
										self.assertEqual(batch.labels.shape[1]       ,3)

										__lowerCamelCase    :    List[str] 	=			tokenizer.prepare_seqaseq_batch(
										    src_texts=SCREAMING_SNAKE_CASE__       ,max_length=3       ,max_target_length=1_0       ,return_tensors='pt')
										self.assertEqual(batch_encoder_only.input_ids.shape[1]       ,3)
										self.assertEqual(batch_encoder_only.attention_mask.shape[1]       ,3)
										self.assertNotIn('decoder_input_ids'       ,SCREAMING_SNAKE_CASE__)
							@unittest.skip('Unfortunately way too slow to build a BPE with SentencePiece.')
							def    lowerCAmelCase					(		self		:      Optional[Any]):
								pass


							def    lowerCAmelCase					(		self		:      Union[str, Any]):
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"):
										__lowerCamelCase    :    str 	=			[AddedToken('<special>'       ,lstrip=SCREAMING_SNAKE_CASE__)]

										__lowerCamelCase    :    Optional[Any] 	=			self.rust_tokenizer_class.from_pretrained(
										    SCREAMING_SNAKE_CASE__       ,additional_special_tokens=SCREAMING_SNAKE_CASE__       ,**SCREAMING_SNAKE_CASE__)
										__lowerCamelCase    :    Optional[Any] 	=			tokenizer_r.encode('Hey this is a <special> token')

										__lowerCamelCase    :    List[Any] 	=			tokenizer_r.encode('<special>'       ,add_special_tokens=SCREAMING_SNAKE_CASE__)[0]

										self.assertTrue(special_token_id in r_output)

										if self.test_slow_tokenizer:
											__lowerCamelCase    :    Any 	=			self.rust_tokenizer_class.from_pretrained(
											    SCREAMING_SNAKE_CASE__       ,additional_special_tokens=SCREAMING_SNAKE_CASE__       ,**SCREAMING_SNAKE_CASE__       ,)
											__lowerCamelCase    :    Optional[int] 	=			self.tokenizer_class.from_pretrained(
											    SCREAMING_SNAKE_CASE__       ,additional_special_tokens=SCREAMING_SNAKE_CASE__       ,**SCREAMING_SNAKE_CASE__)

											__lowerCamelCase    :    str 	=			tokenizer_p.encode('Hey this is a <special> token')

											__lowerCamelCase    :    Optional[int] 	=			tokenizer_cr.encode('Hey this is a <special> token')

											self.assertEqual(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)
											self.assertEqual(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)
											self.assertTrue(special_token_id in p_output)
											self.assertTrue(special_token_id in cr_output)






@require_torch
@require_sentencepiece
@require_tokenizers
class 	A_		(	unittest.TestCase    ):
							_UpperCAmelCase   :  List[Any]										=				'''facebook/nllb-200-distilled-600M'''
							_UpperCAmelCase   :  Optional[Any]										=				[
							    ''' UN Chief Says There Is No Military Solution in Syria''',
							    ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
							]
							_UpperCAmelCase   :  Optional[int]										=				[
							    '''Şeful ONU declară că nu există o soluţie militară în Siria''',
							    '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
							    ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
							    ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
							]
							_UpperCAmelCase   :  Any										=				[
							    256_047,
							    16_297,
							    134_408,
							    8_165,
							    248_066,
							    14_734,
							    950,
							    1_135,
							    105_721,
							    3_573,
							    83,
							    27_352,
							    108,
							    49_486,
							    2,
							]
							@classmethod
							def    lowerCAmelCase					(		cls		:      int):
								__lowerCamelCase    :    NllbTokenizer 	=			NllbTokenizer.from_pretrained(
								    cls.checkpoint_name       ,src_lang='eng_Latn'       ,tgt_lang='ron_Latn')
								__lowerCamelCase    :    str 	=			1
								return cls
							def    lowerCAmelCase					(		self		:      Tuple):
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Arab']       ,2_5_6_0_0_1)
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Latn']       ,2_5_6_0_0_2)
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['fra_Latn']       ,2_5_6_0_5_7)
							def    lowerCAmelCase					(		self		:      Union[str, Any]):
								__lowerCamelCase    :    str 	=			self.tokenizer.batch_encode_plus(self.src_text).input_ids[0]
								self.assertListEqual(self.expected_src_tokens       ,SCREAMING_SNAKE_CASE__)
							def    lowerCAmelCase					(		self		:      List[str]):
								self.assertIn(SCREAMING_SNAKE_CASE__       ,self.tokenizer.all_special_ids)
								# fmt: off
								__lowerCamelCase    :    List[str] 	=			[RO_CODE, 4_2_5_4, 9_8_0_6_8, 1_1_2_9_2_3, 3_9_0_7_2, 3_9_0_9, 7_1_3, 1_0_2_7_6_7, 2_6, 1_7_3_1_4, 3_5_6_4_2, 1_4_6_8_3, 3_3_1_1_8, 2_0_2_2, 6_6_9_8_7, 2, 2_5_6_0_4_7]
								# fmt: on

								__lowerCamelCase    :    Dict 	=			self.tokenizer.decode(SCREAMING_SNAKE_CASE__       ,skip_special_tokens=SCREAMING_SNAKE_CASE__)
								__lowerCamelCase    :    Tuple 	=			self.tokenizer.decode(generated_ids[1:]       ,skip_special_tokens=SCREAMING_SNAKE_CASE__)
								self.assertEqual(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)
								self.assertNotIn(self.tokenizer.eos_token       ,SCREAMING_SNAKE_CASE__)
							def    lowerCAmelCase					(		self		:      List[str]):
								__lowerCamelCase    :    Union[str, Any] 	=			['this is gunna be a long sentence ' * 2_0]
								assert isinstance(src_text[0]       ,SCREAMING_SNAKE_CASE__)
								__lowerCamelCase    :    Union[str, Any] 	=			1_0
								__lowerCamelCase    :    List[Any] 	=			self.tokenizer(SCREAMING_SNAKE_CASE__       ,max_length=SCREAMING_SNAKE_CASE__       ,truncation=SCREAMING_SNAKE_CASE__).input_ids[0]
								self.assertEqual(ids[-1]       ,2)
								self.assertEqual(ids[0]       ,SCREAMING_SNAKE_CASE__)
								self.assertEqual(len(SCREAMING_SNAKE_CASE__)       ,SCREAMING_SNAKE_CASE__)
							def    lowerCAmelCase					(		self		:      List[Any]):
								self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', 'ar_AR'])       ,[2_5_6_2_0_3, 3])
							def    lowerCAmelCase					(		self		:      Tuple):
								__lowerCamelCase    :    int 	=			tempfile.mkdtemp()
								__lowerCamelCase    :    List[Any] 	=			self.tokenizer.fairseq_tokens_to_ids
								self.tokenizer.save_pretrained(SCREAMING_SNAKE_CASE__)
								__lowerCamelCase    :    Any 	=			NllbTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__)
								self.assertDictEqual(new_tok.fairseq_tokens_to_ids       ,SCREAMING_SNAKE_CASE__)
							@require_torch
							def    lowerCAmelCase					(		self		:      str):
								__lowerCamelCase    :    Any 	=			self.tokenizer(
								    self.src_text       ,text_target=self.tgt_text       ,padding=SCREAMING_SNAKE_CASE__       ,truncation=SCREAMING_SNAKE_CASE__       ,max_length=len(self.expected_src_tokens)       ,return_tensors='pt'       ,)
								__lowerCamelCase    :    Any 	=			shift_tokens_right(
								    batch['labels']       ,self.tokenizer.pad_token_id       ,self.tokenizer.lang_code_to_id['ron_Latn'])

								self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)

								self.assertEqual((2, 1_5)       ,batch.input_ids.shape)
								self.assertEqual((2, 1_5)       ,batch.attention_mask.shape)
								__lowerCamelCase    :    Dict 	=			batch.input_ids.tolist()[0]
								self.assertListEqual(self.expected_src_tokens       ,SCREAMING_SNAKE_CASE__)
								self.assertEqual(SCREAMING_SNAKE_CASE__       ,batch.decoder_input_ids[0, 0])  # EOS
								# Test that special tokens are reset
								self.assertEqual(self.tokenizer.prefix_tokens       ,[EN_CODE])
								self.assertEqual(self.tokenizer.suffix_tokens       ,[self.tokenizer.eos_token_id])
							def    lowerCAmelCase					(		self		:      List[Any]):
								__lowerCamelCase    :    Dict 	=			self.tokenizer(self.src_text       ,padding=SCREAMING_SNAKE_CASE__       ,truncation=SCREAMING_SNAKE_CASE__       ,max_length=3       ,return_tensors='pt')
								__lowerCamelCase    :    List[Any] 	=			self.tokenizer(
								    text_target=self.tgt_text       ,padding=SCREAMING_SNAKE_CASE__       ,truncation=SCREAMING_SNAKE_CASE__       ,max_length=1_0       ,return_tensors='pt')
								__lowerCamelCase    :    Optional[Any] 	=			targets['input_ids']
								__lowerCamelCase    :    Optional[Any] 	=			shift_tokens_right(
								    SCREAMING_SNAKE_CASE__       ,self.tokenizer.pad_token_id       ,decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]       ,)

								self.assertEqual(batch.input_ids.shape[1]       ,3)
								self.assertEqual(batch.decoder_input_ids.shape[1]       ,1_0)
							@require_torch
							def    lowerCAmelCase					(		self		:      Dict):
								__lowerCamelCase    :    Dict 	=			self.tokenizer._build_translation_inputs(
								    'A test'       ,return_tensors='pt'       ,src_lang='eng_Latn'       ,tgt_lang='fra_Latn')

								self.assertEqual(
								    nested_simplify(SCREAMING_SNAKE_CASE__)       ,{
								        # A, test, EOS, en_XX
								        'input_ids': [[2_5_6_0_4_7, 7_0, 7_3_5_6, 2]],
								        'attention_mask': [[1, 1, 1, 1]],
								        # ar_AR
								        'forced_bos_token_id': 2_5_6_0_5_7,
								    }       ,)


							@require_torch
							def    lowerCAmelCase					(		self		:      List[Any]):
								__lowerCamelCase    :    Optional[Any] 	=			True
								__lowerCamelCase    :    List[str] 	=			self.tokenizer(
								    'UN Chief says there is no military solution in Syria'       ,src_lang='eng_Latn'       ,tgt_lang='fra_Latn')
								self.assertEqual(
								    inputs.input_ids       ,[1_6_2_9_7, 1_3_4_4_0_8, 2_5_6_5_3, 6_3_7_0, 2_4_8, 2_5_4, 1_0_3_9_2_9, 9_4_9_9_5, 1_0_8, 4_9_4_8_6, 2, 2_5_6_0_4_7])

								__lowerCamelCase    :    Any 	=			False
								__lowerCamelCase    :    Union[str, Any] 	=			self.tokenizer(
								    'UN Chief says there is no military solution in Syria'       ,src_lang='eng_Latn'       ,tgt_lang='fra_Latn')
								self.assertEqual(
								    inputs.input_ids       ,[2_5_6_0_4_7, 1_6_2_9_7, 1_3_4_4_0_8, 2_5_6_5_3, 6_3_7_0, 2_4_8, 2_5_4, 1_0_3_9_2_9, 9_4_9_9_5, 1_0_8, 4_9_4_8_6, 2])







import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
					from test_module.custom_tokenization_fast import CustomTokenizerFast






class 	A_		(	unittest.TestCase    ):
							def    lowerCAmelCase					(		self		:      Optional[Any]):
								# A mock response for an HTTP head request to emulate server down
								__lowerCamelCase    :    List[Any] 	=			mock.Mock()
								__lowerCamelCase    :    Tuple 	=			5_0_0
								__lowerCamelCase    :    Dict 	=			{}
								__lowerCamelCase    :    List[Any] 	=			HTTPError
								__lowerCamelCase    :    int 	=			{}

								# Download this model to make sure it's in the cache.
								__lowerCamelCase    :    Optional[int] 	=			BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert')

								# Under the mock environment we get a 500 error when trying to reach the tokenizer.
								with mock.patch('requests.Session.request'       ,return_value=SCREAMING_SNAKE_CASE__) as mock_head:
									__lowerCamelCase    :    Tuple 	=			BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert')
									# This check we did call the fake head request
									mock_head.assert_called()
							@require_tokenizers
							def    lowerCAmelCase					(		self		:      Any):
								# A mock response for an HTTP head request to emulate server down
								__lowerCamelCase    :    int 	=			mock.Mock()
								__lowerCamelCase    :    List[str] 	=			5_0_0
								__lowerCamelCase    :    Tuple 	=			{}
								__lowerCamelCase    :    List[str] 	=			HTTPError
								__lowerCamelCase    :    int 	=			{}

								# Download this model to make sure it's in the cache.
								__lowerCamelCase    :    Dict 	=			GPTaTokenizerFast.from_pretrained('gpt2')

								# Under the mock environment we get a 500 error when trying to reach the tokenizer.
								with mock.patch('requests.Session.request'       ,return_value=SCREAMING_SNAKE_CASE__) as mock_head:
									__lowerCamelCase    :    Optional[int] 	=			GPTaTokenizerFast.from_pretrained('gpt2')
									# This check we did call the fake head request
									mock_head.assert_called()
							def    lowerCAmelCase					(		self		:      str):
								# This test is for deprecated behavior and can be removed in v5
								try:
									__lowerCamelCase    :    Tuple 	=			tempfile.mktemp()
									with open(SCREAMING_SNAKE_CASE__       ,'wb') as f:
										http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'       ,SCREAMING_SNAKE_CASE__)

									__lowerCamelCase    :    Dict 	=			AlbertTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__)
								finally:
									os.remove(SCREAMING_SNAKE_CASE__)

								# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
								# the current folder and have the right name.
								if os.path.isfile('tokenizer.json'):
									# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
									return
								try:
									with open('tokenizer.json'       ,'wb') as f:
										http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'       ,SCREAMING_SNAKE_CASE__)
									__lowerCamelCase    :    Any 	=			AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2')
									# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
									self.assertEqual(tokenizer.vocab_size       ,1_0_0_0)
									# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

								finally:
									os.remove('tokenizer.json')


							def    lowerCAmelCase					(		self		:      Optional[Any]):
								# This test is for deprecated behavior and can be removed in v5
								__lowerCamelCase    :    str 	=			AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model')






@is_staging_test
class 	A_		(	unittest.TestCase    ):
							_UpperCAmelCase   :  Any										=				['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']
							@classmethod
							def    lowerCAmelCase					(		cls		:      Optional[int]):
								__lowerCamelCase    :    Optional[int] 	=			TOKEN
								HfFolder.save_token(SCREAMING_SNAKE_CASE__)
							@classmethod
							def    lowerCAmelCase					(		cls		:      str):
								try:
									delete_repo(token=cls._token       ,repo_id='test-tokenizer')
								except HTTPError:
									pass

								try:
									delete_repo(token=cls._token       ,repo_id='valid_org/test-tokenizer-org')
								except HTTPError:
									pass

								try:
									delete_repo(token=cls._token       ,repo_id='test-dynamic-tokenizer')
								except HTTPError:
									pass
							def    lowerCAmelCase					(		self		:      List[str]):
								with tempfile.TemporaryDirectory() as tmp_dir:
									__lowerCamelCase    :    Dict 	=			os.path.join(SCREAMING_SNAKE_CASE__       ,'vocab.txt')
									with open(SCREAMING_SNAKE_CASE__       ,'w'       ,encoding='utf-8') as vocab_writer:
										vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))
									__lowerCamelCase    :    Union[str, Any] 	=			BertTokenizer(SCREAMING_SNAKE_CASE__)

								tokenizer.push_to_hub('test-tokenizer'       ,use_auth_token=self._token)
								__lowerCamelCase    :    List[str] 	=			BertTokenizer.from_pretrained(F"{USER}/test-tokenizer")
								self.assertDictEqual(new_tokenizer.vocab       ,tokenizer.vocab)

								# Reset repo
								delete_repo(token=self._token       ,repo_id='test-tokenizer')

								# Push to hub via save_pretrained
								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(SCREAMING_SNAKE_CASE__       ,repo_id='test-tokenizer'       ,push_to_hub=SCREAMING_SNAKE_CASE__       ,use_auth_token=self._token)

								__lowerCamelCase    :    str 	=			BertTokenizer.from_pretrained(F"{USER}/test-tokenizer")
								self.assertDictEqual(new_tokenizer.vocab       ,tokenizer.vocab)
							def    lowerCAmelCase					(		self		:      Any):
								with tempfile.TemporaryDirectory() as tmp_dir:
									__lowerCamelCase    :    Dict 	=			os.path.join(SCREAMING_SNAKE_CASE__       ,'vocab.txt')
									with open(SCREAMING_SNAKE_CASE__       ,'w'       ,encoding='utf-8') as vocab_writer:
										vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))
									__lowerCamelCase    :    Optional[Any] 	=			BertTokenizer(SCREAMING_SNAKE_CASE__)

								tokenizer.push_to_hub('valid_org/test-tokenizer-org'       ,use_auth_token=self._token)
								__lowerCamelCase    :    List[Any] 	=			BertTokenizer.from_pretrained('valid_org/test-tokenizer-org')
								self.assertDictEqual(new_tokenizer.vocab       ,tokenizer.vocab)

								# Reset repo
								delete_repo(token=self._token       ,repo_id='valid_org/test-tokenizer-org')

								# Push to hub via save_pretrained
								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(
									    SCREAMING_SNAKE_CASE__       ,repo_id='valid_org/test-tokenizer-org'       ,push_to_hub=SCREAMING_SNAKE_CASE__       ,use_auth_token=self._token)

								__lowerCamelCase    :    List[Any] 	=			BertTokenizer.from_pretrained('valid_org/test-tokenizer-org')
								self.assertDictEqual(new_tokenizer.vocab       ,tokenizer.vocab)


							@require_tokenizers
							def    lowerCAmelCase					(		self		:      int):
								CustomTokenizer.register_for_auto_class()
								with tempfile.TemporaryDirectory() as tmp_dir:
									__lowerCamelCase    :    Dict 	=			os.path.join(SCREAMING_SNAKE_CASE__       ,'vocab.txt')
									with open(SCREAMING_SNAKE_CASE__       ,'w'       ,encoding='utf-8') as vocab_writer:
										vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))
									__lowerCamelCase    :    Any 	=			CustomTokenizer(SCREAMING_SNAKE_CASE__)

								# No fast custom tokenizer
								tokenizer.push_to_hub('test-dynamic-tokenizer'       ,use_auth_token=self._token)

								__lowerCamelCase    :    Optional[int] 	=			AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"       ,trust_remote_code=SCREAMING_SNAKE_CASE__)
								# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
								self.assertEqual(tokenizer.__class__.__name__       ,'CustomTokenizer')

								# Fast and slow custom tokenizer
								CustomTokenizerFast.register_for_auto_class()
								with tempfile.TemporaryDirectory() as tmp_dir:
									__lowerCamelCase    :    List[str] 	=			os.path.join(SCREAMING_SNAKE_CASE__       ,'vocab.txt')
									with open(SCREAMING_SNAKE_CASE__       ,'w'       ,encoding='utf-8') as vocab_writer:
										vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))

									__lowerCamelCase    :    Dict 	=			BertTokenizerFast.from_pretrained(SCREAMING_SNAKE_CASE__)
									bert_tokenizer.save_pretrained(SCREAMING_SNAKE_CASE__)
									__lowerCamelCase    :    List[Any] 	=			CustomTokenizerFast.from_pretrained(SCREAMING_SNAKE_CASE__)

								tokenizer.push_to_hub('test-dynamic-tokenizer'       ,use_auth_token=self._token)

								__lowerCamelCase    :    Optional[Any] 	=			AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"       ,trust_remote_code=SCREAMING_SNAKE_CASE__)
								# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
								self.assertEqual(tokenizer.__class__.__name__       ,'CustomTokenizerFast')
								__lowerCamelCase    :    Optional[int] 	=			AutoTokenizer.from_pretrained(
								    F"{USER}/test-dynamic-tokenizer"       ,use_fast=SCREAMING_SNAKE_CASE__       ,trust_remote_code=SCREAMING_SNAKE_CASE__)
								# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
								self.assertEqual(tokenizer.__class__.__name__       ,'CustomTokenizer')






class 	A_		(	unittest.TestCase    ):
							def    lowerCAmelCase					(		self		:      List[Any]):
								__lowerCamelCase    :    List[str] 	=			Trie()
								trie.add('Hello 友達')
								self.assertEqual(trie.data       ,{'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}})
								trie.add('Hello')
								trie.data
								self.assertEqual(trie.data       ,{'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}})
							def    lowerCAmelCase					(		self		:      Tuple):
								__lowerCamelCase    :    str 	=			Trie()
								self.assertEqual(trie.split('[CLS] This is a extra_id_100')       ,['[CLS] This is a extra_id_100'])
								trie.add('[CLS]')
								trie.add('extra_id_1')
								trie.add('extra_id_100')
								self.assertEqual(trie.split('[CLS] This is a extra_id_100')       ,['[CLS]', ' This is a ', 'extra_id_100'])
							def    lowerCAmelCase					(		self		:      Optional[Any]):
								__lowerCamelCase    :    Dict 	=			Trie()
								trie.add('A')
								self.assertEqual(trie.split('ABC')       ,['A', 'BC'])
								self.assertEqual(trie.split('BCA')       ,['BC', 'A'])
							def    lowerCAmelCase					(		self		:      Dict):
								__lowerCamelCase    :    Any 	=			Trie()
								trie.add('TOKEN]')
								trie.add('[SPECIAL_TOKEN]')
								self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]')       ,['This is something ', '[SPECIAL_TOKEN]'])
							def    lowerCAmelCase					(		self		:      int):
								__lowerCamelCase    :    List[Any] 	=			Trie()
								trie.add('A')
								trie.add('P')
								trie.add('[SPECIAL_TOKEN]')
								self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]')       ,['This is something ', '[SPECIAL_TOKEN]'])
							def    lowerCAmelCase					(		self		:      Optional[int]):
								__lowerCamelCase    :    Any 	=			Trie()
								trie.add('AB')
								trie.add('B')
								trie.add('C')
								self.assertEqual(trie.split('ABC')       ,['AB', 'C'])
							def    lowerCAmelCase					(		self		:      Optional[int]):
								__lowerCamelCase    :    Dict 	=			Trie()
								trie.add('ABC')
								trie.add('B')
								trie.add('CD')
								self.assertEqual(trie.split('ABCD')       ,['ABC', 'D'])


							def    lowerCAmelCase					(		self		:      List[Any]):
								# Even if the offsets are wrong, we necessarily output correct string
								# parts.
								__lowerCamelCase    :    List[Any] 	=			Trie()
								__lowerCamelCase    :    Any 	=			trie.cut_text('ABC'       ,[0, 0, 2, 1, 2, 3])
								self.assertEqual(SCREAMING_SNAKE_CASE__       ,['AB', 'C'])




