



import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
					from test_module.custom_tokenization_fast import CustomTokenizerFast





class 							UpperCAmelCase				(  unittest.TestCase  ):




	'''simple docstring'''





	def     UpperCamelCase_  (	self :			int			):
					__A             =			0






	@slow
	def     UpperCamelCase_  (	self :			Any			):
					for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
									__A             =			AutoTokenizer.from_pretrained(A			)
									self.assertIsNotNone(A			)
									self.assertIsInstance(A							,(BertTokenizer, BertTokenizerFast)			)
									self.assertGreater(len(A			)							,0			)

					for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
									__A             =			AutoTokenizer.from_pretrained(A			)
									self.assertIsNotNone(A			)
									self.assertIsInstance(A							,(GPTaTokenizer, GPTaTokenizerFast)			)
									self.assertGreater(len(A			)							,0			)






	def     UpperCamelCase_  (	self :			str			):
					__A             =			AutoTokenizer.from_pretrained(A			)
					self.assertIsInstance(A							,(BertTokenizer, BertTokenizerFast)			)
					self.assertEqual(tokenizer.vocab_size							,12			)






	def     UpperCamelCase_  (	self :			List[str]			):
					__A             =			AutoTokenizer.from_pretrained(A			)
					self.assertIsInstance(A							,(RobertaTokenizer, RobertaTokenizerFast)			)
					self.assertEqual(tokenizer.vocab_size							,20			)






	def     UpperCamelCase_  (	self :			List[str]			):
					__A             =			AutoConfig.from_pretrained(A			)
					self.assertIsInstance(A							,A			)
					# Check that tokenizer_type ≠ model_type
					__A             =			AutoTokenizer.from_pretrained(A							,config=A			)
					self.assertIsInstance(A							,(BertTokenizer, BertTokenizerFast)			)
					self.assertEqual(tokenizer.vocab_size							,12			)






	def     UpperCamelCase_  (	self :			List[Any]			):
					with tempfile.TemporaryDirectory() as tmp_dir:
									shutil.copy("./tests/fixtures/vocab.txt"							,os.path.join(A							,"vocab.txt"			)			)

									__A             =			AutoTokenizer.from_pretrained(A							,tokenizer_type="bert"							,use_fast=A			)
									self.assertIsInstance(A							,A			)

					with tempfile.TemporaryDirectory() as tmp_dir:
									shutil.copy("./tests/fixtures/vocab.json"							,os.path.join(A							,"vocab.json"			)			)
									shutil.copy("./tests/fixtures/merges.txt"							,os.path.join(A							,"merges.txt"			)			)

									__A             =			AutoTokenizer.from_pretrained(A							,tokenizer_type="gpt2"							,use_fast=A			)
									self.assertIsInstance(A							,A			)






	@require_tokenizers
	def     UpperCamelCase_  (	self :			Union[str, Any]			):
					with tempfile.TemporaryDirectory() as tmp_dir:
									shutil.copy("./tests/fixtures/vocab.txt"							,os.path.join(A							,"vocab.txt"			)			)

									__A             =			AutoTokenizer.from_pretrained(A							,tokenizer_type="bert"			)
									self.assertIsInstance(A							,A			)

					with tempfile.TemporaryDirectory() as tmp_dir:
									shutil.copy("./tests/fixtures/vocab.json"							,os.path.join(A							,"vocab.json"			)			)
									shutil.copy("./tests/fixtures/merges.txt"							,os.path.join(A							,"merges.txt"			)			)

									__A             =			AutoTokenizer.from_pretrained(A							,tokenizer_type="gpt2"			)
									self.assertIsInstance(A							,A			)






	def     UpperCamelCase_  (	self :			Union[str, Any]			):
					with pytest.raises(A			):
									AutoTokenizer.from_pretrained("./"							,tokenizer_type="xxx"			)






	@require_tokenizers
	def     UpperCamelCase_  (	self :			Dict			):
					for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
									__A             =			tokenizer_class.from_pretrained("wietsedv/bert-base-dutch-cased"			)
									self.assertIsInstance(A							,(BertTokenizer, BertTokenizerFast)			)

									if isinstance(A							,A			):
													self.assertEqual(tokenizer.basic_tokenizer.do_lower_case							,A			)
									else:
													self.assertEqual(tokenizer.do_lower_case							,A			)

									self.assertEqual(tokenizer.model_max_length							,5_12			)






	@require_tokenizers
	def     UpperCamelCase_  (	self :			Any			):
					for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
									with self.assertRaisesRegex(
									    A							,"julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"							,):
													__A             =			tokenizer_class.from_pretrained("julien-c/herlolip-not-exists"			)






	def     UpperCamelCase_  (	self :			Optional[int]			):
					# tests: https://github.com/huggingface/transformers/pull/13251
					# 1. models with `-`, e.g. xlm-roberta -> xlm_roberta
					# 2. models that don't remap 1-1 from model-name to model file, e.g., openai-gpt -> openai
					__A             =			TOKENIZER_MAPPING.values()
					__A             =			[]

					for slow_tok, fast_tok in tokenizers:
									if slow_tok is not None:
													tokenizer_names.append(slow_tok.__name__			)

									if fast_tok is not None:
													tokenizer_names.append(fast_tok.__name__			)

					for tokenizer_name in tokenizer_names:
									# must find the right class
									tokenizer_class_from_name(A			)






	@require_tokenizers
	def     UpperCamelCase_  (	self :			Optional[int]			):
					self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"							,use_fast=A			)							,A			)
					self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"			)							,A			)






	@require_tokenizers
	def     UpperCamelCase_  (	self :			Tuple			):
					__A             =			AutoTokenizer.from_pretrained("distilbert-base-uncased"							,do_lower_case=A			)
					__A             =			"Hello, world. How are you?"
					__A             =			tokenizer.tokenize(A			)
					self.assertEqual("[UNK]"							,tokens[0]			)

					__A             =			AutoTokenizer.from_pretrained("microsoft/mpnet-base"							,do_lower_case=A			)
					__A             =			tokenizer.tokenize(A			)
					self.assertEqual("[UNK]"							,tokens[0]			)






	@require_tokenizers
	def     UpperCamelCase_  (	self :			Optional[int]			):
					__A             =			AutoTokenizer.from_pretrained("robot-test/dummy-tokenizer-fast-with-model-config"			)
					self.assertEqual(type(A			)							,A			)
					self.assertEqual(tokenizer.model_max_length							,5_12			)
					self.assertEqual(tokenizer.vocab_size							,3_00_00			)
					self.assertEqual(tokenizer.unk_token							,"[UNK]"			)
					self.assertEqual(tokenizer.padding_side							,"right"			)
					self.assertEqual(tokenizer.truncation_side							,"right"			)






	def     UpperCamelCase_  (	self :			Optional[Any]			):
					__A             =			AutoTokenizer.from_pretrained(A			)
					self.assertIsInstance(A							,(BertTokenizer, BertTokenizerFast)			)
					with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(A			)
									__A             =			AutoTokenizer.from_pretrained(A			)

					self.assertIsInstance(A							,tokenizer.__class__			)
					self.assertEqual(tokenizera.vocab_size							,12			)






	def     UpperCamelCase_  (	self :			Dict			):
					__A             =			AutoTokenizer.from_pretrained("ctrl"			)
					# There is no fast CTRL so this always gives us a slow tokenizer.
					self.assertIsInstance(A							,A			)






	def     UpperCamelCase_  (	self :			List[str]			):
					# Check we can load the tokenizer config of an online model.
					__A             =			get_tokenizer_config("bert-base-cased"			)
					__A             =			config.pop("_commit_hash"							,A			)
					# If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
					self.assertEqual(A							,{"do_lower_case": False}			)

					# This model does not have a tokenizer_config so we get back an empty dict.
					__A             =			get_tokenizer_config(A			)
					self.assertDictEqual(A							,{}			)

					# A tokenizer saved with `save_pretrained` always creates a tokenizer config.
					__A             =			AutoTokenizer.from_pretrained(A			)
					with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(A			)
									__A             =			get_tokenizer_config(A			)

					# Check the class of the tokenizer was properly saved (note that it always saves the slow class).
					self.assertEqual(config["tokenizer_class"]							,"BertTokenizer"			)






	def     UpperCamelCase_  (	self :			Tuple			):
					try:
									AutoConfig.register("custom"							,A			)

									AutoTokenizer.register(A							,slow_tokenizer_class=A			)
									# Trying to register something existing in the Transformers library will raise an error
									with self.assertRaises(A			):
													AutoTokenizer.register(A							,slow_tokenizer_class=A			)

									__A             =			CustomTokenizer.from_pretrained(A			)
									with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(A			)

													__A             =			AutoTokenizer.from_pretrained(A			)
													self.assertIsInstance(A							,A			)

					finally:
									if "custom" in CONFIG_MAPPING._extra_content:
													del CONFIG_MAPPING._extra_content["custom"]
									if CustomConfig in TOKENIZER_MAPPING._extra_content:
													del TOKENIZER_MAPPING._extra_content[CustomConfig]






	@require_tokenizers
	def     UpperCamelCase_  (	self :			Optional[int]			):
					try:
									AutoConfig.register("custom"							,A			)

									# Can register in two steps
									AutoTokenizer.register(A							,slow_tokenizer_class=A			)
									self.assertEqual(TOKENIZER_MAPPING[CustomConfig]							,(CustomTokenizer, None)			)
									AutoTokenizer.register(A							,fast_tokenizer_class=A			)
									self.assertEqual(TOKENIZER_MAPPING[CustomConfig]							,(CustomTokenizer, CustomTokenizerFast)			)

									del TOKENIZER_MAPPING._extra_content[CustomConfig]
									# Can register in one step
									AutoTokenizer.register(
									    A							,slow_tokenizer_class=A							,fast_tokenizer_class=A			)
									self.assertEqual(TOKENIZER_MAPPING[CustomConfig]							,(CustomTokenizer, CustomTokenizerFast)			)

									# Trying to register something existing in the Transformers library will raise an error
									with self.assertRaises(A			):
													AutoTokenizer.register(A							,fast_tokenizer_class=A			)

									# We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
									# and that model does not have a tokenizer.json
									with tempfile.TemporaryDirectory() as tmp_dir:
													__A             =			BertTokenizerFast.from_pretrained(A			)
													bert_tokenizer.save_pretrained(A			)
													__A             =			CustomTokenizerFast.from_pretrained(A			)

									with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(A			)

													__A             =			AutoTokenizer.from_pretrained(A			)
													self.assertIsInstance(A							,A			)

													__A             =			AutoTokenizer.from_pretrained(A							,use_fast=A			)
													self.assertIsInstance(A							,A			)

					finally:
									if "custom" in CONFIG_MAPPING._extra_content:
													del CONFIG_MAPPING._extra_content["custom"]
									if CustomConfig in TOKENIZER_MAPPING._extra_content:
													del TOKENIZER_MAPPING._extra_content[CustomConfig]






	def     UpperCamelCase_  (	self :			Any			):
					# If remote code is not set, we will time out when asking whether to load the model.
					with self.assertRaises(A			):
									__A             =			AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"			)
					# If remote code is disabled, we can't load this config.
					with self.assertRaises(A			):
									__A             =			AutoTokenizer.from_pretrained(
									    "hf-internal-testing/test_dynamic_tokenizer"							,trust_remote_code=A			)

					__A             =			AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"							,trust_remote_code=A			)
					self.assertTrue(tokenizer.special_attribute_present			)
					# Test tokenizer can be reloaded.
					with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(A			)
									__A             =			AutoTokenizer.from_pretrained(A							,trust_remote_code=A			)
					self.assertTrue(reloaded_tokenizer.special_attribute_present			)

					if is_tokenizers_available():
									self.assertEqual(tokenizer.__class__.__name__							,"NewTokenizerFast"			)
									self.assertEqual(reloaded_tokenizer.__class__.__name__							,"NewTokenizerFast"			)

									# Test we can also load the slow version
									__A             =			AutoTokenizer.from_pretrained(
									    "hf-internal-testing/test_dynamic_tokenizer"							,trust_remote_code=A							,use_fast=A			)
									self.assertTrue(tokenizer.special_attribute_present			)
									self.assertEqual(tokenizer.__class__.__name__							,"NewTokenizer"			)
									# Test tokenizer can be reloaded.
									with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(A			)
													__A             =			AutoTokenizer.from_pretrained(A							,trust_remote_code=A							,use_fast=A			)
									self.assertEqual(reloaded_tokenizer.__class__.__name__							,"NewTokenizer"			)
									self.assertTrue(reloaded_tokenizer.special_attribute_present			)
					else:
									self.assertEqual(tokenizer.__class__.__name__							,"NewTokenizer"			)
									self.assertEqual(reloaded_tokenizer.__class__.__name__							,"NewTokenizer"			)






	@require_tokenizers
	def     UpperCamelCase_  (	self :			Tuple			):





					class 							UpperCAmelCase				(  __SCREAMING_SNAKE_CASE  ):




						'''simple docstring'''





						snake_case_    =					False





					class 							UpperCAmelCase				(  __SCREAMING_SNAKE_CASE  ):




						'''simple docstring'''





						snake_case_    =					NewTokenizer
						snake_case_    =					False

					try:
									AutoConfig.register("custom"							,A			)
									AutoTokenizer.register(A							,slow_tokenizer_class=A			)
									AutoTokenizer.register(A							,fast_tokenizer_class=A			)
									# If remote code is not set, the default is to use local
									__A             =			AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"			)
									self.assertEqual(tokenizer.__class__.__name__							,"NewTokenizerFast"			)
									self.assertFalse(tokenizer.special_attribute_present			)
									__A             =			AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"							,use_fast=A			)
									self.assertEqual(tokenizer.__class__.__name__							,"NewTokenizer"			)
									self.assertFalse(tokenizer.special_attribute_present			)

									# If remote code is disabled, we load the local one.
									__A             =			AutoTokenizer.from_pretrained(
									    "hf-internal-testing/test_dynamic_tokenizer"							,trust_remote_code=A			)
									self.assertEqual(tokenizer.__class__.__name__							,"NewTokenizerFast"			)
									self.assertFalse(tokenizer.special_attribute_present			)
									__A             =			AutoTokenizer.from_pretrained(
									    "hf-internal-testing/test_dynamic_tokenizer"							,trust_remote_code=A							,use_fast=A			)
									self.assertEqual(tokenizer.__class__.__name__							,"NewTokenizer"			)
									self.assertFalse(tokenizer.special_attribute_present			)

									# If remote is enabled, we load from the Hub
									__A             =			AutoTokenizer.from_pretrained(
									    "hf-internal-testing/test_dynamic_tokenizer"							,trust_remote_code=A			)
									self.assertEqual(tokenizer.__class__.__name__							,"NewTokenizerFast"			)
									self.assertTrue(tokenizer.special_attribute_present			)
									__A             =			AutoTokenizer.from_pretrained(
									    "hf-internal-testing/test_dynamic_tokenizer"							,trust_remote_code=A							,use_fast=A			)
									self.assertEqual(tokenizer.__class__.__name__							,"NewTokenizer"			)
									self.assertTrue(tokenizer.special_attribute_present			)

					finally:
									if "custom" in CONFIG_MAPPING._extra_content:
													del CONFIG_MAPPING._extra_content["custom"]
									if CustomConfig in TOKENIZER_MAPPING._extra_content:
													del TOKENIZER_MAPPING._extra_content[CustomConfig]






	def     UpperCamelCase_  (	self :			List[Any]			):
					__A             =			AutoTokenizer.from_pretrained(
					    "hf-internal-testing/test_dynamic_tokenizer_legacy"							,trust_remote_code=A			)
					self.assertTrue(tokenizer.special_attribute_present			)
					if is_tokenizers_available():
									self.assertEqual(tokenizer.__class__.__name__							,"NewTokenizerFast"			)

									# Test we can also load the slow version
									__A             =			AutoTokenizer.from_pretrained(
									    "hf-internal-testing/test_dynamic_tokenizer_legacy"							,trust_remote_code=A							,use_fast=A			)
									self.assertTrue(tokenizer.special_attribute_present			)
									self.assertEqual(tokenizer.__class__.__name__							,"NewTokenizer"			)
					else:
									self.assertEqual(tokenizer.__class__.__name__							,"NewTokenizer"			)






	def     UpperCamelCase_  (	self :			Optional[int]			):
					with self.assertRaisesRegex(
					    A							,"bert-base is not a local folder and is not a valid model identifier"			):
									__A             =			AutoTokenizer.from_pretrained("bert-base"			)






	def     UpperCamelCase_  (	self :			Dict			):
					with self.assertRaisesRegex(
					    A							,R"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"			):
									__A             =			AutoTokenizer.from_pretrained(A							,revision="aaaaaa"			)







	def     UpperCamelCase_  (	self :			Dict			):
					# Make sure we have cached the tokenizer.
					__A             =			AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"			)
					with RequestCounter() as counter:
									__A             =			AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"			)
									self.assertEqual(counter.get_request_count							,0			)
									self.assertEqual(counter.head_request_count							,1			)
									self.assertEqual(counter.other_request_count							,0			)


import json
import os
import unittest

from transformers.models.ctrl.tokenization_ctrl import VOCAB_FILES_NAMES, CTRLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin





class 							UpperCAmelCase				(  __SCREAMING_SNAKE_CASE      ,      unittest.TestCase  ):




	'''simple docstring'''





	snake_case_    =					CTRLTokenizer
	snake_case_    =					False
	snake_case_    =					False






	def     UpperCamelCase_  (	self :			Dict			):
					super().setUp()

					# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
					__A             =			["adapt", "re@@", "a@@", "apt", "c@@", "t", "<unk>"]
					__A             =			dict(zip(A							,range(len(A			)			)			)			)
					__A             =			["#version: 0.2", "a p", "ap t</w>", "r e", "a d", "ad apt</w>", ""]
					__A             =			{"unk_token": "<unk>"}

					__A             =			os.path.join(self.tmpdirname							,VOCAB_FILES_NAMES["vocab_file"]			)
					__A             =			os.path.join(self.tmpdirname							,VOCAB_FILES_NAMES["merges_file"]			)
					with open(self.vocab_file							,"w"							,encoding="utf-8"			) as fp:
									fp.write(json.dumps(A			) + "\n"			)
					with open(self.merges_file							,"w"							,encoding="utf-8"			) as fp:
									fp.write("\n".join(A			)			)






	def     UpperCamelCase_  (	self :			List[str]							,**A :			List[str]			):
					kwargs.update(self.special_tokens_map			)
					return CTRLTokenizer.from_pretrained(self.tmpdirname							,**A			)






	def     UpperCamelCase_  (	self :			Optional[int]							,A :			Tuple			):
					__A             =			"adapt react readapt apt"
					__A             =			"adapt react readapt apt"
					return input_text, output_text







	def     UpperCamelCase_  (	self :			Any			):
					__A             =			CTRLTokenizer(self.vocab_file							,self.merges_file							,**self.special_tokens_map			)
					__A             =			"adapt react readapt apt"
					__A             =			"adapt re@@ a@@ c@@ t re@@ adapt apt".split()
					__A             =			tokenizer.tokenize(A			)
					self.assertListEqual(A							,A			)

					__A             =			tokens + [tokenizer.unk_token]

					__A             =			[0, 1, 2, 4, 5, 1, 0, 3, 6]
					self.assertListEqual(tokenizer.convert_tokens_to_ids(A			)							,A			)
