import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
					from test_module.custom_tokenization_fast import CustomTokenizerFast


class       __SCREAMING_SNAKE_CASE   (				unittest.TestCase):






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												# A mock response for an HTTP head request to emulate server down
												lowerCAmelCase__        =  mock.Mock()
												lowerCAmelCase__        =  5_00
												lowerCAmelCase__        =  {}
												lowerCAmelCase__        =  HTTPError
												lowerCAmelCase__        =  {}

												# Download this model to make sure it's in the cache.
												lowerCAmelCase__        =  BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'				)

												# Under the mock environment we get a 500 error when trying to reach the tokenizer.
												with mock.patch('requests.Session.request'       ,  return_value=__snake_case				) as mock_head:
																		lowerCAmelCase__        =  BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'				)
																		# This check we did call the fake head request
																		mock_head.assert_called()






						@require_tokenizers
						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												# A mock response for an HTTP head request to emulate server down
												lowerCAmelCase__        =  mock.Mock()
												lowerCAmelCase__        =  5_00
												lowerCAmelCase__        =  {}
												lowerCAmelCase__        =  HTTPError
												lowerCAmelCase__        =  {}

												# Download this model to make sure it's in the cache.
												lowerCAmelCase__        =  GPTaTokenizerFast.from_pretrained('gpt2'				)

												# Under the mock environment we get a 500 error when trying to reach the tokenizer.
												with mock.patch('requests.Session.request'       ,  return_value=__snake_case				) as mock_head:
																		lowerCAmelCase__        =  GPTaTokenizerFast.from_pretrained('gpt2'				)
																		# This check we did call the fake head request
																		mock_head.assert_called()






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												# This test is for deprecated behavior and can be removed in v5
												try:
																		lowerCAmelCase__        =  tempfile.mktemp()
																		with open(__snake_case       ,  'wb'				) as f:
																								http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'       ,  __snake_case				)

																		lowerCAmelCase__        =  AlbertTokenizer.from_pretrained(__snake_case				)
												finally:
																		os.remove(__snake_case				)

												# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
												# the current folder and have the right name.
												if os.path.isfile('tokenizer.json'				):
																		# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
																		return
												try:
																		with open('tokenizer.json'       ,  'wb'				) as f:
																								http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'       ,  __snake_case				)
																		lowerCAmelCase__        =  AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2'				)
																		# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
																		self.assertEqual(tokenizer.vocab_size       ,  10_00				)
																		# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

												finally:
																		os.remove('tokenizer.json'				)







						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												# This test is for deprecated behavior and can be removed in v5
												lowerCAmelCase__        =  AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'				)




@is_staging_test
class       __SCREAMING_SNAKE_CASE   (				unittest.TestCase):
						_SCREAMING_SNAKE_CASE		:    Optional[Any]						=   ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']






						@classmethod
						def  UpperCamelCase__ (		cls				):






												"""simple docstring"""
												lowerCAmelCase__        =  TOKEN
												HfFolder.save_token(__snake_case				)






						@classmethod
						def  UpperCamelCase__ (		cls				):






												"""simple docstring"""
												try:
																		delete_repo(token=cls._token       ,  repo_id='test-tokenizer'				)
												except HTTPError:
																		pass

												try:
																		delete_repo(token=cls._token       ,  repo_id='valid_org/test-tokenizer-org'				)
												except HTTPError:
																		pass

												try:
																		delete_repo(token=cls._token       ,  repo_id='test-dynamic-tokenizer'				)
												except HTTPError:
																		pass






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												with tempfile.TemporaryDirectory() as tmp_dir:
																		lowerCAmelCase__        =  os.path.join(__snake_case       ,  'vocab.txt'				)
																		with open(__snake_case       ,  'w'       ,  encoding='utf-8'				) as vocab_writer:
																								vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]				)				)
																		lowerCAmelCase__        =  BertTokenizer(__snake_case				)

												tokenizer.push_to_hub('test-tokenizer'       ,  use_auth_token=self._token				)
												lowerCAmelCase__        =  BertTokenizer.from_pretrained(F"{USER}/test-tokenizer"				)
												self.assertDictEqual(new_tokenizer.vocab       ,  tokenizer.vocab				)

												# Reset repo
												delete_repo(token=self._token       ,  repo_id='test-tokenizer'				)

												# Push to hub via save_pretrained
												with tempfile.TemporaryDirectory() as tmp_dir:
																		tokenizer.save_pretrained(__snake_case       ,  repo_id='test-tokenizer'       ,  push_to_hub=__snake_case       ,  use_auth_token=self._token				)

												lowerCAmelCase__        =  BertTokenizer.from_pretrained(F"{USER}/test-tokenizer"				)
												self.assertDictEqual(new_tokenizer.vocab       ,  tokenizer.vocab				)






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												with tempfile.TemporaryDirectory() as tmp_dir:
																		lowerCAmelCase__        =  os.path.join(__snake_case       ,  'vocab.txt'				)
																		with open(__snake_case       ,  'w'       ,  encoding='utf-8'				) as vocab_writer:
																								vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]				)				)
																		lowerCAmelCase__        =  BertTokenizer(__snake_case				)

												tokenizer.push_to_hub('valid_org/test-tokenizer-org'       ,  use_auth_token=self._token				)
												lowerCAmelCase__        =  BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'				)
												self.assertDictEqual(new_tokenizer.vocab       ,  tokenizer.vocab				)

												# Reset repo
												delete_repo(token=self._token       ,  repo_id='valid_org/test-tokenizer-org'				)

												# Push to hub via save_pretrained
												with tempfile.TemporaryDirectory() as tmp_dir:
																		tokenizer.save_pretrained(
																		    __snake_case       ,  repo_id='valid_org/test-tokenizer-org'       ,  push_to_hub=__snake_case       ,  use_auth_token=self._token				)

												lowerCAmelCase__        =  BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'				)
												self.assertDictEqual(new_tokenizer.vocab       ,  tokenizer.vocab				)







						@require_tokenizers
						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												CustomTokenizer.register_for_auto_class()
												with tempfile.TemporaryDirectory() as tmp_dir:
																		lowerCAmelCase__        =  os.path.join(__snake_case       ,  'vocab.txt'				)
																		with open(__snake_case       ,  'w'       ,  encoding='utf-8'				) as vocab_writer:
																								vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]				)				)
																		lowerCAmelCase__        =  CustomTokenizer(__snake_case				)

												# No fast custom tokenizer
												tokenizer.push_to_hub('test-dynamic-tokenizer'       ,  use_auth_token=self._token				)

												lowerCAmelCase__        =  AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"       ,  trust_remote_code=__snake_case				)
												# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
												self.assertEqual(tokenizer.__class__.__name__       ,  'CustomTokenizer'				)

												# Fast and slow custom tokenizer
												CustomTokenizerFast.register_for_auto_class()
												with tempfile.TemporaryDirectory() as tmp_dir:
																		lowerCAmelCase__        =  os.path.join(__snake_case       ,  'vocab.txt'				)
																		with open(__snake_case       ,  'w'       ,  encoding='utf-8'				) as vocab_writer:
																								vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]				)				)

																		lowerCAmelCase__        =  BertTokenizerFast.from_pretrained(__snake_case				)
																		bert_tokenizer.save_pretrained(__snake_case				)
																		lowerCAmelCase__        =  CustomTokenizerFast.from_pretrained(__snake_case				)

												tokenizer.push_to_hub('test-dynamic-tokenizer'       ,  use_auth_token=self._token				)

												lowerCAmelCase__        =  AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"       ,  trust_remote_code=__snake_case				)
												# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
												self.assertEqual(tokenizer.__class__.__name__       ,  'CustomTokenizerFast'				)
												lowerCAmelCase__        =  AutoTokenizer.from_pretrained(
												    F"{USER}/test-dynamic-tokenizer"       ,  use_fast=__snake_case       ,  trust_remote_code=__snake_case				)
												# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
												self.assertEqual(tokenizer.__class__.__name__       ,  'CustomTokenizer'				)




class       __SCREAMING_SNAKE_CASE   (				unittest.TestCase):






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												lowerCAmelCase__        =  Trie()
												trie.add('Hello 友達'				)
												self.assertEqual(trie.data       ,  {'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}}				)
												trie.add('Hello'				)
												trie.data
												self.assertEqual(trie.data       ,  {'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}}				)






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												lowerCAmelCase__        =  Trie()
												self.assertEqual(trie.split('[CLS] This is a extra_id_100'				)       ,  ['[CLS] This is a extra_id_100']				)
												trie.add('[CLS]'				)
												trie.add('extra_id_1'				)
												trie.add('extra_id_100'				)
												self.assertEqual(trie.split('[CLS] This is a extra_id_100'				)       ,  ['[CLS]', ' This is a ', 'extra_id_100']				)






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												lowerCAmelCase__        =  Trie()
												trie.add('A'				)
												self.assertEqual(trie.split('ABC'				)       ,  ['A', 'BC']				)
												self.assertEqual(trie.split('BCA'				)       ,  ['BC', 'A']				)






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												lowerCAmelCase__        =  Trie()
												trie.add('TOKEN]'				)
												trie.add('[SPECIAL_TOKEN]'				)
												self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'				)       ,  ['This is something ', '[SPECIAL_TOKEN]']				)






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												lowerCAmelCase__        =  Trie()
												trie.add('A'				)
												trie.add('P'				)
												trie.add('[SPECIAL_TOKEN]'				)
												self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'				)       ,  ['This is something ', '[SPECIAL_TOKEN]']				)






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												lowerCAmelCase__        =  Trie()
												trie.add('AB'				)
												trie.add('B'				)
												trie.add('C'				)
												self.assertEqual(trie.split('ABC'				)       ,  ['AB', 'C']				)






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												lowerCAmelCase__        =  Trie()
												trie.add('ABC'				)
												trie.add('B'				)
												trie.add('CD'				)
												self.assertEqual(trie.split('ABCD'				)       ,  ['ABC', 'D']				)







						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												# Even if the offsets are wrong, we necessarily output correct string
												# parts.
												lowerCAmelCase__        =  Trie()
												lowerCAmelCase__        =  trie.cut_text('ABC'       ,  [0, 0, 2, 1, 2, 3]				)
												self.assertEqual(__snake_case       ,  ['AB', 'C']				)













from collections.abc import Iterator, MutableMapping
from dataclasses import dataclass
from typing import Generic, TypeVar

__snake_case      :      Optional[Any]             =       TypeVar("""KEY""")
__snake_case      :      str             =       TypeVar("""VAL""")


@dataclass(frozen=__lowercase						,   slots=__lowercase)
class       __SCREAMING_SNAKE_CASE   (				Generic[KEY, VAL]):
						_SCREAMING_SNAKE_CASE		:    KEY
						_SCREAMING_SNAKE_CASE		:    VAL




class       __SCREAMING_SNAKE_CASE   (				_Item):






						def __init__(		self				):






												"""simple docstring"""
												super().__init__(_UpperCamelCase       ,  _UpperCamelCase				)







						def __bool__(		self				):






												"""simple docstring"""
												return False




__snake_case      :      int             =       _DeletedItem()


class       __SCREAMING_SNAKE_CASE   (				MutableMapping[KEY, VAL]):






						def __init__(		self       ,  _UpperCamelCase = 8       ,  _UpperCamelCase = 0.75				):






												"""simple docstring"""
												lowerCAmelCase__        =  initial_block_size
												lowerCAmelCase__        =  [None] * initial_block_size
												assert 0.0 < capacity_factor < 1.0
												lowerCAmelCase__        =  capacity_factor
												lowerCAmelCase__        =  0






						def  UpperCamelCase__ (		self       ,  _UpperCamelCase				):






												"""simple docstring"""
												return hash(_UpperCamelCase				) % len(self._buckets				)






						def  UpperCamelCase__ (		self       ,  _UpperCamelCase				):






												"""simple docstring"""
												return (ind + 1) % len(self._buckets				)






						def  UpperCamelCase__ (		self       ,  _UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase				):






												"""simple docstring"""
												lowerCAmelCase__        =  self._buckets[ind]
												if not stored:
																		lowerCAmelCase__        =  _Item(_UpperCamelCase       ,  _UpperCamelCase				)
																		self._len += 1
																		return True
												elif stored.key == key:
																		lowerCAmelCase__        =  _Item(_UpperCamelCase       ,  _UpperCamelCase				)
																		return True
												else:
																		return False






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												lowerCAmelCase__        =  len(self._buckets				) * self._capacity_factor
												return len(self				) >= int(_UpperCamelCase				)






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												if len(self._buckets				) <= self._initial_block_size:
																		return False
												lowerCAmelCase__        =  len(self._buckets				) * self._capacity_factor / 2
												return len(self				) < limit






						def  UpperCamelCase__ (		self       ,  _UpperCamelCase				):






												"""simple docstring"""
												lowerCAmelCase__        =  self._buckets
												lowerCAmelCase__        =  [None] * new_size
												lowerCAmelCase__        =  0
												for item in old_buckets:
																		if item:
																								self._add_item(item.key       ,  item.val				)






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												self._resize(len(self._buckets				) * 2				)






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												self._resize(len(self._buckets				) // 2				)






						def  UpperCamelCase__ (		self       ,  _UpperCamelCase				):






												"""simple docstring"""
												lowerCAmelCase__        =  self._get_bucket_index(_UpperCamelCase				)
												for _ in range(len(self._buckets				)				):
																		yield ind
																		lowerCAmelCase__        =  self._get_next_ind(_UpperCamelCase				)






						def  UpperCamelCase__ (		self       ,  _UpperCamelCase       ,  _UpperCamelCase				):






												"""simple docstring"""
												for ind in self._iterate_buckets(_UpperCamelCase				):
																		if self._try_set(_UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase				):
																								break






						def __setitem__(		self       ,  _UpperCamelCase       ,  _UpperCamelCase				):






												"""simple docstring"""
												if self._is_full():
																		self._size_up()

												self._add_item(_UpperCamelCase       ,  _UpperCamelCase				)






						def __delitem__(		self       ,  _UpperCamelCase				):






												"""simple docstring"""
												for ind in self._iterate_buckets(_UpperCamelCase				):
																		lowerCAmelCase__        =  self._buckets[ind]
																		if item is None:
																								raise KeyError(_UpperCamelCase				)
																		if item is _deleted:
																								continue
																		if item.key == key:
																								lowerCAmelCase__        =  _deleted
																								self._len -= 1
																								break
												if self._is_sparse():
																		self._size_down()






						def __getitem__(		self       ,  _UpperCamelCase				):






												"""simple docstring"""
												for ind in self._iterate_buckets(_UpperCamelCase				):
																		lowerCAmelCase__        =  self._buckets[ind]
																		if item is None:
																								break
																		if item is _deleted:
																								continue
																		if item.key == key:
																								return item.val
												raise KeyError(_UpperCamelCase				)






						def __len__(		self				):






												"""simple docstring"""
												return self._len






						def __iter__(		self				):






												"""simple docstring"""
												yield from (item.key for item in self._buckets if item)







						def __repr__(		self				):






												"""simple docstring"""
												lowerCAmelCase__        =  ' ,'.join(
												    F"{item.key}: {item.val}" for item in self._buckets if item				)
												return F"HashMap({val_string})"





