

import argparse
import json
import logging
import os
import sys
from unittest.mock import patch

from transformers.testing_utils import TestCasePlus, get_gpu_count, slow


a		=[
    os.path.join(os.path.dirname(__file__), dirname)
    for dirname in [
        """text-classification""",
        """language-modeling""",
        """summarization""",
        """token-classification""",
        """question-answering""",
    ]
]
sys.path.extend(SRC_DIRS)


if SRC_DIRS is not None:
					import run_clm_flax
					import run_flax_glue
					import run_flax_ner
					import run_mlm_flax
					import run_qa
					import run_summarization_flax
					import run_ta_mlm_flax


logging.basicConfig(level=logging.DEBUG)

a		=logging.getLogger()






def        SCREAMING_SNAKE_CASE__	(  )  ->					Optional[int]:
	__lowerCamelCase    :    List[str] 	=			argparse.ArgumentParser()
	parser.add_argument('-f'      )
	__lowerCamelCase    :    Tuple 	=			parser.parse_args()
	return args.f






def        SCREAMING_SNAKE_CASE__	(  lowerCamelCase__     ,			lowerCamelCase__="eval"      )  ->					Tuple:
	__lowerCamelCase    :    Any 	=			os.path.join(lowerCamelCase__     ,			F"{split}_results.json"      )
	if os.path.exists(lowerCamelCase__      ):
		with open(lowerCamelCase__     ,			'r'      ) as f:
			return json.load(lowerCamelCase__      )
	raise ValueError(F"can't find {path}"      )


a		=logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)






class 	A_		(	SCREAMING_SNAKE_CASE    ):
							def    lowerCAmelCase					(		self		:      Tuple):
								__lowerCamelCase    :    List[str] 	=			self.get_auto_remove_tmp_dir()
								__lowerCamelCase    :    str 	=			F"\n            run_glue.py\n            --model_name_or_path distilbert-base-uncased\n            --output_dir {tmp_dir}\n            --train_file ./tests/fixtures/tests_samples/MRPC/train.csv\n            --validation_file ./tests/fixtures/tests_samples/MRPC/dev.csv\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n            --learning_rate=1e-4\n            --eval_steps=2\n            --warmup_steps=2\n            --seed=42\n            --max_seq_length=128\n            ".split()

								with patch.object(SCREAMING_SNAKE_CASE__       ,'argv'       ,SCREAMING_SNAKE_CASE__):
									run_flax_glue.main()
									__lowerCamelCase    :    int 	=			get_results(SCREAMING_SNAKE_CASE__)
									self.assertGreaterEqual(result['eval_accuracy']       ,0.75)
							@slow
							def    lowerCAmelCase					(		self		:      str):
								__lowerCamelCase    :    List[str] 	=			self.get_auto_remove_tmp_dir()
								__lowerCamelCase    :    Optional[Any] 	=			F"\n            run_clm_flax.py\n            --model_name_or_path distilgpt2\n            --train_file ./tests/fixtures/sample_text.txt\n            --validation_file ./tests/fixtures/sample_text.txt\n            --do_train\n            --do_eval\n            --block_size 128\n            --per_device_train_batch_size 4\n            --per_device_eval_batch_size 4\n            --num_train_epochs 2\n            --logging_steps 2 --eval_steps 2\n            --output_dir {tmp_dir}\n            --overwrite_output_dir\n            ".split()

								with patch.object(SCREAMING_SNAKE_CASE__       ,'argv'       ,SCREAMING_SNAKE_CASE__):
									run_clm_flax.main()
									__lowerCamelCase    :    int 	=			get_results(SCREAMING_SNAKE_CASE__)
									self.assertLess(result['eval_perplexity']       ,1_0_0)
							@slow
							def    lowerCAmelCase					(		self		:      str):
								__lowerCamelCase    :    Tuple 	=			self.get_auto_remove_tmp_dir()
								__lowerCamelCase    :    List[Any] 	=			F"\n            run_summarization.py\n            --model_name_or_path t5-small\n            --train_file tests/fixtures/tests_samples/xsum/sample.json\n            --validation_file tests/fixtures/tests_samples/xsum/sample.json\n            --test_file tests/fixtures/tests_samples/xsum/sample.json\n            --output_dir {tmp_dir}\n            --overwrite_output_dir\n            --num_train_epochs=3\n            --warmup_steps=8\n            --do_train\n            --do_eval\n            --do_predict\n            --learning_rate=2e-4\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n            --predict_with_generate\n        ".split()

								with patch.object(SCREAMING_SNAKE_CASE__       ,'argv'       ,SCREAMING_SNAKE_CASE__):
									run_summarization_flax.main()
									__lowerCamelCase    :    List[Any] 	=			get_results(SCREAMING_SNAKE_CASE__       ,split='test')
									self.assertGreaterEqual(result['test_rouge1']       ,1_0)
									self.assertGreaterEqual(result['test_rouge2']       ,2)
									self.assertGreaterEqual(result['test_rougeL']       ,7)
									self.assertGreaterEqual(result['test_rougeLsum']       ,7)
							@slow
							def    lowerCAmelCase					(		self		:      List[str]):
								__lowerCamelCase    :    Optional[int] 	=			self.get_auto_remove_tmp_dir()
								__lowerCamelCase    :    int 	=			F"\n            run_mlm.py\n            --model_name_or_path distilroberta-base\n            --train_file ./tests/fixtures/sample_text.txt\n            --validation_file ./tests/fixtures/sample_text.txt\n            --output_dir {tmp_dir}\n            --overwrite_output_dir\n            --max_seq_length 128\n            --per_device_train_batch_size 4\n            --per_device_eval_batch_size 4\n            --logging_steps 2 --eval_steps 2\n            --do_train\n            --do_eval\n            --num_train_epochs=1\n        ".split()

								with patch.object(SCREAMING_SNAKE_CASE__       ,'argv'       ,SCREAMING_SNAKE_CASE__):
									run_mlm_flax.main()
									__lowerCamelCase    :    Optional[Any] 	=			get_results(SCREAMING_SNAKE_CASE__)
									self.assertLess(result['eval_perplexity']       ,4_2)
							@slow
							def    lowerCAmelCase					(		self		:      Optional[Any]):
								__lowerCamelCase    :    Tuple 	=			self.get_auto_remove_tmp_dir()
								__lowerCamelCase    :    Union[str, Any] 	=			F"\n            run_t5_mlm_flax.py\n            --model_name_or_path t5-small\n            --train_file ./tests/fixtures/sample_text.txt\n            --validation_file ./tests/fixtures/sample_text.txt\n            --do_train\n            --do_eval\n            --max_seq_length 128\n            --per_device_train_batch_size 4\n            --per_device_eval_batch_size 4\n            --num_train_epochs 2\n            --logging_steps 2 --eval_steps 2\n            --output_dir {tmp_dir}\n            --overwrite_output_dir\n            ".split()

								with patch.object(SCREAMING_SNAKE_CASE__       ,'argv'       ,SCREAMING_SNAKE_CASE__):
									run_ta_mlm_flax.main()
									__lowerCamelCase    :    Union[str, Any] 	=			get_results(SCREAMING_SNAKE_CASE__)
									self.assertGreaterEqual(result['eval_accuracy']       ,0.42)
							@slow
							def    lowerCAmelCase					(		self		:      Tuple):
								# with so little data distributed training needs more epochs to get the score on par with 0/1 gpu
								__lowerCamelCase    :    Optional[int] 	=			7 if get_gpu_count() > 1 else 2

								__lowerCamelCase    :    int 	=			self.get_auto_remove_tmp_dir()
								__lowerCamelCase    :    str 	=			F"\n            run_flax_ner.py\n            --model_name_or_path bert-base-uncased\n            --train_file tests/fixtures/tests_samples/conll/sample.json\n            --validation_file tests/fixtures/tests_samples/conll/sample.json\n            --output_dir {tmp_dir}\n            --overwrite_output_dir\n            --do_train\n            --do_eval\n            --warmup_steps=2\n            --learning_rate=2e-4\n            --logging_steps 2 --eval_steps 2\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=2\n            --num_train_epochs={epochs}\n            --seed 7\n        ".split()

								with patch.object(SCREAMING_SNAKE_CASE__       ,'argv'       ,SCREAMING_SNAKE_CASE__):
									run_flax_ner.main()
									__lowerCamelCase    :    List[Any] 	=			get_results(SCREAMING_SNAKE_CASE__)
									self.assertGreaterEqual(result['eval_accuracy']       ,0.75)
									self.assertGreaterEqual(result['eval_f1']       ,0.3)


							@slow
							def    lowerCAmelCase					(		self		:      Union[str, Any]):
								__lowerCamelCase    :    int 	=			self.get_auto_remove_tmp_dir()
								__lowerCamelCase    :    str 	=			F"\n            run_qa.py\n            --model_name_or_path bert-base-uncased\n            --version_2_with_negative\n            --train_file tests/fixtures/tests_samples/SQUAD/sample.json\n            --validation_file tests/fixtures/tests_samples/SQUAD/sample.json\n            --output_dir {tmp_dir}\n            --overwrite_output_dir\n            --num_train_epochs=3\n            --warmup_steps=2\n            --do_train\n            --do_eval\n            --logging_steps 2 --eval_steps 2\n            --learning_rate=2e-4\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n        ".split()

								with patch.object(SCREAMING_SNAKE_CASE__       ,'argv'       ,SCREAMING_SNAKE_CASE__):
									run_qa.main()
									__lowerCamelCase    :    List[str] 	=			get_results(SCREAMING_SNAKE_CASE__)
									self.assertGreaterEqual(result['eval_f1']       ,3_0)
									self.assertGreaterEqual(result['eval_exact']       ,3_0)







import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
					from test_module.custom_tokenization_fast import CustomTokenizerFast






class 	A_		(	unittest.TestCase    ):
							def    lowerCAmelCase					(		self		:      Optional[Any]):
								# A mock response for an HTTP head request to emulate server down
								__lowerCamelCase    :    List[Any] 	=			mock.Mock()
								__lowerCamelCase    :    Tuple 	=			5_0_0
								__lowerCamelCase    :    Dict 	=			{}
								__lowerCamelCase    :    List[Any] 	=			HTTPError
								__lowerCamelCase    :    int 	=			{}

								# Download this model to make sure it's in the cache.
								__lowerCamelCase    :    Optional[int] 	=			BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert')

								# Under the mock environment we get a 500 error when trying to reach the tokenizer.
								with mock.patch('requests.Session.request'       ,return_value=SCREAMING_SNAKE_CASE__) as mock_head:
									__lowerCamelCase    :    Tuple 	=			BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert')
									# This check we did call the fake head request
									mock_head.assert_called()
							@require_tokenizers
							def    lowerCAmelCase					(		self		:      Any):
								# A mock response for an HTTP head request to emulate server down
								__lowerCamelCase    :    int 	=			mock.Mock()
								__lowerCamelCase    :    List[str] 	=			5_0_0
								__lowerCamelCase    :    Tuple 	=			{}
								__lowerCamelCase    :    List[str] 	=			HTTPError
								__lowerCamelCase    :    int 	=			{}

								# Download this model to make sure it's in the cache.
								__lowerCamelCase    :    Dict 	=			GPTaTokenizerFast.from_pretrained('gpt2')

								# Under the mock environment we get a 500 error when trying to reach the tokenizer.
								with mock.patch('requests.Session.request'       ,return_value=SCREAMING_SNAKE_CASE__) as mock_head:
									__lowerCamelCase    :    Optional[int] 	=			GPTaTokenizerFast.from_pretrained('gpt2')
									# This check we did call the fake head request
									mock_head.assert_called()
							def    lowerCAmelCase					(		self		:      str):
								# This test is for deprecated behavior and can be removed in v5
								try:
									__lowerCamelCase    :    Tuple 	=			tempfile.mktemp()
									with open(SCREAMING_SNAKE_CASE__       ,'wb') as f:
										http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'       ,SCREAMING_SNAKE_CASE__)

									__lowerCamelCase    :    Dict 	=			AlbertTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__)
								finally:
									os.remove(SCREAMING_SNAKE_CASE__)

								# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
								# the current folder and have the right name.
								if os.path.isfile('tokenizer.json'):
									# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
									return
								try:
									with open('tokenizer.json'       ,'wb') as f:
										http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'       ,SCREAMING_SNAKE_CASE__)
									__lowerCamelCase    :    Any 	=			AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2')
									# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
									self.assertEqual(tokenizer.vocab_size       ,1_0_0_0)
									# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

								finally:
									os.remove('tokenizer.json')


							def    lowerCAmelCase					(		self		:      Optional[Any]):
								# This test is for deprecated behavior and can be removed in v5
								__lowerCamelCase    :    str 	=			AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model')






@is_staging_test
class 	A_		(	unittest.TestCase    ):
							_UpperCAmelCase   :  Any										=				['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']
							@classmethod
							def    lowerCAmelCase					(		cls		:      Optional[int]):
								__lowerCamelCase    :    Optional[int] 	=			TOKEN
								HfFolder.save_token(SCREAMING_SNAKE_CASE__)
							@classmethod
							def    lowerCAmelCase					(		cls		:      str):
								try:
									delete_repo(token=cls._token       ,repo_id='test-tokenizer')
								except HTTPError:
									pass

								try:
									delete_repo(token=cls._token       ,repo_id='valid_org/test-tokenizer-org')
								except HTTPError:
									pass

								try:
									delete_repo(token=cls._token       ,repo_id='test-dynamic-tokenizer')
								except HTTPError:
									pass
							def    lowerCAmelCase					(		self		:      List[str]):
								with tempfile.TemporaryDirectory() as tmp_dir:
									__lowerCamelCase    :    Dict 	=			os.path.join(SCREAMING_SNAKE_CASE__       ,'vocab.txt')
									with open(SCREAMING_SNAKE_CASE__       ,'w'       ,encoding='utf-8') as vocab_writer:
										vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))
									__lowerCamelCase    :    Union[str, Any] 	=			BertTokenizer(SCREAMING_SNAKE_CASE__)

								tokenizer.push_to_hub('test-tokenizer'       ,use_auth_token=self._token)
								__lowerCamelCase    :    List[str] 	=			BertTokenizer.from_pretrained(F"{USER}/test-tokenizer")
								self.assertDictEqual(new_tokenizer.vocab       ,tokenizer.vocab)

								# Reset repo
								delete_repo(token=self._token       ,repo_id='test-tokenizer')

								# Push to hub via save_pretrained
								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(SCREAMING_SNAKE_CASE__       ,repo_id='test-tokenizer'       ,push_to_hub=SCREAMING_SNAKE_CASE__       ,use_auth_token=self._token)

								__lowerCamelCase    :    str 	=			BertTokenizer.from_pretrained(F"{USER}/test-tokenizer")
								self.assertDictEqual(new_tokenizer.vocab       ,tokenizer.vocab)
							def    lowerCAmelCase					(		self		:      Any):
								with tempfile.TemporaryDirectory() as tmp_dir:
									__lowerCamelCase    :    Dict 	=			os.path.join(SCREAMING_SNAKE_CASE__       ,'vocab.txt')
									with open(SCREAMING_SNAKE_CASE__       ,'w'       ,encoding='utf-8') as vocab_writer:
										vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))
									__lowerCamelCase    :    Optional[Any] 	=			BertTokenizer(SCREAMING_SNAKE_CASE__)

								tokenizer.push_to_hub('valid_org/test-tokenizer-org'       ,use_auth_token=self._token)
								__lowerCamelCase    :    List[Any] 	=			BertTokenizer.from_pretrained('valid_org/test-tokenizer-org')
								self.assertDictEqual(new_tokenizer.vocab       ,tokenizer.vocab)

								# Reset repo
								delete_repo(token=self._token       ,repo_id='valid_org/test-tokenizer-org')

								# Push to hub via save_pretrained
								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(
									    SCREAMING_SNAKE_CASE__       ,repo_id='valid_org/test-tokenizer-org'       ,push_to_hub=SCREAMING_SNAKE_CASE__       ,use_auth_token=self._token)

								__lowerCamelCase    :    List[Any] 	=			BertTokenizer.from_pretrained('valid_org/test-tokenizer-org')
								self.assertDictEqual(new_tokenizer.vocab       ,tokenizer.vocab)


							@require_tokenizers
							def    lowerCAmelCase					(		self		:      int):
								CustomTokenizer.register_for_auto_class()
								with tempfile.TemporaryDirectory() as tmp_dir:
									__lowerCamelCase    :    Dict 	=			os.path.join(SCREAMING_SNAKE_CASE__       ,'vocab.txt')
									with open(SCREAMING_SNAKE_CASE__       ,'w'       ,encoding='utf-8') as vocab_writer:
										vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))
									__lowerCamelCase    :    Any 	=			CustomTokenizer(SCREAMING_SNAKE_CASE__)

								# No fast custom tokenizer
								tokenizer.push_to_hub('test-dynamic-tokenizer'       ,use_auth_token=self._token)

								__lowerCamelCase    :    Optional[int] 	=			AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"       ,trust_remote_code=SCREAMING_SNAKE_CASE__)
								# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
								self.assertEqual(tokenizer.__class__.__name__       ,'CustomTokenizer')

								# Fast and slow custom tokenizer
								CustomTokenizerFast.register_for_auto_class()
								with tempfile.TemporaryDirectory() as tmp_dir:
									__lowerCamelCase    :    List[str] 	=			os.path.join(SCREAMING_SNAKE_CASE__       ,'vocab.txt')
									with open(SCREAMING_SNAKE_CASE__       ,'w'       ,encoding='utf-8') as vocab_writer:
										vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))

									__lowerCamelCase    :    Dict 	=			BertTokenizerFast.from_pretrained(SCREAMING_SNAKE_CASE__)
									bert_tokenizer.save_pretrained(SCREAMING_SNAKE_CASE__)
									__lowerCamelCase    :    List[Any] 	=			CustomTokenizerFast.from_pretrained(SCREAMING_SNAKE_CASE__)

								tokenizer.push_to_hub('test-dynamic-tokenizer'       ,use_auth_token=self._token)

								__lowerCamelCase    :    Optional[Any] 	=			AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"       ,trust_remote_code=SCREAMING_SNAKE_CASE__)
								# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
								self.assertEqual(tokenizer.__class__.__name__       ,'CustomTokenizerFast')
								__lowerCamelCase    :    Optional[int] 	=			AutoTokenizer.from_pretrained(
								    F"{USER}/test-dynamic-tokenizer"       ,use_fast=SCREAMING_SNAKE_CASE__       ,trust_remote_code=SCREAMING_SNAKE_CASE__)
								# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
								self.assertEqual(tokenizer.__class__.__name__       ,'CustomTokenizer')






class 	A_		(	unittest.TestCase    ):
							def    lowerCAmelCase					(		self		:      List[Any]):
								__lowerCamelCase    :    List[str] 	=			Trie()
								trie.add('Hello 友達')
								self.assertEqual(trie.data       ,{'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}})
								trie.add('Hello')
								trie.data
								self.assertEqual(trie.data       ,{'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}})
							def    lowerCAmelCase					(		self		:      Tuple):
								__lowerCamelCase    :    str 	=			Trie()
								self.assertEqual(trie.split('[CLS] This is a extra_id_100')       ,['[CLS] This is a extra_id_100'])
								trie.add('[CLS]')
								trie.add('extra_id_1')
								trie.add('extra_id_100')
								self.assertEqual(trie.split('[CLS] This is a extra_id_100')       ,['[CLS]', ' This is a ', 'extra_id_100'])
							def    lowerCAmelCase					(		self		:      Optional[Any]):
								__lowerCamelCase    :    Dict 	=			Trie()
								trie.add('A')
								self.assertEqual(trie.split('ABC')       ,['A', 'BC'])
								self.assertEqual(trie.split('BCA')       ,['BC', 'A'])
							def    lowerCAmelCase					(		self		:      Dict):
								__lowerCamelCase    :    Any 	=			Trie()
								trie.add('TOKEN]')
								trie.add('[SPECIAL_TOKEN]')
								self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]')       ,['This is something ', '[SPECIAL_TOKEN]'])
							def    lowerCAmelCase					(		self		:      int):
								__lowerCamelCase    :    List[Any] 	=			Trie()
								trie.add('A')
								trie.add('P')
								trie.add('[SPECIAL_TOKEN]')
								self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]')       ,['This is something ', '[SPECIAL_TOKEN]'])
							def    lowerCAmelCase					(		self		:      Optional[int]):
								__lowerCamelCase    :    Any 	=			Trie()
								trie.add('AB')
								trie.add('B')
								trie.add('C')
								self.assertEqual(trie.split('ABC')       ,['AB', 'C'])
							def    lowerCAmelCase					(		self		:      Optional[int]):
								__lowerCamelCase    :    Dict 	=			Trie()
								trie.add('ABC')
								trie.add('B')
								trie.add('CD')
								self.assertEqual(trie.split('ABCD')       ,['ABC', 'D'])


							def    lowerCAmelCase					(		self		:      List[Any]):
								# Even if the offsets are wrong, we necessarily output correct string
								# parts.
								__lowerCamelCase    :    List[Any] 	=			Trie()
								__lowerCamelCase    :    Any 	=			trie.cut_text('ABC'       ,[0, 0, 2, 1, 2, 3])
								self.assertEqual(SCREAMING_SNAKE_CASE__       ,['AB', 'C'])




