
"""simple docstring"""




import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class     lowercase		( SCREAMING_SNAKE_CASE__	,  unittest.TestCase		):
							lowercase_	:	Optional[Any]	 =CLIPTokenizer
							lowercase_	:	int	 =CLIPTokenizerFast
							lowercase_	:	str	 =True
							lowercase_	:	Dict	 ={}
							lowercase_	:	Any	 =False




							def 			A__				(    self):
												super().setUp()

												# fmt: off
												lowercase						     =						['''l''', '''o''', '''w''', '''e''', '''r''', '''s''', '''t''', '''i''', '''d''', '''n''', '''lo''', '''l</w>''', '''w</w>''', '''r</w>''', '''t</w>''', '''low</w>''', '''er</w>''', '''lowest</w>''', '''newer</w>''', '''wider''', '''<unk>''', '''<|startoftext|>''', '''<|endoftext|>''']
												# fmt: on
												lowercase						     =						dict(zip(A__       ,range(len(A__))))
												lowercase						     =						['''#version: 0.2''', '''l o''', '''lo w</w>''', '''e r</w>''']
												lowercase						     =						{'''unk_token''': '''<unk>'''}

												lowercase						     =						os.path.join(self.tmpdirname       ,VOCAB_FILES_NAMES['''vocab_file'''])
												lowercase						     =						os.path.join(self.tmpdirname       ,VOCAB_FILES_NAMES['''merges_file'''])
												with open(self.vocab_file       ,'''w'''       ,encoding='''utf-8''') as fp:
																	fp.write(json.dumps(A__) + '''\n''')
												with open(self.merges_file       ,'''w'''       ,encoding='''utf-8''') as fp:
																	fp.write('''\n'''.join(A__))




							def 			A__				(    self       ,**A__):
												kwargs.update(self.special_tokens_map)
												return CLIPTokenizer.from_pretrained(self.tmpdirname       ,**A__)




							def 			A__				(    self       ,**A__):
												kwargs.update(self.special_tokens_map)
												return CLIPTokenizerFast.from_pretrained(self.tmpdirname       ,**A__)




							def 			A__				(    self       ,A__):
												lowercase						     =						'''lower newer'''
												lowercase						     =						'''lower newer'''
												return input_text, output_text




							def 			A__				(    self):
												lowercase						     =						CLIPTokenizer(self.vocab_file       ,self.merges_file       ,**self.special_tokens_map)
												lowercase						     =						'''lower newer'''
												lowercase						     =						['''lo''', '''w''', '''er</w>''', '''n''', '''e''', '''w''', '''er</w>''']
												lowercase						     =						tokenizer.tokenize(A__)
												self.assertListEqual(A__       ,A__)

												lowercase						     =						tokens + [tokenizer.unk_token]
												lowercase						     =						[1_0, 2, 1_6, 9, 3, 2, 1_6, 2_0]
												self.assertListEqual(tokenizer.convert_tokens_to_ids(A__)       ,A__)




							@require_ftfy
							def 			A__				(    self):
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'):
																						lowercase						     =						self.tokenizer_class.from_pretrained(A__       ,**A__)
																						lowercase						     =						self.rust_tokenizer_class.from_pretrained(A__       ,**A__)

																						lowercase						     =						'''A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'''
																						lowercase						     =						tokenizer_s.tokenize(A__)
																						lowercase						     =						tokenizer_r.tokenize(A__)

																						self.assertListEqual(A__       ,A__)

																						# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
																						# with Tilde) encoded in 2 different ways
																						lowercase						     =						'''xa\u0303y''' + ''' ''' + '''x\xe3y'''
																						lowercase						     =						tokenizer_s.tokenize(A__)
																						lowercase						     =						tokenizer_r.tokenize(A__)

																						self.assertListEqual(A__       ,A__)

																						# Test that the tokenization is identical on unicode of space type
																						lowercase						     =						[
																						    '''\u0009''',  # (horizontal tab, '\t')
																						    '''\u000B''',  # (vertical tab)
																						    '''\u000C''',  # (form feed)
																						    '''\u0020''',  # (space, ' ')
																						    '''\u200E''',  # (left-to-right mark):w
																						    '''\u200F''',  # (right-to-left mark)
																						]
																						for unicode_seq in spaces_unicodes:
																											lowercase						     =						tokenizer_s.tokenize(A__)
																											lowercase						     =						tokenizer_r.tokenize(A__)

																											self.assertListEqual(A__       ,A__)

																						# Test that the tokenization is identical on unicode of line break type
																						lowercase						     =						[
																						    '''\u000A''',  # (line feed, '\n')
																						    '''\r\n''',  # (carriage return and line feed, '\r\n')
																						    '''\u000D''',  # (carriage return, '\r')
																						    '''\r''',  # (carriage return, '\r')
																						    '''\u000D''',  # (carriage return, '\r')
																						    '''\u2028''',  # (line separator)
																						    '''\u2029''',  # (paragraph separator)
																						    # "\u0085", # (next line)
																						]

																						# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
																						# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
																						# space (and thus into an empty list).

																						for unicode_seq in line_break_unicodes:
																											lowercase						     =						tokenizer_s.tokenize(A__)
																											lowercase						     =						tokenizer_r.tokenize(A__)

																											self.assertListEqual(A__       ,A__)




							def 			A__				(    self):
												# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'):
																						lowercase						     =						'''hello'''  # `hello` is a token in the vocabulary of `pretrained_name`
																						lowercase						     =						f'{text_of_1_token} {text_of_1_token}'

																						lowercase						     =						self.rust_tokenizer_class.from_pretrained(
																						    A__       ,use_fast=A__       ,)
																						lowercase						     =						tokenizer_r(A__       ,return_offsets_mapping=A__       ,add_special_tokens=A__)
																						self.assertEqual(encoding.offset_mapping[0]       ,(0, len(A__)))
																						self.assertEqual(
																						    encoding.offset_mapping[1]       ,(len(A__) + 1, len(A__) + 1 + len(A__))       ,)

																						lowercase						     =						f' {text}'

																						lowercase						     =						self.rust_tokenizer_class.from_pretrained(
																						    A__       ,use_fast=A__       ,)
																						lowercase						     =						tokenizer_r(A__       ,return_offsets_mapping=A__       ,add_special_tokens=A__)
																						self.assertEqual(encoding.offset_mapping[0]       ,(1, 1 + len(A__)))
																						self.assertEqual(
																						    encoding.offset_mapping[1]       ,(1 + len(A__) + 1, 1 + len(A__) + 1 + len(A__))       ,)




							def 			A__				(    self):
												# Test related to the breaking change introduced in transformers v4.17.0
												# We need to check that an error in raised when the user try to load a previous version of the tokenizer.
												with self.assertRaises(A__) as context:
																	self.rust_tokenizer_class.from_pretrained('''robot-test/old-clip-tokenizer''')

												self.assertTrue(
												    context.exception.args[0].startswith(
												        '''The `backend_tokenizer` provided does not match the expected format.'''))




							@require_ftfy
							def 			A__				(    self):
												super().test_tokenization_python_rust_equals()





							def 			A__				(    self):
												# CLIP always lower cases letters
												pass






import unittest
from dataclasses import dataclass

import pytest

from accelerate.commands.config.config_args import SageMakerConfig
from accelerate.utils import ComputeEnvironment
from accelerate.utils.launch import _convert_nargs_to_dict







@dataclass
class     lowercase		( SCREAMING_SNAKE_CASE__		):
							lowercase_	:	int	 =ComputeEnvironment.AMAZON_SAGEMAKER
							lowercase_	:	Optional[int]	 =True
							lowercase_	:	Any	 ='''ml.p3.2xlarge'''
							lowercase_	:	Any	 ='''accelerate_sagemaker_execution_role'''
							lowercase_	:	Union[str, Any]	 ='''hf-sm'''
							lowercase_	:	Any	 ='''us-east-1'''
							lowercase_	:	List[str]	 =1
							lowercase_	:	Any	 ='''accelerate-sagemaker-1'''
							lowercase_	:	Union[str, Any]	 ='''1.6'''
							lowercase_	:	Any	 ='''4.4'''
							lowercase_	:	Any	 ='''train.py'''
							lowercase_	:	int	 =[
							    '''--model_name_or_path''',
							    '''bert''',
							    '''--do_train''',
							    '''False''',
							    '''--epochs''',
							    '''3''',
							    '''--learning_rate''',
							    '''5e-5''',
							    '''--max_steps''',
							    '''50.5''',
							]
							lowercase_	:	List[Any]	 =[
							    '''--model_name_or_path''',
							    '''bert''',
							    '''--do_train''',
							    '''--do_test''',
							    '''False''',
							    '''--do_predict''',
							    '''--epochs''',
							    '''3''',
							    '''--learning_rate''',
							    '''5e-5''',
							    '''--max_steps''',
							    '''50.5''',
							]







class     lowercase		( unittest.TestCase		):




							def 			A__				(    self):
												# If no defaults are changed, `to_kwargs` returns an empty dict.
												lowercase						     =						_convert_nargs_to_dict(MockLaunchConfig.success_training_script_args)
												assert isinstance(converted_args['''model_name_or_path''']       ,A__)
												assert isinstance(converted_args['''do_train''']       ,A__)
												assert isinstance(converted_args['''epochs''']       ,A__)
												assert isinstance(converted_args['''learning_rate''']       ,A__)
												assert isinstance(converted_args['''max_steps''']       ,A__)

												with pytest.raises(A__):
																	_convert_nargs_to_dict(MockLaunchConfig.fail_training_script_args)




