


import itertools
import json
import os
import unittest

from transformers import AddedToken, RobertaTokenizer, RobertaTokenizerFast
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class 	_a							(      UpperCamelCase__					,     unittest.TestCase      ):
							_lowercase     :      List[Any]  				=			RobertaTokenizer
							_lowercase     :      int  				=			RobertaTokenizerFast
							_lowercase     :      str  				=			True
							_lowercase     :      int  				=			{'''cls_token''': '''<s>'''}

							def 				lowerCamelCase_			(							self:     Any							)      ->       Any:





									"""simple docstring"""







									super().setUp()

									# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
									lowercase__						=				[
									    '''l''',
									    '''o''',
									    '''w''',
									    '''e''',
									    '''r''',
									    '''s''',
									    '''t''',
									    '''i''',
									    '''d''',
									    '''n''',
									    '''\u0120''',
									    '''\u0120l''',
									    '''\u0120n''',
									    '''\u0120lo''',
									    '''\u0120low''',
									    '''er''',
									    '''\u0120lowest''',
									    '''\u0120newer''',
									    '''\u0120wider''',
									    '''<unk>''',
									]
									lowercase__						=				dict(zip(UpperCamelCase_   ,						range(len(UpperCamelCase_							)							)							)							)
									lowercase__						=				['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
									lowercase__						=				{'''unk_token''': '''<unk>'''}

									lowercase__						=				os.path.join(self.tmpdirname   ,						VOCAB_FILES_NAMES['''vocab_file''']							)
									lowercase__						=				os.path.join(self.tmpdirname   ,						VOCAB_FILES_NAMES['''merges_file''']							)
									with open(self.vocab_file   ,						'''w'''   ,						encoding='''utf-8'''							) as fp:
											fp.write(json.dumps(UpperCamelCase_							) + '''\n'''							)
									with open(self.merges_file   ,						'''w'''   ,						encoding='''utf-8'''							) as fp:
											fp.write('''\n'''.join(UpperCamelCase_							)							)

							def 				lowerCamelCase_			(							self:     Tuple   ,						**UpperCamelCase_:     Optional[Any]							)      ->       int:





									"""simple docstring"""







									kwargs.update(self.special_tokens_map							)
									return self.tokenizer_class.from_pretrained(self.tmpdirname   ,						**UpperCamelCase_							)

							def 				lowerCamelCase_			(							self:     str   ,						**UpperCamelCase_:     Optional[int]							)      ->       Tuple:





									"""simple docstring"""







									kwargs.update(self.special_tokens_map							)
									return RobertaTokenizerFast.from_pretrained(self.tmpdirname   ,						**UpperCamelCase_							)

							def 				lowerCamelCase_			(							self:     int   ,						UpperCamelCase_:     Optional[int]							)      ->       List[Any]:





									"""simple docstring"""







									lowercase__						=				'''lower newer'''
									lowercase__						=				'''lower newer'''
									return input_text, output_text

							def 				lowerCamelCase_			(							self:     List[Any]							)      ->       Union[str, Any]:





									"""simple docstring"""







									lowercase__						=				self.tokenizer_class(self.vocab_file   ,						self.merges_file   ,						**self.special_tokens_map							)
									lowercase__						=				'''lower newer'''
									lowercase__						=				['''l''', '''o''', '''w''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
									lowercase__						=				tokenizer.tokenize(UpperCamelCase_							)  # , add_prefix_space=True)
									self.assertListEqual(UpperCamelCase_   ,						UpperCamelCase_							)

									lowercase__						=				tokens + [tokenizer.unk_token]
									lowercase__						=				[0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
									self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase_							)   ,						UpperCamelCase_							)

							def 				lowerCamelCase_			(							self:     Optional[int]							)      ->       List[Any]:





									"""simple docstring"""







									lowercase__						=				self.get_tokenizer()

									self.assertListEqual(tokenizer.encode('''Hello world!'''   ,						add_special_tokens=UpperCamelCase_							)   ,						[0, 31_414, 232, 328, 2]							)
									self.assertListEqual(
									    tokenizer.encode('''Hello world! cécé herlolip 418'''   ,						add_special_tokens=UpperCamelCase_							)   ,						[0, 31_414, 232, 328, 740, 1_140, 12_695, 69, 46_078, 1_588, 2]   ,						)

							@slow
							def 				lowerCamelCase_			(							self:     Tuple							)      ->       Union[str, Any]:





									"""simple docstring"""







									lowercase__						=				self.tokenizer_class.from_pretrained('''roberta-base'''							)

									lowercase__						=				tokenizer.encode('''sequence builders'''   ,						add_special_tokens=UpperCamelCase_							)
									lowercase__						=				tokenizer.encode('''multi-sequence build'''   ,						add_special_tokens=UpperCamelCase_							)

									lowercase__						=				tokenizer.encode(
									    '''sequence builders'''   ,						add_special_tokens=UpperCamelCase_   ,						add_prefix_space=UpperCamelCase_							)
									lowercase__						=				tokenizer.encode(
									    '''sequence builders'''   ,						'''multi-sequence build'''   ,						add_special_tokens=UpperCamelCase_   ,						add_prefix_space=UpperCamelCase_							)

									lowercase__						=				tokenizer.build_inputs_with_special_tokens(UpperCamelCase_							)
									lowercase__						=				tokenizer.build_inputs_with_special_tokens(UpperCamelCase_   ,						UpperCamelCase_							)

									assert encoded_sentence == encoded_text_from_decode
									assert encoded_pair == encoded_pair_from_decode

							def 				lowerCamelCase_			(							self:     Tuple							)      ->       Optional[int]:





									"""simple docstring"""







									lowercase__						=				self.get_tokenizer()

									lowercase__						=				'''Encode this sequence.'''
									lowercase__						=				tokenizer.byte_encoder[''' '''.encode('''utf-8'''							)[0]]

									# Testing encoder arguments
									lowercase__						=				tokenizer.encode(UpperCamelCase_   ,						add_special_tokens=UpperCamelCase_   ,						add_prefix_space=UpperCamelCase_							)
									lowercase__						=				tokenizer.convert_ids_to_tokens(encoded[0]							)[0]
									self.assertNotEqual(UpperCamelCase_   ,						UpperCamelCase_							)

									lowercase__						=				tokenizer.encode(UpperCamelCase_   ,						add_special_tokens=UpperCamelCase_   ,						add_prefix_space=UpperCamelCase_							)
									lowercase__						=				tokenizer.convert_ids_to_tokens(encoded[0]							)[0]
									self.assertEqual(UpperCamelCase_   ,						UpperCamelCase_							)

									tokenizer.add_special_tokens({'''bos_token''': '''<s>'''}							)
									lowercase__						=				tokenizer.encode(UpperCamelCase_   ,						add_special_tokens=UpperCamelCase_							)
									lowercase__						=				tokenizer.convert_ids_to_tokens(encoded[1]							)[0]
									self.assertNotEqual(UpperCamelCase_   ,						UpperCamelCase_							)

									# Testing spaces after special tokens
									lowercase__						=				'''<mask>'''
									tokenizer.add_special_tokens(
									    {'''mask_token''': AddedToken(UpperCamelCase_   ,						lstrip=UpperCamelCase_   ,						rstrip=UpperCamelCase_							)}							)  # mask token has a left space
									lowercase__						=				tokenizer.convert_tokens_to_ids(UpperCamelCase_							)

									lowercase__						=				'''Encode <mask> sequence'''
									lowercase__						=				'''Encode <mask>sequence'''

									lowercase__						=				tokenizer.encode(UpperCamelCase_							)
									lowercase__						=				encoded.index(UpperCamelCase_							)
									lowercase__						=				tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]							)[0]
									self.assertEqual(UpperCamelCase_   ,						UpperCamelCase_							)

									lowercase__						=				tokenizer.encode(UpperCamelCase_							)
									lowercase__						=				encoded.index(UpperCamelCase_							)
									lowercase__						=				tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]							)[0]
									self.assertNotEqual(UpperCamelCase_   ,						UpperCamelCase_							)

							def 				lowerCamelCase_			(							self:     Union[str, Any]							)      ->       Optional[int]:





									"""simple docstring"""







									pass

							def 				lowerCamelCase_			(							self:     List[Any]							)      ->       Optional[int]:





									"""simple docstring"""







									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'							):
													lowercase__						=				self.rust_tokenizer_class.from_pretrained(UpperCamelCase_   ,						**UpperCamelCase_							)
													lowercase__						=				self.tokenizer_class.from_pretrained(UpperCamelCase_   ,						**UpperCamelCase_							)
													lowercase__						=				'''A, <mask> AllenNLP sentence.'''
													lowercase__						=				tokenizer_r.encode_plus(UpperCamelCase_   ,						add_special_tokens=UpperCamelCase_   ,						return_token_type_ids=UpperCamelCase_							)
													lowercase__						=				tokenizer_p.encode_plus(UpperCamelCase_   ,						add_special_tokens=UpperCamelCase_   ,						return_token_type_ids=UpperCamelCase_							)

													# token_type_ids should put 0 everywhere
													self.assertEqual(sum(tokens_r['''token_type_ids''']							)   ,						sum(tokens_p['''token_type_ids''']							)							)

													# attention_mask should put 1 everywhere, so sum over length should be 1
													self.assertEqual(
													    sum(tokens_r['''attention_mask''']							) / len(tokens_r['''attention_mask''']							)   ,						sum(tokens_p['''attention_mask''']							) / len(tokens_p['''attention_mask''']							)   ,						)

													lowercase__						=				tokenizer_r.convert_ids_to_tokens(tokens_r['''input_ids''']							)
													lowercase__						=				tokenizer_p.convert_ids_to_tokens(tokens_p['''input_ids''']							)

													# Rust correctly handles the space before the mask while python doesnt
													self.assertSequenceEqual(tokens_p['''input_ids''']   ,						[0, 250, 6, 50_264, 3_823, 487, 21_992, 3_645, 4, 2]							)
													self.assertSequenceEqual(tokens_r['''input_ids''']   ,						[0, 250, 6, 50_264, 3_823, 487, 21_992, 3_645, 4, 2]							)

													self.assertSequenceEqual(
													    UpperCamelCase_   ,						['''<s>''', '''A''', ''',''', '''<mask>''', '''ĠAllen''', '''N''', '''LP''', '''Ġsentence''', '''.''', '''</s>''']							)
													self.assertSequenceEqual(
													    UpperCamelCase_   ,						['''<s>''', '''A''', ''',''', '''<mask>''', '''ĠAllen''', '''N''', '''LP''', '''Ġsentence''', '''.''', '''</s>''']							)

							def 				lowerCamelCase_			(							self:     Optional[int]							)      ->       Optional[int]:





									"""simple docstring"""







									for trim_offsets, add_prefix_space in itertools.product([True, False]   ,						repeat=2							):
											lowercase__						=				self.rust_tokenizer_class.from_pretrained(
											    self.tmpdirname   ,						use_fast=UpperCamelCase_   ,						add_prefix_space=UpperCamelCase_   ,						trim_offsets=UpperCamelCase_							)

											lowercase__						=				json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()							)
											lowercase__						=				json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()							)

											self.assertEqual(pre_tokenizer_state['''add_prefix_space''']   ,						UpperCamelCase_							)

											self.assertEqual(post_processor_state['''add_prefix_space''']   ,						UpperCamelCase_							)
											self.assertEqual(post_processor_state['''trim_offsets''']   ,						UpperCamelCase_							)


							def 				lowerCamelCase_			(							self:     int							)      ->       int:





									"""simple docstring"""







									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'							):
													lowercase__						=				'''hello'''  # `hello` is a token in the vocabulary of `pretrained_name`
													lowercase__						=				f'{text_of_1_token} {text_of_1_token}'

													lowercase__						=				self.rust_tokenizer_class.from_pretrained(
													    UpperCamelCase_   ,						use_fast=UpperCamelCase_   ,						add_prefix_space=UpperCamelCase_   ,						trim_offsets=UpperCamelCase_							)
													lowercase__						=				tokenizer_r(UpperCamelCase_   ,						return_offsets_mapping=UpperCamelCase_   ,						add_special_tokens=UpperCamelCase_							)
													self.assertEqual(encoding.offset_mapping[0]   ,						(0, len(UpperCamelCase_							))							)
													self.assertEqual(
													    encoding.offset_mapping[1]   ,						(len(UpperCamelCase_							) + 1, len(UpperCamelCase_							) + 1 + len(UpperCamelCase_							))   ,						)

													lowercase__						=				self.rust_tokenizer_class.from_pretrained(
													    UpperCamelCase_   ,						use_fast=UpperCamelCase_   ,						add_prefix_space=UpperCamelCase_   ,						trim_offsets=UpperCamelCase_							)
													lowercase__						=				tokenizer_r(UpperCamelCase_   ,						return_offsets_mapping=UpperCamelCase_   ,						add_special_tokens=UpperCamelCase_							)
													self.assertEqual(encoding.offset_mapping[0]   ,						(0, len(UpperCamelCase_							))							)
													self.assertEqual(
													    encoding.offset_mapping[1]   ,						(len(UpperCamelCase_							) + 1, len(UpperCamelCase_							) + 1 + len(UpperCamelCase_							))   ,						)

													lowercase__						=				self.rust_tokenizer_class.from_pretrained(
													    UpperCamelCase_   ,						use_fast=UpperCamelCase_   ,						add_prefix_space=UpperCamelCase_   ,						trim_offsets=UpperCamelCase_							)
													lowercase__						=				tokenizer_r(UpperCamelCase_   ,						return_offsets_mapping=UpperCamelCase_   ,						add_special_tokens=UpperCamelCase_							)
													self.assertEqual(encoding.offset_mapping[0]   ,						(0, len(UpperCamelCase_							))							)
													self.assertEqual(
													    encoding.offset_mapping[1]   ,						(len(UpperCamelCase_							), len(UpperCamelCase_							) + 1 + len(UpperCamelCase_							))   ,						)

													lowercase__						=				self.rust_tokenizer_class.from_pretrained(
													    UpperCamelCase_   ,						use_fast=UpperCamelCase_   ,						add_prefix_space=UpperCamelCase_   ,						trim_offsets=UpperCamelCase_							)
													lowercase__						=				tokenizer_r(UpperCamelCase_   ,						return_offsets_mapping=UpperCamelCase_   ,						add_special_tokens=UpperCamelCase_							)
													self.assertEqual(encoding.offset_mapping[0]   ,						(0, len(UpperCamelCase_							))							)
													self.assertEqual(
													    encoding.offset_mapping[1]   ,						(len(UpperCamelCase_							), len(UpperCamelCase_							) + 1 + len(UpperCamelCase_							))   ,						)

													lowercase__						=				f' {text}'

													# tokenizer_r = self.rust_tokenizer_class.from_pretrained(
													#     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
													# )
													# encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
													# self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
													# self.assertEqual(
													#     encoding.offset_mapping[1],
													#     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
													# )

													lowercase__						=				self.rust_tokenizer_class.from_pretrained(
													    UpperCamelCase_   ,						use_fast=UpperCamelCase_   ,						add_prefix_space=UpperCamelCase_   ,						trim_offsets=UpperCamelCase_							)
													lowercase__						=				tokenizer_r(UpperCamelCase_   ,						return_offsets_mapping=UpperCamelCase_   ,						add_special_tokens=UpperCamelCase_							)
													self.assertEqual(encoding.offset_mapping[0]   ,						(1, 1 + len(UpperCamelCase_							))							)
													self.assertEqual(
													    encoding.offset_mapping[1]   ,						(1 + len(UpperCamelCase_							) + 1, 1 + len(UpperCamelCase_							) + 1 + len(UpperCamelCase_							))   ,						)

													lowercase__						=				self.rust_tokenizer_class.from_pretrained(
													    UpperCamelCase_   ,						use_fast=UpperCamelCase_   ,						add_prefix_space=UpperCamelCase_   ,						trim_offsets=UpperCamelCase_							)
													lowercase__						=				tokenizer_r(UpperCamelCase_   ,						return_offsets_mapping=UpperCamelCase_   ,						add_special_tokens=UpperCamelCase_							)
													self.assertEqual(encoding.offset_mapping[0]   ,						(0, 1 + len(UpperCamelCase_							))							)
													self.assertEqual(
													    encoding.offset_mapping[1]   ,						(1 + len(UpperCamelCase_							), 1 + len(UpperCamelCase_							) + 1 + len(UpperCamelCase_							))   ,						)

													lowercase__						=				self.rust_tokenizer_class.from_pretrained(
													    UpperCamelCase_   ,						use_fast=UpperCamelCase_   ,						add_prefix_space=UpperCamelCase_   ,						trim_offsets=UpperCamelCase_							)
													lowercase__						=				tokenizer_r(UpperCamelCase_   ,						return_offsets_mapping=UpperCamelCase_   ,						add_special_tokens=UpperCamelCase_							)
													self.assertEqual(encoding.offset_mapping[0]   ,						(0, 1 + len(UpperCamelCase_							))							)
													self.assertEqual(
													    encoding.offset_mapping[1]   ,						(1 + len(UpperCamelCase_							), 1 + len(UpperCamelCase_							) + 1 + len(UpperCamelCase_							))   ,						)






import argparse
from collections import defaultdict

import yaml


lowerCAmelCase	=							'docs/source/en/_toctree.yml'
def       _a			(				SCREAMING_SNAKE_CASE					):


		"""simple docstring"""




		lowercase__						=				defaultdict(SCREAMING_SNAKE_CASE					)
		lowercase__						=				[]
		lowercase__						=				[]
		for doc in doc_list:
				if "local" in doc:
						counts[doc["local"]] += 1

				if doc["title"].lower() == "overview":
						overview_doc.append({'''local''': doc['''local'''], '''title''': doc['''title''']}					)
				else:
						new_doc_list.append(SCREAMING_SNAKE_CASE					)

		lowercase__						=				new_doc_list
		lowercase__						=				[key for key, value in counts.items() if value > 1]

		lowercase__						=				[]
		for duplicate_key in duplicates:
				lowercase__						=				list({doc['''title'''] for doc in doc_list if doc['''local'''] == duplicate_key}					)
				if len(SCREAMING_SNAKE_CASE					) > 1:
						raise ValueError(
						    f'{duplicate_key} is present several times in the documentation table of content at '
						    '''`docs/source/en/_toctree.yml` with different *Title* values. Choose one of those and remove the '''
						    '''others.'''					)
				# Only add this once
				new_doc.append({'''local''': duplicate_key, '''title''': titles[0]}					)

		# Add none duplicate-keys
		new_doc.extend([doc for doc in doc_list if '''local''' not in counts or counts[doc['''local''']] == 1]					)
		lowercase__						=				sorted(SCREAMING_SNAKE_CASE						,    key=lambda SCREAMING_SNAKE_CASE					: s["title"].lower()					)

		# "overview" gets special treatment and is always first
		if len(SCREAMING_SNAKE_CASE					) > 1:
				raise ValueError('''{doc_list} has two \'overview\' docs which is not allowed.'''					)

		overview_doc.extend(SCREAMING_SNAKE_CASE					)

		# Sort
		return overview_doc
def       _a			(				SCREAMING_SNAKE_CASE=False					):


		"""simple docstring"""




		with open(SCREAMING_SNAKE_CASE						,    encoding='''utf-8'''					) as f:
				lowercase__						=				yaml.safe_load(f.read()					)

		# Get to the API doc
		lowercase__						=				0
		while content[api_idx]["title"] != "API":
				api_idx += 1
		lowercase__						=				content[api_idx]['''sections''']

		# Then to the model doc
		lowercase__						=				0
		while api_doc[scheduler_idx]["title"] != "Schedulers":
				scheduler_idx += 1

		lowercase__						=				api_doc[scheduler_idx]['''sections''']
		lowercase__						=				clean_doc_toc(SCREAMING_SNAKE_CASE					)

		lowercase__						=				False
		if new_scheduler_doc != scheduler_doc:
				lowercase__						=				True
				if overwrite:
						lowercase__						=				new_scheduler_doc

		if diff:
				if overwrite:
						lowercase__						=				api_doc
						with open(SCREAMING_SNAKE_CASE						,    '''w'''						,    encoding='''utf-8'''					) as f:
								f.write(yaml.dump(SCREAMING_SNAKE_CASE						,    allow_unicode=SCREAMING_SNAKE_CASE					)					)
				else:
						raise ValueError(
						    '''The model doc part of the table of content is not properly sorted, run `make style` to fix this.'''					)
def       _a			(				SCREAMING_SNAKE_CASE=False					):


		"""simple docstring"""




		with open(SCREAMING_SNAKE_CASE						,    encoding='''utf-8'''					) as f:
				lowercase__						=				yaml.safe_load(f.read()					)

		# Get to the API doc
		lowercase__						=				0
		while content[api_idx]["title"] != "API":
				api_idx += 1
		lowercase__						=				content[api_idx]['''sections''']

		# Then to the model doc
		lowercase__						=				0
		while api_doc[pipeline_idx]["title"] != "Pipelines":
				pipeline_idx += 1

		lowercase__						=				False
		lowercase__						=				api_doc[pipeline_idx]['''sections''']
		lowercase__						=				[]

		# sort sub pipeline docs
		for pipeline_doc in pipeline_docs:
				if "section" in pipeline_doc:
						lowercase__						=				pipeline_doc['''section''']
						lowercase__						=				clean_doc_toc(SCREAMING_SNAKE_CASE					)
						if overwrite:
								lowercase__						=				new_sub_pipeline_doc
				new_pipeline_docs.append(SCREAMING_SNAKE_CASE					)

		# sort overall pipeline doc
		lowercase__						=				clean_doc_toc(SCREAMING_SNAKE_CASE					)

		if new_pipeline_docs != pipeline_docs:
				lowercase__						=				True
				if overwrite:
						lowercase__						=				new_pipeline_docs

		if diff:
				if overwrite:
						lowercase__						=				api_doc
						with open(SCREAMING_SNAKE_CASE						,    '''w'''						,    encoding='''utf-8'''					) as f:
								f.write(yaml.dump(SCREAMING_SNAKE_CASE						,    allow_unicode=SCREAMING_SNAKE_CASE					)					)
				else:
						raise ValueError(
						    '''The model doc part of the table of content is not properly sorted, run `make style` to fix this.'''					)


if __name__ == "__main__":
			lowerCAmelCase	=							argparse.ArgumentParser()
			parser.add_argument('--fix_and_overwrite', action='store_true', help='Whether to fix inconsistencies.')
			lowerCAmelCase	=							parser.parse_args()

			check_scheduler_doc(args.fix_and_overwrite)
			check_pipeline_doc(args.fix_and_overwrite)


