


import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class 	_a							(      __A					,     unittest.TestCase      ):
							_lowercase     :      str  				=			MobileBertTokenizer
							_lowercase     :      Tuple  				=			MobileBertTokenizerFast
							_lowercase     :      Optional[int]  				=			True
							_lowercase     :      Tuple  				=			True
							_lowercase     :      Optional[int]  				=			filter_non_english
							_lowercase     :      List[str]  				=			'google/mobilebert-uncased'

							def 				lowerCamelCase_			(							self:     Optional[Any]							)      ->       List[str]:





									"""simple docstring"""







									super().setUp()

									lowercase__						=				[
									    '''[UNK]''',
									    '''[CLS]''',
									    '''[SEP]''',
									    '''[PAD]''',
									    '''[MASK]''',
									    '''want''',
									    '''##want''',
									    '''##ed''',
									    '''wa''',
									    '''un''',
									    '''runn''',
									    '''##ing''',
									    ''',''',
									    '''low''',
									    '''lowest''',
									]
									lowercase__						=				os.path.join(self.tmpdirname   ,						VOCAB_FILES_NAMES['''vocab_file''']							)
									with open(self.vocab_file   ,						'''w'''   ,						encoding='''utf-8'''							) as vocab_writer:
											vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]							)							)

									lowercase__						=				[
									    (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
									    for tokenizer_def in self.tokenizers_list
									]

							def 				lowerCamelCase_			(							self:     Tuple   ,						UpperCamelCase_:     Union[str, Any]							)      ->       Tuple:





									"""simple docstring"""







									lowercase__						=				'''UNwant\u00E9d,running'''
									lowercase__						=				'''unwanted, running'''
									return input_text, output_text

							def 				lowerCamelCase_			(							self:     Tuple							)      ->       Tuple:





									"""simple docstring"""







									lowercase__						=				self.tokenizer_class(self.vocab_file							)

									lowercase__						=				tokenizer.tokenize('''UNwant\u00E9d,running'''							)
									self.assertListEqual(__lowercase   ,						['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']							)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowercase							)   ,						[9, 6, 7, 12, 10, 11]							)

							def 				lowerCamelCase_			(							self:     str							)      ->       str:





									"""simple docstring"""







									if not self.test_rust_tokenizer:
											return

									lowercase__						=				self.get_tokenizer()
									lowercase__						=				self.get_rust_tokenizer()

									lowercase__						=				'''UNwant\u00E9d,running'''

									lowercase__						=				tokenizer.tokenize(__lowercase							)
									lowercase__						=				rust_tokenizer.tokenize(__lowercase							)
									self.assertListEqual(__lowercase   ,						__lowercase							)

									lowercase__						=				tokenizer.encode(__lowercase   ,						add_special_tokens=__lowercase							)
									lowercase__						=				rust_tokenizer.encode(__lowercase   ,						add_special_tokens=__lowercase							)
									self.assertListEqual(__lowercase   ,						__lowercase							)

									lowercase__						=				self.get_rust_tokenizer()
									lowercase__						=				tokenizer.encode(__lowercase							)
									lowercase__						=				rust_tokenizer.encode(__lowercase							)
									self.assertListEqual(__lowercase   ,						__lowercase							)

									# With lower casing
									lowercase__						=				self.get_tokenizer(do_lower_case=__lowercase							)
									lowercase__						=				self.get_rust_tokenizer(do_lower_case=__lowercase							)

									lowercase__						=				'''UNwant\u00E9d,running'''

									lowercase__						=				tokenizer.tokenize(__lowercase							)
									lowercase__						=				rust_tokenizer.tokenize(__lowercase							)
									self.assertListEqual(__lowercase   ,						__lowercase							)

									lowercase__						=				tokenizer.encode(__lowercase   ,						add_special_tokens=__lowercase							)
									lowercase__						=				rust_tokenizer.encode(__lowercase   ,						add_special_tokens=__lowercase							)
									self.assertListEqual(__lowercase   ,						__lowercase							)

									lowercase__						=				self.get_rust_tokenizer()
									lowercase__						=				tokenizer.encode(__lowercase							)
									lowercase__						=				rust_tokenizer.encode(__lowercase							)
									self.assertListEqual(__lowercase   ,						__lowercase							)

							def 				lowerCamelCase_			(							self:     Any							)      ->       Tuple:





									"""simple docstring"""







									lowercase__						=				BasicTokenizer()

									self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''							)   ,						['''ah''', '''\u535A''', '''\u63A8''', '''zz''']							)

							def 				lowerCamelCase_			(							self:     Optional[Any]							)      ->       Optional[Any]:





									"""simple docstring"""







									lowercase__						=				BasicTokenizer(do_lower_case=__lowercase							)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''							)   ,						['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)   ,						['''hello''']							)

							def 				lowerCamelCase_			(							self:     Union[str, Any]							)      ->       List[str]:





									"""simple docstring"""







									lowercase__						=				BasicTokenizer(do_lower_case=__lowercase   ,						strip_accents=__lowercase							)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)   ,						['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)   ,						['''h\u00E9llo''']							)

							def 				lowerCamelCase_			(							self:     List[str]							)      ->       Union[str, Any]:





									"""simple docstring"""







									lowercase__						=				BasicTokenizer(do_lower_case=__lowercase   ,						strip_accents=__lowercase							)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)   ,						['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)   ,						['''hello''']							)

							def 				lowerCamelCase_			(							self:     Optional[Any]							)      ->       str:





									"""simple docstring"""







									lowercase__						=				BasicTokenizer(do_lower_case=__lowercase							)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)   ,						['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)   ,						['''hello''']							)

							def 				lowerCamelCase_			(							self:     Tuple							)      ->       Optional[int]:





									"""simple docstring"""







									lowercase__						=				BasicTokenizer(do_lower_case=__lowercase							)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''							)   ,						['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)

							def 				lowerCamelCase_			(							self:     Optional[int]							)      ->       Union[str, Any]:





									"""simple docstring"""







									lowercase__						=				BasicTokenizer(do_lower_case=__lowercase   ,						strip_accents=__lowercase							)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)   ,						['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)

							def 				lowerCamelCase_			(							self:     Optional[int]							)      ->       Union[str, Any]:





									"""simple docstring"""







									lowercase__						=				BasicTokenizer(do_lower_case=__lowercase   ,						strip_accents=__lowercase							)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)   ,						['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)

							def 				lowerCamelCase_			(							self:     str							)      ->       str:





									"""simple docstring"""







									lowercase__						=				BasicTokenizer(do_lower_case=__lowercase   ,						never_split=['''[UNK]''']							)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''							)   ,						['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']							)

							def 				lowerCamelCase_			(							self:     Union[str, Any]							)      ->       Tuple:





									"""simple docstring"""







									lowercase__						=				['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

									lowercase__						=				{}
									for i, token in enumerate(__lowercase							):
											lowercase__						=				i
									lowercase__						=				WordpieceTokenizer(vocab=__lowercase   ,						unk_token='''[UNK]'''							)

									self.assertListEqual(tokenizer.tokenize(''''''							)   ,						[]							)

									self.assertListEqual(tokenizer.tokenize('''unwanted running'''							)   ,						['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']							)

									self.assertListEqual(tokenizer.tokenize('''unwantedX running'''							)   ,						['''[UNK]''', '''runn''', '''##ing''']							)

							def 				lowerCamelCase_			(							self:     str							)      ->       Any:





									"""simple docstring"""







									self.assertTrue(_is_whitespace(''' '''							)							)
									self.assertTrue(_is_whitespace('''\t'''							)							)
									self.assertTrue(_is_whitespace('''\r'''							)							)
									self.assertTrue(_is_whitespace('''\n'''							)							)
									self.assertTrue(_is_whitespace('''\u00A0'''							)							)

									self.assertFalse(_is_whitespace('''A'''							)							)
									self.assertFalse(_is_whitespace('''-'''							)							)

							def 				lowerCamelCase_			(							self:     Tuple							)      ->       List[Any]:





									"""simple docstring"""







									self.assertTrue(_is_control('''\u0005'''							)							)

									self.assertFalse(_is_control('''A'''							)							)
									self.assertFalse(_is_control(''' '''							)							)
									self.assertFalse(_is_control('''\t'''							)							)
									self.assertFalse(_is_control('''\r'''							)							)

							def 				lowerCamelCase_			(							self:     Dict							)      ->       str:





									"""simple docstring"""







									self.assertTrue(_is_punctuation('''-'''							)							)
									self.assertTrue(_is_punctuation('''$'''							)							)
									self.assertTrue(_is_punctuation('''`'''							)							)
									self.assertTrue(_is_punctuation('''.'''							)							)

									self.assertFalse(_is_punctuation('''A'''							)							)
									self.assertFalse(_is_punctuation(''' '''							)							)

							def 				lowerCamelCase_			(							self:     int							)      ->       Any:





									"""simple docstring"""







									lowercase__						=				self.get_tokenizer()
									lowercase__						=				self.get_rust_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(__lowercase							) for t in ['''Test''', '''\xad''', '''test''']]   ,						[['''[UNK]'''], [], ['''[UNK]''']]							)

									self.assertListEqual(
									    [rust_tokenizer.tokenize(__lowercase							) for t in ['''Test''', '''\xad''', '''test''']]   ,						[['''[UNK]'''], [], ['''[UNK]''']]							)

							@slow
							def 				lowerCamelCase_			(							self:     Tuple							)      ->       Dict:





									"""simple docstring"""







									lowercase__						=				self.tokenizer_class.from_pretrained('''google/mobilebert-uncased'''							)

									lowercase__						=				tokenizer.encode('''sequence builders'''   ,						add_special_tokens=__lowercase							)
									lowercase__						=				tokenizer.encode('''multi-sequence build'''   ,						add_special_tokens=__lowercase							)

									lowercase__						=				tokenizer.build_inputs_with_special_tokens(__lowercase							)
									lowercase__						=				tokenizer.build_inputs_with_special_tokens(__lowercase   ,						__lowercase							)

									assert encoded_sentence == [101] + text + [102]
									assert encoded_pair == [101] + text + [102] + text_a + [102]

							def 				lowerCamelCase_			(							self:     Optional[Any]							)      ->       Any:





									"""simple docstring"""







									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'							):
													lowercase__						=				self.rust_tokenizer_class.from_pretrained(__lowercase   ,						**__lowercase							)

													lowercase__						=				f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
													lowercase__						=				tokenizer_r.encode_plus(
													    __lowercase   ,						return_attention_mask=__lowercase   ,						return_token_type_ids=__lowercase   ,						return_offsets_mapping=__lowercase   ,						add_special_tokens=__lowercase   ,						)

													lowercase__						=				tokenizer_r.do_lower_case if hasattr(__lowercase   ,						'''do_lower_case'''							) else False
													lowercase__						=				(
													    [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), '''A'''),
													        ((1, 2), ''','''),
													        ((3, 5), '''na'''),
													        ((5, 6), '''##ï'''),
													        ((6, 8), '''##ve'''),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), '''Allen'''),
													        ((21, 23), '''##NL'''),
													        ((23, 24), '''##P'''),
													        ((25, 33), '''sentence'''),
													        ((33, 34), '''.'''),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													    if not do_lower_case
													    else [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), '''a'''),
													        ((1, 2), ''','''),
													        ((3, 8), '''naive'''),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), '''allen'''),
													        ((21, 23), '''##nl'''),
													        ((23, 24), '''##p'''),
													        ((25, 33), '''sentence'''),
													        ((33, 34), '''.'''),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													)

													self.assertEqual(
													    [e[1] for e in expected_results]   ,						tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']							)							)
													self.assertEqual([e[0] for e in expected_results]   ,						tokens['''offset_mapping''']							)


							def 				lowerCamelCase_			(							self:     str							)      ->       Dict:





									"""simple docstring"""







									lowercase__						=				['''的''', '''人''', '''有''']
									lowercase__						=				''''''.join(__lowercase							)
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'							):
													lowercase__						=				True
													lowercase__						=				self.tokenizer_class.from_pretrained(__lowercase   ,						**__lowercase							)
													lowercase__						=				self.rust_tokenizer_class.from_pretrained(__lowercase   ,						**__lowercase							)

													lowercase__						=				tokenizer_p.encode(__lowercase   ,						add_special_tokens=__lowercase							)
													lowercase__						=				tokenizer_r.encode(__lowercase   ,						add_special_tokens=__lowercase							)

													lowercase__						=				tokenizer_r.convert_ids_to_tokens(__lowercase							)
													lowercase__						=				tokenizer_p.convert_ids_to_tokens(__lowercase							)

													# it is expected that each Chinese character is not preceded by "##"
													self.assertListEqual(__lowercase   ,						__lowercase							)
													self.assertListEqual(__lowercase   ,						__lowercase							)

													lowercase__						=				False
													lowercase__						=				self.rust_tokenizer_class.from_pretrained(__lowercase   ,						**__lowercase							)
													lowercase__						=				self.tokenizer_class.from_pretrained(__lowercase   ,						**__lowercase							)

													lowercase__						=				tokenizer_r.encode(__lowercase   ,						add_special_tokens=__lowercase							)
													lowercase__						=				tokenizer_p.encode(__lowercase   ,						add_special_tokens=__lowercase							)

													lowercase__						=				tokenizer_r.convert_ids_to_tokens(__lowercase							)
													lowercase__						=				tokenizer_p.convert_ids_to_tokens(__lowercase							)

													# it is expected that only the first Chinese character is not preceded by "##".
													lowercase__						=				[
													    f'##{token}' if idx != 0 else token for idx, token in enumerate(__lowercase							)
													]
													self.assertListEqual(__lowercase   ,						__lowercase							)
													self.assertListEqual(__lowercase   ,						__lowercase							)






from abc import ABC, abstractmethod
from typing import List, Optional


class 	_a							(      UpperCamelCase__      ):

							def __init__(							self:     Optional[int]							)      ->       Union[str, Any]:





									"""simple docstring"""







									self.test()

							def 				lowerCamelCase_			(							self:     List[Any]							)      ->       List[Any]:





									"""simple docstring"""







									lowercase__						=				0
									lowercase__						=				False
									while not completed:
											if counter == 1:
													self.reset()
											lowercase__						=				self.advance()
											if not self.does_advance(UpperCamelCase_							):
													raise Exception(
													    '''Custom Constraint is not defined correctly. self.does_advance(self.advance()) must be true.'''							)

											lowercase__					,		lowercase__					,		lowercase__						=				self.update(UpperCamelCase_							)
											counter += 1

											if counter > 10_000:
													raise Exception('''update() does not fulfill the constraint.'''							)

									if self.remaining() != 0:
											raise Exception('''Custom Constraint is not defined correctly.'''							)

							@abstractmethod
							def 				lowerCamelCase_			(							self:     int							)      ->       Any:





									"""simple docstring"""







									raise NotImplementedError(
									    f'{self.__class__} is an abstract class. Only classes inheriting this class can be called.'							)

							@abstractmethod
							def 				lowerCamelCase_			(							self:     Tuple   ,						UpperCamelCase_:     int							)      ->       Optional[int]:





									"""simple docstring"""







									raise NotImplementedError(
									    f'{self.__class__} is an abstract class. Only classes inheriting this class can be called.'							)

							@abstractmethod
							def 				lowerCamelCase_			(							self:     Optional[int]   ,						UpperCamelCase_:     int							)      ->       str:





									"""simple docstring"""







									raise NotImplementedError(
									    f'{self.__class__} is an abstract class. Only classes inheriting this class can be called.'							)

							@abstractmethod
							def 				lowerCamelCase_			(							self:     Tuple							)      ->       List[Any]:





									"""simple docstring"""







									raise NotImplementedError(
									    f'{self.__class__} is an abstract class. Only classes inheriting this class can be called.'							)

							@abstractmethod
							def 				lowerCamelCase_			(							self:     Any							)      ->       Tuple:





									"""simple docstring"""







									raise NotImplementedError(
									    f'{self.__class__} is an abstract class. Only classes inheriting this class can be called.'							)


							@abstractmethod
							def 				lowerCamelCase_			(							self:     Tuple   ,						UpperCamelCase_:     int=False							)      ->       Any:





									"""simple docstring"""







									raise NotImplementedError(
									    f'{self.__class__} is an abstract class. Only classes inheriting this class can be called.'							)


class 	_a							(      UpperCamelCase__      ):

							def __init__(							self:     str   ,						UpperCamelCase_:     List[int]							)      ->       Tuple:





									"""simple docstring"""







									super(UpperCamelCase_   ,						self							).__init__()

									if not isinstance(UpperCamelCase_   ,						UpperCamelCase_							) or len(UpperCamelCase_							) == 0:
											raise ValueError(f'`token_ids` has to be a non-empty list, but is {token_ids}.'							)
									if any((not isinstance(UpperCamelCase_   ,						UpperCamelCase_							) or token_id < 0) for token_id in token_ids							):
											raise ValueError(f'Each list in `token_ids` has to be a list of positive integers, but is {token_ids}.'							)

									lowercase__						=				token_ids

									lowercase__						=				len(self.token_ids							)
									lowercase__						=				-1  # the index of the currently fulfilled step
									lowercase__						=				False

							def 				lowerCamelCase_			(							self:     Tuple							)      ->       Tuple:





									"""simple docstring"""







									if self.completed:
											return None
									return self.token_ids[self.fulfilled_idx + 1]

							def 				lowerCamelCase_			(							self:     Union[str, Any]   ,						UpperCamelCase_:     int							)      ->       Optional[Any]:





									"""simple docstring"""







									if not isinstance(UpperCamelCase_   ,						UpperCamelCase_							):
											raise ValueError(f'`token_id` has to be an `int`, but is {token_id} of type {type(UpperCamelCase_							)}'							)

									if self.completed:
											return False

									return token_id == self.token_ids[self.fulfilled_idx + 1]

							def 				lowerCamelCase_			(							self:     Optional[int]   ,						UpperCamelCase_:     int							)      ->       Dict:





									"""simple docstring"""







									if not isinstance(UpperCamelCase_   ,						UpperCamelCase_							):
											raise ValueError(f'`token_id` has to be an `int`, but is {token_id} of type {type(UpperCamelCase_							)}'							)

									lowercase__						=				False
									lowercase__						=				False
									lowercase__						=				False

									if self.does_advance(UpperCamelCase_							):
											self.fulfilled_idx += 1
											lowercase__						=				True
											if self.fulfilled_idx == (self.seqlen - 1):
													lowercase__						=				True
											lowercase__						=				completed
									else:
											# failed to make progress.
											lowercase__						=				True
											self.reset()
									return stepped, completed, reset

							def 				lowerCamelCase_			(							self:     Tuple							)      ->       int:





									"""simple docstring"""







									lowercase__						=				False
									lowercase__						=				0

							def 				lowerCamelCase_			(							self:     Any							)      ->       Dict:





									"""simple docstring"""







									return self.seqlen - (self.fulfilled_idx + 1)


							def 				lowerCamelCase_			(							self:     Any   ,						UpperCamelCase_:     Union[str, Any]=False							)      ->       Tuple:





									"""simple docstring"""







									lowercase__						=				PhrasalConstraint(self.token_ids							)

									if stateful:
											lowercase__						=				self.seqlen
											lowercase__						=				self.fulfilled_idx
											lowercase__						=				self.completed

									return new_constraint


class 	_a							:

							def __init__(							self:     Union[str, Any]   ,						UpperCamelCase_:     List[List[int]]   ,						UpperCamelCase_:     int=True							)      ->       int:





									"""simple docstring"""







									lowercase__						=				max([len(UpperCamelCase_							) for one in nested_token_ids]							)

									lowercase__						=				{}
									for token_ids in nested_token_ids:
											lowercase__						=				root
											for tidx, token_id in enumerate(UpperCamelCase_							):
													if token_id not in level:
															lowercase__						=				{}

													lowercase__						=				level[token_id]

									if no_subsets and self.has_subsets(UpperCamelCase_   ,						UpperCamelCase_							):
											raise ValueError(
											    '''Each list in `nested_token_ids` can\'t be a complete subset of another list, but is'''
											    f' {nested_token_ids}.'							)

									lowercase__						=				root

							def 				lowerCamelCase_			(							self:     int   ,						UpperCamelCase_:     str							)      ->       Any:





									"""simple docstring"""







									lowercase__						=				self.trie

									for current_token in current_seq:
											lowercase__						=				start[current_token]

									lowercase__						=				list(start.keys()							)

									return next_tokens

							def 				lowerCamelCase_			(							self:     Any   ,						UpperCamelCase_:     Tuple							)      ->       Optional[Any]:





									"""simple docstring"""







									lowercase__						=				self.next_tokens(UpperCamelCase_							)

									return len(UpperCamelCase_							) == 0

							def 				lowerCamelCase_			(							self:     List[str]   ,						UpperCamelCase_:     Optional[Any]							)      ->       Any:





									"""simple docstring"""







									lowercase__						=				list(root.values()							)
									if len(UpperCamelCase_							) == 0:
											return 1
									else:
											return sum([self.count_leaves(UpperCamelCase_							) for nn in next_nodes]							)


							def 				lowerCamelCase_			(							self:     Any   ,						UpperCamelCase_:     Union[str, Any]   ,						UpperCamelCase_:     int							)      ->       Tuple:





									"""simple docstring"""







									lowercase__						=				self.count_leaves(UpperCamelCase_							)
									return len(UpperCamelCase_							) != leaf_count


class 	_a							(      UpperCamelCase__      ):

							def __init__(							self:     Optional[int]   ,						UpperCamelCase_:     List[List[int]]							)      ->       List[Any]:





									"""simple docstring"""







									super(UpperCamelCase_   ,						self							).__init__()

									if not isinstance(UpperCamelCase_   ,						UpperCamelCase_							) or len(UpperCamelCase_							) == 0:
											raise ValueError(f'`nested_token_ids` has to be a non-empty list, but is {nested_token_ids}.'							)
									if any(not isinstance(UpperCamelCase_   ,						UpperCamelCase_							) for token_ids in nested_token_ids							):
											raise ValueError(f'`nested_token_ids` has to be a list of lists, but is {nested_token_ids}.'							)
									if any(
									    any((not isinstance(UpperCamelCase_   ,						UpperCamelCase_							) or token_id < 0) for token_id in token_ids							)
									    for token_ids in nested_token_ids							):
											raise ValueError(
											    f'Each list in `nested_token_ids` has to be a list of positive integers, but is {nested_token_ids}.'							)

									lowercase__						=				DisjunctiveTrie(UpperCamelCase_							)
									lowercase__						=				nested_token_ids

									lowercase__						=				self.trie.max_height
									lowercase__						=				[]
									lowercase__						=				False

							def 				lowerCamelCase_			(							self:     Union[str, Any]							)      ->       str:





									"""simple docstring"""







									lowercase__						=				self.trie.next_tokens(self.current_seq							)

									if len(UpperCamelCase_							) == 0:
											return None
									else:
											return token_list

							def 				lowerCamelCase_			(							self:     str   ,						UpperCamelCase_:     int							)      ->       Tuple:





									"""simple docstring"""







									if not isinstance(UpperCamelCase_   ,						UpperCamelCase_							):
											raise ValueError(f'`token_id` is supposed to be type `int`, but is {token_id} of type {type(UpperCamelCase_							)}'							)

									lowercase__						=				self.trie.next_tokens(self.current_seq							)

									return token_id in next_tokens

							def 				lowerCamelCase_			(							self:     int   ,						UpperCamelCase_:     int							)      ->       Dict:





									"""simple docstring"""







									if not isinstance(UpperCamelCase_   ,						UpperCamelCase_							):
											raise ValueError(f'`token_id` is supposed to be type `int`, but is {token_id} of type {type(UpperCamelCase_							)}'							)

									lowercase__						=				False
									lowercase__						=				False
									lowercase__						=				False

									if self.does_advance(UpperCamelCase_							):
											self.current_seq.append(UpperCamelCase_							)
											lowercase__						=				True
									else:
											lowercase__						=				True
											self.reset()

									lowercase__						=				self.trie.reached_leaf(self.current_seq							)
									lowercase__						=				completed

									return stepped, completed, reset

							def 				lowerCamelCase_			(							self:     List[str]							)      ->       List[Any]:





									"""simple docstring"""







									lowercase__						=				False
									lowercase__						=				[]

							def 				lowerCamelCase_			(							self:     List[str]							)      ->       Optional[Any]:





									"""simple docstring"""







									if self.completed:
											# since this can be completed without reaching max height
											return 0
									else:
											return self.seqlen - len(self.current_seq							)


							def 				lowerCamelCase_			(							self:     Any   ,						UpperCamelCase_:     Optional[Any]=False							)      ->       str:





									"""simple docstring"""







									lowercase__						=				DisjunctiveConstraint(self.token_ids							)

									if stateful:
											lowercase__						=				self.seqlen
											lowercase__						=				self.current_seq
											lowercase__						=				self.completed

									return new_constraint


class 	_a							:

							def __init__(							self:     int   ,						UpperCamelCase_:     List[Constraint]							)      ->       Dict:





									"""simple docstring"""







									lowercase__						=				constraints

									# max # of steps required to fulfill a given constraint
									lowercase__						=				max([c.seqlen for c in constraints]							)
									lowercase__						=				len(UpperCamelCase_							)
									lowercase__						=				False

									self.init_state()

							def 				lowerCamelCase_			(							self:     List[str]							)      ->       str:





									"""simple docstring"""







									lowercase__						=				[]
									lowercase__						=				None
									lowercase__						=				[constraint.copy(stateful=UpperCamelCase_							) for constraint in self.constraints]

							def 				lowerCamelCase_			(							self:     str							)      ->       Any:





									"""simple docstring"""







									lowercase__						=				0
									if self.inprogress_constraint:
											# extra points for having a constraint mid-fulfilled
											add += self.max_seqlen - self.inprogress_constraint.remaining()

									return (len(self.complete_constraints							) * self.max_seqlen) + add

							def 				lowerCamelCase_			(							self:     Union[str, Any]							)      ->       List[Any]:





									"""simple docstring"""







									lowercase__						=				[]
									if self.inprogress_constraint is None:
											for constraint in self.pending_constraints:  # "pending" == "unfulfilled yet"
													lowercase__						=				constraint.advance()
													if isinstance(UpperCamelCase_   ,						UpperCamelCase_							):
															token_list.append(UpperCamelCase_							)
													elif isinstance(UpperCamelCase_   ,						UpperCamelCase_							):
															token_list.extend(UpperCamelCase_							)
									else:
											lowercase__						=				self.inprogress_constraint.advance()
											if isinstance(UpperCamelCase_   ,						UpperCamelCase_							):
													token_list.append(UpperCamelCase_							)
											elif isinstance(UpperCamelCase_   ,						UpperCamelCase_							):
													token_list.extend(UpperCamelCase_							)

									if len(UpperCamelCase_							) == 0:
											return None
									else:
											return token_list

							def 				lowerCamelCase_			(							self:     Optional[Any]   ,						UpperCamelCase_:     Optional[List[int]]							)      ->       Optional[int]:





									"""simple docstring"""







									self.init_state()

									if token_ids is not None:
											for token in token_ids:
													# completes or steps **one** constraint
													lowercase__					,		lowercase__						=				self.add(UpperCamelCase_							)

													# the entire list of constraints are fulfilled
													if self.completed:
															break

							def 				lowerCamelCase_			(							self:     Optional[Any]   ,						UpperCamelCase_:     int							)      ->       int:





									"""simple docstring"""







									if not isinstance(UpperCamelCase_   ,						UpperCamelCase_							):
											raise ValueError(f'`token_id` should be an `int`, but is `{token_id}`.'							)

									lowercase__					,		lowercase__						=				False, False

									if self.completed:
											lowercase__						=				True
											lowercase__						=				False
											return complete, stepped

									if self.inprogress_constraint is not None:
											# In the middle of fulfilling a constraint. If the `token_id` *does* makes an incremental progress to current
											# job, simply update the state

											lowercase__					,		lowercase__					,		lowercase__						=				self.inprogress_constraint.update(UpperCamelCase_							)
											if reset:
													# 1. If the next token breaks the progress, then we must restart.
													#     e.g. constraint = "I love pies" and sequence so far is "I love" but `token_id` == "books".

													#     But that doesn't mean we self.init_state(), since we only reset the state for this particular
													#     constraint, not the full list of constraints.

													self.pending_constraints.append(self.inprogress_constraint.copy(stateful=UpperCamelCase_							)							)
													lowercase__						=				None

											if complete:
													# 2. If the next token completes the constraint, move it to completed list, set
													#     inprogress to None. If there are no pending constraints either, then this full list of constraints
													#     is complete.

													self.complete_constraints.append(self.inprogress_constraint							)
													lowercase__						=				None

													if len(self.pending_constraints							) == 0:
															# we're done!
															lowercase__						=				True

									else:
											# Not in the middle of fulfilling a constraint. So does this `token_id` helps us step towards any of our list
											# of constraints?

											for cidx, pending_constraint in enumerate(self.pending_constraints							):
													if pending_constraint.does_advance(UpperCamelCase_							):
															lowercase__					,		lowercase__					,		lowercase__						=				pending_constraint.update(UpperCamelCase_							)

															if not stepped:
																	raise Exception(
																	    '''`constraint.update(token_id)` is not yielding incremental progress, '''
																	    '''even though `constraint.does_advance(token_id)` is true.'''							)

															if complete:
																	self.complete_constraints.append(UpperCamelCase_							)
																	lowercase__						=				None

															if not complete and stepped:
																	lowercase__						=				pending_constraint

															if complete or stepped:
																	# If we made any progress at all, then it's at least not a "pending constraint".

																	lowercase__						=				(
																	    self.pending_constraints[:cidx] + self.pending_constraints[cidx + 1 :]
																	)

																	if len(self.pending_constraints							) == 0 and self.inprogress_constraint is None:
																			# If there's no longer any pending after this and no inprogress either, then we must be
																			# complete.

																			lowercase__						=				True

																	break  # prevent accidentally stepping through multiple constraints with just one token.

									return complete, stepped


							def 				lowerCamelCase_			(							self:     Any   ,						UpperCamelCase_:     Optional[Any]=True							)      ->       Dict:





									"""simple docstring"""







									lowercase__						=				ConstraintListState(self.constraints							)  # we actually never though self.constraints objects
									# throughout this process. So it's at initialization state.

									if stateful:
											lowercase__						=				[
											    constraint.copy(stateful=UpperCamelCase_							) for constraint in self.complete_constraints
											]
											if self.inprogress_constraint is not None:
													lowercase__						=				self.inprogress_constraint.copy(stateful=UpperCamelCase_							)
											lowercase__						=				[constraint.copy() for constraint in self.pending_constraints]

									return new_state


