"""simple docstring"""







def   __UpperCAmelCase    (   UpperCAmelCase_   :    str      ,							UpperCAmelCase_   :    int  )							->      list:

	'''simple docstring'''





	__snake_case      :							str						=			word.split()

	def justify(UpperCAmelCase_   :    list      ,							UpperCAmelCase_   :    int      ,							UpperCAmelCase_   :    int  ) -> str:
		__snake_case      :							Tuple						=			max_width - width
		__snake_case      :							List[str]						=			len(UpperCAmelCase_  )
		if len(UpperCAmelCase_  ) == 1:
			# if there is only word in line
			# just insert overall_spaces_count for the remainder of line
			return line[0] + " " * overall_spaces_count
		else:
			__snake_case      :							List[str]						=			words_count - 1
			# num_spaces_between_words_list[i] : tells you to insert
			# num_spaces_between_words_list[i] spaces
			# after word on line[i]
			__snake_case      :							Tuple						=			spaces_to_insert_between_words * [
			    overall_spaces_count // spaces_to_insert_between_words
			]
			__snake_case      :							Tuple						=			(
			    overall_spaces_count % spaces_to_insert_between_words
			)
			# distribute spaces via round robin to the left words
			for i in range(UpperCAmelCase_  ):
				num_spaces_between_words_list[i] += 1
			__snake_case      :							List[str]						=			[]
			for i in range(UpperCAmelCase_  ):
				# add the word
				aligned_words_list.append(line[i]  )
				# add the spaces to insert
				aligned_words_list.append(num_spaces_between_words_list[i] * ' '  )
			# just add the last word to the sentence
			aligned_words_list.append(line[-1]  )
			# join the aligned words list to form a justified line
			return "".join(UpperCAmelCase_  )

	__snake_case      :							str						=			[]
	__snake_case      :							list[str]						=			[]
	__snake_case      :							Optional[int]						=			0
	for word in words:
		if width + len(UpperCAmelCase_  ) + len(UpperCAmelCase_  ) <= max_width:
			# keep adding words until we can fill out max_width
			# width = sum of length of all words (without overall_spaces_count)
			# len(word) = length of current word
			# len(line) = number of overall_spaces_count to insert between words
			line.append(UpperCAmelCase_  )
			width += len(UpperCAmelCase_  )
		else:
			# justify the line and add it to result
			answer.append(justify(UpperCAmelCase_      ,							UpperCAmelCase_      ,							UpperCAmelCase_  )  )
			# reset new line and new width
			__snake_case		,    __snake_case      :							Dict						=			[word], len(UpperCAmelCase_  )
	__snake_case      :							Dict						=			max_width - width - len(UpperCAmelCase_  )
	answer.append(' '.join(UpperCAmelCase_  ) + (remaining_spaces + 1) * ' '  )
	return answer


if __name__ == "__main__":
	from doctest import testmod

	testmod()






"""simple docstring"""







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


_a					:					Any=			logging.get_logger(__name__)

_a					:					str=			{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_a					:					Optional[Any]=			[
    "small",
    "small-base",
    "medium",
    "medium-base",
    "intermediate",
    "intermediate-base",
    "large",
    "large-base",
    "xlarge",
    "xlarge-base",
]

_a					:					List[Any]=			{
    "vocab_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt",
        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt",
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt",
        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt",
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json",
        "funnel-transformer/small-base": (
            "https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json",
        "funnel-transformer/large-base": (
            "https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"
        ),
    },
}
_a					:					str=			{f'''funnel-transformer/{name}''': 512 for name in _model_names}
_a					:					List[Any]=			{f'''funnel-transformer/{name}''': {"do_lower_case": True} for name in _model_names}

class 		UpperCamelCase      (   lowercase						):
				UpperCAmelCase    :				List[str]   =		VOCAB_FILES_NAMES
				UpperCAmelCase    :				Tuple   =		PRETRAINED_VOCAB_FILES_MAP
				UpperCAmelCase    :				int   =		PRETRAINED_INIT_CONFIGURATION
				UpperCAmelCase    :				Tuple   =		FunnelTokenizer
				UpperCAmelCase    :				Tuple   =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				UpperCAmelCase    :				int   =		2



				def __init__(self    :				int     ,  _A    :				Any=None     ,  _A    :				Union[str, Any]=None     ,  _A    :				Union[str, Any]=True     ,  _A    :				List[str]="<unk>"     ,  _A    :				Any="<sep>"     ,  _A    :				Dict="<pad>"     ,  _A    :				Tuple="<cls>"     ,  _A    :				Dict="<mask>"     ,  _A    :				Optional[Any]="<s>"     ,  _A    :				List[Any]="</s>"     ,  _A    :				Optional[int]=True     ,  _A    :				Dict=True     ,  _A    :				Tuple=None     ,  _A    :				int="##"     ,  **_A    :				Any     ,  )   ->     str:
					super().__init__(
					    _A     ,  tokenizer_file=_A     ,  do_lower_case=_A     ,  unk_token=_A     ,  sep_token=_A     ,  pad_token=_A     ,  cls_token=_A     ,  mask_token=_A     ,  bos_token=_A     ,  eos_token=_A     ,  clean_text=_A     ,  tokenize_chinese_chars=_A     ,  strip_accents=_A     ,  wordpieces_prefix=_A     ,  **_A     ,  )

					__snake_case      :							Optional[int]						=			json.loads(self.backend_tokenizer.normalizer.__getstate__())
					if (
					    normalizer_state.get('lowercase'     ,  _A) != do_lower_case
					    or normalizer_state.get('strip_accents'     ,  _A) != strip_accents
					    or normalizer_state.get('handle_chinese_chars'     ,  _A) != tokenize_chinese_chars
					):
						__snake_case      :							List[str]						=			getattr(_A     ,  normalizer_state.pop('type'))
						__snake_case      :							int						=			do_lower_case
						__snake_case      :							Optional[int]						=			strip_accents
						__snake_case      :							str						=			tokenize_chinese_chars
						__snake_case      :							Optional[int]						=			normalizer_class(**_A)

					__snake_case      :							str						=			do_lower_case



				def  _lowercase					(self    :				Optional[Any]     ,  _A    :				Dict     ,  _A    :				Tuple=None)   ->     Any:
					__snake_case      :							List[Any]						=			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

					if token_ids_a:
						output += token_ids_a + [self.sep_token_id]

					return output



				def  _lowercase					(self    :				str     ,  _A    :				List[int]     ,  _A    :				Optional[List[int]] = None)   ->     List[int]:
					__snake_case      :							Union[str, Any]						=			[self.sep_token_id]
					__snake_case      :							Union[str, Any]						=			[self.cls_token_id]
					if token_ids_a is None:
						return len(cls) * [self.cls_token_type_id] + len(token_ids_a + sep) * [0]
					return len(cls) * [self.cls_token_type_id] + len(token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]






				def  _lowercase					(self    :				Tuple     ,  _A    :				str     ,  _A    :				Optional[str] = None)   ->     Tuple[str]:
					__snake_case      :							int						=			self._tokenizer.model.save(_A     ,  name=_A)
					return tuple(_A)





