


'''simple docstring'''



from manim import *






class 	UpperCamelCase__(       lowerCAmelCase		):




				def 					a__(		self      :			str				)->					int:

										"""simple docstring"""




										UpperCAmelCase					    =     Rectangle(height=0.5				,   width=0.5				)
										UpperCAmelCase					    =     Rectangle(height=0.46				,   width=0.46				).set_stroke(width=0				)

										UpperCAmelCase					    =     [mem.copy() for i in range(6				)]
										UpperCAmelCase					    =     [mem.copy() for i in range(6				)]
										UpperCAmelCase					    =     VGroup(*lowerCAmelCase				).arrange(lowerCAmelCase				,   buff=0				)
										UpperCAmelCase					    =     VGroup(*lowerCAmelCase				).arrange(lowerCAmelCase				,   buff=0				)
										UpperCAmelCase					    =     VGroup(lowerCAmelCase				,   lowerCAmelCase				).arrange(lowerCAmelCase				,   buff=0				)
										UpperCAmelCase					    =     Text('''CPU'''				,   font_size=24				)
										UpperCAmelCase					    =     Group(lowerCAmelCase				,   lowerCAmelCase				).arrange(lowerCAmelCase				,   buff=0.5				,   aligned_edge=lowerCAmelCase				)
										cpu.move_to([-2.5, -0.5, 0]				)
										self.add(lowerCAmelCase				)

										UpperCAmelCase					    =     [mem.copy() for i in range(1				)]
										UpperCAmelCase					    =     VGroup(*lowerCAmelCase				).arrange(lowerCAmelCase				,   buff=0				)
										UpperCAmelCase					    =     Text('''GPU'''				,   font_size=24				)
										UpperCAmelCase					    =     Group(lowerCAmelCase				,   lowerCAmelCase				).arrange(lowerCAmelCase				,   buff=0.5				,   aligned_edge=lowerCAmelCase				)
										gpu.align_to(lowerCAmelCase				,   lowerCAmelCase				)
										gpu.set_x(gpu.get_x() - 1				)

										self.add(lowerCAmelCase				)

										UpperCAmelCase					    =     [mem.copy() for i in range(6				)]
										UpperCAmelCase					    =     VGroup(*lowerCAmelCase				).arrange(lowerCAmelCase				,   buff=0				)

										UpperCAmelCase					    =     Text('''Model'''				,   font_size=24				)
										UpperCAmelCase					    =     Group(lowerCAmelCase				,   lowerCAmelCase				).arrange(lowerCAmelCase				,   buff=0.5				,   aligned_edge=lowerCAmelCase				)
										model.move_to([3, -1.0, 0]				)

										self.play(
										    Create(lowerCAmelCase				,   run_time=1				)				,   Create(lowerCAmelCase				,   run_time=1				)				,   Create(lowerCAmelCase				,   run_time=1				)				,   )

										UpperCAmelCase					    =     MarkupText(
										    F"""First, an empty model skeleton is loaded\ninto <span fgcolor='{YELLOW}'>memory</span> without using much RAM."""				,   font_size=24				,   )

										UpperCAmelCase					    =     Square(side_length=2.2				)
										key.move_to([-5, 2, 0]				)

										UpperCAmelCase					    =     MarkupText(
										    F"""<b>Key:</b>\n\n<span fgcolor='{YELLOW}'>●</span> Empty Model"""				,   font_size=18				,   )

										key_text.move_to([-5, 2.4, 0]				)

										step_a.move_to([2, 2, 0]				)
										self.play(Write(lowerCAmelCase				,   run_time=2.5				)				,   Write(lowerCAmelCase				)				,   Write(lowerCAmelCase				)				)

										self.add(lowerCAmelCase				)

										UpperCAmelCase					    =     []
										UpperCAmelCase					    =     []
										UpperCAmelCase					    =     []
										for i, rect in enumerate(lowerCAmelCase				):
																UpperCAmelCase					    =     Rectangle(height=0.46				,   width=0.46				).set_stroke(width=0.0				).set_fill(lowerCAmelCase				,   opacity=0.7				)
																cpu_target.move_to(lowerCAmelCase				)
																cpu_target.generate_target()
																UpperCAmelCase					    =     0.46 / 4
																UpperCAmelCase					    =     0.46 / 3

																if i == 0:
																						cpu_target.target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT				)				,   buff=0.02				,   direction=lowerCAmelCase				)
																						cpu_target.target.set_x(cpu_target.target.get_x() + 0.1				)
																elif i == 3:
																						cpu_target.target.next_to(cpu_targs[0].target				,   direction=lowerCAmelCase				,   buff=0.0				)
																else:
																						cpu_target.target.next_to(cpu_targs[i - 1].target				,   direction=lowerCAmelCase				,   buff=0.0				)
																cpu_targs.append(lowerCAmelCase				)

																first_animations.append(rect.animate(run_time=0.5				).set_stroke(lowerCAmelCase				)				)
																second_animations.append(MoveToTarget(lowerCAmelCase				,   run_time=1.5				)				)

										self.play(*lowerCAmelCase				)
										self.play(*lowerCAmelCase				)

										self.wait()









'''simple docstring'''



import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
	from transformers.pipelines.conversational import Conversation


_lowercase      :      str            =					logging.get_logger(__name__)

_lowercase      :      Any            =					{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

_lowercase      :      Tuple            =					{
    """vocab_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/vocab.json""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/vocab.json""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/vocab.json""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/vocab.json""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/merges.txt""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/merges.txt""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/merges.txt""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/merges.txt""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/tokenizer.json""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/tokenizer.json""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/tokenizer.json""",
    },
}

_lowercase      :      Union[str, Any]            =					{
    """gpt2""": 1024,
    """gpt2-medium""": 1024,
    """gpt2-large""": 1024,
    """gpt2-xl""": 1024,
    """distilgpt2""": 1024,
}






class 	UpperCamelCase__(       lowerCAmelCase	):
				__magic_name__    :				Dict  		=	VOCAB_FILES_NAMES
				__magic_name__    :				str  		=	PRETRAINED_VOCAB_FILES_MAP
				__magic_name__    :				Any  		=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__magic_name__    :				List[Any]  		=	["input_ids", "attention_mask"]
				__magic_name__    :				List[Any]  		=	GPTaTokenizer




				def __init__(		self      :			Tuple				,   lowerCAmelCase      :			List[Any]=None				,   lowerCAmelCase      :			Dict=None				,   lowerCAmelCase      :			Dict=None				,   lowerCAmelCase      :			Tuple="<|endoftext|>"				,   lowerCAmelCase      :			Union[str, Any]="<|endoftext|>"				,   lowerCAmelCase      :			Union[str, Any]="<|endoftext|>"				,   lowerCAmelCase      :			Optional[int]=False				,   **lowerCAmelCase      :			Tuple				,   )->					int:

										"""simple docstring"""




										super().__init__(
										    lowerCAmelCase				,   lowerCAmelCase				,   tokenizer_file=lowerCAmelCase				,   unk_token=lowerCAmelCase				,   bos_token=lowerCAmelCase				,   eos_token=lowerCAmelCase				,   add_prefix_space=lowerCAmelCase				,   **lowerCAmelCase				,   )

										UpperCAmelCase					    =     kwargs.pop('''add_bos_token'''				,   lowerCAmelCase				)

										UpperCAmelCase					    =     json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()				)
										if pre_tok_state.get('''add_prefix_space'''				,   lowerCAmelCase				) != add_prefix_space:
																UpperCAmelCase					    =     getattr(lowerCAmelCase				,   pre_tok_state.pop('''type'''				)				)
																UpperCAmelCase					    =     add_prefix_space
																UpperCAmelCase					    =     pre_tok_class(**lowerCAmelCase				)

										UpperCAmelCase					    =     add_prefix_space




				def 					a__(		self      :			Union[str, Any]				,   *lowerCAmelCase      :			Union[str, Any]				,   **lowerCAmelCase      :			Dict				)->					BatchEncoding:

										"""simple docstring"""




										UpperCAmelCase					    =     kwargs.get('''is_split_into_words'''				,   lowerCAmelCase				)
										assert self.add_prefix_space or not is_split_into_words, (
										    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
										    "to use it with pretokenized inputs."
										)

										return super()._batch_encode_plus(*lowerCAmelCase				,   **lowerCAmelCase				)




				def 					a__(		self      :			List[str]				,   *lowerCAmelCase      :			Any				,   **lowerCAmelCase      :			Tuple				)->					BatchEncoding:

										"""simple docstring"""




										UpperCAmelCase					    =     kwargs.get('''is_split_into_words'''				,   lowerCAmelCase				)

										assert self.add_prefix_space or not is_split_into_words, (
										    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
										    "to use it with pretokenized inputs."
										)

										return super()._encode_plus(*lowerCAmelCase				,   **lowerCAmelCase				)




				def 					a__(		self      :			str				,   lowerCAmelCase      :			str				,   lowerCAmelCase      :			Optional[str] = None				)->					Tuple[str]:

										"""simple docstring"""




										UpperCAmelCase					    =     self._tokenizer.model.save(lowerCAmelCase				,   name=lowerCAmelCase				)
										return tuple(lowerCAmelCase				)





				def 					a__(		self      :			List[Any]				,   lowerCAmelCase      :			"Conversation"				)->					List[int]:

										"""simple docstring"""




										UpperCAmelCase					    =     []
										for is_user, text in conversation.iter_texts():
																input_ids.extend(self.encode(lowerCAmelCase				,   add_special_tokens=lowerCAmelCase				) + [self.eos_token_id]				)

										if len(lowerCAmelCase				) > self.model_max_length:
																UpperCAmelCase					    =     input_ids[-self.model_max_length :]
										return input_ids





