

from manim import *





class  __UpperCAmelCase					(	lowerCamelCase__						):



   def 	__magic_name__   (							self							:					int						):
     UpperCAmelCase	:							List[str]						=			Rectangle(height=0.5,		width=0.5						)
     UpperCAmelCase	:							Dict						=			Rectangle(height=0.4_6,		width=0.4_6						).set_stroke(width=0						)

     UpperCAmelCase	:							str						=			Rectangle(height=0.2_5,		width=0.2_5						)

     UpperCAmelCase	:							Any						=			[mem.copy() for i in range(6						)]
     UpperCAmelCase	:							Any						=			[mem.copy() for i in range(6						)]
     UpperCAmelCase	:							Tuple						=			VGroup(*__A						).arrange(__A,		buff=0						)
     UpperCAmelCase	:							Any						=			VGroup(*__A						).arrange(__A,		buff=0						)
     UpperCAmelCase	:							Optional[Any]						=			VGroup(__A,		__A						).arrange(__A,		buff=0						)
     UpperCAmelCase	:							Dict						=			Text('''CPU''',		font_size=2_4						)
     UpperCAmelCase	:							str						=			Group(__A,		__A						).arrange(__A,		buff=0.5,		aligned_edge=__A						)
     cpu.move_to([-2.5, -0.5, 0]						)
     self.add(__A						)

     UpperCAmelCase	:							str						=			[mem.copy() for i in range(4						)]
     UpperCAmelCase	:							Tuple						=			VGroup(*__A						).arrange(__A,		buff=0						)
     UpperCAmelCase	:							Union[str, Any]						=			Text('''GPU''',		font_size=2_4						)
     UpperCAmelCase	:							Union[str, Any]						=			Group(__A,		__A						).arrange(__A,		buff=0.5,		aligned_edge=__A						)
     gpu.move_to([-1, -1, 0]						)
     self.add(__A						)

     UpperCAmelCase	:							Optional[Any]						=			[mem.copy() for i in range(6						)]
     UpperCAmelCase	:							List[Any]						=			VGroup(*__A						).arrange(__A,		buff=0						)

     UpperCAmelCase	:							Tuple						=			Text('''Model''',		font_size=2_4						)
     UpperCAmelCase	:							Dict						=			Group(__A,		__A						).arrange(__A,		buff=0.5,		aligned_edge=__A						)
     model.move_to([3, -1.0, 0]						)
     self.add(__A						)

     UpperCAmelCase	:							Union[str, Any]						=			[]
     UpperCAmelCase	:							Any						=			[]

     for i, rect in enumerate(__A						):
       UpperCAmelCase	:							Optional[int]						=			fill.copy().set_fill(__A,		opacity=0.8						)
       target.move_to(__A						)
       model_arr.append(__A						)

       UpperCAmelCase	:							str						=			Rectangle(height=0.4_6,		width=0.4_6						).set_stroke(width=0.0						).set_fill(__A,		opacity=0.8						)
       cpu_target.move_to(cpu_left_col_base[i]						)
       model_cpu_arr.append(__A						)

     self.add(*__A,		*__A						)

     UpperCAmelCase	:							List[Any]						=			[meta_mem.copy() for i in range(6						)]
     UpperCAmelCase	:							Optional[int]						=			[meta_mem.copy() for i in range(6						)]
     UpperCAmelCase	:							str						=			VGroup(*__A						).arrange(__A,		buff=0						)
     UpperCAmelCase	:							Tuple						=			VGroup(*__A						).arrange(__A,		buff=0						)
     UpperCAmelCase	:							Union[str, Any]						=			VGroup(__A,		__A						).arrange(__A,		buff=0						)
     UpperCAmelCase	:							Optional[Any]						=			Text('''Disk''',		font_size=2_4						)
     UpperCAmelCase	:							Union[str, Any]						=			Group(__A,		__A						).arrange(__A,		buff=0.5,		aligned_edge=__A						)
     disk.move_to([-4, -1.2_5, 0]						)
     self.add(__A,		__A						)

     UpperCAmelCase	:							Optional[Any]						=			Square(side_length=2.2						)
     key.move_to([-5, 2, 0]						)

     UpperCAmelCase	:							Dict						=			MarkupText(
         F'''<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model''',		font_size=1_8,		)

     key_text.move_to([-5, 2.4, 0]						)

     self.add(__A,		__A						)

     UpperCAmelCase	:							List[Any]						=			MarkupText(
         F'''<span fgcolor=\'{BLUE}\'>●</span> Checkpoint''',		font_size=1_8,		)

     blue_text.next_to(__A,		DOWN * 2.4,		aligned_edge=key_text.get_left()						)
     self.add(__A						)

     UpperCAmelCase	:							Tuple						=			MarkupText(
         F'''Now watch as an input is passed through the model\nand how the memory is utilized and handled.''',		font_size=2_4,		)
     step_a.move_to([2, 2, 0]						)

     self.play(Write(__A						)						)

     UpperCAmelCase	:							Dict						=			Square(0.3						)
     input.set_fill(__A,		opacity=1.0						)
     input.set_stroke(width=0.0						)
     input.next_to(model_base[0],		__A,		buff=0.5						)

     self.play(Write(__A						)						)

     input.generate_target()
     input.target.next_to(model_arr[0],		direction=__A,		buff=0.0_2						)
     self.play(MoveToTarget(__A						)						)

     self.play(FadeOut(__A						)						)

     UpperCAmelCase	:							Tuple						=			Arrow(start=__A,		end=__A,		color=__A,		buff=0.5						)
     a.next_to(model_arr[0].get_left(),		__A,		buff=0.2						)

     model_cpu_arr[0].generate_target()
     model_cpu_arr[0].target.move_to(gpu_rect[0]						)

     UpperCAmelCase	:							Union[str, Any]						=			MarkupText(
         F'''As the input reaches a layer, the hook triggers\nand weights are moved from the CPU\nto the GPU and back.''',		font_size=2_4,		)
     step_a.move_to([2, 2, 0]						)

     self.play(Write(__A,		run_time=3						)						)

     UpperCAmelCase	:							List[Any]						=			{'''run_time''': 1, '''fade_in''': True, '''fade_out''': True, '''buff''': 0.0_2}

     self.play(
         Write(__A						),		Circumscribe(model_arr[0],		color=__A,		**__A						),		Circumscribe(model_cpu_arr[0],		color=__A,		**__A						),		Circumscribe(gpu_rect[0],		color=__A,		**__A						),		)
     self.play(MoveToTarget(model_cpu_arr[0]						)						)

     UpperCAmelCase	:							Tuple						=			a.copy()
     for i in range(6						):
       a_c.next_to(model_arr[i].get_right() + 0.0_2,		__A,		buff=0.2						)

       input.generate_target()
       input.target.move_to(model_arr[i].get_right() + 0.0_2						)

       UpperCAmelCase	:							Dict						=			AnimationGroup(
           FadeOut(__A,		run_time=0.5						),		MoveToTarget(__A,		run_time=0.5						),		FadeIn(__A,		run_time=0.5						),		lag_ratio=0.2						)

       self.play(__A						)

       model_cpu_arr[i].generate_target()
       model_cpu_arr[i].target.move_to(cpu_left_col_base[i]						)

       if i < 5:
         model_cpu_arr[i + 1].generate_target()
         model_cpu_arr[i + 1].target.move_to(gpu_rect[0]						)
         if i >= 1:
           UpperCAmelCase	:							List[str]						=			0.7

         self.play(
             Circumscribe(model_arr[i],		**__A						),		Circumscribe(cpu_left_col_base[i],		**__A						),		Circumscribe(cpu_left_col_base[i + 1],		color=__A,		**__A						),		Circumscribe(gpu_rect[0],		color=__A,		**__A						),		Circumscribe(model_arr[i + 1],		color=__A,		**__A						),		)
         if i < 1:
           self.play(
               MoveToTarget(model_cpu_arr[i]						),		MoveToTarget(model_cpu_arr[i + 1]						),		)
         else:
           self.play(
               MoveToTarget(model_cpu_arr[i],		run_time=0.7						),		MoveToTarget(model_cpu_arr[i + 1],		run_time=0.7						),		)
       else:
         model_cpu_arr[i].generate_target()
         model_cpu_arr[i].target.move_to(cpu_left_col_base[-1]						)
         input.generate_target()
         input.target.next_to(model_arr[-1].get_right(),		RIGHT + 0.0_2,		buff=0.2						)

         self.play(
             Circumscribe(model_arr[-1],		color=__A,		**__A						),		Circumscribe(cpu_left_col_base[-1],		color=__A,		**__A						),		Circumscribe(gpu_rect[0],		color=__A,		**__A						),		)

         self.play(MoveToTarget(model_cpu_arr[i]						)						)

       UpperCAmelCase	:							int						=			a_c
       UpperCAmelCase	:							Tuple						=			a_c.copy()

     input.generate_target()
     input.target.next_to(model_base[-1],		RIGHT + 0.0_2,		buff=0.5						)
     self.play(
         FadeOut(__A						),		FadeOut(__A,		run_time=0.5						),		)

     UpperCAmelCase	:							str						=			MarkupText(F'''Inference on a model too large for GPU memory\nis successfully completed.''',		font_size=2_4						)
     step_a.move_to([2, 2, 0]						)

     self.play(Write(__A,		run_time=3						),		MoveToTarget(__A						)						)

     self.wait()








import torch
from torch import nn





class  __UpperCAmelCase					(	nn.Module						):



   def __init__(							self							:					List[Any],		__A							:					List[Any],		__A							:					Optional[Any],		__A							:					int,		__A							:					List[Any],		__A							:					int=1,		__A							:					List[str]=False						):
     super().__init__()

     UpperCAmelCase	:							Union[str, Any]						=			n_token
     UpperCAmelCase	:							List[str]						=			d_embed
     UpperCAmelCase	:							Dict						=			d_proj

     UpperCAmelCase	:							List[Any]						=			cutoffs + [n_token]
     UpperCAmelCase	:							Dict						=			[0] + self.cutoffs
     UpperCAmelCase	:							int						=			div_val

     UpperCAmelCase	:							Union[str, Any]						=			self.cutoffs[0]
     UpperCAmelCase	:							str						=			len(self.cutoffs						) - 1
     UpperCAmelCase	:							Optional[int]						=			self.shortlist_size + self.n_clusters

     if self.n_clusters > 0:
       UpperCAmelCase	:							str						=			nn.Parameter(torch.zeros(self.n_clusters,		self.d_embed						)						)
       UpperCAmelCase	:							List[str]						=			nn.Parameter(torch.zeros(self.n_clusters						)						)

     UpperCAmelCase	:							Dict						=			nn.ModuleList()
     UpperCAmelCase	:							Optional[int]						=			nn.ParameterList()

     if div_val == 1:
       for i in range(len(self.cutoffs						)						):
         if d_proj != d_embed:
           self.out_projs.append(nn.Parameter(torch.FloatTensor(__A,		__A						)						)						)
         else:
           self.out_projs.append(__A						)

       self.out_layers.append(nn.Linear(__A,		__A						)						)
     else:
       for i in range(len(self.cutoffs						)						):
         UpperCAmelCase			,						UpperCAmelCase	:							Dict						=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
         UpperCAmelCase	:							str						=			d_embed // (div_val**i)

         self.out_projs.append(nn.Parameter(torch.FloatTensor(__A,		__A						)						)						)

         self.out_layers.append(nn.Linear(__A,		r_idx - l_idx						)						)

     UpperCAmelCase	:							Optional[int]						=			keep_order



   def 	__magic_name__   (							self							:					Union[str, Any],		__A							:					List[str],		__A							:					Any,		__A							:					Dict,		__A							:					Optional[Any]						):
     if proj is None:
       UpperCAmelCase	:							List[Any]						=			nn.functional.linear(__A,		__A,		bias=__A						)
     else:
       # if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
       UpperCAmelCase	:							Union[str, Any]						=			nn.functional.linear(__A,		proj.t().contiguous()						)
       UpperCAmelCase	:							Optional[int]						=			nn.functional.linear(__A,		__A,		bias=__A						)
       # else:
       #     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
       #     if bias is not None:
       #         logit = logit + bias

     return logit



   def 	__magic_name__   (							self							:					int,		__A							:					int,		__A							:					List[Any]=None,		__A							:					Dict=False						):

     if labels is not None:
       # Shift so that tokens < n predict n
       UpperCAmelCase	:							List[Any]						=			hidden[..., :-1, :].contiguous()
       UpperCAmelCase	:							Any						=			labels[..., 1:].contiguous()
       UpperCAmelCase	:							Optional[Any]						=			hidden.view(-1,		hidden.size(-1						)						)
       UpperCAmelCase	:							Union[str, Any]						=			labels.view(-1						)
       if hidden.size(0						) != labels.size(0						):
         raise RuntimeError('''Input and labels should have the same size in the batch dimension.'''						)
     else:
       UpperCAmelCase	:							str						=			hidden.view(-1,		hidden.size(-1						)						)

     if self.n_clusters == 0:
       UpperCAmelCase	:							List[str]						=			self._compute_logit(__A,		self.out_layers[0].weight,		self.out_layers[0].bias,		self.out_projs[0]						)
       if labels is not None:
         UpperCAmelCase	:							Optional[int]						=			labels != -1_0_0
         UpperCAmelCase	:							Dict						=			torch.zeros_like(__A,		dtype=hidden.dtype,		device=hidden.device						)
         UpperCAmelCase	:							Any						=			(
             -nn.functional.log_softmax(__A,		dim=-1						)[mask].gather(1,		labels[mask].unsqueeze(1						)						).squeeze(1						)
         )
       else:
         UpperCAmelCase	:							Any						=			nn.functional.log_softmax(__A,		dim=-1						)
     else:
       # construct weights and biases
       UpperCAmelCase			,						UpperCAmelCase	:							Union[str, Any]						=			[], []
       for i in range(len(self.cutoffs						)						):
         if self.div_val == 1:
           UpperCAmelCase			,						UpperCAmelCase	:							List[Any]						=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
           UpperCAmelCase	:							List[str]						=			self.out_layers[0].weight[l_idx:r_idx]
           UpperCAmelCase	:							Optional[int]						=			self.out_layers[0].bias[l_idx:r_idx]
         else:
           UpperCAmelCase	:							List[Any]						=			self.out_layers[i].weight
           UpperCAmelCase	:							Dict						=			self.out_layers[i].bias

         if i == 0:
           UpperCAmelCase	:							List[str]						=			torch.cat([weight_i, self.cluster_weight],		dim=0						)
           UpperCAmelCase	:							List[Any]						=			torch.cat([bias_i, self.cluster_bias],		dim=0						)

         weights.append(__A						)
         biases.append(__A						)

       UpperCAmelCase			,						UpperCAmelCase			,						UpperCAmelCase	:							str						=			weights[0], biases[0], self.out_projs[0]

       UpperCAmelCase	:							Dict						=			self._compute_logit(__A,		__A,		__A,		__A						)
       UpperCAmelCase	:							int						=			nn.functional.log_softmax(__A,		dim=1						)

       if labels is None:
         UpperCAmelCase	:							Optional[int]						=			hidden.new_empty((head_logit.size(0						), self.n_token)						)
       else:
         UpperCAmelCase	:							Union[str, Any]						=			torch.zeros_like(__A,		dtype=hidden.dtype,		device=hidden.device						)

       UpperCAmelCase	:							Union[str, Any]						=			0
       UpperCAmelCase	:							Any						=			[0] + self.cutoffs
       for i in range(len(__A						) - 1						):
         UpperCAmelCase			,						UpperCAmelCase	:							Optional[int]						=			cutoff_values[i], cutoff_values[i + 1]

         if labels is not None:
           UpperCAmelCase	:							List[str]						=			(labels >= l_idx) & (labels < r_idx)
           UpperCAmelCase	:							Tuple						=			mask_i.nonzero().squeeze()

           if indices_i.numel() == 0:
             continue

           UpperCAmelCase	:							Any						=			labels.index_select(0,		__A						) - l_idx
           UpperCAmelCase	:							Dict						=			head_logprob.index_select(0,		__A						)
           UpperCAmelCase	:							List[str]						=			hidden.index_select(0,		__A						)
         else:
           UpperCAmelCase	:							Tuple						=			hidden

         if i == 0:
           if labels is not None:
             UpperCAmelCase	:							Union[str, Any]						=			head_logprob_i.gather(1,		target_i[:, None]						).squeeze(1						)
           else:
             UpperCAmelCase	:							Optional[int]						=			head_logprob[:, : self.cutoffs[0]]
         else:
           UpperCAmelCase			,						UpperCAmelCase			,						UpperCAmelCase	:							List[str]						=			weights[i], biases[i], self.out_projs[i]

           UpperCAmelCase	:							List[str]						=			self._compute_logit(__A,		__A,		__A,		__A						)
           UpperCAmelCase	:							Dict						=			nn.functional.log_softmax(__A,		dim=1						)
           UpperCAmelCase	:							int						=			self.cutoffs[0] + i - 1  # No probability for the head cluster
           if labels is not None:
             UpperCAmelCase	:							Union[str, Any]						=			head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
                 1,		target_i[:, None]						).squeeze(1						)
           else:
             UpperCAmelCase	:							int						=			head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
             UpperCAmelCase	:							Optional[Any]						=			logprob_i

         if labels is not None:
           if (hasattr(self,		'''keep_order'''						) and self.keep_order) or keep_order:
             out.index_copy_(0,		__A,		-logprob_i						)
           else:
             out[offset : offset + logprob_i.size(0						)].copy_(-logprob_i						)
           offset += logprob_i.size(0						)

     return out





   def 	__magic_name__   (							self							:					Tuple,		__A							:					List[str]						):
     if self.n_clusters == 0:
       UpperCAmelCase	:							int						=			self._compute_logit(__A,		self.out_layers[0].weight,		self.out_layers[0].bias,		self.out_projs[0]						)
       return nn.functional.log_softmax(__A,		dim=-1						)
     else:
       # construct weights and biases
       UpperCAmelCase			,						UpperCAmelCase	:							Any						=			[], []
       for i in range(len(self.cutoffs						)						):
         if self.div_val == 1:
           UpperCAmelCase			,						UpperCAmelCase	:							Any						=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
           UpperCAmelCase	:							List[Any]						=			self.out_layers[0].weight[l_idx:r_idx]
           UpperCAmelCase	:							List[str]						=			self.out_layers[0].bias[l_idx:r_idx]
         else:
           UpperCAmelCase	:							List[str]						=			self.out_layers[i].weight
           UpperCAmelCase	:							Dict						=			self.out_layers[i].bias

         if i == 0:
           UpperCAmelCase	:							Dict						=			torch.cat([weight_i, self.cluster_weight],		dim=0						)
           UpperCAmelCase	:							str						=			torch.cat([bias_i, self.cluster_bias],		dim=0						)

         weights.append(__A						)
         biases.append(__A						)

       UpperCAmelCase			,						UpperCAmelCase			,						UpperCAmelCase	:							Tuple						=			weights[0], biases[0], self.out_projs[0]
       UpperCAmelCase	:							int						=			self._compute_logit(__A,		__A,		__A,		__A						)

       UpperCAmelCase	:							Optional[int]						=			hidden.new_empty((head_logit.size(0						), self.n_token)						)
       UpperCAmelCase	:							Dict						=			nn.functional.log_softmax(__A,		dim=1						)

       UpperCAmelCase	:							List[str]						=			[0] + self.cutoffs
       for i in range(len(__A						) - 1						):
         UpperCAmelCase			,						UpperCAmelCase	:							Optional[Any]						=			cutoff_values[i], cutoff_values[i + 1]

         if i == 0:
           UpperCAmelCase	:							Any						=			head_logprob[:, : self.cutoffs[0]]
         else:
           UpperCAmelCase			,						UpperCAmelCase			,						UpperCAmelCase	:							int						=			weights[i], biases[i], self.out_projs[i]

           UpperCAmelCase	:							Tuple						=			self._compute_logit(__A,		__A,		__A,		__A						)
           UpperCAmelCase	:							List[Any]						=			nn.functional.log_softmax(__A,		dim=1						)

           UpperCAmelCase	:							Optional[int]						=			head_logprob[:, -i] + tail_logprob_i
           UpperCAmelCase	:							Optional[Any]						=			logprob_i

       return out





