



'''simple docstring'''






from manim import *

class 							a__( lowerCamelCase__     ):







				def       lowercase_	(				self			:  str     ):
						a						: Dict					       =					Rectangle(height=0.5						,      width=0.5     )
						a						: Union[str, Any]					       =					Rectangle(height=0.46						,      width=0.46     ).set_stroke(width=0     )

						a						: Optional[Any]					       =					[mem.copy() for i in range(6     )]
						a						: Dict					       =					[mem.copy() for i in range(6     )]
						a						: List[Any]					       =					VGroup(*__snake_case     ).arrange(__snake_case						,      buff=0     )
						a						: int					       =					VGroup(*__snake_case     ).arrange(__snake_case						,      buff=0     )
						a						: Union[str, Any]					       =					VGroup(__snake_case						,      __snake_case     ).arrange(__snake_case						,      buff=0     )
						a						: Any					       =					Text('CPU'						,      font_size=24     )
						a						: int					       =					Group(__snake_case						,      __snake_case     ).arrange(__snake_case						,      buff=0.5						,      aligned_edge=__snake_case     )
						cpu.move_to([-2.5, -0.5, 0]     )
						self.add(__snake_case     )

						a						: int					       =					[mem.copy() for i in range(4     )]
						a						: str					       =					VGroup(*__snake_case     ).arrange(__snake_case						,      buff=0     )
						a						: List[Any]					       =					Text('GPU'						,      font_size=24     )
						a						: Optional[int]					       =					Group(__snake_case						,      __snake_case     ).arrange(__snake_case						,      buff=0.5						,      aligned_edge=__snake_case     )
						gpu.move_to([-1, -1, 0]     )
						self.add(__snake_case     )

						a						: List[str]					       =					[mem.copy() for i in range(6     )]
						a						: List[str]					       =					VGroup(*__snake_case     ).arrange(__snake_case						,      buff=0     )

						a						: int					       =					Text('Model'						,      font_size=24     )
						a						: Dict					       =					Group(__snake_case						,      __snake_case     ).arrange(__snake_case						,      buff=0.5						,      aligned_edge=__snake_case     )
						model.move_to([3, -1.0, 0]     )
						self.add(__snake_case     )

						a						: Optional[Any]					       =					[]
						for i, rect in enumerate(__snake_case     ):
								rect.set_stroke(__snake_case     )
								# target = fill.copy().set_fill(YELLOW, opacity=0.7)
								# target.move_to(rect)
								# self.add(target)

								a						: Optional[int]					       =					Rectangle(height=0.46 / 4						,      width=0.46 / 3     ).set_stroke(width=0.0     ).set_fill(__snake_case						,      opacity=0.7     )

								if i == 0:
										cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT     )						,      buff=0.02						,      direction=__snake_case     )
										cpu_target.set_x(cpu_target.get_x() + 0.1     )
								elif i == 3:
										cpu_target.next_to(cpu_targs[0]						,      direction=__snake_case						,      buff=0.0     )
								else:
										cpu_target.next_to(cpu_targs[i - 1]						,      direction=__snake_case						,      buff=0.0     )
								self.add(__snake_case     )
								cpu_targs.append(__snake_case     )

						a						: List[Any]					       =					[mem.copy() for i in range(6     )]
						a						: Optional[int]					       =					VGroup(*__snake_case     ).arrange(__snake_case						,      buff=0     )

						a						: Tuple					       =					Text('Loaded Checkpoint'						,      font_size=24     )
						a						: str					       =					Group(__snake_case						,      __snake_case     ).arrange(__snake_case						,      aligned_edge=__snake_case						,      buff=0.4     )
						checkpoint.move_to([3, 0.5, 0]     )

						a						: int					       =					Square(side_length=2.2     )
						key.move_to([-5, 2, 0]     )

						a						: Union[str, Any]					       =					MarkupText(
						    F"""<b>Key:</b>\n\n<span fgcolor='{YELLOW}'>●</span> Empty Model"""						,      font_size=18						,      )

						key_text.move_to([-5, 2.4, 0]     )

						self.add(__snake_case						,      __snake_case     )

						a						: Tuple					       =					MarkupText(
						    F"""<span fgcolor='{BLUE}'>●</span> Checkpoint"""						,      font_size=18						,      )

						blue_text.next_to(__snake_case						,      DOWN * 2.4						,      aligned_edge=key_text.get_left()     )

						a						: Optional[int]					       =					MarkupText(
						    F"""Next, a <i><span fgcolor=\"{BLUE}\">second</span></i> model is loaded into memory,\nwith the weights of a <span fgcolor=\"{BLUE}\">single shard</span>."""						,      font_size=24						,      )
						step_a.move_to([2, 2, 0]     )
						self.play(Write(__snake_case     )						,      Write(__snake_case     )     )

						self.play(Write(__snake_case						,      run_time=1     )						,      Create(__snake_case						,      run_time=1     )     )

						a						: List[str]					       =					[]
						a						: Dict					       =					[]
						for i, rect in enumerate(__snake_case     ):
								a						: Optional[int]					       =					fill.copy().set_fill(__snake_case						,      opacity=0.7     )
								target.move_to(__snake_case     )
								first_animations.append(GrowFromCenter(__snake_case						,      run_time=1     )     )

								a						: List[Any]					       =					target.copy()
								cpu_target.generate_target()
								if i < 5:
										cpu_target.target.move_to(cpu_left_col_base[i + 1]     )
								else:
										cpu_target.target.move_to(cpu_right_col_base[i - 5]     )
								second_animations.append(MoveToTarget(__snake_case						,      run_time=1.5     )     )

						self.play(*__snake_case     )
						self.play(*__snake_case     )
						self.wait()




'''simple docstring'''






import torch

from transformers import AutoModel

class 							a__( torch.nn.Module     ):







    def __init__(				self			:  Any						,      __snake_case			:  str="sayef/fsner-bert-base-uncased"     ):
      super(__snake_case						,      self     ).__init__()

      a						: List[str]					       =					AutoModel.from_pretrained(__snake_case						,      return_dict=__snake_case     )
      a						: Optional[Any]					       =					torch.nn.CosineSimilarity(3						,      1e-0_8     )
      a						: Tuple					       =					torch.nn.Softmax(dim=1     )







    def       lowercase_	(				self			:  List[str]						,      **__snake_case			:  int     ):
      return self.bert(**__snake_case     ).last_hidden_state







    def       lowercase_	(				self			:  Optional[int]						,      __snake_case			:  Union[str, Any]     ):
      return token_embeddings.sum(2						,      keepdim=__snake_case     )







    def       lowercase_	(				self			:  str						,      __snake_case			:  Tuple						,      __snake_case			:  List[Any]						,      __snake_case			:  Optional[Any]=1     ):
      return self.softmax(T * self.cos(__snake_case						,      __snake_case     )     )







    def       lowercase_	(				self			:  int						,      __snake_case			:  int						,      __snake_case			:  Tuple     ):
      a						: List[Any]					       =					W_supports['sizes'].tolist()
      a						: Any					       =					W_supports['start_token_id'].item()
      a						: int					       =					W_supports['end_token_id'].item()

      del W_supports["sizes"]
      del W_supports["start_token_id"]
      del W_supports["end_token_id"]

      a						: Optional[Any]					       =					self.BERT(**__snake_case     )
      a						: Tuple					       =					self.BERT(**__snake_case     )

      a						: Dict					       =					None
      a						: Optional[Any]					       =					None

      a						: Union[str, Any]					       =					W_supports['input_ids'] == start_token_id
      a						: str					       =					W_supports['input_ids'] == end_token_id

      for i, size in enumerate(__snake_case     ):
        if i == 0:
          a						: Optional[int]					       =					0
        else:
          a						: Tuple					       =					support_sizes[i - 1]

        a						: Tuple					       =					S[s : s + size][start_token_masks[s : s + size]]
        a						: int					       =					S[s : s + size][end_token_masks[s : s + size]]

        a						: Any					       =					torch.matmul(q[i]						,      s_start.T     ).sum(1     ).softmax(0     )
        a						: List[str]					       =					torch.matmul(q[i]						,      s_end.T     ).sum(1     ).softmax(0     )

        if p_starts is not None:
          a						: List[Any]					       =					torch.vstack((p_starts, p_start)     )
          a						: List[Any]					       =					torch.vstack((p_ends, p_end)     )
        else:
          a						: List[Any]					       =					p_start
          a						: Optional[int]					       =					p_end

      return p_starts, p_ends