




"""simple docstring"""


import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
    BertModel,
    BertPreTrainedModel,
)


A_      =    logging.getLogger(__name__)






class  lowercase(					__a			):







 '''simple docstring'''





 def 			UpperCamelCase_       (    self:		Optional[Any],				a_:		List[str],				a_:		int,				a_:		Any=None,				a_:		str=None		):




   '''simple docstring'''



   _snake_case							:		Optional[int]								=  self.layer[current_layer](a_,				a_,				head_mask[current_layer]		)

   _snake_case							:		Optional[int]								=  layer_outputs[0]

   return hidden_states







@add_start_docstrings(
    "The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top."	,      __a	,      )
class  lowercase(					__a			):







 '''simple docstring'''





 def __init__(    self:		str,				a_:		Tuple		):




   '''simple docstring'''



   super().__init__(a_		)

   _snake_case							:		Tuple								=  BertEncoderWithPabee(a_		)

   self.init_weights()
   _snake_case							:		Tuple								=  0
   _snake_case							:		str								=  0
   _snake_case							:		Any								=  0

   _snake_case							:		Tuple								=  0


 def 			UpperCamelCase_       (    self:		int,				a_:		Union[str, Any]		):




   '''simple docstring'''



   _snake_case							:		Any								=  threshold


 def 			UpperCamelCase_       (    self:		List[Any],				a_:		str		):




   '''simple docstring'''



   _snake_case							:		Dict								=  patience


 def 			UpperCamelCase_       (    self:		Optional[int]		):




   '''simple docstring'''



   _snake_case							:		Optional[int]								=  0
   _snake_case							:		Any								=  0


 def 			UpperCamelCase_       (    self:		str		):




   '''simple docstring'''



   _snake_case							:		Optional[int]								=  self.inference_layers_num / self.inference_instances_num
   _snake_case							:		Dict								=  (
       f"*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="
       f" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"
   )
   print(a_		)


 @add_start_docstrings_to_model_forward(a_		)
 def 			UpperCamelCase_       (    self:		Any,				a_:		str=None,				a_:		Tuple=None,				a_:		Dict=None,				a_:		List[str]=None,				a_:		Optional[Any]=None,				a_:		Optional[int]=None,				a_:		List[str]=None,				a_:		List[str]=None,				a_:		Tuple=None,				a_:		Any=None,				a_:		Optional[int]=False,				):




   '''simple docstring'''



   if input_ids is not None and inputs_embeds is not None:
     raise ValueError("""You cannot specify both input_ids and inputs_embeds at the same time"""		)
   elif input_ids is not None:
     _snake_case							:		Tuple								=  input_ids.size()
   elif inputs_embeds is not None:
     _snake_case							:		int								=  inputs_embeds.size()[:-1]
   else:
     raise ValueError("""You have to specify either input_ids or inputs_embeds"""		)

   _snake_case							:		Any								=  input_ids.device if input_ids is not None else inputs_embeds.device

   if attention_mask is None:
     _snake_case							:		List[Any]								=  torch.ones(a_,				device=a_		)
   if token_type_ids is None:
     _snake_case							:		Optional[int]								=  torch.zeros(a_,				dtype=torch.long,				device=a_		)

   # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
   # ourselves in which case we just need to make it broadcastable to all heads.
   _snake_case							:		torch.Tensor								=  self.get_extended_attention_mask(a_,				a_,				a_		)

   # If a 2D ou 3D attention mask is provided for the cross-attention
   # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
   if self.config.is_decoder and encoder_hidden_states is not None:
     _snake_case		, _snake_case		, _snake_case							:		Any								=  encoder_hidden_states.size()
     _snake_case							:		Optional[Any]								=  (encoder_batch_size, encoder_sequence_length)
     if encoder_attention_mask is None:
       _snake_case							:		Union[str, Any]								=  torch.ones(a_,				device=a_		)
     _snake_case							:		str								=  self.invert_attention_mask(a_		)
   else:
     _snake_case							:		Optional[Any]								=  None

   # Prepare head mask if needed
   # 1.0 in head_mask indicate we keep the head
   # attention_probs has shape bsz x n_heads x N x N
   # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
   # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
   _snake_case							:		List[Any]								=  self.get_head_mask(a_,				self.config.num_hidden_layers		)

   _snake_case							:		Tuple								=  self.embeddings(
       input_ids=a_,				position_ids=a_,				token_type_ids=a_,				inputs_embeds=a_		)
   _snake_case							:		List[str]								=  embedding_output

   if self.training:
     _snake_case							:		Optional[Any]								=  []
     for i in range(self.config.num_hidden_layers		):
       _snake_case							:		List[Any]								=  self.encoder.adaptive_forward(
           a_,				current_layer=a_,				attention_mask=a_,				head_mask=a_		)

       _snake_case							:		str								=  self.pooler(a_		)
       _snake_case							:		str								=  output_layers[i](output_dropout(a_		)		)
       res.append(a_		)
   elif self.patience == 0:  # Use all layers for inference
     _snake_case							:		Optional[int]								=  self.encoder(
         a_,				attention_mask=a_,				head_mask=a_,				encoder_hidden_states=a_,				encoder_attention_mask=a_,				)
     _snake_case							:		Union[str, Any]								=  self.pooler(encoder_outputs[0]		)
     _snake_case							:		Tuple								=  [output_layers[self.config.num_hidden_layers - 1](a_		)]
   else:
     _snake_case							:		List[str]								=  0
     _snake_case							:		Dict								=  None
     _snake_case							:		Any								=  0
     for i in range(self.config.num_hidden_layers		):
       calculated_layer_num += 1
       _snake_case							:		int								=  self.encoder.adaptive_forward(
           a_,				current_layer=a_,				attention_mask=a_,				head_mask=a_		)

       _snake_case							:		str								=  self.pooler(a_		)
       _snake_case							:		Tuple								=  output_layers[i](a_		)
       if regression:
         _snake_case							:		List[str]								=  logits.detach()
         if patient_result is not None:
           _snake_case							:		Optional[Any]								=  patient_result.detach()
         if (patient_result is not None) and torch.abs(patient_result - labels		) < self.regression_threshold:
           patient_counter += 1
         else:
           _snake_case							:		List[Any]								=  0
       else:
         _snake_case							:		Optional[Any]								=  logits.detach().argmax(dim=1		)
         if patient_result is not None:
           _snake_case							:		Optional[Any]								=  patient_result.detach().argmax(dim=1		)
         if (patient_result is not None) and torch.all(labels.eq(a_		)		):
           patient_counter += 1
         else:
           _snake_case							:		Union[str, Any]								=  0

       _snake_case							:		Dict								=  logits
       if patient_counter == self.patience:
         break
     _snake_case							:		List[Any]								=  [patient_result]
     self.inference_layers_num += calculated_layer_num
     self.inference_instances_num += 1

   return res







@add_start_docstrings(
    "Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of\n    the pooled output) e.g. for GLUE tasks. "	,      __a	,      )
class  lowercase(					__a			):







 '''simple docstring'''





 def __init__(    self:		Tuple,				a_:		Any		):




   '''simple docstring'''



   super().__init__(a_		)
   _snake_case							:		Tuple								=  config.num_labels

   _snake_case							:		Optional[Any]								=  BertModelWithPabee(a_		)
   _snake_case							:		int								=  nn.Dropout(config.hidden_dropout_prob		)
   _snake_case							:		Any								=  nn.ModuleList(
       [nn.Linear(config.hidden_size,				self.config.num_labels		) for _ in range(config.num_hidden_layers		)]		)

   self.init_weights()


 @add_start_docstrings_to_model_forward(a_		)
 def 			UpperCamelCase_       (    self:		Union[str, Any],				a_:		Union[str, Any]=None,				a_:		int=None,				a_:		int=None,				a_:		int=None,				a_:		Any=None,				a_:		str=None,				a_:		List[Any]=None,				):




   '''simple docstring'''



   _snake_case							:		int								=  self.bert(
       input_ids=a_,				attention_mask=a_,				token_type_ids=a_,				position_ids=a_,				head_mask=a_,				inputs_embeds=a_,				output_dropout=self.dropout,				output_layers=self.classifiers,				regression=self.num_labels == 1,				)

   _snake_case							:		Any								=  (logits[-1],)

   if labels is not None:
     _snake_case							:		Optional[Any]								=  None
     _snake_case							:		Union[str, Any]								=  0
     for ix, logits_item in enumerate(a_		):
       if self.num_labels == 1:
         #  We are doing regression
         _snake_case							:		str								=  MSELoss()
         _snake_case							:		Optional[Any]								=  loss_fct(logits_item.view(-1		),				labels.view(-1		)		)
       else:
         _snake_case							:		Any								=  CrossEntropyLoss()
         _snake_case							:		List[str]								=  loss_fct(logits_item.view(-1,				self.num_labels		),				labels.view(-1		)		)
       if total_loss is None:
         _snake_case							:		List[str]								=  loss
       else:
         total_loss += loss * (ix + 1)
       total_weights += ix + 1
     _snake_case							:		List[Any]								=  (total_loss / total_weights,) + outputs

   return outputs








"""simple docstring"""


def  UpperCAmelCase__	(snake_case__							: int   ):
  """simple docstring"""

  _snake_case							:		Optional[Any]								=  (1 + 24 * n) ** 0.5
  return ((1 + root) / 6) % 1 == 0


def  UpperCAmelCase__	(snake_case__							: int = 50_00   ):
  """simple docstring"""

  _snake_case							:		List[str]								=  [(i * (3 * i - 1)) // 2 for i in range(1		,       snake_case__   )]
  for i, pentagonal_i in enumerate(snake_case__   ):
    for j in range(snake_case__		,       len(snake_case__   )   ):
      _snake_case							:		Dict								=  pentagonal_nums[j]
      _snake_case							:		Optional[Any]								=  pentagonal_i + pentagonal_j
      _snake_case							:		List[str]								=  pentagonal_j - pentagonal_i
      if is_pentagonal(snake_case__   ) and is_pentagonal(snake_case__   ):
        return b

  return -1


if __name__ == "__main__":
     print(F'''{solution() = }''')


