


'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


a_   :      Dict				    =      {
    "configuration_albert": ["ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP", "AlbertConfig", "AlbertOnnxConfig"],
}

try:
 if not is_sentencepiece_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 a_   :      List[Any]				    =      ["AlbertTokenizer"]

try:
 if not is_tokenizers_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 a_   :      Union[str, Any]				    =      ["AlbertTokenizerFast"]

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 a_   :      int				    =      [
     "ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST",
     "AlbertForMaskedLM",
     "AlbertForMultipleChoice",
     "AlbertForPreTraining",
     "AlbertForQuestionAnswering",
     "AlbertForSequenceClassification",
     "AlbertForTokenClassification",
     "AlbertModel",
     "AlbertPreTrainedModel",
     "load_tf_weights_in_albert",
 ]

try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 a_   :      Optional[int]				    =      [
     "TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST",
     "TFAlbertForMaskedLM",
     "TFAlbertForMultipleChoice",
     "TFAlbertForPreTraining",
     "TFAlbertForQuestionAnswering",
     "TFAlbertForSequenceClassification",
     "TFAlbertForTokenClassification",
     "TFAlbertMainLayer",
     "TFAlbertModel",
     "TFAlbertPreTrainedModel",
 ]

try:
 if not is_flax_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 a_   :      Optional[Any]				    =      [
     "FlaxAlbertForMaskedLM",
     "FlaxAlbertForMultipleChoice",
     "FlaxAlbertForPreTraining",
     "FlaxAlbertForQuestionAnswering",
     "FlaxAlbertForSequenceClassification",
     "FlaxAlbertForTokenClassification",
     "FlaxAlbertModel",
     "FlaxAlbertPreTrainedModel",
 ]

if TYPE_CHECKING:
 from .configuration_albert import ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, AlbertConfig, AlbertOnnxConfig

 try:
  if not is_sentencepiece_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_albert import AlbertTokenizer

 try:
  if not is_tokenizers_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_albert_fast import AlbertTokenizerFast

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_albert import (
      ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      AlbertForMaskedLM,
      AlbertForMultipleChoice,
      AlbertForPreTraining,
      AlbertForQuestionAnswering,
      AlbertForSequenceClassification,
      AlbertForTokenClassification,
      AlbertModel,
      AlbertPreTrainedModel,
      load_tf_weights_in_albert,
  )

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_albert import (
      TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFAlbertForMaskedLM,
      TFAlbertForMultipleChoice,
      TFAlbertForPreTraining,
      TFAlbertForQuestionAnswering,
      TFAlbertForSequenceClassification,
      TFAlbertForTokenClassification,
      TFAlbertMainLayer,
      TFAlbertModel,
      TFAlbertPreTrainedModel,
  )

 try:
  if not is_flax_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_flax_albert import (
      FlaxAlbertForMaskedLM,
      FlaxAlbertForMultipleChoice,
      FlaxAlbertForPreTraining,
      FlaxAlbertForQuestionAnswering,
      FlaxAlbertForSequenceClassification,
      FlaxAlbertForTokenClassification,
      FlaxAlbertModel,
      FlaxAlbertPreTrainedModel,
  )
else:
 import sys

 a_   :      Tuple				    =      _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)





'''simple docstring'''






import unicodedata
from dataclasses import dataclass
from typing import Optional, Union

import numpy as np

from transformers.data.data_collator import DataCollatorMixin
from transformers.file_utils import PaddingStrategy
from transformers.tokenization_utils_base import PreTrainedTokenizerBase






def 		_A			(lowerCAmelCase__					:int			,  lowerCAmelCase__					:Any			,  lowerCAmelCase__					:Optional[Any]			,  lowerCAmelCase__					:Dict						)				->					Optional[int]:




     '''simple docstring'''



     if isinstance(lowerCAmelCase__			,  lowerCAmelCase__						):
          _a      =							np.full((len(lowerCAmelCase__						), sequence_length, 2)			,  lowerCAmelCase__						)
     else:
          _a      =							np.full((len(lowerCAmelCase__						), sequence_length)			,  lowerCAmelCase__						)

     for i, tensor in enumerate(lowerCAmelCase__						):
          if padding_side == "right":
               if isinstance(lowerCAmelCase__			,  lowerCAmelCase__						):
                    _a      =							tensor[:sequence_length]
               else:
                    _a      =							tensor[:sequence_length]
          else:
               if isinstance(lowerCAmelCase__			,  lowerCAmelCase__						):
                    _a      =							tensor[:sequence_length]
               else:
                    _a      =							tensor[:sequence_length]

     return out_tensor.tolist()






def 		_A			(lowerCAmelCase__					:Any						)				->					Union[str, Any]:




     '''simple docstring'''



     _a      =							ord(lowerCAmelCase__						)
     if (cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or (cp >= 91 and cp <= 96) or (cp >= 1_23 and cp <= 1_26):
          return True
     _a      =							unicodedata.category(lowerCAmelCase__						)
     if cat.startswith('P'						):
          return True
     return False






@dataclass
class 	a       (       _SCREAMING_SNAKE_CASE    ):
 _lowerCAmelCase	   =		42
 _lowerCAmelCase	   =		True
 _lowerCAmelCase	   =		None
 _lowerCAmelCase	   =		None
 _lowerCAmelCase	   =		-1_0_0
 _lowerCAmelCase	   =		"pt"
 def 							__UpperCAmelCase						(	self ,			__magic_name__  )   ->							Any:
      import torch

      _a      =							'label' if 'label' in features[0].keys() else 'labels'
      _a      =							[feature[label_name] for feature in features] if label_name in features[0].keys() else None
      _a      =							self.tokenizer.pad(
          __magic_name__ ,			padding=self.padding ,			max_length=self.max_length ,			pad_to_multiple_of=self.pad_to_multiple_of ,			return_tensors='pt' if labels is None else None ,			)

      if labels is None:
           return batch

      _a      =							torch.tensor(batch['entity_ids']  ).shape[1]
      _a      =							self.tokenizer.padding_side
      if padding_side == "right":
           _a      =							[
               list(__magic_name__  ) + [self.label_pad_token_id] * (sequence_length - len(__magic_name__  )) for label in labels
           ]
      else:
           _a      =							[
               [self.label_pad_token_id] * (sequence_length - len(__magic_name__  )) + list(__magic_name__  ) for label in labels
           ]

      _a      =							[feature['ner_tags'] for feature in features]
      _a      =							padding_tensor(__magic_name__ ,			-1 ,			__magic_name__ ,			__magic_name__  )
      _a      =							[feature['original_entity_spans'] for feature in features]
      _a      =							padding_tensor(__magic_name__ ,			(-1, -1) ,			__magic_name__ ,			__magic_name__  )
      _a      =							{k: torch.tensor(__magic_name__ ,			dtype=torch.intaa  ) for k, v in batch.items()}

      return batch

