
import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowercase__		:List[str]			  =   logging.get_logger(__name__)


lowercase__		:Optional[Any]			  =   {"vocab_file": "vocab.json", "merges_file": "merges.txt"}

lowercase__		:int			  =   {
    "vocab_file": {
        "allenai/longformer-base-4096": "https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json",
        "allenai/longformer-large-4096": (
            "https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"
        ),
        "allenai/longformer-large-4096-finetuned-triviaqa": (
            "https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"
        ),
        "allenai/longformer-base-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"
        ),
        "allenai/longformer-large-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"
        ),
    },
    "merges_file": {
        "allenai/longformer-base-4096": "https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt",
        "allenai/longformer-large-4096": (
            "https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"
        ),
        "allenai/longformer-large-4096-finetuned-triviaqa": (
            "https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"
        ),
        "allenai/longformer-base-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"
        ),
        "allenai/longformer-large-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"
        ),
    },
}

lowercase__		:Optional[int]			  =   {
    "allenai/longformer-base-4096": 4096,
    "allenai/longformer-large-4096": 4096,
    "allenai/longformer-large-4096-finetuned-triviaqa": 4096,
    "allenai/longformer-base-4096-extra.pos.embd.only": 4096,
    "allenai/longformer-large-4096-extra.pos.embd.only": 4096,
}



@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def 	UpperCamelCase   (							):
					'''simple docstring'''
					lowercase						     =						(
					    list(range(ord('''!'''		)			,      ord('''~'''		) + 1		)		) + list(range(ord('''¡'''		)			,      ord('''¬'''		) + 1		)		) + list(range(ord('''®'''		)			,      ord('''ÿ'''		) + 1		)		)
					)
					lowercase						     =						bs[:]
					lowercase						     =						0
					for b in range(2**8		):
										if b not in bs:
															bs.append(lowerCAmelCase__		)
															cs.append(2**8 + n		)
															n += 1
					lowercase						     =						[chr(lowerCAmelCase__		) for n in cs]
					return dict(zip(lowerCAmelCase__			,      lowerCAmelCase__		)		)



def 	UpperCamelCase   (							lowerCAmelCase__		):
					'''simple docstring'''
					lowercase						     =						set()
					lowercase						     =						word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)		)
										lowercase						     =						char
					return pairs







class     lowercase		( lowerCAmelCase_		):
							lowercase_	:	Optional[Any]	 =VOCAB_FILES_NAMES
							lowercase_	:	Optional[int]	 =PRETRAINED_VOCAB_FILES_MAP
							lowercase_	:	List[Any]	 =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase_	:	List[Any]	 =['input_ids', 'attention_mask']




							def __init__(    self       ,A__       ,A__       ,A__="replace"       ,A__="<s>"       ,A__="</s>"       ,A__="</s>"       ,A__="<s>"       ,A__="<unk>"       ,A__="<pad>"       ,A__="<mask>"       ,A__=False       ,**A__       ,):
												lowercase						     =						AddedToken(__lowerCAmelCase       ,lstrip=__lowerCAmelCase       ,rstrip=__lowerCAmelCase) if isinstance(__lowerCAmelCase       ,__lowerCAmelCase) else bos_token
												lowercase						     =						AddedToken(__lowerCAmelCase       ,lstrip=__lowerCAmelCase       ,rstrip=__lowerCAmelCase) if isinstance(__lowerCAmelCase       ,__lowerCAmelCase) else eos_token
												lowercase						     =						AddedToken(__lowerCAmelCase       ,lstrip=__lowerCAmelCase       ,rstrip=__lowerCAmelCase) if isinstance(__lowerCAmelCase       ,__lowerCAmelCase) else sep_token
												lowercase						     =						AddedToken(__lowerCAmelCase       ,lstrip=__lowerCAmelCase       ,rstrip=__lowerCAmelCase) if isinstance(__lowerCAmelCase       ,__lowerCAmelCase) else cls_token
												lowercase						     =						AddedToken(__lowerCAmelCase       ,lstrip=__lowerCAmelCase       ,rstrip=__lowerCAmelCase) if isinstance(__lowerCAmelCase       ,__lowerCAmelCase) else unk_token
												lowercase						     =						AddedToken(__lowerCAmelCase       ,lstrip=__lowerCAmelCase       ,rstrip=__lowerCAmelCase) if isinstance(__lowerCAmelCase       ,__lowerCAmelCase) else pad_token

												# Mask token behave like a normal word, i.e. include the space before it
												lowercase						     =						AddedToken(__lowerCAmelCase       ,lstrip=__lowerCAmelCase       ,rstrip=__lowerCAmelCase) if isinstance(__lowerCAmelCase       ,__lowerCAmelCase) else mask_token

												super().__init__(
												    errors=__lowerCAmelCase       ,bos_token=__lowerCAmelCase       ,eos_token=__lowerCAmelCase       ,unk_token=__lowerCAmelCase       ,sep_token=__lowerCAmelCase       ,cls_token=__lowerCAmelCase       ,pad_token=__lowerCAmelCase       ,mask_token=__lowerCAmelCase       ,add_prefix_space=__lowerCAmelCase       ,**__lowerCAmelCase       ,)

												with open(__lowerCAmelCase       ,encoding='''utf-8''') as vocab_handle:
																	lowercase						     =						json.load(__lowerCAmelCase)
												lowercase						     =						{v: k for k, v in self.encoder.items()}
												lowercase						     =						errors  # how to handle errors in decoding
												lowercase						     =						bytes_to_unicode()
												lowercase						     =						{v: k for k, v in self.byte_encoder.items()}
												with open(__lowerCAmelCase       ,encoding='''utf-8''') as merges_handle:
																	lowercase						     =						merges_handle.read().split('''\n''')[1:-1]
												lowercase						     =						[tuple(merge.split()) for merge in bpe_merges]
												lowercase						     =						dict(zip(__lowerCAmelCase       ,range(len(__lowerCAmelCase))))
												lowercase						     =						{}
												lowercase						     =						add_prefix_space

												# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
												lowercase						     =						re.compile(r'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+''')




							@property
							def 			A__				(    self):
												return len(self.encoder)




							def 			A__				(    self):
												return dict(self.encoder       ,**self.added_tokens_encoder)




							def 			A__				(    self       ,A__):
												if token in self.cache:
																	return self.cache[token]
												lowercase						     =						tuple(__lowerCAmelCase)
												lowercase						     =						get_pairs(__lowerCAmelCase)

												if not pairs:
																	return token

												while True:
																	lowercase						     =						min(__lowerCAmelCase       ,key=lambda A__: self.bpe_ranks.get(__lowerCAmelCase       ,float('''inf''')))
																	if bigram not in self.bpe_ranks:
																						break
																	lowercase     ,						lowercase						     =						bigram
																	lowercase						     =						[]
																	lowercase						     =						0
																	while i < len(__lowerCAmelCase):
																						try:
																											lowercase						     =						word.index(__lowerCAmelCase       ,__lowerCAmelCase)
																						except ValueError:
																											new_word.extend(word[i:])
																											break
																						else:
																											new_word.extend(word[i:j])
																											lowercase						     =						j

																						if word[i] == first and i < len(__lowerCAmelCase) - 1 and word[i + 1] == second:
																											new_word.append(first + second)
																											i += 2
																						else:
																											new_word.append(word[i])
																											i += 1
																	lowercase						     =						tuple(__lowerCAmelCase)
																	lowercase						     =						new_word
																	if len(__lowerCAmelCase) == 1:
																						break
																	else:
																						lowercase						     =						get_pairs(__lowerCAmelCase)
												lowercase						     =						''' '''.join(__lowerCAmelCase)
												lowercase						     =						word
												return word




							def 			A__				(    self       ,A__):
												lowercase						     =						[]
												for token in re.findall(self.pat       ,__lowerCAmelCase):
																	lowercase						     =						''''''.join(
																	    self.byte_encoder[b] for b in token.encode('''utf-8'''))  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																	bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__lowerCAmelCase).split(''' '''))
												return bpe_tokens




							def 			A__				(    self       ,A__):
												return self.encoder.get(__lowerCAmelCase       ,self.encoder.get(self.unk_token))




							def 			A__				(    self       ,A__):
												return self.decoder.get(__lowerCAmelCase)




							def 			A__				(    self       ,A__):
												lowercase						     =						''''''.join(__lowerCAmelCase)
												lowercase						     =						bytearray([self.byte_decoder[c] for c in text]).decode('''utf-8'''       ,errors=self.errors)
												return text




							def 			A__				(    self       ,A__       ,A__ = None):
												if not os.path.isdir(__lowerCAmelCase):
																	logger.error(f'Vocabulary path ({save_directory}) should be a directory')
																	return
												lowercase						     =						os.path.join(
												    __lowerCAmelCase       ,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])
												lowercase						     =						os.path.join(
												    __lowerCAmelCase       ,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file'''])

												with open(__lowerCAmelCase       ,'''w'''       ,encoding='''utf-8''') as f:
																	f.write(json.dumps(self.encoder       ,indent=2       ,sort_keys=__lowerCAmelCase       ,ensure_ascii=__lowerCAmelCase) + '''\n''')

												lowercase						     =						0
												with open(__lowerCAmelCase       ,'''w'''       ,encoding='''utf-8''') as writer:
																	writer.write('''#version: 0.2\n''')
																	for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,key=lambda A__: kv[1]):
																						if index != token_index:
																											logger.warning(
																											    f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
																											    ''' Please check that the tokenizer is not corrupted!''')
																											lowercase						     =						token_index
																						writer.write(''' '''.join(__lowerCAmelCase) + '''\n''')
																						index += 1

												return vocab_file, merge_file




							def 			A__				(    self       ,A__       ,A__ = None):
												if token_ids_a is None:
																	return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
												lowercase						     =						[self.cls_token_id]
												lowercase						     =						[self.sep_token_id]
												return cls + token_ids_a + sep + sep + token_ids_a + sep




							def 			A__				(    self       ,A__       ,A__ = None       ,A__ = False):
												if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=__lowerCAmelCase       ,token_ids_a=__lowerCAmelCase       ,already_has_special_tokens=__lowerCAmelCase)

												if token_ids_a is None:
																	return [1] + ([0] * len(__lowerCAmelCase)) + [1]
												return [1] + ([0] * len(__lowerCAmelCase)) + [1, 1] + ([0] * len(__lowerCAmelCase)) + [1]




							def 			A__				(    self       ,A__       ,A__ = None):
												lowercase						     =						[self.sep_token_id]
												lowercase						     =						[self.cls_token_id]

												if token_ids_a is None:
																	return len(cls + token_ids_a + sep) * [0]
												return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]





							def 			A__				(    self       ,A__       ,A__=False       ,**A__):
												lowercase						     =						kwargs.pop('''add_prefix_space'''       ,self.add_prefix_space)
												if (is_split_into_words or add_prefix_space) and (len(__lowerCAmelCase) > 0 and not text[0].isspace()):
																	lowercase						     =						''' ''' + text
												return (text, kwargs)






import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    MODEL_FOR_PRETRAINING_MAPPING,
							    AlbertForMaskedLM,
							    AlbertForMultipleChoice,
							    AlbertForPreTraining,
							    AlbertForQuestionAnswering,
							    AlbertForSequenceClassification,
							    AlbertForTokenClassification,
							    AlbertModel,
							)
							from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST







class     lowercase		:




							def __init__(    self       ,A__       ,A__=1_3       ,A__=7       ,A__=True       ,A__=True       ,A__=True       ,A__=True       ,A__=9_9       ,A__=1_6       ,A__=3_6       ,A__=6       ,A__=6       ,A__=6       ,A__=3_7       ,A__="gelu"       ,A__=0.1       ,A__=0.1       ,A__=5_1_2       ,A__=1_6       ,A__=2       ,A__=0.02       ,A__=3       ,A__=4       ,A__=None       ,):
												lowercase						     =						parent
												lowercase						     =						batch_size
												lowercase						     =						seq_length
												lowercase						     =						is_training
												lowercase						     =						use_input_mask
												lowercase						     =						use_token_type_ids
												lowercase						     =						use_labels
												lowercase						     =						vocab_size
												lowercase						     =						embedding_size
												lowercase						     =						hidden_size
												lowercase						     =						num_hidden_layers
												lowercase						     =						num_hidden_groups
												lowercase						     =						num_attention_heads
												lowercase						     =						intermediate_size
												lowercase						     =						hidden_act
												lowercase						     =						hidden_dropout_prob
												lowercase						     =						attention_probs_dropout_prob
												lowercase						     =						max_position_embeddings
												lowercase						     =						type_vocab_size
												lowercase						     =						type_sequence_label_size
												lowercase						     =						initializer_range
												lowercase						     =						num_labels
												lowercase						     =						num_choices
												lowercase						     =						scope




							def 			A__				(    self):
												lowercase						     =						ids_tensor([self.batch_size, self.seq_length]       ,self.vocab_size)

												lowercase						     =						None
												if self.use_input_mask:
																	lowercase						     =						random_attention_mask([self.batch_size, self.seq_length])

												lowercase						     =						None
												if self.use_token_type_ids:
																	lowercase						     =						ids_tensor([self.batch_size, self.seq_length]       ,self.type_vocab_size)

												lowercase						     =						None
												lowercase						     =						None
												lowercase						     =						None
												if self.use_labels:
																	lowercase						     =						ids_tensor([self.batch_size]       ,self.type_sequence_label_size)
																	lowercase						     =						ids_tensor([self.batch_size, self.seq_length]       ,self.num_labels)
																	lowercase						     =						ids_tensor([self.batch_size]       ,self.num_choices)

												lowercase						     =						self.get_config()

												return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




							def 			A__				(    self):
												return AlbertConfig(
												    vocab_size=self.vocab_size       ,hidden_size=self.hidden_size       ,num_hidden_layers=self.num_hidden_layers       ,num_attention_heads=self.num_attention_heads       ,intermediate_size=self.intermediate_size       ,hidden_act=self.hidden_act       ,hidden_dropout_prob=self.hidden_dropout_prob       ,attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,max_position_embeddings=self.max_position_embeddings       ,type_vocab_size=self.type_vocab_size       ,initializer_range=self.initializer_range       ,num_hidden_groups=self.num_hidden_groups       ,)




							def 			A__				(    self       ,A__       ,A__       ,A__       ,A__       ,A__       ,A__       ,A__):
												lowercase						     =						AlbertModel(config=A__)
												model.to(A__)
												model.eval()
												lowercase						     =						model(A__       ,attention_mask=A__       ,token_type_ids=A__)
												lowercase						     =						model(A__       ,token_type_ids=A__)
												lowercase						     =						model(A__)
												self.parent.assertEqual(result.last_hidden_state.shape       ,(self.batch_size, self.seq_length, self.hidden_size))
												self.parent.assertEqual(result.pooler_output.shape       ,(self.batch_size, self.hidden_size))




							def 			A__				(    self       ,A__       ,A__       ,A__       ,A__       ,A__       ,A__       ,A__):
												lowercase						     =						AlbertForPreTraining(config=A__)
												model.to(A__)
												model.eval()
												lowercase						     =						model(
												    A__       ,attention_mask=A__       ,token_type_ids=A__       ,labels=A__       ,sentence_order_label=A__       ,)
												self.parent.assertEqual(result.prediction_logits.shape       ,(self.batch_size, self.seq_length, self.vocab_size))
												self.parent.assertEqual(result.sop_logits.shape       ,(self.batch_size, config.num_labels))




							def 			A__				(    self       ,A__       ,A__       ,A__       ,A__       ,A__       ,A__       ,A__):
												lowercase						     =						AlbertForMaskedLM(config=A__)
												model.to(A__)
												model.eval()
												lowercase						     =						model(A__       ,attention_mask=A__       ,token_type_ids=A__       ,labels=A__)
												self.parent.assertEqual(result.logits.shape       ,(self.batch_size, self.seq_length, self.vocab_size))




							def 			A__				(    self       ,A__       ,A__       ,A__       ,A__       ,A__       ,A__       ,A__):
												lowercase						     =						AlbertForQuestionAnswering(config=A__)
												model.to(A__)
												model.eval()
												lowercase						     =						model(
												    A__       ,attention_mask=A__       ,token_type_ids=A__       ,start_positions=A__       ,end_positions=A__       ,)
												self.parent.assertEqual(result.start_logits.shape       ,(self.batch_size, self.seq_length))
												self.parent.assertEqual(result.end_logits.shape       ,(self.batch_size, self.seq_length))




							def 			A__				(    self       ,A__       ,A__       ,A__       ,A__       ,A__       ,A__       ,A__):
												lowercase						     =						self.num_labels
												lowercase						     =						AlbertForSequenceClassification(A__)
												model.to(A__)
												model.eval()
												lowercase						     =						model(A__       ,attention_mask=A__       ,token_type_ids=A__       ,labels=A__)
												self.parent.assertEqual(result.logits.shape       ,(self.batch_size, self.num_labels))




							def 			A__				(    self       ,A__       ,A__       ,A__       ,A__       ,A__       ,A__       ,A__):
												lowercase						     =						self.num_labels
												lowercase						     =						AlbertForTokenClassification(config=A__)
												model.to(A__)
												model.eval()
												lowercase						     =						model(A__       ,attention_mask=A__       ,token_type_ids=A__       ,labels=A__)
												self.parent.assertEqual(result.logits.shape       ,(self.batch_size, self.seq_length, self.num_labels))




							def 			A__				(    self       ,A__       ,A__       ,A__       ,A__       ,A__       ,A__       ,A__):
												lowercase						     =						self.num_choices
												lowercase						     =						AlbertForMultipleChoice(config=A__)
												model.to(A__)
												model.eval()
												lowercase						     =						input_ids.unsqueeze(1).expand(-1       ,self.num_choices       ,-1).contiguous()
												lowercase						     =						token_type_ids.unsqueeze(1).expand(-1       ,self.num_choices       ,-1).contiguous()
												lowercase						     =						input_mask.unsqueeze(1).expand(-1       ,self.num_choices       ,-1).contiguous()
												lowercase						     =						model(
												    A__       ,attention_mask=A__       ,token_type_ids=A__       ,labels=A__       ,)
												self.parent.assertEqual(result.logits.shape       ,(self.batch_size, self.num_choices))





							def 			A__				(    self):
												lowercase						     =						self.prepare_config_and_inputs()
												(
												    (
												    lowercase
												)     ,						(
												    lowercase
												)     ,						(
												    lowercase
												)     ,						(
												    lowercase
												)     ,						(
												    lowercase
												)     ,						(
												    lowercase
												)     ,						(
												    lowercase
												)     ,						
												)						     =						config_and_inputs
												lowercase						     =						{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
												return config, inputs_dict







@require_torch
class     lowercase		( SCREAMING_SNAKE_CASE__	,  SCREAMING_SNAKE_CASE__	,  unittest.TestCase		):
							lowercase_	:	Union[str, Any]	 =(
							    (
							        AlbertModel,
							        AlbertForPreTraining,
							        AlbertForMaskedLM,
							        AlbertForMultipleChoice,
							        AlbertForSequenceClassification,
							        AlbertForTokenClassification,
							        AlbertForQuestionAnswering,
							    )
							    if is_torch_available()
							    else ()
							)
							lowercase_	:	int	 =(
							    {
							        '''feature-extraction''': AlbertModel,
							        '''fill-mask''': AlbertForMaskedLM,
							        '''question-answering''': AlbertForQuestionAnswering,
							        '''text-classification''': AlbertForSequenceClassification,
							        '''token-classification''': AlbertForTokenClassification,
							        '''zero-shot''': AlbertForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)
							lowercase_	:	str	 =True




							def 			A__				(    self       ,A__       ,A__       ,A__=False):
												lowercase						     =						super()._prepare_for_class(A__       ,A__       ,return_labels=A__)

												if return_labels:
																	if model_class in get_values(A__):
																						lowercase						     =						torch.zeros(
																						    (self.model_tester.batch_size, self.model_tester.seq_length)       ,dtype=torch.long       ,device=A__)
																						lowercase						     =						torch.zeros(
																						    self.model_tester.batch_size       ,dtype=torch.long       ,device=A__)
												return inputs_dict




							def 			A__				(    self):
												lowercase						     =						AlbertModelTester(self)
												lowercase						     =						ConfigTester(self       ,config_class=A__       ,hidden_size=3_7)




							def 			A__				(    self):
												self.config_tester.run_common_tests()




							def 			A__				(    self):
												lowercase						     =						self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_model(*A__)




							def 			A__				(    self):
												lowercase						     =						self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_pretraining(*A__)




							def 			A__				(    self):
												lowercase						     =						self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_masked_lm(*A__)




							def 			A__				(    self):
												lowercase						     =						self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_multiple_choice(*A__)




							def 			A__				(    self):
												lowercase						     =						self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_question_answering(*A__)




							def 			A__				(    self):
												lowercase						     =						self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_sequence_classification(*A__)




							def 			A__				(    self):
												lowercase						     =						self.model_tester.prepare_config_and_inputs()
												for type in ["absolute", "relative_key", "relative_key_query"]:
																	lowercase						     =						type
																	self.model_tester.create_and_check_model(*A__)





							@slow
							def 			A__				(    self):
												for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																	lowercase						     =						AlbertModel.from_pretrained(A__)
																	self.assertIsNotNone(A__)







@require_torch
class     lowercase		( unittest.TestCase		):




							@slow
							def 			A__				(    self):
												lowercase						     =						AlbertModel.from_pretrained('''albert-base-v2''')
												lowercase						     =						torch.tensor([[0, 3_4_5, 2_3_2, 3_2_8, 7_4_0, 1_4_0, 1_6_9_5, 6_9, 6_0_7_8, 1_5_8_8, 2]])
												lowercase						     =						torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])
												with torch.no_grad():
																	lowercase						     =						model(A__       ,attention_mask=A__)[0]
												lowercase						     =						torch.Size((1, 1_1, 7_6_8))
												self.assertEqual(output.shape       ,A__)
												lowercase						     =						torch.tensor(
												    [[[-0.6513, 1.5035, -0.2766], [-0.6515, 1.5046, -0.2780], [-0.6512, 1.5049, -0.2784]]])

												self.assertTrue(torch.allclose(output[:, 1:4, 1:4]       ,A__       ,atol=1E-4))




