

'''simple docstring'''



import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
		from test_module.custom_tokenization_fast import CustomTokenizerFast



class lowerCamelCase			(  unittest.TestCase						):



			'''simple docstring'''






			def lowercase__						(				self	:						List[Any]			)      ->		Dict:





									'''simple docstring'''
									A__					:       List[Any]     =0


			@slow
			def lowercase__						(				self	:						List[str]			)      ->		List[Any]:





									'''simple docstring'''
									for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
															A__					:       str     =AutoTokenizer.from_pretrained(_lowercase			)
															self.assertIsNotNone(_lowercase			)
															self.assertIsInstance(_lowercase		,      (BertTokenizer, BertTokenizerFast)			)
															self.assertGreater(len(_lowercase			)		,      0			)

									for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
															A__					:       Optional[Any]     =AutoTokenizer.from_pretrained(_lowercase			)
															self.assertIsNotNone(_lowercase			)
															self.assertIsInstance(_lowercase		,      (GPTaTokenizer, GPTaTokenizerFast)			)
															self.assertGreater(len(_lowercase			)		,      0			)


			def lowercase__						(				self	:						Optional[int]			)      ->		Dict:





									'''simple docstring'''
									A__					:       Dict     =AutoTokenizer.from_pretrained(_lowercase			)
									self.assertIsInstance(_lowercase		,      (BertTokenizer, BertTokenizerFast)			)
									self.assertEqual(tokenizer.vocab_size		,      12			)


			def lowercase__						(				self	:						Tuple			)      ->		str:





									'''simple docstring'''
									A__					:       Tuple     =AutoTokenizer.from_pretrained(_lowercase			)
									self.assertIsInstance(_lowercase		,      (RobertaTokenizer, RobertaTokenizerFast)			)
									self.assertEqual(tokenizer.vocab_size		,      20			)


			def lowercase__						(				self	:						List[str]			)      ->		List[str]:





									'''simple docstring'''
									A__					:       Dict     =AutoConfig.from_pretrained(_lowercase			)
									self.assertIsInstance(_lowercase		,      _lowercase			)
									# Check that tokenizer_type ≠ model_type
									A__					:       Union[str, Any]     =AutoTokenizer.from_pretrained(_lowercase		,      config=_lowercase			)
									self.assertIsInstance(_lowercase		,      (BertTokenizer, BertTokenizerFast)			)
									self.assertEqual(tokenizer.vocab_size		,      12			)


			def lowercase__						(				self	:						Any			)      ->		str:





									'''simple docstring'''
									with tempfile.TemporaryDirectory() as tmp_dir:
															shutil.copy("""./tests/fixtures/vocab.txt"""		,      os.path.join(_lowercase		,      """vocab.txt"""			)			)

															A__					:       Optional[Any]     =AutoTokenizer.from_pretrained(_lowercase		,      tokenizer_type="""bert"""		,      use_fast=_lowercase			)
															self.assertIsInstance(_lowercase		,      _lowercase			)

									with tempfile.TemporaryDirectory() as tmp_dir:
															shutil.copy("""./tests/fixtures/vocab.json"""		,      os.path.join(_lowercase		,      """vocab.json"""			)			)
															shutil.copy("""./tests/fixtures/merges.txt"""		,      os.path.join(_lowercase		,      """merges.txt"""			)			)

															A__					:       Tuple     =AutoTokenizer.from_pretrained(_lowercase		,      tokenizer_type="""gpt2"""		,      use_fast=_lowercase			)
															self.assertIsInstance(_lowercase		,      _lowercase			)


			@require_tokenizers
			def lowercase__						(				self	:						int			)      ->		Union[str, Any]:





									'''simple docstring'''
									with tempfile.TemporaryDirectory() as tmp_dir:
															shutil.copy("""./tests/fixtures/vocab.txt"""		,      os.path.join(_lowercase		,      """vocab.txt"""			)			)

															A__					:       Optional[Any]     =AutoTokenizer.from_pretrained(_lowercase		,      tokenizer_type="""bert"""			)
															self.assertIsInstance(_lowercase		,      _lowercase			)

									with tempfile.TemporaryDirectory() as tmp_dir:
															shutil.copy("""./tests/fixtures/vocab.json"""		,      os.path.join(_lowercase		,      """vocab.json"""			)			)
															shutil.copy("""./tests/fixtures/merges.txt"""		,      os.path.join(_lowercase		,      """merges.txt"""			)			)

															A__					:       int     =AutoTokenizer.from_pretrained(_lowercase		,      tokenizer_type="""gpt2"""			)
															self.assertIsInstance(_lowercase		,      _lowercase			)


			def lowercase__						(				self	:						List[Any]			)      ->		List[Any]:





									'''simple docstring'''
									with pytest.raises(_lowercase			):
															AutoTokenizer.from_pretrained("""./"""		,      tokenizer_type="""xxx"""			)


			@require_tokenizers
			def lowercase__						(				self	:						Optional[int]			)      ->		Union[str, Any]:





									'''simple docstring'''
									for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
															A__					:       List[Any]     =tokenizer_class.from_pretrained("""wietsedv/bert-base-dutch-cased"""			)
															self.assertIsInstance(_lowercase		,      (BertTokenizer, BertTokenizerFast)			)

															if isinstance(_lowercase		,      _lowercase			):
																					self.assertEqual(tokenizer.basic_tokenizer.do_lower_case		,      _lowercase			)
															else:
																					self.assertEqual(tokenizer.do_lower_case		,      _lowercase			)

															self.assertEqual(tokenizer.model_max_length		,      5_12			)


			@require_tokenizers
			def lowercase__						(				self	:						int			)      ->		List[str]:





									'''simple docstring'''
									for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
															with self.assertRaisesRegex(
															    _lowercase		,      """julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"""		,      ):
																					A__					:       List[Any]     =tokenizer_class.from_pretrained("""julien-c/herlolip-not-exists"""			)


			def lowercase__						(				self	:						Optional[Any]			)      ->		List[str]:





									'''simple docstring'''
									# tests: https://github.com/huggingface/transformers/pull/13251
									# 1. models with `-`, e.g. xlm-roberta -> xlm_roberta
									# 2. models that don't remap 1-1 from model-name to model file, e.g., openai-gpt -> openai
									A__					:       List[str]     =TOKENIZER_MAPPING.values()
									A__					:       Optional[int]     =[]

									for slow_tok, fast_tok in tokenizers:
															if slow_tok is not None:
																					tokenizer_names.append(slow_tok.__name__			)

															if fast_tok is not None:
																					tokenizer_names.append(fast_tok.__name__			)

									for tokenizer_name in tokenizer_names:
															# must find the right class
															tokenizer_class_from_name(_lowercase			)


			@require_tokenizers
			def lowercase__						(				self	:						Optional[Any]			)      ->		Any:





									'''simple docstring'''
									self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""		,      use_fast=_lowercase			)		,      _lowercase			)
									self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""			)		,      _lowercase			)


			@require_tokenizers
			def lowercase__						(				self	:						Tuple			)      ->		Union[str, Any]:





									'''simple docstring'''
									A__					:       Optional[Any]     =AutoTokenizer.from_pretrained("""distilbert-base-uncased"""		,      do_lower_case=_lowercase			)
									A__					:       Tuple     ="""Hello, world. How are you?"""
									A__					:       List[Any]     =tokenizer.tokenize(_lowercase			)
									self.assertEqual("""[UNK]"""		,      tokens[0]			)

									A__					:       List[str]     =AutoTokenizer.from_pretrained("""microsoft/mpnet-base"""		,      do_lower_case=_lowercase			)
									A__					:       str     =tokenizer.tokenize(_lowercase			)
									self.assertEqual("""[UNK]"""		,      tokens[0]			)


			@require_tokenizers
			def lowercase__						(				self	:						int			)      ->		Optional[Any]:





									'''simple docstring'''
									A__					:       Dict     =AutoTokenizer.from_pretrained("""robot-test/dummy-tokenizer-fast-with-model-config"""			)
									self.assertEqual(type(_lowercase			)		,      _lowercase			)
									self.assertEqual(tokenizer.model_max_length		,      5_12			)
									self.assertEqual(tokenizer.vocab_size		,      3_00_00			)
									self.assertEqual(tokenizer.unk_token		,      """[UNK]"""			)
									self.assertEqual(tokenizer.padding_side		,      """right"""			)
									self.assertEqual(tokenizer.truncation_side		,      """right"""			)


			def lowercase__						(				self	:						List[str]			)      ->		str:





									'''simple docstring'''
									A__					:       str     =AutoTokenizer.from_pretrained(_lowercase			)
									self.assertIsInstance(_lowercase		,      (BertTokenizer, BertTokenizerFast)			)
									with tempfile.TemporaryDirectory() as tmp_dir:
															tokenizer.save_pretrained(_lowercase			)
															A__					:       List[str]     =AutoTokenizer.from_pretrained(_lowercase			)

									self.assertIsInstance(_lowercase		,      tokenizer.__class__			)
									self.assertEqual(tokenizera.vocab_size		,      12			)


			def lowercase__						(				self	:						Optional[int]			)      ->		List[str]:





									'''simple docstring'''
									A__					:       Tuple     =AutoTokenizer.from_pretrained("""ctrl"""			)
									# There is no fast CTRL so this always gives us a slow tokenizer.
									self.assertIsInstance(_lowercase		,      _lowercase			)


			def lowercase__						(				self	:						Dict			)      ->		Tuple:





									'''simple docstring'''
									# Check we can load the tokenizer config of an online model.
									A__					:       List[Any]     =get_tokenizer_config("""bert-base-cased"""			)
									A__					:       Any     =config.pop("""_commit_hash"""		,      _lowercase			)
									# If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
									self.assertEqual(_lowercase		,      {"""do_lower_case""": False}			)

									# This model does not have a tokenizer_config so we get back an empty dict.
									A__					:       Optional[Any]     =get_tokenizer_config(_lowercase			)
									self.assertDictEqual(_lowercase		,      {}			)

									# A tokenizer saved with `save_pretrained` always creates a tokenizer config.
									A__					:       Any     =AutoTokenizer.from_pretrained(_lowercase			)
									with tempfile.TemporaryDirectory() as tmp_dir:
															tokenizer.save_pretrained(_lowercase			)
															A__					:       int     =get_tokenizer_config(_lowercase			)

									# Check the class of the tokenizer was properly saved (note that it always saves the slow class).
									self.assertEqual(config["""tokenizer_class"""]		,      """BertTokenizer"""			)


			def lowercase__						(				self	:						str			)      ->		int:





									'''simple docstring'''
									try:
															AutoConfig.register("""custom"""		,      _lowercase			)

															AutoTokenizer.register(_lowercase		,      slow_tokenizer_class=_lowercase			)
															# Trying to register something existing in the Transformers library will raise an error
															with self.assertRaises(_lowercase			):
																					AutoTokenizer.register(_lowercase		,      slow_tokenizer_class=_lowercase			)

															A__					:       Optional[int]     =CustomTokenizer.from_pretrained(_lowercase			)
															with tempfile.TemporaryDirectory() as tmp_dir:
																					tokenizer.save_pretrained(_lowercase			)

																					A__					:       str     =AutoTokenizer.from_pretrained(_lowercase			)
																					self.assertIsInstance(_lowercase		,      _lowercase			)

									finally:
															if "custom" in CONFIG_MAPPING._extra_content:
																					del CONFIG_MAPPING._extra_content["custom"]
															if CustomConfig in TOKENIZER_MAPPING._extra_content:
																					del TOKENIZER_MAPPING._extra_content[CustomConfig]


			@require_tokenizers
			def lowercase__						(				self	:						str			)      ->		Tuple:





									'''simple docstring'''
									try:
															AutoConfig.register("""custom"""		,      _lowercase			)

															# Can register in two steps
															AutoTokenizer.register(_lowercase		,      slow_tokenizer_class=_lowercase			)
															self.assertEqual(TOKENIZER_MAPPING[CustomConfig]		,      (CustomTokenizer, None)			)
															AutoTokenizer.register(_lowercase		,      fast_tokenizer_class=_lowercase			)
															self.assertEqual(TOKENIZER_MAPPING[CustomConfig]		,      (CustomTokenizer, CustomTokenizerFast)			)

															del TOKENIZER_MAPPING._extra_content[CustomConfig]
															# Can register in one step
															AutoTokenizer.register(
															    _lowercase		,      slow_tokenizer_class=_lowercase		,      fast_tokenizer_class=_lowercase			)
															self.assertEqual(TOKENIZER_MAPPING[CustomConfig]		,      (CustomTokenizer, CustomTokenizerFast)			)

															# Trying to register something existing in the Transformers library will raise an error
															with self.assertRaises(_lowercase			):
																					AutoTokenizer.register(_lowercase		,      fast_tokenizer_class=_lowercase			)

															# We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
															# and that model does not have a tokenizer.json
															with tempfile.TemporaryDirectory() as tmp_dir:
																					A__					:       Tuple     =BertTokenizerFast.from_pretrained(_lowercase			)
																					bert_tokenizer.save_pretrained(_lowercase			)
																					A__					:       Dict     =CustomTokenizerFast.from_pretrained(_lowercase			)

															with tempfile.TemporaryDirectory() as tmp_dir:
																					tokenizer.save_pretrained(_lowercase			)

																					A__					:       Any     =AutoTokenizer.from_pretrained(_lowercase			)
																					self.assertIsInstance(_lowercase		,      _lowercase			)

																					A__					:       Optional[int]     =AutoTokenizer.from_pretrained(_lowercase		,      use_fast=_lowercase			)
																					self.assertIsInstance(_lowercase		,      _lowercase			)

									finally:
															if "custom" in CONFIG_MAPPING._extra_content:
																					del CONFIG_MAPPING._extra_content["custom"]
															if CustomConfig in TOKENIZER_MAPPING._extra_content:
																					del TOKENIZER_MAPPING._extra_content[CustomConfig]


			def lowercase__						(				self	:						Optional[int]			)      ->		Union[str, Any]:





									'''simple docstring'''
									# If remote code is not set, we will time out when asking whether to load the model.
									with self.assertRaises(_lowercase			):
															A__					:       List[Any]     =AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""			)
									# If remote code is disabled, we can't load this config.
									with self.assertRaises(_lowercase			):
															A__					:       List[Any]     =AutoTokenizer.from_pretrained(
															    """hf-internal-testing/test_dynamic_tokenizer"""		,      trust_remote_code=_lowercase			)

									A__					:       int     =AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""		,      trust_remote_code=_lowercase			)
									self.assertTrue(tokenizer.special_attribute_present			)
									# Test tokenizer can be reloaded.
									with tempfile.TemporaryDirectory() as tmp_dir:
															tokenizer.save_pretrained(_lowercase			)
															A__					:       Any     =AutoTokenizer.from_pretrained(_lowercase		,      trust_remote_code=_lowercase			)
									self.assertTrue(reloaded_tokenizer.special_attribute_present			)

									if is_tokenizers_available():
															self.assertEqual(tokenizer.__class__.__name__		,      """NewTokenizerFast"""			)
															self.assertEqual(reloaded_tokenizer.__class__.__name__		,      """NewTokenizerFast"""			)

															# Test we can also load the slow version
															A__					:       List[str]     =AutoTokenizer.from_pretrained(
															    """hf-internal-testing/test_dynamic_tokenizer"""		,      trust_remote_code=_lowercase		,      use_fast=_lowercase			)
															self.assertTrue(tokenizer.special_attribute_present			)
															self.assertEqual(tokenizer.__class__.__name__		,      """NewTokenizer"""			)
															# Test tokenizer can be reloaded.
															with tempfile.TemporaryDirectory() as tmp_dir:
																					tokenizer.save_pretrained(_lowercase			)
																					A__					:       Dict     =AutoTokenizer.from_pretrained(_lowercase		,      trust_remote_code=_lowercase		,      use_fast=_lowercase			)
															self.assertEqual(reloaded_tokenizer.__class__.__name__		,      """NewTokenizer"""			)
															self.assertTrue(reloaded_tokenizer.special_attribute_present			)
									else:
															self.assertEqual(tokenizer.__class__.__name__		,      """NewTokenizer"""			)
															self.assertEqual(reloaded_tokenizer.__class__.__name__		,      """NewTokenizer"""			)


			@require_tokenizers
			def lowercase__						(				self	:						Union[str, Any]			)      ->		List[Any]:





									'''simple docstring'''



									class lowerCamelCase			(  lowercase_						):



												'''simple docstring'''






												__snake_case        =	False



									class lowerCamelCase			(  lowercase_						):



												'''simple docstring'''






												__snake_case        =	NewTokenizer
												__snake_case        =	False

									try:
															AutoConfig.register("""custom"""		,      _lowercase			)
															AutoTokenizer.register(_lowercase		,      slow_tokenizer_class=_lowercase			)
															AutoTokenizer.register(_lowercase		,      fast_tokenizer_class=_lowercase			)
															# If remote code is not set, the default is to use local
															A__					:       Union[str, Any]     =AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""			)
															self.assertEqual(tokenizer.__class__.__name__		,      """NewTokenizerFast"""			)
															self.assertFalse(tokenizer.special_attribute_present			)
															A__					:       Dict     =AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""		,      use_fast=_lowercase			)
															self.assertEqual(tokenizer.__class__.__name__		,      """NewTokenizer"""			)
															self.assertFalse(tokenizer.special_attribute_present			)

															# If remote code is disabled, we load the local one.
															A__					:       Tuple     =AutoTokenizer.from_pretrained(
															    """hf-internal-testing/test_dynamic_tokenizer"""		,      trust_remote_code=_lowercase			)
															self.assertEqual(tokenizer.__class__.__name__		,      """NewTokenizerFast"""			)
															self.assertFalse(tokenizer.special_attribute_present			)
															A__					:       List[str]     =AutoTokenizer.from_pretrained(
															    """hf-internal-testing/test_dynamic_tokenizer"""		,      trust_remote_code=_lowercase		,      use_fast=_lowercase			)
															self.assertEqual(tokenizer.__class__.__name__		,      """NewTokenizer"""			)
															self.assertFalse(tokenizer.special_attribute_present			)

															# If remote is enabled, we load from the Hub
															A__					:       List[str]     =AutoTokenizer.from_pretrained(
															    """hf-internal-testing/test_dynamic_tokenizer"""		,      trust_remote_code=_lowercase			)
															self.assertEqual(tokenizer.__class__.__name__		,      """NewTokenizerFast"""			)
															self.assertTrue(tokenizer.special_attribute_present			)
															A__					:       Tuple     =AutoTokenizer.from_pretrained(
															    """hf-internal-testing/test_dynamic_tokenizer"""		,      trust_remote_code=_lowercase		,      use_fast=_lowercase			)
															self.assertEqual(tokenizer.__class__.__name__		,      """NewTokenizer"""			)
															self.assertTrue(tokenizer.special_attribute_present			)

									finally:
															if "custom" in CONFIG_MAPPING._extra_content:
																					del CONFIG_MAPPING._extra_content["custom"]
															if CustomConfig in TOKENIZER_MAPPING._extra_content:
																					del TOKENIZER_MAPPING._extra_content[CustomConfig]


			def lowercase__						(				self	:						List[Any]			)      ->		int:





									'''simple docstring'''
									A__					:       Union[str, Any]     =AutoTokenizer.from_pretrained(
									    """hf-internal-testing/test_dynamic_tokenizer_legacy"""		,      trust_remote_code=_lowercase			)
									self.assertTrue(tokenizer.special_attribute_present			)
									if is_tokenizers_available():
															self.assertEqual(tokenizer.__class__.__name__		,      """NewTokenizerFast"""			)

															# Test we can also load the slow version
															A__					:       Tuple     =AutoTokenizer.from_pretrained(
															    """hf-internal-testing/test_dynamic_tokenizer_legacy"""		,      trust_remote_code=_lowercase		,      use_fast=_lowercase			)
															self.assertTrue(tokenizer.special_attribute_present			)
															self.assertEqual(tokenizer.__class__.__name__		,      """NewTokenizer"""			)
									else:
															self.assertEqual(tokenizer.__class__.__name__		,      """NewTokenizer"""			)


			def lowercase__						(				self	:						int			)      ->		int:





									'''simple docstring'''
									with self.assertRaisesRegex(
									    _lowercase		,      """bert-base is not a local folder and is not a valid model identifier"""			):
															A__					:       Dict     =AutoTokenizer.from_pretrained("""bert-base"""			)


			def lowercase__						(				self	:						Dict			)      ->		Tuple:





									'''simple docstring'''
									with self.assertRaisesRegex(
									    _lowercase		,      R"""aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"""			):
															A__					:       int     =AutoTokenizer.from_pretrained(_lowercase		,      revision="""aaaaaa"""			)



			def lowercase__						(				self	:						Dict			)      ->		Optional[Any]:





									'''simple docstring'''
									# Make sure we have cached the tokenizer.
									A__					:       str     =AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""			)
									with RequestCounter() as counter:
															A__					:       Dict     =AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""			)
															self.assertEqual(counter.get_request_count		,      0			)
															self.assertEqual(counter.head_request_count		,      1			)
															self.assertEqual(counter.other_request_count		,      0			)




'''simple docstring'''



import torch
from torch import nn



class lowerCamelCase			(  nn.Module						):



			'''simple docstring'''






			def __init__(				self	:						Any		,      lowerCAmelCase_	:						List[str]		,      lowerCAmelCase_	:						List[Any]		,      lowerCAmelCase_	:						Dict		,      lowerCAmelCase_	:						str		,      lowerCAmelCase_	:						Optional[int]=1		,      lowerCAmelCase_	:						str=False			)      ->		List[str]:





									'''simple docstring'''
									super().__init__()

									A__					:       Any     =n_token
									A__					:       int     =d_embed
									A__					:       Any     =d_proj

									A__					:       Tuple     =cutoffs + [n_token]
									A__					:       Optional[Any]     =[0] + self.cutoffs
									A__					:       Dict     =div_val

									A__					:       str     =self.cutoffs[0]
									A__					:       Optional[Any]     =len(self.cutoffs			) - 1
									A__					:       List[Any]     =self.shortlist_size + self.n_clusters

									if self.n_clusters > 0:
															A__					:       Any     =nn.Parameter(torch.zeros(self.n_clusters		,      self.d_embed			)			)
															A__					:       str     =nn.Parameter(torch.zeros(self.n_clusters			)			)

									A__					:       Union[str, Any]     =nn.ModuleList()
									A__					:       Optional[int]     =nn.ParameterList()

									if div_val == 1:
															for i in range(len(self.cutoffs			)			):
																					if d_proj != d_embed:
																											self.out_projs.append(nn.Parameter(torch.FloatTensor(lowerCAmelCase_		,      lowerCAmelCase_			)			)			)
																					else:
																											self.out_projs.append(lowerCAmelCase_			)

															self.out_layers.append(nn.Linear(lowerCAmelCase_		,      lowerCAmelCase_			)			)
									else:
															for i in range(len(self.cutoffs			)			):
																					A__   ,						A__					:       Optional[int]     =self.cutoff_ends[i], self.cutoff_ends[i + 1]
																					A__					:       Tuple     =d_embed // (div_val**i)

																					self.out_projs.append(nn.Parameter(torch.FloatTensor(lowerCAmelCase_		,      lowerCAmelCase_			)			)			)

																					self.out_layers.append(nn.Linear(lowerCAmelCase_		,      r_idx - l_idx			)			)

									A__					:       Optional[int]     =keep_order


			def lowercase__						(				self	:						Dict		,      lowerCAmelCase_	:						int		,      lowerCAmelCase_	:						Tuple		,      lowerCAmelCase_	:						Union[str, Any]		,      lowerCAmelCase_	:						Any			)      ->		Union[str, Any]:





									'''simple docstring'''
									if proj is None:
															A__					:       Optional[int]     =nn.functional.linear(lowerCAmelCase_		,      lowerCAmelCase_		,      bias=lowerCAmelCase_			)
									else:
															# if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
															A__					:       Optional[int]     =nn.functional.linear(lowerCAmelCase_		,      proj.t().contiguous()			)
															A__					:       Union[str, Any]     =nn.functional.linear(lowerCAmelCase_		,      lowerCAmelCase_		,      bias=lowerCAmelCase_			)
															# else:
															#     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
															#     if bias is not None:
															#         logit = logit + bias

									return logit


			def lowercase__						(				self	:						Tuple		,      lowerCAmelCase_	:						List[str]		,      lowerCAmelCase_	:						List[Any]=None		,      lowerCAmelCase_	:						Dict=False			)      ->		Optional[int]:





									'''simple docstring'''

									if labels is not None:
															# Shift so that tokens < n predict n
															A__					:       Optional[Any]     =hidden[..., :-1, :].contiguous()
															A__					:       List[Any]     =labels[..., 1:].contiguous()
															A__					:       Optional[int]     =hidden.view(-1		,      hidden.size(-1			)			)
															A__					:       str     =labels.view(-1			)
															if hidden.size(0			) != labels.size(0			):
																					raise RuntimeError("""Input and labels should have the same size in the batch dimension."""			)
									else:
															A__					:       Optional[int]     =hidden.view(-1		,      hidden.size(-1			)			)

									if self.n_clusters == 0:
															A__					:       Optional[Any]     =self._compute_logit(lowerCAmelCase_		,      self.out_layers[0].weight		,      self.out_layers[0].bias		,      self.out_projs[0]			)
															if labels is not None:
																					A__					:       Tuple     =labels != -1_00
																					A__					:       int     =torch.zeros_like(lowerCAmelCase_		,      dtype=hidden.dtype		,      device=hidden.device			)
																					A__					:       Union[str, Any]     =(
																					    -nn.functional.log_softmax(lowerCAmelCase_		,      dim=-1			)[mask].gather(1		,      labels[mask].unsqueeze(1			)			).squeeze(1			)
																					)
															else:
																					A__					:       List[Any]     =nn.functional.log_softmax(lowerCAmelCase_		,      dim=-1			)
									else:
															# construct weights and biases
															A__   ,						A__					:       Any     =[], []
															for i in range(len(self.cutoffs			)			):
																					if self.div_val == 1:
																											A__   ,						A__					:       Optional[int]     =self.cutoff_ends[i], self.cutoff_ends[i + 1]
																											A__					:       int     =self.out_layers[0].weight[l_idx:r_idx]
																											A__					:       List[str]     =self.out_layers[0].bias[l_idx:r_idx]
																					else:
																											A__					:       List[str]     =self.out_layers[i].weight
																											A__					:       Union[str, Any]     =self.out_layers[i].bias

																					if i == 0:
																											A__					:       Tuple     =torch.cat([weight_i, self.cluster_weight]		,      dim=0			)
																											A__					:       List[str]     =torch.cat([bias_i, self.cluster_bias]		,      dim=0			)

																					weights.append(lowerCAmelCase_			)
																					biases.append(lowerCAmelCase_			)

															A__   ,						A__   ,						A__					:       Tuple     =weights[0], biases[0], self.out_projs[0]

															A__					:       List[Any]     =self._compute_logit(lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_			)
															A__					:       int     =nn.functional.log_softmax(lowerCAmelCase_		,      dim=1			)

															if labels is None:
																					A__					:       Union[str, Any]     =hidden.new_empty((head_logit.size(0			), self.n_token)			)
															else:
																					A__					:       Union[str, Any]     =torch.zeros_like(lowerCAmelCase_		,      dtype=hidden.dtype		,      device=hidden.device			)

															A__					:       Any     =0
															A__					:       Tuple     =[0] + self.cutoffs
															for i in range(len(lowerCAmelCase_			) - 1			):
																					A__   ,						A__					:       Tuple     =cutoff_values[i], cutoff_values[i + 1]

																					if labels is not None:
																											A__					:       Tuple     =(labels >= l_idx) & (labels < r_idx)
																											A__					:       Any     =mask_i.nonzero().squeeze()

																											if indices_i.numel() == 0:
																																	continue

																											A__					:       int     =labels.index_select(0		,      lowerCAmelCase_			) - l_idx
																											A__					:       List[str]     =head_logprob.index_select(0		,      lowerCAmelCase_			)
																											A__					:       str     =hidden.index_select(0		,      lowerCAmelCase_			)
																					else:
																											A__					:       Optional[Any]     =hidden

																					if i == 0:
																											if labels is not None:
																																	A__					:       Optional[Any]     =head_logprob_i.gather(1		,      target_i[:, None]			).squeeze(1			)
																											else:
																																	A__					:       Union[str, Any]     =head_logprob[:, : self.cutoffs[0]]
																					else:
																											A__   ,						A__   ,						A__					:       Dict     =weights[i], biases[i], self.out_projs[i]

																											A__					:       List[Any]     =self._compute_logit(lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_			)
																											A__					:       List[Any]     =nn.functional.log_softmax(lowerCAmelCase_		,      dim=1			)
																											A__					:       Optional[Any]     =self.cutoffs[0] + i - 1  # No probability for the head cluster
																											if labels is not None:
																																	A__					:       Union[str, Any]     =head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
																																	    1		,      target_i[:, None]			).squeeze(1			)
																											else:
																																	A__					:       List[str]     =head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
																																	A__					:       Tuple     =logprob_i

																					if labels is not None:
																											if (hasattr(self		,      """keep_order"""			) and self.keep_order) or keep_order:
																																	out.index_copy_(0		,      lowerCAmelCase_		,      -logprob_i			)
																											else:
																																	out[offset : offset + logprob_i.size(0			)].copy_(-logprob_i			)
																											offset += logprob_i.size(0			)

									return out



			def lowercase__						(				self	:						List[str]		,      lowerCAmelCase_	:						Optional[Any]			)      ->		Any:





									'''simple docstring'''
									if self.n_clusters == 0:
															A__					:       List[str]     =self._compute_logit(lowerCAmelCase_		,      self.out_layers[0].weight		,      self.out_layers[0].bias		,      self.out_projs[0]			)
															return nn.functional.log_softmax(lowerCAmelCase_		,      dim=-1			)
									else:
															# construct weights and biases
															A__   ,						A__					:       List[str]     =[], []
															for i in range(len(self.cutoffs			)			):
																					if self.div_val == 1:
																											A__   ,						A__					:       int     =self.cutoff_ends[i], self.cutoff_ends[i + 1]
																											A__					:       List[str]     =self.out_layers[0].weight[l_idx:r_idx]
																											A__					:       List[Any]     =self.out_layers[0].bias[l_idx:r_idx]
																					else:
																											A__					:       Dict     =self.out_layers[i].weight
																											A__					:       Any     =self.out_layers[i].bias

																					if i == 0:
																											A__					:       List[str]     =torch.cat([weight_i, self.cluster_weight]		,      dim=0			)
																											A__					:       Tuple     =torch.cat([bias_i, self.cluster_bias]		,      dim=0			)

																					weights.append(lowerCAmelCase_			)
																					biases.append(lowerCAmelCase_			)

															A__   ,						A__   ,						A__					:       Optional[int]     =weights[0], biases[0], self.out_projs[0]
															A__					:       Any     =self._compute_logit(lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_			)

															A__					:       Dict     =hidden.new_empty((head_logit.size(0			), self.n_token)			)
															A__					:       Dict     =nn.functional.log_softmax(lowerCAmelCase_		,      dim=1			)

															A__					:       Tuple     =[0] + self.cutoffs
															for i in range(len(lowerCAmelCase_			) - 1			):
																					A__   ,						A__					:       List[Any]     =cutoff_values[i], cutoff_values[i + 1]

																					if i == 0:
																											A__					:       Tuple     =head_logprob[:, : self.cutoffs[0]]
																					else:
																											A__   ,						A__   ,						A__					:       Any     =weights[i], biases[i], self.out_projs[i]

																											A__					:       Dict     =self._compute_logit(lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_			)
																											A__					:       str     =nn.functional.log_softmax(lowerCAmelCase_		,      dim=1			)

																											A__					:       str     =head_logprob[:, -i] + tail_logprob_i
																											A__					:       List[Any]     =logprob_i

															return out

