


'''simple docstring'''



import tensorflow as tf

from ...tf_utils import shape_list






class 	UpperCamelCase__(       tf.keras.layers.Layer	):




				def __init__(		self      :			Union[str, Any]				,   lowerCAmelCase      :			Optional[Any]				,   lowerCAmelCase      :			Any				,   lowerCAmelCase      :			Dict				,   lowerCAmelCase      :			Tuple				,   lowerCAmelCase      :			List[str]=1				,   lowerCAmelCase      :			List[str]=False				,   **lowerCAmelCase      :			Dict				)->					int:

										"""simple docstring"""




										super().__init__(**lowerCAmelCase				)

										UpperCAmelCase					    =     vocab_size
										UpperCAmelCase					    =     d_embed
										UpperCAmelCase					    =     d_proj

										UpperCAmelCase					    =     cutoffs + [vocab_size]
										UpperCAmelCase					    =     [0] + self.cutoffs
										UpperCAmelCase					    =     div_val

										UpperCAmelCase					    =     self.cutoffs[0]
										UpperCAmelCase					    =     len(self.cutoffs				) - 1
										UpperCAmelCase					    =     self.shortlist_size + self.n_clusters
										UpperCAmelCase					    =     keep_order

										UpperCAmelCase					    =     []
										UpperCAmelCase					    =     []




				def 					a__(		self      :			Union[str, Any]				,   lowerCAmelCase      :			Dict				)->					str:

										"""simple docstring"""




										if self.n_clusters > 0:
																UpperCAmelCase					    =     self.add_weight(
																    shape=(self.n_clusters, self.d_embed)				,   initializer='''zeros'''				,   trainable=lowerCAmelCase				,   name='''cluster_weight'''				)
																UpperCAmelCase					    =     self.add_weight(
																    shape=(self.n_clusters,)				,   initializer='''zeros'''				,   trainable=lowerCAmelCase				,   name='''cluster_bias'''				)

										if self.div_val == 1:
																for i in range(len(self.cutoffs				)				):
																						if self.d_proj != self.d_embed:
																												UpperCAmelCase					    =     self.add_weight(
																												    shape=(self.d_embed, self.d_proj)				,   initializer='''zeros'''				,   trainable=lowerCAmelCase				,   name=F"""out_projs_._{i}"""				,   )
																												self.out_projs.append(lowerCAmelCase				)
																						else:
																												self.out_projs.append(lowerCAmelCase				)
																						UpperCAmelCase					    =     self.add_weight(
																						    shape=(self.vocab_size, self.d_embed)				,   initializer='''zeros'''				,   trainable=lowerCAmelCase				,   name=F"""out_layers_._{i}_._weight"""				,   )
																						UpperCAmelCase					    =     self.add_weight(
																						    shape=(self.vocab_size,)				,   initializer='''zeros'''				,   trainable=lowerCAmelCase				,   name=F"""out_layers_._{i}_._bias"""				,   )
																						self.out_layers.append((weight, bias)				)
										else:
																for i in range(len(self.cutoffs				)				):
																						UpperCAmelCase    ,    UpperCAmelCase					    =     self.cutoff_ends[i], self.cutoff_ends[i + 1]
																						UpperCAmelCase					    =     self.d_embed // (self.div_val**i)

																						UpperCAmelCase					    =     self.add_weight(
																						    shape=(d_emb_i, self.d_proj)				,   initializer='''zeros'''				,   trainable=lowerCAmelCase				,   name=F"""out_projs_._{i}"""				)
																						self.out_projs.append(lowerCAmelCase				)
																						UpperCAmelCase					    =     self.add_weight(
																						    shape=(r_idx - l_idx, d_emb_i)				,   initializer='''zeros'''				,   trainable=lowerCAmelCase				,   name=F"""out_layers_._{i}_._weight"""				,   )
																						UpperCAmelCase					    =     self.add_weight(
																						    shape=(r_idx - l_idx,)				,   initializer='''zeros'''				,   trainable=lowerCAmelCase				,   name=F"""out_layers_._{i}_._bias"""				,   )
																						self.out_layers.append((weight, bias)				)
										super().build(lowerCAmelCase				)




				@staticmethod
				def 					a__(		lowerCAmelCase      :			int				,   lowerCAmelCase      :			Union[str, Any]				,   lowerCAmelCase      :			Any				,   lowerCAmelCase      :			str=None				)->					Any:

										"""simple docstring"""




										UpperCAmelCase					    =     x
										if proj is not None:
																UpperCAmelCase					    =     tf.einsum('''ibd,ed->ibe'''				,   lowerCAmelCase				,   lowerCAmelCase				)
										return tf.einsum('''ibd,nd->ibn'''				,   lowerCAmelCase				,   lowerCAmelCase				) + b




				@staticmethod
				def 					a__(		lowerCAmelCase      :			Optional[int]				,   lowerCAmelCase      :			Union[str, Any]				)->					Optional[Any]:

										"""simple docstring"""




										UpperCAmelCase					    =     shape_list(lowerCAmelCase				)
										UpperCAmelCase					    =     tf.range(lp_size[0]				,   dtype=target.dtype				)
										UpperCAmelCase					    =     tf.stack([r, target]				,   1				)
										return tf.gather_nd(lowerCAmelCase				,   lowerCAmelCase				)





				def 					a__(		self      :			Optional[Any]				,   lowerCAmelCase      :			Dict				,   lowerCAmelCase      :			List[Any]				,   lowerCAmelCase      :			Union[str, Any]=True				,   lowerCAmelCase      :			Optional[int]=False				)->					Union[str, Any]:

										"""simple docstring"""




										UpperCAmelCase					    =     0
										if self.n_clusters == 0:
																UpperCAmelCase					    =     self._logit(lowerCAmelCase				,   self.out_layers[0][0]				,   self.out_layers[0][1]				,   self.out_projs[0]				)
																if target is not None:
																						UpperCAmelCase					    =     tf.nn.sparse_softmax_cross_entropy_with_logits(labels=lowerCAmelCase				,   logits=lowerCAmelCase				)
																UpperCAmelCase					    =     tf.nn.log_softmax(lowerCAmelCase				,   axis=-1				)
										else:
																UpperCAmelCase					    =     shape_list(lowerCAmelCase				)
																UpperCAmelCase					    =     []
																UpperCAmelCase					    =     tf.zeros(hidden_sizes[:2]				)
																for i in range(len(self.cutoffs				)				):
																						UpperCAmelCase    ,    UpperCAmelCase					    =     self.cutoff_ends[i], self.cutoff_ends[i + 1]
																						if target is not None:
																												UpperCAmelCase					    =     (target >= l_idx) & (target < r_idx)
																												UpperCAmelCase					    =     tf.where(lowerCAmelCase				)
																												UpperCAmelCase					    =     tf.boolean_mask(lowerCAmelCase				,   lowerCAmelCase				) - l_idx

																						if self.div_val == 1:
																												UpperCAmelCase					    =     self.out_layers[0][0][l_idx:r_idx]
																												UpperCAmelCase					    =     self.out_layers[0][1][l_idx:r_idx]
																						else:
																												UpperCAmelCase					    =     self.out_layers[i][0]
																												UpperCAmelCase					    =     self.out_layers[i][1]

																						if i == 0:
																												UpperCAmelCase					    =     tf.concat([cur_W, self.cluster_weight]				,   0				)
																												UpperCAmelCase					    =     tf.concat([cur_b, self.cluster_bias]				,   0				)

																												UpperCAmelCase					    =     self._logit(lowerCAmelCase				,   lowerCAmelCase				,   lowerCAmelCase				,   self.out_projs[0]				)
																												UpperCAmelCase					    =     tf.nn.log_softmax(lowerCAmelCase				)
																												out.append(head_logprob[..., : self.cutoffs[0]]				)
																												if target is not None:
																																		UpperCAmelCase					    =     tf.boolean_mask(lowerCAmelCase				,   lowerCAmelCase				)
																																		UpperCAmelCase					    =     self._gather_logprob(lowerCAmelCase				,   lowerCAmelCase				)
																						else:
																												UpperCAmelCase					    =     self._logit(lowerCAmelCase				,   lowerCAmelCase				,   lowerCAmelCase				,   self.out_projs[i]				)
																												UpperCAmelCase					    =     tf.nn.log_softmax(lowerCAmelCase				)
																												UpperCAmelCase					    =     self.cutoffs[0] + i - 1  # No probability for the head cluster
																												UpperCAmelCase					    =     head_logprob[..., cluster_prob_idx, None] + tail_logprob
																												out.append(lowerCAmelCase				)
																												if target is not None:
																																		UpperCAmelCase					    =     tf.boolean_mask(lowerCAmelCase				,   lowerCAmelCase				)
																																		UpperCAmelCase					    =     tf.boolean_mask(lowerCAmelCase				,   lowerCAmelCase				)
																																		UpperCAmelCase					    =     self._gather_logprob(lowerCAmelCase				,   lowerCAmelCase				)
																																		cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1]
																						if target is not None:
																												loss += tf.scatter_nd(lowerCAmelCase				,   -cur_logprob				,   shape_list(lowerCAmelCase				)				)
																UpperCAmelCase					    =     tf.concat(lowerCAmelCase				,   axis=-1				)

										if target is not None:
																if return_mean:
																						UpperCAmelCase					    =     tf.reduce_mean(lowerCAmelCase				)
																# Add the training-time loss value to the layer using `self.add_loss()`.
																self.add_loss(lowerCAmelCase				)

																# Log the loss as a metric (we could log arbitrary metrics,
																# including different metrics for training and inference.
																self.add_metric(lowerCAmelCase				,   name=self.name				,   aggregation='''mean''' if return_mean else ''''''				)

										return out









'''simple docstring'''



import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
	from transformers.pipelines.conversational import Conversation


_lowercase      :      str            =					logging.get_logger(__name__)

_lowercase      :      Any            =					{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

_lowercase      :      Tuple            =					{
    """vocab_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/vocab.json""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/vocab.json""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/vocab.json""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/vocab.json""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/merges.txt""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/merges.txt""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/merges.txt""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/merges.txt""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/tokenizer.json""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/tokenizer.json""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/tokenizer.json""",
    },
}

_lowercase      :      Union[str, Any]            =					{
    """gpt2""": 1024,
    """gpt2-medium""": 1024,
    """gpt2-large""": 1024,
    """gpt2-xl""": 1024,
    """distilgpt2""": 1024,
}






class 	UpperCamelCase__(       lowerCAmelCase	):
				__magic_name__    :				Dict  		=	VOCAB_FILES_NAMES
				__magic_name__    :				str  		=	PRETRAINED_VOCAB_FILES_MAP
				__magic_name__    :				Any  		=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__magic_name__    :				List[Any]  		=	["input_ids", "attention_mask"]
				__magic_name__    :				List[Any]  		=	GPTaTokenizer




				def __init__(		self      :			Tuple				,   lowerCAmelCase      :			List[Any]=None				,   lowerCAmelCase      :			Dict=None				,   lowerCAmelCase      :			Dict=None				,   lowerCAmelCase      :			Tuple="<|endoftext|>"				,   lowerCAmelCase      :			Union[str, Any]="<|endoftext|>"				,   lowerCAmelCase      :			Union[str, Any]="<|endoftext|>"				,   lowerCAmelCase      :			Optional[int]=False				,   **lowerCAmelCase      :			Tuple				,   )->					int:

										"""simple docstring"""




										super().__init__(
										    lowerCAmelCase				,   lowerCAmelCase				,   tokenizer_file=lowerCAmelCase				,   unk_token=lowerCAmelCase				,   bos_token=lowerCAmelCase				,   eos_token=lowerCAmelCase				,   add_prefix_space=lowerCAmelCase				,   **lowerCAmelCase				,   )

										UpperCAmelCase					    =     kwargs.pop('''add_bos_token'''				,   lowerCAmelCase				)

										UpperCAmelCase					    =     json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()				)
										if pre_tok_state.get('''add_prefix_space'''				,   lowerCAmelCase				) != add_prefix_space:
																UpperCAmelCase					    =     getattr(lowerCAmelCase				,   pre_tok_state.pop('''type'''				)				)
																UpperCAmelCase					    =     add_prefix_space
																UpperCAmelCase					    =     pre_tok_class(**lowerCAmelCase				)

										UpperCAmelCase					    =     add_prefix_space




				def 					a__(		self      :			Union[str, Any]				,   *lowerCAmelCase      :			Union[str, Any]				,   **lowerCAmelCase      :			Dict				)->					BatchEncoding:

										"""simple docstring"""




										UpperCAmelCase					    =     kwargs.get('''is_split_into_words'''				,   lowerCAmelCase				)
										assert self.add_prefix_space or not is_split_into_words, (
										    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
										    "to use it with pretokenized inputs."
										)

										return super()._batch_encode_plus(*lowerCAmelCase				,   **lowerCAmelCase				)




				def 					a__(		self      :			List[str]				,   *lowerCAmelCase      :			Any				,   **lowerCAmelCase      :			Tuple				)->					BatchEncoding:

										"""simple docstring"""




										UpperCAmelCase					    =     kwargs.get('''is_split_into_words'''				,   lowerCAmelCase				)

										assert self.add_prefix_space or not is_split_into_words, (
										    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
										    "to use it with pretokenized inputs."
										)

										return super()._encode_plus(*lowerCAmelCase				,   **lowerCAmelCase				)




				def 					a__(		self      :			str				,   lowerCAmelCase      :			str				,   lowerCAmelCase      :			Optional[str] = None				)->					Tuple[str]:

										"""simple docstring"""




										UpperCAmelCase					    =     self._tokenizer.model.save(lowerCAmelCase				,   name=lowerCAmelCase				)
										return tuple(lowerCAmelCase				)





				def 					a__(		self      :			List[Any]				,   lowerCAmelCase      :			"Conversation"				)->					List[int]:

										"""simple docstring"""




										UpperCAmelCase					    =     []
										for is_user, text in conversation.iter_texts():
																input_ids.extend(self.encode(lowerCAmelCase				,   add_special_tokens=lowerCAmelCase				) + [self.eos_token_id]				)

										if len(lowerCAmelCase				) > self.model_max_length:
																UpperCAmelCase					    =     input_ids[-self.model_max_length :]
										return input_ids





