

'''simple docstring'''



import numpy as np
import torch
from torch.utils.data import Dataset

from utils import logger



class lowerCamelCase			(  lowercase_						):



			'''simple docstring'''






			def __init__(				self	:						List[Any]		,      lowerCAmelCase_	:						Optional[Any]		,      lowerCAmelCase_	:						Optional[int]			)      ->		int:





									'''simple docstring'''
									A__					:       Optional[Any]     =params

									A__					:       int     =np.array(lowerCAmelCase_			)
									A__					:       str     =np.array([len(lowerCAmelCase_			) for t in data]			)

									self.check()
									self.remove_long_sequences()
									self.remove_empty_sequences()
									self.remove_unknown_sequences()
									self.check()
									self.print_statistics()


			def __getitem__(				self	:						Union[str, Any]		,      lowerCAmelCase_	:						List[Any]			)      ->		int:





									'''simple docstring'''
									return (self.token_ids[index], self.lengths[index])


			def __len__(				self	:						List[str]			)      ->		List[Any]:





									'''simple docstring'''
									return len(self.lengths			)


			def lowercase__						(				self	:						Any			)      ->		int:





									'''simple docstring'''
									assert len(self.token_ids			) == len(self.lengths			)
									assert all(self.lengths[i] == len(self.token_ids[i]			) for i in range(len(self.lengths			)			)			)


			def lowercase__						(				self	:						Union[str, Any]			)      ->		int:





									'''simple docstring'''
									A__					:       Optional[Any]     =self.params.max_model_input_size
									A__					:       int     =self.lengths > max_len
									logger.info(f"Splitting {sum(lowerCAmelCase_			)} too long sequences."			)

									def divide_chunks(lowerCAmelCase_	:						Union[str, Any]		,      lowerCAmelCase_	:						int			):
															return [l[i : i + n] for i in range(0		,      len(lowerCAmelCase_			)		,      lowerCAmelCase_			)]

									A__					:       int     =[]
									A__					:       Optional[Any]     =[]
									if self.params.mlm:
															A__   ,						A__					:       int     =self.params.special_tok_ids["""cls_token"""], self.params.special_tok_ids["""sep_token"""]
									else:
															A__   ,						A__					:       List[Any]     =self.params.special_tok_ids["""bos_token"""], self.params.special_tok_ids["""eos_token"""]

									for seq_, len_ in zip(self.token_ids		,      self.lengths			):
															assert (seq_[0] == cls_id) and (seq_[-1] == sep_id), seq_
															if len_ <= max_len:
																					new_tok_ids.append(seq_			)
																					new_lengths.append(len_			)
															else:
																					A__					:       str     =[]
																					for sub_s in divide_chunks(seq_		,      max_len - 2			):
																											if sub_s[0] != cls_id:
																																	A__					:       Dict     =np.insert(lowerCAmelCase_		,      0		,      lowerCAmelCase_			)
																											if sub_s[-1] != sep_id:
																																	A__					:       Dict     =np.insert(lowerCAmelCase_		,      len(lowerCAmelCase_			)		,      lowerCAmelCase_			)
																											assert len(lowerCAmelCase_			) <= max_len
																											assert (sub_s[0] == cls_id) and (sub_s[-1] == sep_id), sub_s
																											sub_seqs.append(lowerCAmelCase_			)

																					new_tok_ids.extend(lowerCAmelCase_			)
																					new_lengths.extend([len(lowerCAmelCase_			) for l in sub_seqs]			)

									A__					:       int     =np.array(lowerCAmelCase_			)
									A__					:       Dict     =np.array(lowerCAmelCase_			)


			def lowercase__						(				self	:						List[Any]			)      ->		Optional[int]:





									'''simple docstring'''
									A__					:       int     =len(self			)
									A__					:       Optional[Any]     =self.lengths > 11
									A__					:       Optional[Any]     =self.token_ids[indices]
									A__					:       Any     =self.lengths[indices]
									A__					:       Optional[int]     =len(self			)
									logger.info(f"Remove {init_size - new_size} too short (<=11 tokens) sequences."			)


			def lowercase__						(				self	:						Optional[int]			)      ->		Tuple:





									'''simple docstring'''
									if "unk_token" not in self.params.special_tok_ids:
															return
									else:
															A__					:       Union[str, Any]     =self.params.special_tok_ids["""unk_token"""]
									A__					:       Dict     =len(self			)
									A__					:       Optional[int]     =np.array([np.count_nonzero(a == unk_token_id			) for a in self.token_ids]			)
									A__					:       List[str]     =(unk_occs / self.lengths) < 0.5
									A__					:       Optional[int]     =self.token_ids[indices]
									A__					:       int     =self.lengths[indices]
									A__					:       List[str]     =len(self			)
									logger.info(f"Remove {init_size - new_size} sequences with a high level of unknown tokens (50%)."			)


			def lowercase__						(				self	:						Optional[Any]			)      ->		int:





									'''simple docstring'''
									if not self.params.is_master:
															return
									logger.info(f"{len(self			)} sequences"			)
									# data_len = sum(self.lengths)
									# nb_unique_tokens = len(Counter(list(chain(*self.token_ids))))
									# logger.info(f'{data_len} tokens ({nb_unique_tokens} unique)')

									# unk_idx = self.params.special_tok_ids['unk_token']
									# nb_unknown = sum([(t==unk_idx).sum() for t in self.token_ids])
									# logger.info(f'{nb_unknown} unknown tokens (covering {100*nb_unknown/data_len:.2f}% of the data)')



			def lowercase__						(				self	:						Any		,      lowerCAmelCase_	:						str			)      ->		Any:





									'''simple docstring'''
									A__					:       int     =[t[0] for t in batch]
									A__					:       Optional[int]     =[t[1] for t in batch]
									assert len(lowerCAmelCase_			) == len(lowerCAmelCase_			)

									# Max for paddings
									A__					:       Optional[int]     =max(lowerCAmelCase_			)

									# Pad token ids
									if self.params.mlm:
															A__					:       Tuple     =self.params.special_tok_ids["""pad_token"""]
									else:
															A__					:       str     =self.params.special_tok_ids["""unk_token"""]
									A__					:       Any     =[list(t.astype(lowerCAmelCase_			)			) + [pad_idx] * (max_seq_len_ - len(lowerCAmelCase_			)) for t in token_ids]
									assert len(tk_			) == len(lowerCAmelCase_			)
									assert all(len(lowerCAmelCase_			) == max_seq_len_ for t in tk_			)

									A__					:       Tuple     =torch.tensor(tk_			)  # (bs, max_seq_len_)
									A__					:       List[str]     =torch.tensor(lowerCAmelCase_			)  # (bs)
									return tk_t, lg_t




'''simple docstring'''



from ...configuration_utils import PretrainedConfig
from ...utils import logging


__snake_case			:     Union[str, Any]										=      logging.get_logger(__name__)

__snake_case			:     Optional[int]										=      {
    'bigcode/gpt_bigcode-santacoder': 'https://huggingface.co/bigcode/gpt_bigcode-santacoder/resolve/main/config.json',
}



class lowerCamelCase			(  lowercase_						):



			'''simple docstring'''






			__snake_case        =	'gpt_bigcode'
			__snake_case        =	['past_key_values']
			__snake_case        =	{
			    'hidden_size': 'n_embd',
			    'max_position_embeddings': 'n_positions',
			    'num_attention_heads': 'n_head',
			    'num_hidden_layers': 'n_layer',
			}


			def __init__(				self	:						Optional[Any]		,      lowerCAmelCase_	:						List[str]=5_02_57		,      lowerCAmelCase_	:						str=10_24		,      lowerCAmelCase_	:						str=7_68		,      lowerCAmelCase_	:						str=12		,      lowerCAmelCase_	:						int=12		,      lowerCAmelCase_	:						Dict=None		,      lowerCAmelCase_	:						List[Any]="gelu_pytorch_tanh"		,      lowerCAmelCase_	:						Union[str, Any]=0.1		,      lowerCAmelCase_	:						Optional[Any]=0.1		,      lowerCAmelCase_	:						Optional[int]=0.1		,      lowerCAmelCase_	:						Dict=1e-5		,      lowerCAmelCase_	:						str=0.02		,      lowerCAmelCase_	:						List[Any]=True		,      lowerCAmelCase_	:						str=True		,      lowerCAmelCase_	:						str=5_02_56		,      lowerCAmelCase_	:						Dict=5_02_56		,      lowerCAmelCase_	:						Union[str, Any]=True		,      lowerCAmelCase_	:						str=True		,      lowerCAmelCase_	:						Optional[Any]=True		,      **lowerCAmelCase_	:						Optional[Any]		,      )      ->		Tuple:





									'''simple docstring'''
									A__					:       Optional[Any]     =vocab_size
									A__					:       Optional[Any]     =n_positions
									A__					:       List[str]     =n_embd
									A__					:       str     =n_layer
									A__					:       Optional[int]     =n_head
									A__					:       Optional[int]     =n_inner
									A__					:       int     =activation_function
									A__					:       int     =resid_pdrop
									A__					:       int     =embd_pdrop
									A__					:       Dict     =attn_pdrop
									A__					:       Any     =layer_norm_epsilon
									A__					:       List[Any]     =initializer_range
									A__					:       Dict     =scale_attn_weights
									A__					:       Any     =use_cache
									A__					:       List[Any]     =attention_softmax_in_fpaa
									A__					:       Optional[int]     =scale_attention_softmax_in_fpaa
									A__					:       Dict     =multi_query

									A__					:       List[str]     =bos_token_id
									A__					:       Any     =eos_token_id

									super().__init__(bos_token_id=lowerCAmelCase_		,      eos_token_id=lowerCAmelCase_		,      **lowerCAmelCase_			)

