


"""simple docstring"""







import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


lowerCAmelCase__				    =       logging.get_logger(__name__)

lowerCAmelCase__				    =       {'''vocab_file''': '''spiece.model'''}

lowerCAmelCase__				    =       {
    '''vocab_file''': {
        '''TsinghuaAI/CPM-Generate''': '''https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model''',
    }
}


class 				SCREAMING_SNAKE_CASE__    (		lowercase    ):
							"""simple docstring"""


							def __init__(						self		,					snake_case__		,					snake_case__=False		,					snake_case__=True		,					snake_case__=False		,					snake_case__="<s>"		,					snake_case__="</s>"		,					snake_case__="<unk>"		,					snake_case__="<sep>"		,					snake_case__="<pad>"		,					snake_case__="<cls>"		,					snake_case__="<mask>"		,					snake_case__=["<eop>", "<eod>"]		,					snake_case__ = None		,					**snake_case__		,					):






									"""simple docstring"""





									lowerCAmelCase					:					Dict								=							AddedToken(snake_case__		,					lstrip=snake_case__		,					rstrip=snake_case__						) if isinstance(snake_case__		,					snake_case__						) else mask_token

									lowerCAmelCase					:					Optional[int]								=							{} if sp_model_kwargs is None else sp_model_kwargs

									super().__init__(
									    do_lower_case=snake_case__		,					remove_space=snake_case__		,					keep_accents=snake_case__		,					bos_token=snake_case__		,					eos_token=snake_case__		,					unk_token=snake_case__		,					sep_token=snake_case__		,					pad_token=snake_case__		,					cls_token=snake_case__		,					mask_token=snake_case__		,					additional_special_tokens=snake_case__		,					sp_model_kwargs=self.sp_model_kwargs		,					**snake_case__		,					)

									lowerCAmelCase					:					Optional[int]								=							3

									lowerCAmelCase					:					Tuple								=							do_lower_case
									lowerCAmelCase					:					int								=							remove_space
									lowerCAmelCase					:					List[str]								=							keep_accents
									lowerCAmelCase					:					int								=							vocab_file

									lowerCAmelCase					:					Union[str, Any]								=							spm.SentencePieceProcessor(**self.sp_model_kwargs						)
									self.sp_model.Load(snake_case__						)

									try:
											import jieba
									except ModuleNotFoundError as error:
											raise error.__class__(
											    "You need to install jieba to use CpmTokenizer or CpmTokenizerFast. "
											    "See https://pypi.org/project/jieba/ for installation."						)
									lowerCAmelCase					:					Any								=							jieba
									lowerCAmelCase					:					Union[str, Any]								=							str.maketrans(" \n"		,					"\u2582\u2583"						)

							@property
							# Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
							def 	lowercase__							(						self						):






									"""simple docstring"""





									return len(self.sp_model						)

							def 	lowercase__							(						self						):






									"""simple docstring"""





									lowerCAmelCase					:					int								=							{self.convert_ids_to_tokens(snake_case__						): i for i in range(self.vocab_size						)}
									vocab.update(self.added_tokens_encoder						)
									return vocab

							def __getstate__(						self						):






									"""simple docstring"""





									lowerCAmelCase					:					Optional[Any]								=							self.__dict__.copy()
									lowerCAmelCase					:					Any								=							None
									return state

							def __setstate__(						self		,					snake_case__						):






									"""simple docstring"""





									lowerCAmelCase					:					List[str]								=							d

									# for backward compatibility
									if not hasattr(self		,					"sp_model_kwargs"						):
											lowerCAmelCase					:					Optional[Any]								=							{}

									lowerCAmelCase					:					Optional[int]								=							spm.SentencePieceProcessor(**self.sp_model_kwargs						)
									self.sp_model.Load(self.vocab_file						)

							def 	lowercase__							(						self		,					snake_case__						):






									"""simple docstring"""





									if self.remove_space:
											lowerCAmelCase					:					str								=							" ".join(inputs.strip().split()						)
									else:
											lowerCAmelCase					:					Any								=							inputs
									lowerCAmelCase					:					List[Any]								=							outputs.replace("``"		,					"\""						).replace("''"		,					"\""						)

									if not self.keep_accents:
											lowerCAmelCase					:					List[str]								=							unicodedata.normalize("NFKD"		,					snake_case__						)
											lowerCAmelCase					:					Optional[int]								=							"".join([c for c in outputs if not unicodedata.combining(snake_case__						)]						)
									if self.do_lower_case:
											lowerCAmelCase					:					Any								=							outputs.lower()

									return outputs

							def 	lowercase__							(						self		,					snake_case__						):






									"""simple docstring"""





									lowerCAmelCase					:					Dict								=							self.preprocess_text(snake_case__						)
									lowerCAmelCase					:					Optional[int]								=							self.sp_model.encode(snake_case__		,					out_type=snake_case__						)
									lowerCAmelCase					:					str								=							[]
									for piece in pieces:
											if len(snake_case__						) > 1 and piece[-1] == str(","						) and piece[-2].isdigit():
													lowerCAmelCase					:					Dict								=							self.sp_model.EncodeAsPieces(piece[:-1].replace(snake_case__		,					""						)						)
													if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
															if len(cur_pieces[0]						) == 1:
																	lowerCAmelCase					:					Optional[Any]								=							cur_pieces[1:]
															else:
																	lowerCAmelCase					:					List[Any]								=							cur_pieces[0][1:]
													cur_pieces.append(piece[-1]						)
													new_pieces.extend(snake_case__						)
											else:
													new_pieces.append(snake_case__						)

									return new_pieces

							def 	lowercase__							(						self		,					snake_case__						):






									"""simple docstring"""





									return self.sp_model.PieceToId(snake_case__						)

							def 	lowercase__							(						self		,					snake_case__						):






									"""simple docstring"""





									return self.sp_model.IdToPiece(snake_case__						)

							def 	lowercase__							(						self		,					snake_case__						):






									"""simple docstring"""





									lowerCAmelCase					:					List[Any]								=							"".join(snake_case__						).replace(snake_case__		,					" "						).strip()
									return out_string

							def 	lowercase__							(						self		,					snake_case__		,					snake_case__ = None						):






									"""simple docstring"""





									lowerCAmelCase					:					Any								=							[self.sep_token_id]
									lowerCAmelCase					:					Tuple								=							[self.cls_token_id]
									if token_ids_a is None:
											return token_ids_a + sep + cls
									return token_ids_a + sep + token_ids_a + sep + cls

							def 	lowercase__							(						self		,					snake_case__		,					snake_case__ = None		,					snake_case__ = False						):






									"""simple docstring"""





									if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=snake_case__		,					token_ids_a=snake_case__		,					already_has_special_tokens=snake_case__						)

									if token_ids_a is not None:
											return ([0] * len(snake_case__						)) + [1] + ([0] * len(snake_case__						)) + [1, 1]
									return ([0] * len(snake_case__						)) + [1, 1]

							def 	lowercase__							(						self		,					snake_case__		,					snake_case__ = None						):






									"""simple docstring"""





									lowerCAmelCase					:					Union[str, Any]								=							[self.sep_token_id]
									lowerCAmelCase					:					Union[str, Any]								=							[2]

									if token_ids_a is None:
											return len(token_ids_a + sep						) * [0] + cls_segment_id
									return len(token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1] + cls_segment_id

							def 	lowercase__							(						self		,					snake_case__		,					snake_case__ = None						):






									"""simple docstring"""





									if not os.path.isdir(snake_case__						):
											logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""						)
											return
									lowerCAmelCase					:					str								=							os.path.join(
									    snake_case__		,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)

									if os.path.abspath(self.vocab_file						) != os.path.abspath(snake_case__						) and os.path.isfile(self.vocab_file						):
											copyfile(self.vocab_file		,					snake_case__						)
									elif not os.path.isfile(self.vocab_file						):
											with open(snake_case__		,					"wb"						) as fi:
													lowerCAmelCase					:					Optional[Any]								=							self.sp_model.serialized_model_proto()
													fi.write(snake_case__						)

									return (out_vocab_file,)







							def 	lowercase__							(						self		,					*snake_case__		,					**snake_case__						):






									"""simple docstring"""





									lowerCAmelCase					:					Optional[int]								=							super()._decode(*snake_case__		,					**snake_case__						)
									lowerCAmelCase					:					Dict								=							text.replace(" "		,					""						).replace("\u2582"		,					" "						).replace("\u2583"		,					"\n"						)
									return text









"""simple docstring"""







from ...configuration_utils import PretrainedConfig


lowerCAmelCase__				    =       {
    '''google/tapas-base-finetuned-sqa''': (
        '''https://huggingface.co/google/tapas-base-finetuned-sqa/resolve/main/config.json'''
    ),
    '''google/tapas-base-finetuned-wtq''': (
        '''https://huggingface.co/google/tapas-base-finetuned-wtq/resolve/main/config.json'''
    ),
    '''google/tapas-base-finetuned-wikisql-supervised''': (
        '''https://huggingface.co/google/tapas-base-finetuned-wikisql-supervised/resolve/main/config.json'''
    ),
    '''google/tapas-base-finetuned-tabfact''': (
        '''https://huggingface.co/google/tapas-base-finetuned-tabfact/resolve/main/config.json'''
    ),
}


class 				SCREAMING_SNAKE_CASE__    (		lowercase    ):
       """simple docstring"""


       a  :    Any       ="tapas"

       def __init__(						self		,					snake_case__=30_522		,					snake_case__=768		,					snake_case__=12		,					snake_case__=12		,					snake_case__=3_072		,					snake_case__="gelu"		,					snake_case__=0.1		,					snake_case__=0.1		,					snake_case__=1_024		,					snake_case__=[3, 256, 256, 2, 256, 256, 10]		,					snake_case__=0.02		,					snake_case__=1e-12		,					snake_case__=0		,					snake_case__=10.0		,					snake_case__=0		,					snake_case__=1.0		,					snake_case__=None		,					snake_case__=1.0		,					snake_case__=False		,					snake_case__=None		,					snake_case__=1.0		,					snake_case__=1.0		,					snake_case__=False		,					snake_case__=False		,					snake_case__="ratio"		,					snake_case__=None		,					snake_case__=None		,					snake_case__=64		,					snake_case__=32		,					snake_case__=False		,					snake_case__=True		,					snake_case__=False		,					snake_case__=False		,					snake_case__=True		,					snake_case__=False		,					snake_case__=None		,					snake_case__=None		,					**snake_case__		,					):






         """simple docstring"""





         super().__init__(pad_token_id=snake_case__		,					**snake_case__						)

         # BERT hyperparameters (with updated max_position_embeddings and type_vocab_sizes)
         lowerCAmelCase					:					List[Any]								=							vocab_size
         lowerCAmelCase					:					List[str]								=							hidden_size
         lowerCAmelCase					:					Optional[Any]								=							num_hidden_layers
         lowerCAmelCase					:					List[str]								=							num_attention_heads
         lowerCAmelCase					:					Optional[Any]								=							hidden_act
         lowerCAmelCase					:					List[Any]								=							intermediate_size
         lowerCAmelCase					:					Optional[Any]								=							hidden_dropout_prob
         lowerCAmelCase					:					str								=							attention_probs_dropout_prob
         lowerCAmelCase					:					Any								=							max_position_embeddings
         lowerCAmelCase					:					Dict								=							type_vocab_sizes
         lowerCAmelCase					:					Union[str, Any]								=							initializer_range
         lowerCAmelCase					:					str								=							layer_norm_eps

         # Fine-tuning task hyperparameters
         lowerCAmelCase					:					Dict								=							positive_label_weight
         lowerCAmelCase					:					Union[str, Any]								=							num_aggregation_labels
         lowerCAmelCase					:					Optional[Any]								=							aggregation_loss_weight
         lowerCAmelCase					:					List[Any]								=							use_answer_as_supervision
         lowerCAmelCase					:					Dict								=							answer_loss_importance
         lowerCAmelCase					:					List[Any]								=							use_normalized_answer_loss
         lowerCAmelCase					:					List[str]								=							huber_loss_delta
         lowerCAmelCase					:					Optional[int]								=							temperature
         lowerCAmelCase					:					Optional[int]								=							aggregation_temperature
         lowerCAmelCase					:					Any								=							use_gumbel_for_cells
         lowerCAmelCase					:					Union[str, Any]								=							use_gumbel_for_aggregation
         lowerCAmelCase					:					Union[str, Any]								=							average_approximation_function
         lowerCAmelCase					:					int								=							cell_selection_preference
         lowerCAmelCase					:					Dict								=							answer_loss_cutoff
         lowerCAmelCase					:					Optional[int]								=							max_num_rows
         lowerCAmelCase					:					Union[str, Any]								=							max_num_columns
         lowerCAmelCase					:					Any								=							average_logits_per_cell
         lowerCAmelCase					:					List[Any]								=							select_one_column
         lowerCAmelCase					:					Tuple								=							allow_empty_column_selection
         lowerCAmelCase					:					str								=							init_cell_selection_weights_to_zero
         lowerCAmelCase					:					List[Any]								=							reset_position_index_per_cell
         lowerCAmelCase					:					Optional[Any]								=							disable_per_token_loss

         # Aggregation hyperparameters
         lowerCAmelCase					:					List[str]								=							aggregation_labels
         lowerCAmelCase					:					List[str]								=							no_aggregation_label_index

         if isinstance(self.aggregation_labels		,					snake_case__						):
           lowerCAmelCase					:					Union[str, Any]								=							{int(snake_case__						): v for k, v in aggregation_labels.items()}





