import os

import jsonlines
import numpy as np
from tqdm import tqdm


__lowercase										= 2048
__lowercase										= 4096
__lowercase										= 42
__lowercase										= os.environ.pop('''PROCESS_TRAIN''', '''false''')
__lowercase										= {'''null''': 0, '''short''': 1, '''long''': 2, '''yes''': 3, '''no''': 4}



def lowerCamelCase  (       SCREAMING_SNAKE_CASE     ):
				'''simple docstring'''







				def choose_first(SCREAMING_SNAKE_CASE  ,      SCREAMING_SNAKE_CASE=False     ):
								assert isinstance(SCREAMING_SNAKE_CASE  ,      SCREAMING_SNAKE_CASE     )
								if len(SCREAMING_SNAKE_CASE     ) == 1:
												__UpperCamelCase					:Tuple        =		answer[0]
												return {k: [answer[k]] for k in answer} if is_long_answer else answer
								for a in answer:
												if is_long_answer:
																__UpperCamelCase					:Dict        =		{k: [a[k]] for k in a}
												if len(a['''start_token''']     ) > 0:
																break
								return a

				__UpperCamelCase					:List[Any]        =		{'''id''': example['''id''']}
				__UpperCamelCase					:List[str]        =		example['''annotations''']
				__UpperCamelCase					:Dict        =		annotation['''yes_no_answer''']
				if 0 in yes_no_answer or 1 in yes_no_answer:
								__UpperCamelCase					:Any        =		['''yes'''] if 1 in yes_no_answer else ['''no''']
								__UpperCamelCase					:Any        =		[]
								__UpperCamelCase					:List[Any]        =		[]
								__UpperCamelCase					:Optional[Any]        =		['''<cls>''']
				else:
								__UpperCamelCase					:Optional[Any]        =		['''short''']
								__UpperCamelCase					:List[str]        =		choose_first(annotation['''short_answers''']     )
								if len(out['''start_token''']     ) == 0:
												# answer will be long if short is not available
												__UpperCamelCase					:Optional[int]        =		['''long''']
												__UpperCamelCase					:int        =		choose_first(annotation['''long_answer''']  ,      is_long_answer=SCREAMING_SNAKE_CASE     )
												__UpperCamelCase					:Any        =		[]
								answer.update(SCREAMING_SNAKE_CASE     )

				# disregard some samples
				if len(answer['''start_token''']     ) > 1 or answer["start_token"] == answer["end_token"]:
								__UpperCamelCase					:str        =		True
				else:
								__UpperCamelCase					:Any        =		False

				__UpperCamelCase					:Dict        =		['''start_token''', '''end_token''', '''start_byte''', '''end_byte''', '''text''']
				if not all(isinstance(answer[k]  ,      SCREAMING_SNAKE_CASE     ) for k in cols     ):
								raise ValueError('''Issue in ID'''  ,      example['''id''']     )

				return answer



def lowerCamelCase  (       SCREAMING_SNAKE_CASE  ,      SCREAMING_SNAKE_CASE=False     ):
				'''simple docstring'''







				__UpperCamelCase					:Union[str, Any]        =		_get_single_answer(SCREAMING_SNAKE_CASE     )
				# bytes are of no use
				del answer["start_byte"]
				del answer["end_byte"]

				# handle yes_no answers explicitly
				if answer["category"][0] in ["yes", "no"]:  # category is list with one element
								__UpperCamelCase					:int        =		example['''document''']['''tokens''']
								__UpperCamelCase					:Tuple        =		[]
								for i in range(len(doc['''token''']     )     ):
												if not doc["is_html"][i]:
																context.append(doc['''token'''][i]     )
								return {
								    "context": " ".join(SCREAMING_SNAKE_CASE     ),
								    "answer": {
								        "start_token": -100,  # ignore index in cross-entropy
								        "end_token": -100,  # ignore index in cross-entropy
								        "category": answer["category"],
								        "span": answer["category"],  # extra
								    },
								}

				# later, help in removing all no answers
				if answer["start_token"] == [-1]:
								return {
								    "context": "None",
								    "answer": {
								        "start_token": -1,
								        "end_token": -1,
								        "category": "null",
								        "span": "None",  # extra
								    },
								}

				# handling normal samples

				__UpperCamelCase					:Union[str, Any]        =		['''start_token''', '''end_token''']
				answer.update({k: answer[k][0] if len(answer[k]     ) > 0 else answer[k] for k in cols}     )  # e.g. [10] == 10

				__UpperCamelCase					:List[str]        =		example['''document''']['''tokens''']
				__UpperCamelCase					:int        =		answer['''start_token''']
				__UpperCamelCase					:str        =		answer['''end_token''']

				__UpperCamelCase					:int        =		[]
				for i in range(len(doc['''token''']     )     ):
								if not doc["is_html"][i]:
												context.append(doc['''token'''][i]     )
								else:
												if answer["start_token"] > i:
																start_token -= 1
												if answer["end_token"] > i:
																end_token -= 1
				__UpperCamelCase					:int        =		''' '''.join(context[start_token:end_token]     )

				# checking above code
				if assertion:
								__UpperCamelCase					:Dict        =		doc['''is_html'''][answer['''start_token'''] : answer['''end_token''']]
								__UpperCamelCase					:Dict        =		doc['''token'''][answer['''start_token'''] : answer['''end_token''']]
								__UpperCamelCase					:Optional[int]        =		''' '''.join([old[i] for i in range(len(SCREAMING_SNAKE_CASE     )     ) if not is_html[i]]     )
								if new != old:
												print('''ID:'''  ,      example['''id''']     )
												print('''New:'''  ,      SCREAMING_SNAKE_CASE  ,      end='''\n'''     )
												print('''Old:'''  ,      SCREAMING_SNAKE_CASE  ,      end='''\n\n'''     )

				return {
				    "context": " ".join(SCREAMING_SNAKE_CASE     ),
				    "answer": {
				        "start_token": start_token,
				        "end_token": end_token - 1,  # this makes it inclusive
				        "category": answer["category"],  # either long or short
				        "span": new,  # extra
				    },
				}



def lowerCamelCase  (       SCREAMING_SNAKE_CASE  ,      SCREAMING_SNAKE_CASE  ,      SCREAMING_SNAKE_CASE=2_048  ,      SCREAMING_SNAKE_CASE=4_096  ,      SCREAMING_SNAKE_CASE=True     ):
				'''simple docstring'''







				__UpperCamelCase					:Any        =		get_context_and_ans(SCREAMING_SNAKE_CASE  ,      assertion=SCREAMING_SNAKE_CASE     )
				__UpperCamelCase					:Union[str, Any]        =		out['''answer''']

				# later, removing these samples
				if answer["start_token"] == -1:
								return {
								    "example_id": example["id"],
								    "input_ids": [[-1]],
								    "labels": {
								        "start_token": [-1],
								        "end_token": [-1],
								        "category": ["null"],
								    },
								}

				__UpperCamelCase					:List[Any]        =		tokenizer(example['''question''']['''text''']  ,      out['''context''']     ).input_ids
				__UpperCamelCase					:Any        =		input_ids.index(tokenizer.sep_token_id     ) + 1

				# return yes/no
				if answer["category"][0] in ["yes", "no"]:  # category is list with one element
								__UpperCamelCase					:List[str]        =		[]
								__UpperCamelCase					:Dict        =		[]
								__UpperCamelCase					:Any        =		input_ids[:q_len]
								__UpperCamelCase					:Optional[int]        =		range(SCREAMING_SNAKE_CASE  ,      len(SCREAMING_SNAKE_CASE     )  ,      max_length - doc_stride     )
								for i in doc_start_indices:
												__UpperCamelCase					:List[str]        =		i + max_length - q_len
												__UpperCamelCase					:List[Any]        =		input_ids[i:end_index]
												inputs.append(q_indices + slice     )
												category.append(answer['''category'''][0]     )
												if slice[-1] == tokenizer.sep_token_id:
																break

								return {
								    "example_id": example["id"],
								    "input_ids": inputs,
								    "labels": {
								        "start_token": [-100] * len(SCREAMING_SNAKE_CASE     ),
								        "end_token": [-100] * len(SCREAMING_SNAKE_CASE     ),
								        "category": category,
								    },
								}

				__UpperCamelCase					:List[Any]        =		out['''context'''].split()
				__UpperCamelCase					:Optional[Any]        =		splitted_context[answer['''end_token''']]
				__UpperCamelCase					:Optional[int]        =		len(
				    tokenizer(
				        ''' '''.join(splitted_context[: answer['''start_token''']]     )  ,      add_special_tokens=SCREAMING_SNAKE_CASE  ,      ).input_ids     )
				__UpperCamelCase					:List[Any]        =		len(
				    tokenizer(''' '''.join(splitted_context[: answer['''end_token''']]     )  ,      add_special_tokens=SCREAMING_SNAKE_CASE     ).input_ids     )

				answer["start_token"] += q_len
				answer["end_token"] += q_len

				# fixing end token
				__UpperCamelCase					:str        =		len(tokenizer(SCREAMING_SNAKE_CASE  ,      add_special_tokens=SCREAMING_SNAKE_CASE     ).input_ids     )
				if num_sub_tokens > 1:
								answer["end_token"] += num_sub_tokens - 1

				__UpperCamelCase					:Union[str, Any]        =		input_ids[answer['''start_token'''] : answer['''end_token'''] + 1]  # right & left are inclusive
				__UpperCamelCase					:Optional[int]        =		answer['''start_token''']
				__UpperCamelCase					:Dict        =		answer['''end_token''']

				if assertion:
								__UpperCamelCase					:Optional[int]        =		tokenizer.decode(SCREAMING_SNAKE_CASE     )
								if answer["span"] != new:
												print('''ISSUE IN TOKENIZATION'''     )
												print('''OLD:'''  ,      answer['''span''']     )
												print('''NEW:'''  ,      SCREAMING_SNAKE_CASE  ,      end='''\n\n'''     )

				if len(SCREAMING_SNAKE_CASE     ) <= max_length:
								return {
								    "example_id": example["id"],
								    "input_ids": [input_ids],
								    "labels": {
								        "start_token": [answer["start_token"]],
								        "end_token": [answer["end_token"]],
								        "category": answer["category"],
								    },
								}

				__UpperCamelCase					:Dict        =		input_ids[:q_len]
				__UpperCamelCase					:Optional[Any]        =		range(SCREAMING_SNAKE_CASE  ,      len(SCREAMING_SNAKE_CASE     )  ,      max_length - doc_stride     )

				__UpperCamelCase					:Union[str, Any]        =		[]
				__UpperCamelCase					:Dict        =		[]
				__UpperCamelCase					:Union[str, Any]        =		[]
				__UpperCamelCase					:Dict        =		[]  # null, yes, no, long, short
				for i in doc_start_indices:
								__UpperCamelCase					:int        =		i + max_length - q_len
								__UpperCamelCase					:Optional[int]        =		input_ids[i:end_index]
								inputs.append(q_indices + slice     )
								assert len(inputs[-1]     ) <= max_length, "Issue in truncating length"

								if start_token >= i and end_token <= end_index - 1:
												__UpperCamelCase					:List[str]        =		start_token - i + q_len
												__UpperCamelCase					:str        =		end_token - i + q_len
												answers_category.append(answer['''category'''][0]     )  # ["short"] -> "short"
								else:
												__UpperCamelCase					:int        =		-100
												__UpperCamelCase					:Optional[Any]        =		-100
												answers_category.append('''null'''     )
								__UpperCamelCase					:Optional[int]        =		inputs[-1][start_token : end_token + 1]

								answers_start_token.append(SCREAMING_SNAKE_CASE     )
								answers_end_token.append(SCREAMING_SNAKE_CASE     )
								if assertion:
												if new != old and new != [tokenizer.cls_token_id]:
																print('''ISSUE in strided for ID:'''  ,      example['''id''']     )
																print('''New:'''  ,      tokenizer.decode(SCREAMING_SNAKE_CASE     )     )
																print('''Old:'''  ,      tokenizer.decode(SCREAMING_SNAKE_CASE     )  ,      end='''\n\n'''     )
								if slice[-1] == tokenizer.sep_token_id:
												break

				return {
				    "example_id": example["id"],
				    "input_ids": inputs,
				    "labels": {
				        "start_token": answers_start_token,
				        "end_token": answers_end_token,
				        "category": answers_category,
				    },
				}



def lowerCamelCase  (       SCREAMING_SNAKE_CASE  ,      SCREAMING_SNAKE_CASE  ,      SCREAMING_SNAKE_CASE=2_048  ,      SCREAMING_SNAKE_CASE=4_096  ,      SCREAMING_SNAKE_CASE=False     ):
				'''simple docstring'''







				__UpperCamelCase					:Dict        =		get_strided_contexts_and_ans(
				    SCREAMING_SNAKE_CASE  ,      SCREAMING_SNAKE_CASE  ,      doc_stride=SCREAMING_SNAKE_CASE  ,      max_length=SCREAMING_SNAKE_CASE  ,      assertion=SCREAMING_SNAKE_CASE  ,      )

				return example



def lowerCamelCase  (       SCREAMING_SNAKE_CASE  ,      SCREAMING_SNAKE_CASE     ):
				'''simple docstring'''







				with jsonlines.open(SCREAMING_SNAKE_CASE  ,      '''a'''     ) as writer:
								for example in tqdm(SCREAMING_SNAKE_CASE  ,      total=len(SCREAMING_SNAKE_CASE     )  ,      desc='''Saving samples ... '''     ):
												__UpperCamelCase					:Optional[Any]        =		example['''labels''']
												for ids, start, end, cat in zip(
												    example['''input_ids''']  ,      labels['''start_token''']  ,      labels['''end_token''']  ,      labels['''category''']  ,      ):
																if start == -1 and end == -1:
																				continue  # leave waste samples with no answer
																if cat == "null" and np.random.rand() < 0.6:
																				continue  # removing 50 % samples
																writer.write(
																    {
																        '''input_ids''': ids,
																        '''start_token''': start,
																        '''end_token''': end,
																        '''category''': CATEGORY_MAPPING[cat],
																    }     )


if __name__ == "__main__":
				from datasets import load_dataset

				from transformers import BigBirdTokenizer

				__lowercase										= load_dataset('''natural_questions''')
				__lowercase										= BigBirdTokenizer.from_pretrained('''google/bigbird-roberta-base''')

				__lowercase										= data['''train''' if PROCESS_TRAIN == '''true''' else '''validation''']

				__lowercase										= {
				    '''tokenizer''': tokenizer,
				    '''doc_stride''': DOC_STRIDE,
				    '''max_length''': MAX_LENGTH,
				    '''assertion''': False,
				}
				__lowercase										= data.map(prepare_inputs, fn_kwargs=fn_kwargs)
				__lowercase										= data.remove_columns(['''annotations''', '''document''', '''id''', '''question'''])
				print(data)

				np.random.seed(SEED)
				__lowercase										= '''nq-training.jsonl''' if PROCESS_TRAIN == '''true''' else '''nq-validation.jsonl'''
				save_to_disk(data, file_name=cache_file_name)






from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
    import numpy as np
    import tensorflow as tf

    from transformers import (
        TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        FlaubertConfig,
        TFFlaubertForMultipleChoice,
        TFFlaubertForQuestionAnsweringSimple,
        TFFlaubertForSequenceClassification,
        TFFlaubertForTokenClassification,
        TFFlaubertModel,
        TFFlaubertWithLMHeadModel,
    )


class      lowerCamelCase_					:






    '''simple docstring'''






    def __init__(     self			,			__lowercase			,			)	->     Optional[Any]:
        __UpperCamelCase					:Optional[int]        =		parent
        __UpperCamelCase					:Optional[int]        =		13
        __UpperCamelCase					:Dict        =		7
        __UpperCamelCase					:Optional[int]        =		True
        __UpperCamelCase					:List[str]        =		True
        __UpperCamelCase					:List[Any]        =		True
        __UpperCamelCase					:Union[str, Any]        =		True
        __UpperCamelCase					:Any        =		True
        __UpperCamelCase					:Optional[int]        =		False
        __UpperCamelCase					:Any        =		False
        __UpperCamelCase					:str        =		False
        __UpperCamelCase					:Optional[Any]        =		2
        __UpperCamelCase					:Optional[int]        =		99
        __UpperCamelCase					:Any        =		0
        __UpperCamelCase					:List[Any]        =		32
        __UpperCamelCase					:int        =		2
        __UpperCamelCase					:Optional[Any]        =		4
        __UpperCamelCase					:Dict        =		0.1
        __UpperCamelCase					:Optional[Any]        =		0.1
        __UpperCamelCase					:str        =		512
        __UpperCamelCase					:Any        =		16
        __UpperCamelCase					:str        =		2
        __UpperCamelCase					:Dict        =		0.02
        __UpperCamelCase					:List[Any]        =		3
        __UpperCamelCase					:Optional[int]        =		4
        __UpperCamelCase					:Tuple        =		'''last'''
        __UpperCamelCase					:Any        =		True
        __UpperCamelCase					:Union[str, Any]        =		None
        __UpperCamelCase					:Tuple        =		0







    def       UpperCamelCase__       (     self)	->     Dict:
        __UpperCamelCase					:List[str]        =		ids_tensor([self.batch_size, self.seq_length]			,			self.vocab_size)
        __UpperCamelCase					:Any        =		random_attention_mask([self.batch_size, self.seq_length]			,			dtype=tf.floataa)

        __UpperCamelCase					:Tuple        =		None
        if self.use_input_lengths:
            __UpperCamelCase					:List[str]        =		(
                ids_tensor([self.batch_size]			,			vocab_size=2) + self.seq_length - 2
            )  # small variation of seq_length

        __UpperCamelCase					:Any        =		None
        if self.use_token_type_ids:
            __UpperCamelCase					:int        =		ids_tensor([self.batch_size, self.seq_length]			,			self.n_langs)

        __UpperCamelCase					:Dict        =		None
        __UpperCamelCase					:List[str]        =		None
        __UpperCamelCase					:Union[str, Any]        =		None
        if self.use_labels:
            __UpperCamelCase					:Dict        =		ids_tensor([self.batch_size]			,			self.type_sequence_label_size)
            __UpperCamelCase					:int        =		ids_tensor([self.batch_size, self.seq_length]			,			self.num_labels)
            __UpperCamelCase					:List[str]        =		ids_tensor([self.batch_size]			,			2			,			dtype=tf.floataa)
            __UpperCamelCase					:Optional[Any]        =		ids_tensor([self.batch_size]			,			self.num_choices)

        __UpperCamelCase					:Optional[int]        =		FlaubertConfig(
            vocab_size=self.vocab_size			,			n_special=self.n_special			,			emb_dim=self.hidden_size			,			n_layers=self.num_hidden_layers			,			n_heads=self.num_attention_heads			,			dropout=self.hidden_dropout_prob			,			attention_dropout=self.attention_probs_dropout_prob			,			gelu_activation=self.gelu_activation			,			sinusoidal_embeddings=self.sinusoidal_embeddings			,			asm=self.asm			,			causal=self.causal			,			n_langs=self.n_langs			,			max_position_embeddings=self.max_position_embeddings			,			initializer_range=self.initializer_range			,			summary_type=self.summary_type			,			use_proj=self.use_proj			,			bos_token_id=self.bos_token_id			,			)

        return (
            config,
            input_ids,
            token_type_ids,
            input_lengths,
            sequence_labels,
            token_labels,
            is_impossible_labels,
            choice_labels,
            input_mask,
        )







    def       UpperCamelCase__       (     self			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			)	->     Optional[Any]:
        __UpperCamelCase					:str        =		TFFlaubertModel(config=__lowercase)
        __UpperCamelCase					:List[str]        =		{'''input_ids''': input_ids, '''lengths''': input_lengths, '''langs''': token_type_ids}
        __UpperCamelCase					:Optional[int]        =		model(__lowercase)

        __UpperCamelCase					:Optional[int]        =		[input_ids, input_mask]
        __UpperCamelCase					:Union[str, Any]        =		model(__lowercase)
        self.parent.assertEqual(result.last_hidden_state.shape			,			(self.batch_size, self.seq_length, self.hidden_size))







    def       UpperCamelCase__       (     self			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			)	->     int:
        __UpperCamelCase					:str        =		TFFlaubertWithLMHeadModel(__lowercase)

        __UpperCamelCase					:List[str]        =		{'''input_ids''': input_ids, '''lengths''': input_lengths, '''langs''': token_type_ids}
        __UpperCamelCase					:Optional[int]        =		model(__lowercase)

        self.parent.assertEqual(result.logits.shape			,			(self.batch_size, self.seq_length, self.vocab_size))







    def       UpperCamelCase__       (     self			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			)	->     Any:
        __UpperCamelCase					:Any        =		TFFlaubertForQuestionAnsweringSimple(__lowercase)

        __UpperCamelCase					:str        =		{'''input_ids''': input_ids, '''lengths''': input_lengths}

        __UpperCamelCase					:int        =		model(__lowercase)

        self.parent.assertEqual(result.start_logits.shape			,			(self.batch_size, self.seq_length))
        self.parent.assertEqual(result.end_logits.shape			,			(self.batch_size, self.seq_length))







    def       UpperCamelCase__       (     self			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			)	->     Any:
        __UpperCamelCase					:int        =		TFFlaubertForSequenceClassification(__lowercase)

        __UpperCamelCase					:List[Any]        =		{'''input_ids''': input_ids, '''lengths''': input_lengths}

        __UpperCamelCase					:Any        =		model(__lowercase)

        self.parent.assertEqual(result.logits.shape			,			(self.batch_size, self.type_sequence_label_size))







    def       UpperCamelCase__       (     self			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			)	->     Tuple:
        __UpperCamelCase					:Optional[int]        =		self.num_labels
        __UpperCamelCase					:int        =		TFFlaubertForTokenClassification(config=__lowercase)
        __UpperCamelCase					:Union[str, Any]        =		{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
        __UpperCamelCase					:Optional[Any]        =		model(__lowercase)
        self.parent.assertEqual(result.logits.shape			,			(self.batch_size, self.seq_length, self.num_labels))







    def       UpperCamelCase__       (     self			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			)	->     List[Any]:
        __UpperCamelCase					:Any        =		self.num_choices
        __UpperCamelCase					:Any        =		TFFlaubertForMultipleChoice(config=__lowercase)
        __UpperCamelCase					:List[str]        =		tf.tile(tf.expand_dims(__lowercase			,			1)			,			(1, self.num_choices, 1))
        __UpperCamelCase					:Optional[Any]        =		tf.tile(tf.expand_dims(__lowercase			,			1)			,			(1, self.num_choices, 1))
        __UpperCamelCase					:Tuple        =		tf.tile(tf.expand_dims(__lowercase			,			1)			,			(1, self.num_choices, 1))
        __UpperCamelCase					:Any        =		{
            '''input_ids''': multiple_choice_inputs_ids,
            '''attention_mask''': multiple_choice_input_mask,
            '''token_type_ids''': multiple_choice_token_type_ids,
        }
        __UpperCamelCase					:List[Any]        =		model(__lowercase)
        self.parent.assertEqual(result.logits.shape			,			(self.batch_size, self.num_choices))







    def       UpperCamelCase__       (     self)	->     Dict:
        __UpperCamelCase					:Optional[int]        =		self.prepare_config_and_inputs()
        (
            (
            __UpperCamelCase
        )  ,	(
            __UpperCamelCase
        )  ,	(
            __UpperCamelCase
        )  ,	(
            __UpperCamelCase
        )  ,	(
            __UpperCamelCase
        )  ,	(
            __UpperCamelCase
        )  ,	(
            __UpperCamelCase
        )  ,	(
            __UpperCamelCase
        )  ,	(
            __UpperCamelCase
        )  ,	
        )					:Any        =		config_and_inputs
        __UpperCamelCase					:int        =		{
            '''input_ids''': input_ids,
            '''token_type_ids''': token_type_ids,
            '''langs''': token_type_ids,
            '''lengths''': input_lengths,
        }
        return config, inputs_dict


@require_tf
class      lowerCamelCase_					(       UpperCAmelCase_						,     UpperCAmelCase_						,     unittest.TestCase			):






    '''simple docstring'''






    a__     :						List[Any]      =				(
        (
            TFFlaubertModel,
            TFFlaubertWithLMHeadModel,
            TFFlaubertForSequenceClassification,
            TFFlaubertForQuestionAnsweringSimple,
            TFFlaubertForTokenClassification,
            TFFlaubertForMultipleChoice,
        )
        if is_tf_available()
        else ()
    )
    a__     :						Tuple      =				(
        (TFFlaubertWithLMHeadModel,) if is_tf_available() else ()
    )  # TODO (PVP): Check other models whether language generation is also applicable
    a__     :						Tuple      =				(
        {
            """feature-extraction""": TFFlaubertModel,
            """fill-mask""": TFFlaubertWithLMHeadModel,
            """question-answering""": TFFlaubertForQuestionAnsweringSimple,
            """text-classification""": TFFlaubertForSequenceClassification,
            """token-classification""": TFFlaubertForTokenClassification,
            """zero-shot""": TFFlaubertForSequenceClassification,
        }
        if is_tf_available()
        else {}
    )
    a__     :						Tuple      =				False
    a__     :						Any      =				False







    def       UpperCamelCase__       (     self			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase)	->     Any:
        if (
            pipeline_test_casse_name == "QAPipelineTests"
            and tokenizer_name is not None
            and not tokenizer_name.endswith('''Fast''')
        ):
            # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
            # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
            # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
            return True

        return False







    def       UpperCamelCase__       (     self)	->     Union[str, Any]:
        __UpperCamelCase					:int        =		TFFlaubertModelTester(self)
        __UpperCamelCase					:Dict        =		ConfigTester(self			,			config_class=__lowercase			,			emb_dim=37)







    def       UpperCamelCase__       (     self)	->     str:
        self.config_tester.run_common_tests()







    def       UpperCamelCase__       (     self)	->     Any:
        __UpperCamelCase					:int        =		self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_flaubert_model(*__lowercase)







    def       UpperCamelCase__       (     self)	->     Union[str, Any]:
        __UpperCamelCase					:Optional[int]        =		self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_flaubert_lm_head(*__lowercase)







    def       UpperCamelCase__       (     self)	->     Dict:
        __UpperCamelCase					:Optional[Any]        =		self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_flaubert_qa(*__lowercase)







    def       UpperCamelCase__       (     self)	->     Tuple:
        __UpperCamelCase					:Optional[Any]        =		self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_flaubert_sequence_classif(*__lowercase)







    def       UpperCamelCase__       (     self)	->     Optional[Any]:
        __UpperCamelCase					:Dict        =		self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_flaubert_for_token_classification(*__lowercase)







    def       UpperCamelCase__       (     self)	->     str:
        __UpperCamelCase					:Union[str, Any]        =		self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_flaubert_for_multiple_choice(*__lowercase)







    @slow
    def       UpperCamelCase__       (     self)	->     Union[str, Any]:
        for model_name in TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            __UpperCamelCase					:Any        =		TFFlaubertModel.from_pretrained(__lowercase)
            self.assertIsNotNone(__lowercase)


@require_tf
@require_sentencepiece
@require_tokenizers
class      lowerCamelCase_					(       unittest.TestCase			):






    '''simple docstring'''






    @slow
    def       UpperCamelCase__       (     self)	->     Optional[Any]:
        __UpperCamelCase					:List[Any]        =		TFFlaubertModel.from_pretrained('''jplu/tf-flaubert-small-cased''')

        __UpperCamelCase					:Tuple        =		tf.convert_to_tensor(
            [[0, 158, 735, 2_592, 1_424, 6_727, 82, 1]]			,			dtype=tf.intaa			,			)  # "J'aime flaubert !"

        __UpperCamelCase					:Union[str, Any]        =		model(__lowercase)[0]
        __UpperCamelCase					:str        =		tf.TensorShape((1, 8, 512))
        self.assertEqual(output.shape			,			__lowercase)
        # compare the actual values for a slice.
        __UpperCamelCase					:Optional[int]        =		tf.convert_to_tensor(
            [
                [
                    [-1.8_76_87_73, -1.56_65_55, 0.27_07_24_18],
                    [-1.6_92_00_38, -0.5_87_35_05, 1.9_32_95_99],
                    [-2.9_56_39_85, -1.6_99_38_35, 1.7_97_20_52],
                ]
            ]			,			dtype=tf.floataa			,			)

        self.assertTrue(np.allclose(output[:, :3, :3].numpy()			,			expected_slice.numpy()			,			atol=1E-4))





