import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
    from transformers.pipelines.conversational import Conversation


__lowercase										= logging.get_logger(__name__)

__lowercase										= {'''vocab_file''': '''vocab.txt''', '''emoji_file''': '''emoji.json'''}

__lowercase										= {
    '''vocab_file''': {
        '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt''',
    },
    '''emoji_file''': {
        '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json''',
    },
}

__lowercase										= {
    '''abeja/gpt-neox-japanese-2.7b''': 2048,
}



def lowerCamelCase  (       SCREAMING_SNAKE_CASE  ,      SCREAMING_SNAKE_CASE     ):
    '''simple docstring'''







    with open(SCREAMING_SNAKE_CASE  ,      '''r'''  ,      encoding='''utf-8'''     ) as f:
        __UpperCamelCase					:Any        =		json.loads(f.read()     )

    __UpperCamelCase					:Any        =		collections.OrderedDict()
    __UpperCamelCase					:List[str]        =		collections.OrderedDict()
    __UpperCamelCase					:List[str]        =		collections.OrderedDict()
    with open(SCREAMING_SNAKE_CASE  ,      '''r'''  ,      encoding='''utf-8'''     ) as f:
        __UpperCamelCase					:Any        =		f.readlines()
    __UpperCamelCase					:List[Any]        =		[[t.rstrip('''\n'''     )] if (t == ''',''' or ''',''' not in t) else t.rstrip('''\n'''     ).split(''','''     ) for t in token]
    for idx, b in enumerate(SCREAMING_SNAKE_CASE     ):
        __UpperCamelCase					:List[Any]        =		b
        __UpperCamelCase					:Optional[int]        =		idx
        for wd in b:
            __UpperCamelCase					:Optional[Any]        =		idx

    return vocab, raw_vocab, ids_to_tokens, emoji


class      lowerCamelCase_					(       UpperCAmelCase_			):






    '''simple docstring'''






    a__     :						Tuple      =				VOCAB_FILES_NAMES
    a__     :						Optional[int]      =				PRETRAINED_VOCAB_FILES_MAP
    a__     :						List[Any]      =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    a__     :						Optional[int]      =				["""input_ids""", """attention_mask"""]







    def __init__(     self			,			__lowercase			,			__lowercase			,			__lowercase="<|endoftext|>"			,			__lowercase="<|endoftext|>"			,			__lowercase="<|startoftext|>"			,			__lowercase="<|endoftext|>"			,			__lowercase=False			,			**__lowercase			,			)	->     Union[str, Any]:
        super().__init__(
            unk_token=__lowercase			,			pad_token=__lowercase			,			bos_token=__lowercase			,			eos_token=__lowercase			,			do_clean_text=__lowercase			,			**__lowercase			,			)
        if not os.path.isfile(__lowercase):
            raise ValueError(
                f"""Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"""
                ''' model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`''')
        if not os.path.isfile(__lowercase):
            raise ValueError(
                f"""Can't find a emoji file at path '{emoji_file}'. To load the emoji information from a Google"""
                ''' pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`''')
        __UpperCamelCase					:List[str]        =		do_clean_text
        __UpperCamelCase  ,	__UpperCamelCase  ,	__UpperCamelCase  ,	__UpperCamelCase					:Any        =		load_vocab_and_emoji(__lowercase			,			__lowercase)
        __UpperCamelCase					:Optional[int]        =		SubWordJapaneseTokenizer(
            vocab=self.vocab			,			ids_to_tokens=self.ids_to_tokens			,			emoji=self.emoji)







    @property
    def       UpperCamelCase__       (     self)	->     Optional[Any]:
        # self.vocab contains support for character fluctuation unique to Japanese, and has a large number of vocab
        return len(self.raw_vocab)







    def       UpperCamelCase__       (     self)	->     int:
        return dict(self.raw_vocab			,			**self.added_tokens_encoder)







    def       UpperCamelCase__       (     self			,			__lowercase)	->     Any:
        return self.subword_tokenizer.tokenize(__lowercase			,			clean=self.do_clean_text)







    def       UpperCamelCase__       (     self			,			__lowercase)	->     List[str]:
        return self.vocab.get(__lowercase			,			self.vocab.get(self.unk_token))







    def       UpperCamelCase__       (     self			,			__lowercase)	->     Tuple:
        return self.subword_tokenizer.convert_id_to_token(__lowercase)







    def       UpperCamelCase__       (     self			,			__lowercase)	->     Optional[Any]:
        __UpperCamelCase					:Optional[Any]        =		''''''.join(__lowercase).strip()
        return out_string







    def       UpperCamelCase__       (     self			,			__lowercase)	->     List[int]:
        __UpperCamelCase					:int        =		[]
        for is_user, text in conversation.iter_texts():
            input_ids.extend(self.encode(__lowercase			,			add_special_tokens=__lowercase) + [self.eos_token_id])

        if len(__lowercase) > self.model_max_length:
            __UpperCamelCase					:Any        =		input_ids[-self.model_max_length :]
        return input_ids







    def       UpperCamelCase__       (     self			,			__lowercase			,			__lowercase = None)	->     Tuple[str]:
        __UpperCamelCase					:str        =		0
        if os.path.isdir(__lowercase):
            __UpperCamelCase					:Union[str, Any]        =		os.path.join(
                __lowercase			,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])
            __UpperCamelCase					:Any        =		os.path.join(
                __lowercase			,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''emoji_file'''])
        else:
            __UpperCamelCase					:Dict        =		(
                (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory + VOCAB_FILES_NAMES['''vocab_file''']
            )
            __UpperCamelCase					:Optional[Any]        =		(
                (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory + VOCAB_FILES_NAMES['''emoji_file''']
            )
        with open(__lowercase			,			'''w'''			,			encoding='''utf-8''') as writer:
            for token_index, token in self.ids_to_tokens.items():
                if index != token_index:
                    logger.warning(
                        f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
                        ''' Please check that the vocabulary is not corrupted!''')
                    __UpperCamelCase					:str        =		token_index
                writer.write(''','''.join(__lowercase) + '''\n''')
                index += 1
        with open(__lowercase			,			'''w'''			,			encoding='''utf-8''') as writer:
            json.dump(self.emoji			,			__lowercase)
        return vocab_file, emoji_file


class      lowerCamelCase_					(       UpperCAmelCase_			):






    '''simple docstring'''






    def __init__(     self			,			__lowercase			,			__lowercase			,			__lowercase)	->     List[Any]:
        __UpperCamelCase					:Tuple        =		vocab  # same as swe
        __UpperCamelCase					:Any        =		ids_to_tokens  # same as bpe
        __UpperCamelCase					:Tuple        =		emoji
        __UpperCamelCase					:Dict        =		np.max([len(__lowercase) for w in self.vocab.keys()])
        __UpperCamelCase					:Tuple        =		re.compile(r'''(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)''')
        __UpperCamelCase					:str        =		re.compile(r'''[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*''')
        __UpperCamelCase					:List[Any]        =		re.compile(r'''[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}''')
        __UpperCamelCase					:Optional[int]        =		re.compile(
            r'''([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*''')
        __UpperCamelCase					:Optional[Any]        =		re.compile(
            r'''(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*''')
        __UpperCamelCase					:Dict        =		re.compile(
            r'''((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*''')
        __UpperCamelCase					:Union[str, Any]        =		'''─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿'''
        __UpperCamelCase					:Optional[int]        =		'''▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟'''
        __UpperCamelCase					:Dict        =		str.maketrans({k: '''<BLOCK>''' for k in keisen + blocks})







    def __len__(     self)	->     Dict:
        return len(self.ids_to_tokens)







    def       UpperCamelCase__       (     self			,			__lowercase)	->     Dict:
        __UpperCamelCase					:Union[str, Any]        =		self.content_repattera.sub('''<URL>'''			,			__lowercase)
        __UpperCamelCase					:List[Any]        =		self.content_repattera.sub('''<EMAIL>'''			,			__lowercase)
        __UpperCamelCase					:Dict        =		self.content_repattera.sub('''<TEL>'''			,			__lowercase)
        __UpperCamelCase					:Optional[int]        =		self.content_repattera.sub('''<DATE>'''			,			__lowercase)
        __UpperCamelCase					:List[Any]        =		self.content_repattera.sub('''<DATE>'''			,			__lowercase)
        __UpperCamelCase					:Union[str, Any]        =		self.content_repattera.sub('''<PRICE>'''			,			__lowercase)
        __UpperCamelCase					:Any        =		content.translate(self.content_transa)
        while "<BLOCK><BLOCK>" in content:
            __UpperCamelCase					:Optional[int]        =		content.replace('''<BLOCK><BLOCK>'''			,			'''<BLOCK>''')
        return content







    def       UpperCamelCase__       (     self			,			__lowercase			,			__lowercase=False)	->     List[Any]:
        __UpperCamelCase					:List[str]        =		text.replace(''' '''			,			'''<SP>''')
        __UpperCamelCase					:Optional[Any]        =		text.replace('''　'''			,			'''<SP>''')
        __UpperCamelCase					:Union[str, Any]        =		text.replace('''\r\n'''			,			'''<BR>''')
        __UpperCamelCase					:Any        =		text.replace('''\n'''			,			'''<BR>''')
        __UpperCamelCase					:int        =		text.replace('''\r'''			,			'''<BR>''')
        __UpperCamelCase					:List[str]        =		text.replace('''\t'''			,			'''<TAB>''')
        __UpperCamelCase					:Union[str, Any]        =		text.replace('''—'''			,			'''ー''')
        __UpperCamelCase					:List[str]        =		text.replace('''−'''			,			'''ー''')
        for k, v in self.emoji["emoji"].items():
            if k in text:
                __UpperCamelCase					:Dict        =		text.replace(__lowercase			,			__lowercase)
        if clean:
            __UpperCamelCase					:Optional[Any]        =		self.clean_text(__lowercase)

        def check_simbol(__lowercase):
            __UpperCamelCase					:Optional[int]        =		x.encode()
            if len(__lowercase) == 1 and len(__lowercase) == 2:
                __UpperCamelCase					:Optional[int]        =		(int(e[0]) << 8) + int(e[1])
                if (
                    (c >= 0xC_2A1 and c <= 0xC_2BF)
                    or (c >= 0xC_780 and c <= 0xC_783)
                    or (c >= 0xC_AB9 and c <= 0xC_BBF)
                    or (c >= 0xC_C80 and c <= 0xC_DA2)
                ):
                    return True
            return False

        def checkuae(__lowercase):
            __UpperCamelCase					:List[str]        =		x.encode()
            if len(__lowercase) == 1 and len(__lowercase) == 3:
                __UpperCamelCase					:List[str]        =		(int(e[0]) << 16) + (int(e[1]) << 8) + int(e[2])
                if c >= 0xE28_080 and c <= 0xE2B_07F:
                    return True
            return False

        __UpperCamelCase					:List[str]        =		0
        __UpperCamelCase					:Any        =		[]
        while pos < len(__lowercase):
            __UpperCamelCase					:Any        =		min(len(__lowercase)			,			pos + self.maxlen + 1) if text[pos] == '''<''' else pos + 3
            __UpperCamelCase					:Any        =		[]  # (token_id, token, pos)
            for e in range(__lowercase			,			__lowercase			,			-1):
                __UpperCamelCase					:List[str]        =		text[pos:e]
                if wd in self.vocab:
                    if wd[0] == "<" and len(__lowercase) > 2:
                        __UpperCamelCase					:Optional[int]        =		[(self.vocab[wd], wd, e)]
                        break
                    else:
                        candidates.append((self.vocab[wd], wd, e))
            if len(__lowercase) > 0:
                # the smallest token_id is adopted
                __UpperCamelCase  ,	__UpperCamelCase  ,	__UpperCamelCase					:Dict        =		sorted(__lowercase			,			key=lambda __lowercase: x[0])[0]
                result.append(__lowercase)
                __UpperCamelCase					:Union[str, Any]        =		e
            else:
                __UpperCamelCase					:int        =		pos + 1
                __UpperCamelCase					:Optional[Any]        =		text[pos:end]
                if check_simbol(__lowercase):
                    result.append('''<KIGOU>''')
                elif checkuae(__lowercase):
                    result.append('''<U2000U2BFF>''')
                else:
                    for i in wd.encode('''utf-8'''):
                        result.append('''<|byte%d|>''' % i)
                __UpperCamelCase					:Tuple        =		end
        return result







    def       UpperCamelCase__       (     self			,			__lowercase			,			__lowercase="\n")	->     Optional[int]:
        __UpperCamelCase					:Tuple        =		[]
        __UpperCamelCase					:List[str]        =		[]
        __UpperCamelCase					:Optional[int]        =		self.ids_to_tokens[index][0]
        if word[:6] == "<|byte" and word[-2:] == "|>":
            byte_tokens.append(int(word[6:-2]))
        else:
            if len(__lowercase) > 0:
                words.append(bytearray(__lowercase).decode('''utf-8'''			,			errors='''replace'''))
                __UpperCamelCase					:Optional[int]        =		[]
            if word[:7] == "<|emoji" and word[-2:] == "|>":
                words.append(self.emoji['''emoji_inv'''][word])
            elif word == "<SP>":
                words.append(''' ''')
            elif word == "<BR>":
                words.append(__lowercase)
            elif word == "<TAB>":
                words.append('''\t''')
            elif word == "<BLOCK>":
                words.append('''▀''')
            elif word == "<KIGOU>":
                words.append('''ǀ''')
            elif word == "<U2000U2BFF>":
                words.append('''‖''')
            else:
                words.append(__lowercase)
        if len(__lowercase) > 0:
            words.append(bytearray(__lowercase).decode('''utf-8'''			,			errors='''replace'''))
        __UpperCamelCase					:Union[str, Any]        =		''''''.join(__lowercase)
        return text






from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
    import numpy as np
    import tensorflow as tf

    from transformers import (
        TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        FlaubertConfig,
        TFFlaubertForMultipleChoice,
        TFFlaubertForQuestionAnsweringSimple,
        TFFlaubertForSequenceClassification,
        TFFlaubertForTokenClassification,
        TFFlaubertModel,
        TFFlaubertWithLMHeadModel,
    )


class      lowerCamelCase_					:






    '''simple docstring'''






    def __init__(     self			,			__lowercase			,			)	->     Optional[Any]:
        __UpperCamelCase					:Optional[int]        =		parent
        __UpperCamelCase					:Optional[int]        =		13
        __UpperCamelCase					:Dict        =		7
        __UpperCamelCase					:Optional[int]        =		True
        __UpperCamelCase					:List[str]        =		True
        __UpperCamelCase					:List[Any]        =		True
        __UpperCamelCase					:Union[str, Any]        =		True
        __UpperCamelCase					:Any        =		True
        __UpperCamelCase					:Optional[int]        =		False
        __UpperCamelCase					:Any        =		False
        __UpperCamelCase					:str        =		False
        __UpperCamelCase					:Optional[Any]        =		2
        __UpperCamelCase					:Optional[int]        =		99
        __UpperCamelCase					:Any        =		0
        __UpperCamelCase					:List[Any]        =		32
        __UpperCamelCase					:int        =		2
        __UpperCamelCase					:Optional[Any]        =		4
        __UpperCamelCase					:Dict        =		0.1
        __UpperCamelCase					:Optional[Any]        =		0.1
        __UpperCamelCase					:str        =		512
        __UpperCamelCase					:Any        =		16
        __UpperCamelCase					:str        =		2
        __UpperCamelCase					:Dict        =		0.02
        __UpperCamelCase					:List[Any]        =		3
        __UpperCamelCase					:Optional[int]        =		4
        __UpperCamelCase					:Tuple        =		'''last'''
        __UpperCamelCase					:Any        =		True
        __UpperCamelCase					:Union[str, Any]        =		None
        __UpperCamelCase					:Tuple        =		0







    def       UpperCamelCase__       (     self)	->     Dict:
        __UpperCamelCase					:List[str]        =		ids_tensor([self.batch_size, self.seq_length]			,			self.vocab_size)
        __UpperCamelCase					:Any        =		random_attention_mask([self.batch_size, self.seq_length]			,			dtype=tf.floataa)

        __UpperCamelCase					:Tuple        =		None
        if self.use_input_lengths:
            __UpperCamelCase					:List[str]        =		(
                ids_tensor([self.batch_size]			,			vocab_size=2) + self.seq_length - 2
            )  # small variation of seq_length

        __UpperCamelCase					:Any        =		None
        if self.use_token_type_ids:
            __UpperCamelCase					:int        =		ids_tensor([self.batch_size, self.seq_length]			,			self.n_langs)

        __UpperCamelCase					:Dict        =		None
        __UpperCamelCase					:List[str]        =		None
        __UpperCamelCase					:Union[str, Any]        =		None
        if self.use_labels:
            __UpperCamelCase					:Dict        =		ids_tensor([self.batch_size]			,			self.type_sequence_label_size)
            __UpperCamelCase					:int        =		ids_tensor([self.batch_size, self.seq_length]			,			self.num_labels)
            __UpperCamelCase					:List[str]        =		ids_tensor([self.batch_size]			,			2			,			dtype=tf.floataa)
            __UpperCamelCase					:Optional[Any]        =		ids_tensor([self.batch_size]			,			self.num_choices)

        __UpperCamelCase					:Optional[int]        =		FlaubertConfig(
            vocab_size=self.vocab_size			,			n_special=self.n_special			,			emb_dim=self.hidden_size			,			n_layers=self.num_hidden_layers			,			n_heads=self.num_attention_heads			,			dropout=self.hidden_dropout_prob			,			attention_dropout=self.attention_probs_dropout_prob			,			gelu_activation=self.gelu_activation			,			sinusoidal_embeddings=self.sinusoidal_embeddings			,			asm=self.asm			,			causal=self.causal			,			n_langs=self.n_langs			,			max_position_embeddings=self.max_position_embeddings			,			initializer_range=self.initializer_range			,			summary_type=self.summary_type			,			use_proj=self.use_proj			,			bos_token_id=self.bos_token_id			,			)

        return (
            config,
            input_ids,
            token_type_ids,
            input_lengths,
            sequence_labels,
            token_labels,
            is_impossible_labels,
            choice_labels,
            input_mask,
        )







    def       UpperCamelCase__       (     self			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			)	->     Optional[Any]:
        __UpperCamelCase					:str        =		TFFlaubertModel(config=__lowercase)
        __UpperCamelCase					:List[str]        =		{'''input_ids''': input_ids, '''lengths''': input_lengths, '''langs''': token_type_ids}
        __UpperCamelCase					:Optional[int]        =		model(__lowercase)

        __UpperCamelCase					:Optional[int]        =		[input_ids, input_mask]
        __UpperCamelCase					:Union[str, Any]        =		model(__lowercase)
        self.parent.assertEqual(result.last_hidden_state.shape			,			(self.batch_size, self.seq_length, self.hidden_size))







    def       UpperCamelCase__       (     self			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			)	->     int:
        __UpperCamelCase					:str        =		TFFlaubertWithLMHeadModel(__lowercase)

        __UpperCamelCase					:List[str]        =		{'''input_ids''': input_ids, '''lengths''': input_lengths, '''langs''': token_type_ids}
        __UpperCamelCase					:Optional[int]        =		model(__lowercase)

        self.parent.assertEqual(result.logits.shape			,			(self.batch_size, self.seq_length, self.vocab_size))







    def       UpperCamelCase__       (     self			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			)	->     Any:
        __UpperCamelCase					:Any        =		TFFlaubertForQuestionAnsweringSimple(__lowercase)

        __UpperCamelCase					:str        =		{'''input_ids''': input_ids, '''lengths''': input_lengths}

        __UpperCamelCase					:int        =		model(__lowercase)

        self.parent.assertEqual(result.start_logits.shape			,			(self.batch_size, self.seq_length))
        self.parent.assertEqual(result.end_logits.shape			,			(self.batch_size, self.seq_length))







    def       UpperCamelCase__       (     self			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			)	->     Any:
        __UpperCamelCase					:int        =		TFFlaubertForSequenceClassification(__lowercase)

        __UpperCamelCase					:List[Any]        =		{'''input_ids''': input_ids, '''lengths''': input_lengths}

        __UpperCamelCase					:Any        =		model(__lowercase)

        self.parent.assertEqual(result.logits.shape			,			(self.batch_size, self.type_sequence_label_size))







    def       UpperCamelCase__       (     self			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			)	->     Tuple:
        __UpperCamelCase					:Optional[int]        =		self.num_labels
        __UpperCamelCase					:int        =		TFFlaubertForTokenClassification(config=__lowercase)
        __UpperCamelCase					:Union[str, Any]        =		{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
        __UpperCamelCase					:Optional[Any]        =		model(__lowercase)
        self.parent.assertEqual(result.logits.shape			,			(self.batch_size, self.seq_length, self.num_labels))







    def       UpperCamelCase__       (     self			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			)	->     List[Any]:
        __UpperCamelCase					:Any        =		self.num_choices
        __UpperCamelCase					:Any        =		TFFlaubertForMultipleChoice(config=__lowercase)
        __UpperCamelCase					:List[str]        =		tf.tile(tf.expand_dims(__lowercase			,			1)			,			(1, self.num_choices, 1))
        __UpperCamelCase					:Optional[Any]        =		tf.tile(tf.expand_dims(__lowercase			,			1)			,			(1, self.num_choices, 1))
        __UpperCamelCase					:Tuple        =		tf.tile(tf.expand_dims(__lowercase			,			1)			,			(1, self.num_choices, 1))
        __UpperCamelCase					:Any        =		{
            '''input_ids''': multiple_choice_inputs_ids,
            '''attention_mask''': multiple_choice_input_mask,
            '''token_type_ids''': multiple_choice_token_type_ids,
        }
        __UpperCamelCase					:List[Any]        =		model(__lowercase)
        self.parent.assertEqual(result.logits.shape			,			(self.batch_size, self.num_choices))







    def       UpperCamelCase__       (     self)	->     Dict:
        __UpperCamelCase					:Optional[int]        =		self.prepare_config_and_inputs()
        (
            (
            __UpperCamelCase
        )  ,	(
            __UpperCamelCase
        )  ,	(
            __UpperCamelCase
        )  ,	(
            __UpperCamelCase
        )  ,	(
            __UpperCamelCase
        )  ,	(
            __UpperCamelCase
        )  ,	(
            __UpperCamelCase
        )  ,	(
            __UpperCamelCase
        )  ,	(
            __UpperCamelCase
        )  ,	
        )					:Any        =		config_and_inputs
        __UpperCamelCase					:int        =		{
            '''input_ids''': input_ids,
            '''token_type_ids''': token_type_ids,
            '''langs''': token_type_ids,
            '''lengths''': input_lengths,
        }
        return config, inputs_dict


@require_tf
class      lowerCamelCase_					(       UpperCAmelCase_						,     UpperCAmelCase_						,     unittest.TestCase			):






    '''simple docstring'''






    a__     :						List[Any]      =				(
        (
            TFFlaubertModel,
            TFFlaubertWithLMHeadModel,
            TFFlaubertForSequenceClassification,
            TFFlaubertForQuestionAnsweringSimple,
            TFFlaubertForTokenClassification,
            TFFlaubertForMultipleChoice,
        )
        if is_tf_available()
        else ()
    )
    a__     :						Tuple      =				(
        (TFFlaubertWithLMHeadModel,) if is_tf_available() else ()
    )  # TODO (PVP): Check other models whether language generation is also applicable
    a__     :						Tuple      =				(
        {
            """feature-extraction""": TFFlaubertModel,
            """fill-mask""": TFFlaubertWithLMHeadModel,
            """question-answering""": TFFlaubertForQuestionAnsweringSimple,
            """text-classification""": TFFlaubertForSequenceClassification,
            """token-classification""": TFFlaubertForTokenClassification,
            """zero-shot""": TFFlaubertForSequenceClassification,
        }
        if is_tf_available()
        else {}
    )
    a__     :						Tuple      =				False
    a__     :						Any      =				False







    def       UpperCamelCase__       (     self			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase			,			__lowercase)	->     Any:
        if (
            pipeline_test_casse_name == "QAPipelineTests"
            and tokenizer_name is not None
            and not tokenizer_name.endswith('''Fast''')
        ):
            # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
            # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
            # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
            return True

        return False







    def       UpperCamelCase__       (     self)	->     Union[str, Any]:
        __UpperCamelCase					:int        =		TFFlaubertModelTester(self)
        __UpperCamelCase					:Dict        =		ConfigTester(self			,			config_class=__lowercase			,			emb_dim=37)







    def       UpperCamelCase__       (     self)	->     str:
        self.config_tester.run_common_tests()







    def       UpperCamelCase__       (     self)	->     Any:
        __UpperCamelCase					:int        =		self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_flaubert_model(*__lowercase)







    def       UpperCamelCase__       (     self)	->     Union[str, Any]:
        __UpperCamelCase					:Optional[int]        =		self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_flaubert_lm_head(*__lowercase)







    def       UpperCamelCase__       (     self)	->     Dict:
        __UpperCamelCase					:Optional[Any]        =		self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_flaubert_qa(*__lowercase)







    def       UpperCamelCase__       (     self)	->     Tuple:
        __UpperCamelCase					:Optional[Any]        =		self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_flaubert_sequence_classif(*__lowercase)







    def       UpperCamelCase__       (     self)	->     Optional[Any]:
        __UpperCamelCase					:Dict        =		self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_flaubert_for_token_classification(*__lowercase)







    def       UpperCamelCase__       (     self)	->     str:
        __UpperCamelCase					:Union[str, Any]        =		self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_flaubert_for_multiple_choice(*__lowercase)







    @slow
    def       UpperCamelCase__       (     self)	->     Union[str, Any]:
        for model_name in TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            __UpperCamelCase					:Any        =		TFFlaubertModel.from_pretrained(__lowercase)
            self.assertIsNotNone(__lowercase)


@require_tf
@require_sentencepiece
@require_tokenizers
class      lowerCamelCase_					(       unittest.TestCase			):






    '''simple docstring'''






    @slow
    def       UpperCamelCase__       (     self)	->     Optional[Any]:
        __UpperCamelCase					:List[Any]        =		TFFlaubertModel.from_pretrained('''jplu/tf-flaubert-small-cased''')

        __UpperCamelCase					:Tuple        =		tf.convert_to_tensor(
            [[0, 158, 735, 2_592, 1_424, 6_727, 82, 1]]			,			dtype=tf.intaa			,			)  # "J'aime flaubert !"

        __UpperCamelCase					:Union[str, Any]        =		model(__lowercase)[0]
        __UpperCamelCase					:str        =		tf.TensorShape((1, 8, 512))
        self.assertEqual(output.shape			,			__lowercase)
        # compare the actual values for a slice.
        __UpperCamelCase					:Optional[int]        =		tf.convert_to_tensor(
            [
                [
                    [-1.8_76_87_73, -1.56_65_55, 0.27_07_24_18],
                    [-1.6_92_00_38, -0.5_87_35_05, 1.9_32_95_99],
                    [-2.9_56_39_85, -1.6_99_38_35, 1.7_97_20_52],
                ]
            ]			,			dtype=tf.floataa			,			)

        self.assertTrue(np.allclose(output[:, :3, :3].numpy()			,			expected_slice.numpy()			,			atol=1E-4))





