
import os
from dataclasses import dataclass, field
from io import BytesIO
from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union

import numpy as np
import pyarrow as pa

from .. import config
from ..download.streaming_download_manager import xopen, xsplitext
from ..table import array_cast
from ..utils.py_utils import no_op_if_value_is_null, string_to_dict


if TYPE_CHECKING:
					from .features import FeatureType


SCREAMING_SNAKE_CASE			,   SCREAMING_SNAKE_CASE			,   SCREAMING_SNAKE_CASE							:Dict										=    False, False, False





@dataclass
class 							UpperCAmelCase				:




	'''simple docstring'''





	snake_case_    =					None
	snake_case_    =					True
	snake_case_    =					True
	snake_case_    =					None
	# Automatically constructed
	snake_case_    =					"dict"
	snake_case_    =					pa.struct({"bytes": pa.binary(), "path": pa.string()}  )
	snake_case_    =					field(default="Audio"      ,      init=__SCREAMING_SNAKE_CASE      ,      repr=__SCREAMING_SNAKE_CASE  )






	def __call__(	self :			Any			):
					return self.pa_type






	def     UpperCamelCase_  (	self :			List[Any]							,A :			Union[str, bytes, dict]			):
					try:
									import soundfile as sf  # soundfile is a dependency of librosa, needed to decode audio files.
					except ImportError as err:
									raise ImportError("To support encoding audio data, please install 'soundfile'."			) from err
					if isinstance(A							,A			):
									return {"bytes": None, "path": value}
					elif isinstance(A							,A			):
									return {"bytes": value, "path": None}
					elif "array" in value:
									# convert the audio array to wav bytes
									__A             =			BytesIO()
									sf.write(A							,value["array"]							,value["sampling_rate"]							,format="wav"			)
									return {"bytes": buffer.getvalue(), "path": None}
					elif value.get("path"			) is not None and os.path.isfile(value["path"]			):
									# we set "bytes": None to not duplicate the data if they're already available locally
									if value["path"].endswith("pcm"			):
													# "PCM" only has raw audio bytes
													if value.get("sampling_rate"			) is None:
																	# At least, If you want to convert "PCM-byte" to "WAV-byte", you have to know sampling rate
																	raise KeyError("To use PCM files, please specify a 'sampling_rate' in Audio object"			)
													if value.get("bytes"			):
																	# If we already had PCM-byte, we don`t have to make "read file, make bytes" (just use it!)
																	__A             =			np.frombuffer(value["bytes"]							,dtype=np.intaa			).astype(np.floataa			) / 3_27_67
													else:
																	__A             =			np.memmap(value["path"]							,dtype="h"							,mode="r"			).astype(np.floataa			) / 3_27_67

													__A             =			BytesIO(bytes()			)
													sf.write(A							,A							,value["sampling_rate"]							,format="wav"			)
													return {"bytes": buffer.getvalue(), "path": None}
									else:
													return {"bytes": None, "path": value.get("path"			)}
					elif value.get("bytes"			) is not None or value.get("path"			) is not None:
									# store the audio bytes, and path is used to infer the audio format using the file extension
									return {"bytes": value.get("bytes"			), "path": value.get("path"			)}
					else:
									raise ValueError(
									    f'''An audio sample should have one of \'path\' or \'bytes\' but they are missing or None in {value}.'''			)






	def     UpperCamelCase_  (	self :			Any							,A :			dict							,A :			Optional[Dict[str, Union[str, bool, None]]] = None			):
					if not self.decode:
									raise RuntimeError("Decoding is disabled for this feature. Please use Audio(decode=True) instead."			)

					__A      ,							__A             =			(value["path"], BytesIO(value["bytes"]			)) if value["bytes"] is not None else (value["path"], None)
					if path is None and file is None:
									raise ValueError(f'''An audio sample should have one of \'path\' or \'bytes\' but both are None in {value}.'''			)

					try:
									import librosa
									import soundfile as sf
					except ImportError as err:
									raise ImportError("To support decoding audio files, please install 'librosa' and 'soundfile'."			) from err

					__A             =			xsplitext(A			)[1][1:].lower() if path is not None else None
					if not config.IS_OPUS_SUPPORTED and audio_format == "opus":
									raise RuntimeError(
									    "Decoding 'opus' files requires system library 'libsndfile'>=1.0.31, "
									    "You can try to update `soundfile` python library: `pip install \"soundfile>=0.12.1\"`. "			)
					elif not config.IS_MP3_SUPPORTED and audio_format == "mp3":
									raise RuntimeError(
									    "Decoding 'mp3' files requires system library 'libsndfile'>=1.1.0, "
									    "You can try to update `soundfile` python library: `pip install \"soundfile>=0.12.1\"`. "			)

					if file is None:
									__A             =			token_per_repo_id or {}
									__A             =			path.split("::"			)[-1]
									try:
													__A             =			string_to_dict(A							,config.HUB_DATASETS_URL			)["repo_id"]
													__A             =			token_per_repo_id[repo_id]
									except (ValueError, KeyError):
													__A             =			None

									with xopen(A							,"rb"							,use_auth_token=A			) as f:
													__A      ,							__A             =			sf.read(A			)

					else:
									__A      ,							__A             =			sf.read(A			)

					__A             =			array.T
					if self.mono:
									__A             =			librosa.to_mono(A			)
					if self.sampling_rate and self.sampling_rate != sampling_rate:
									__A             =			librosa.resample(A							,orig_sr=A							,target_sr=self.sampling_rate			)
									__A             =			self.sampling_rate

					return {"path": path, "array": array, "sampling_rate": sampling_rate}






	def     UpperCamelCase_  (	self :			Optional[Any]			):
					from .features import Value

					if self.decode:
									raise ValueError("Cannot flatten a decoded Audio feature."			)
					return {
					    "bytes": Value("binary"			),
					    "path": Value("string"			),
					}






	def     UpperCamelCase_  (	self :			Tuple							,A :			Union[pa.StringArray, pa.StructArray]			):
					if pa.types.is_string(storage.type			):
									__A             =			pa.array([None] * len(A			)							,type=pa.binary()			)
									__A             =			pa.StructArray.from_arrays([bytes_array, storage]							,["bytes", "path"]							,mask=storage.is_null()			)
					elif pa.types.is_binary(storage.type			):
									__A             =			pa.array([None] * len(A			)							,type=pa.string()			)
									__A             =			pa.StructArray.from_arrays([storage, path_array]							,["bytes", "path"]							,mask=storage.is_null()			)
					elif pa.types.is_struct(storage.type			) and storage.type.get_all_field_indices("array"			):
									__A             =			pa.array([Audio().encode_example(A			) if x is not None else None for x in storage.to_pylist()]			)
					elif pa.types.is_struct(storage.type			):
									if storage.type.get_field_index("bytes"			) >= 0:
													__A             =			storage.field("bytes"			)
									else:
													__A             =			pa.array([None] * len(A			)							,type=pa.binary()			)
									if storage.type.get_field_index("path"			) >= 0:
													__A             =			storage.field("path"			)
									else:
													__A             =			pa.array([None] * len(A			)							,type=pa.string()			)
									__A             =			pa.StructArray.from_arrays([bytes_array, path_array]							,["bytes", "path"]							,mask=storage.is_null()			)
					return array_cast(A							,self.pa_type			)







	def     UpperCamelCase_  (	self :			int							,A :			pa.StructArray			):

					@no_op_if_value_is_null
					def path_to_bytes(A :			int			):
									with xopen(A							,"rb"			) as f:
													__A             =			f.read()
									return bytes_

					__A             =			pa.array(
					    [
					        (path_to_bytes(x["path"]			) if x["bytes"] is None else x["bytes"]) if x is not None else None
					        for x in storage.to_pylist()
					    ]							,type=pa.binary()							,)
					__A             =			pa.array(
					    [os.path.basename(A			) if path is not None else None for path in storage.field("path"			).to_pylist()]							,type=pa.string()							,)
					__A             =			pa.StructArray.from_arrays([bytes_array, path_array]							,["bytes", "path"]							,mask=bytes_array.is_null()			)
					return array_cast(A							,self.pa_type			)


import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin





@custom_tokenizers
class 							UpperCAmelCase				(  __SCREAMING_SNAKE_CASE      ,      unittest.TestCase  ):




	'''simple docstring'''





	snake_case_    =					BertJapaneseTokenizer
	snake_case_    =					False
	snake_case_    =					True






	def     UpperCamelCase_  (	self :			List[Any]			):
					super().setUp()

					__A             =			[
					    "[UNK]",
					    "[CLS]",
					    "[SEP]",
					    "こんにちは",
					    "こん",
					    "にちは",
					    "ばんは",
					    "##こん",
					    "##にちは",
					    "##ばんは",
					    "世界",
					    "##世界",
					    "、",
					    "##、",
					    "。",
					    "##。",
					]

					__A             =			os.path.join(self.tmpdirname							,VOCAB_FILES_NAMES["vocab_file"]			)
					with open(self.vocab_file							,"w"							,encoding="utf-8"			) as vocab_writer:
									vocab_writer.write("".join([x + "\n" for x in vocab_tokens]			)			)






	def     UpperCamelCase_  (	self :			Union[str, Any]							,A :			Optional[Any]			):
					__A             =			"こんにちは、世界。 \nこんばんは、世界。"
					__A             =			"こんにちは 、 世界 。 こんばんは 、 世界 。"
					return input_text, output_text






	def     UpperCamelCase_  (	self :			Any							,A :			Optional[int]			):
					__A      ,							__A             =			self.get_input_output_texts(A			)
					__A             =			tokenizer.encode(A							,add_special_tokens=A			)
					__A             =			tokenizer.decode(A							,clean_up_tokenization_spaces=A			)
					return text, ids






	def     UpperCamelCase_  (	self :			int			):
					pass  # TODO add if relevant






	def     UpperCamelCase_  (	self :			int			):
					pass  # TODO add if relevant






	def     UpperCamelCase_  (	self :			Optional[int]			):
					pass  # TODO add if relevant






	def     UpperCamelCase_  (	self :			List[Any]			):
					__A             =			self.tokenizer_class(self.vocab_file			)

					__A             =			tokenizer.tokenize("こんにちは、世界。\nこんばんは、世界。"			)
					self.assertListEqual(A							,["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]			)
					self.assertListEqual(tokenizer.convert_tokens_to_ids(A			)							,[3, 12, 10, 14, 4, 9, 12, 10, 14]			)






	def     UpperCamelCase_  (	self :			int			):
					__A             =			self.tokenizer_class(self.vocab_file							,word_tokenizer_type="mecab"			)
					self.assertIsNotNone(A			)

					__A             =			"こんにちは、世界。\nこんばんは、世界。"
					__A             =			tokenizer.tokenize(A			)
					self.assertListEqual(A							,["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]			)
					self.assertListEqual(tokenizer.convert_tokens_to_ids(A			)							,[3, 12, 10, 14, 4, 9, 12, 10, 14]			)

					__A             =			os.path.join(self.tmpdirname							,"tokenizer.bin"			)
					with open(A							,"wb"			) as handle:
									pickle.dump(A							,A			)

					with open(A							,"rb"			) as handle:
									__A             =			pickle.load(A			)

					__A             =			tokenizer_new.tokenize(A			)

					self.assertListEqual(A							,A			)






	def     UpperCamelCase_  (	self :			Any			):
					__A             =			MecabTokenizer(mecab_dic="ipadic"			)

					self.assertListEqual(
					    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)							,["アップルストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]							,)






	def     UpperCamelCase_  (	self :			List[str]			):
					try:
									__A             =			MecabTokenizer(mecab_dic="unidic_lite"			)
					except ModuleNotFoundError:
									return

					self.assertListEqual(
					    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)							,["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]							,)






	def     UpperCamelCase_  (	self :			Tuple			):
					try:
									__A             =			MecabTokenizer(mecab_dic="unidic"			)
					except ModuleNotFoundError:
									return

					self.assertListEqual(
					    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)							,["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]							,)






	def     UpperCamelCase_  (	self :			Tuple			):
					__A             =			MecabTokenizer(do_lower_case=A							,mecab_dic="ipadic"			)

					self.assertListEqual(
					    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)							,["アップルストア", "で", "iphone", "8", "が", "発売", "さ", "れ", "た", "。"]							,)






	def     UpperCamelCase_  (	self :			Any			):
					try:
									__A             =			MecabTokenizer(
									    do_lower_case=A							,normalize_text=A							,mecab_option="-d /usr/local/lib/mecab/dic/jumandic"			)
					except RuntimeError:
									# if dict doesn't exist in the system, previous code raises this error.
									return

					self.assertListEqual(
					    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)							,["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れた", "\u3000", "。"]							,)






	def     UpperCamelCase_  (	self :			int			):
					__A             =			MecabTokenizer(normalize_text=A							,mecab_dic="ipadic"			)

					self.assertListEqual(
					    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)							,["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れ", "た", "　", "。"]							,)






	@require_sudachi
	def     UpperCamelCase_  (	self :			int			):
					__A             =			self.tokenizer_class(self.vocab_file							,word_tokenizer_type="sudachi"			)
					self.assertIsNotNone(A			)

					__A             =			"こんにちは、世界。\nこんばんは、世界。"
					__A             =			tokenizer.tokenize(A			)
					self.assertListEqual(A							,["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]			)
					self.assertListEqual(tokenizer.convert_tokens_to_ids(A			)							,[3, 12, 10, 14, 4, 9, 12, 10, 14]			)

					__A             =			os.path.join(self.tmpdirname							,"tokenizer.bin"			)
					with open(A							,"wb"			) as handle:
									pickle.dump(A							,A			)

					with open(A							,"rb"			) as handle:
									__A             =			pickle.load(A			)

					__A             =			tokenizer_new.tokenize(A			)

					self.assertListEqual(A							,A			)






	@require_sudachi
	def     UpperCamelCase_  (	self :			Optional[int]			):
					__A             =			SudachiTokenizer(sudachi_dict_type="core"			)

					self.assertListEqual(
					    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)							,[" ", "\t", "アップル", "ストア", "で", "iPhone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]							,)






	@require_sudachi
	def     UpperCamelCase_  (	self :			List[Any]			):
					__A             =			SudachiTokenizer(sudachi_dict_type="core"							,sudachi_split_mode="A"			)

					self.assertListEqual(tokenizer.tokenize("外国人参政権"			)							,["外国", "人", "参政", "権"]			)






	@require_sudachi
	def     UpperCamelCase_  (	self :			int			):
					__A             =			SudachiTokenizer(sudachi_dict_type="core"							,sudachi_split_mode="B"			)

					self.assertListEqual(tokenizer.tokenize("外国人参政権"			)							,["外国人", "参政権"]			)






	@require_sudachi
	def     UpperCamelCase_  (	self :			Tuple			):
					__A             =			SudachiTokenizer(sudachi_dict_type="core"							,sudachi_split_mode="C"			)

					self.assertListEqual(tokenizer.tokenize("外国人参政権"			)							,["外国人参政権"]			)






	@require_sudachi
	def     UpperCamelCase_  (	self :			int			):
					__A             =			SudachiTokenizer(do_lower_case=A							,sudachi_dict_type="core"			)

					self.assertListEqual(
					    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)							,[" ", "\t", "アップル", "ストア", "で", "iphone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]							,)






	@require_sudachi
	def     UpperCamelCase_  (	self :			List[str]			):
					__A             =			SudachiTokenizer(normalize_text=A							,sudachi_dict_type="core"			)

					self.assertListEqual(
					    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)							,[" ", "\t", "ｱｯﾌﾟﾙ", "ストア", "で", "iPhone", "８", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", "\u3000", "。", " ", " "]							,)






	@require_sudachi
	def     UpperCamelCase_  (	self :			str			):
					__A             =			SudachiTokenizer(trim_whitespace=A							,sudachi_dict_type="core"			)

					self.assertListEqual(
					    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)							,["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]							,)






	@require_jumanpp
	def     UpperCamelCase_  (	self :			Optional[Any]			):
					__A             =			self.tokenizer_class(self.vocab_file							,word_tokenizer_type="jumanpp"			)
					self.assertIsNotNone(A			)

					__A             =			"こんにちは、世界。\nこんばんは、世界。"
					__A             =			tokenizer.tokenize(A			)
					self.assertListEqual(A							,["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]			)
					self.assertListEqual(tokenizer.convert_tokens_to_ids(A			)							,[3, 12, 10, 14, 4, 9, 12, 10, 14]			)

					__A             =			os.path.join(self.tmpdirname							,"tokenizer.bin"			)
					with open(A							,"wb"			) as handle:
									pickle.dump(A							,A			)

					with open(A							,"rb"			) as handle:
									__A             =			pickle.load(A			)

					__A             =			tokenizer_new.tokenize(A			)

					self.assertListEqual(A							,A			)






	@require_jumanpp
	def     UpperCamelCase_  (	self :			int			):
					__A             =			JumanppTokenizer()

					self.assertListEqual(
					    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)							,["アップル", "ストア", "で", "iPhone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]							,)






	@require_jumanpp
	def     UpperCamelCase_  (	self :			str			):
					__A             =			JumanppTokenizer(do_lower_case=A			)

					self.assertListEqual(
					    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)							,["アップル", "ストア", "で", "iphone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]							,)






	@require_jumanpp
	def     UpperCamelCase_  (	self :			Any			):
					__A             =			JumanppTokenizer(normalize_text=A			)

					self.assertListEqual(
					    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)							,["ｱ", "ｯ", "ﾌ", "ﾟ", "ﾙ", "ストア", "で", "iPhone", "８", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]							,)






	@require_jumanpp
	def     UpperCamelCase_  (	self :			List[str]			):
					__A             =			JumanppTokenizer(trim_whitespace=A			)

					self.assertListEqual(
					    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)							,["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れた", "。"]							,)






	@require_jumanpp
	def     UpperCamelCase_  (	self :			Dict			):
					__A             =			JumanppTokenizer()

					self.assertListEqual(
					    tokenizer.tokenize("ありがとうございますm(_ _)ｍ見つけるのが大変です。"			)							,["ありがとう", "ございます", "m(_ _)m", "見つける", "の", "が", "大変です", "。"]							,)






	def     UpperCamelCase_  (	self :			str			):
					__A             =			["[UNK]", "[CLS]", "[SEP]", "こんにちは", "こん", "にちは", "ばんは", "##こん", "##にちは", "##ばんは"]

					__A             =			{}
					for i, token in enumerate(A			):
									__A             =			i
					__A             =			WordpieceTokenizer(vocab=A							,unk_token="[UNK]"			)

					self.assertListEqual(tokenizer.tokenize(""			)							,[]			)

					self.assertListEqual(tokenizer.tokenize("こんにちは"			)							,["こんにちは"]			)

					self.assertListEqual(tokenizer.tokenize("こんばんは"			)							,["こん", "##ばんは"]			)

					self.assertListEqual(tokenizer.tokenize("こんばんは こんばんにちは こんにちは"			)							,["こん", "##ばんは", "[UNK]", "こんにちは"]			)






	def     UpperCamelCase_  (	self :			Any			):
					__A             =			BertJapaneseTokenizer.from_pretrained("nlp-waseda/roberta-base-japanese-with-auto-jumanpp"			)
					__A             =			tokenizer.subword_tokenizer

					__A             =			subword_tokenizer.tokenize("国境 の 長い トンネル を 抜ける と 雪国 であった 。"			)
					self.assertListEqual(A							,["▁国境", "▁の", "▁長い", "▁トンネル", "▁を", "▁抜ける", "▁と", "▁雪", "国", "▁であった", "▁。"]			)

					__A             =			subword_tokenizer.tokenize("こんばんは こんばん にち は こんにちは"			)
					self.assertListEqual(A							,["▁こん", "ばん", "は", "▁こん", "ばん", "▁に", "ち", "▁は", "▁こんにちは"]			)







	def     UpperCamelCase_  (	self :			int			):
					__A             =			self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese"			)

					__A             =			tokenizer.encode("ありがとう。"							,add_special_tokens=A			)
					__A             =			tokenizer.encode("どういたしまして。"							,add_special_tokens=A			)

					__A             =			tokenizer.build_inputs_with_special_tokens(A			)
					__A             =			tokenizer.build_inputs_with_special_tokens(A							,A			)

					# 2 is for "[CLS]", 3 is for "[SEP]"
					assert encoded_sentence == [2] + text + [3]
					assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 							UpperCAmelCase				(  __SCREAMING_SNAKE_CASE      ,      unittest.TestCase  ):




	'''simple docstring'''





	snake_case_    =					BertJapaneseTokenizer
	snake_case_    =					False






	def     UpperCamelCase_  (	self :			Any			):
					super().setUp()

					__A             =			["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

					__A             =			os.path.join(self.tmpdirname							,VOCAB_FILES_NAMES["vocab_file"]			)
					with open(self.vocab_file							,"w"							,encoding="utf-8"			) as vocab_writer:
									vocab_writer.write("".join([x + "\n" for x in vocab_tokens]			)			)






	def     UpperCamelCase_  (	self :			int							,**A :			str			):
					return BertJapaneseTokenizer.from_pretrained(self.tmpdirname							,subword_tokenizer_type="character"							,**A			)






	def     UpperCamelCase_  (	self :			List[str]							,A :			str			):
					__A             =			"こんにちは、世界。 \nこんばんは、世界。"
					__A             =			"こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"
					return input_text, output_text






	def     UpperCamelCase_  (	self :			str			):
					pass  # TODO add if relevant






	def     UpperCamelCase_  (	self :			Optional[Any]			):
					pass  # TODO add if relevant






	def     UpperCamelCase_  (	self :			Any			):
					pass  # TODO add if relevant






	def     UpperCamelCase_  (	self :			str			):
					__A             =			self.tokenizer_class(self.vocab_file							,subword_tokenizer_type="character"			)

					__A             =			tokenizer.tokenize("こんにちは、世界。 \nこんばんは、世界。"			)
					self.assertListEqual(
					    A							,["こ", "ん", "に", "ち", "は", "、", "世", "界", "。", "こ", "ん", "ば", "ん", "は", "、", "世", "界", "。"]			)
					self.assertListEqual(
					    tokenizer.convert_tokens_to_ids(A			)							,[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]			)






	def     UpperCamelCase_  (	self :			Union[str, Any]			):
					__A             =			["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

					__A             =			{}
					for i, token in enumerate(A			):
									__A             =			i
					__A             =			CharacterTokenizer(vocab=A							,unk_token="[UNK]"			)

					self.assertListEqual(tokenizer.tokenize(""			)							,[]			)

					self.assertListEqual(tokenizer.tokenize("こんにちは"			)							,["こ", "ん", "に", "ち", "は"]			)

					self.assertListEqual(tokenizer.tokenize("こんにちほ"			)							,["こ", "ん", "に", "ち", "[UNK]"]			)







	def     UpperCamelCase_  (	self :			Tuple			):
					__A             =			self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese-char"			)

					__A             =			tokenizer.encode("ありがとう。"							,add_special_tokens=A			)
					__A             =			tokenizer.encode("どういたしまして。"							,add_special_tokens=A			)

					__A             =			tokenizer.build_inputs_with_special_tokens(A			)
					__A             =			tokenizer.build_inputs_with_special_tokens(A							,A			)

					# 2 is for "[CLS]", 3 is for "[SEP]"
					assert encoded_sentence == [2] + text + [3]
					assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 							UpperCAmelCase				(  unittest.TestCase  ):




	'''simple docstring'''





	def     UpperCamelCase_  (	self :			int			):
					__A             =			"cl-tohoku/bert-base-japanese"
					__A             =			AutoTokenizer.from_pretrained(A			)
					self.assertIsInstance(A							,A			)





class 							UpperCAmelCase				(  unittest.TestCase  ):




	'''simple docstring'''





	def     UpperCamelCase_  (	self :			Tuple			):
					__A             =			"cl-tohoku/bert-base-japanese"
					with self.assertLogs("transformers"							,level="WARNING"			) as cm:
									BertTokenizer.from_pretrained(A			)
									self.assertTrue(
									    cm.records[0].message.startswith(
									        "The tokenizer class you load from this checkpoint is not the same type as the class this function"
									        " is called from."			)			)
					__A             =			"bert-base-cased"
					with self.assertLogs("transformers"							,level="WARNING"			) as cm:
									BertJapaneseTokenizer.from_pretrained(A			)
									self.assertTrue(
									    cm.records[0].message.startswith(
									        "The tokenizer class you load from this checkpoint is not the same type as the class this function"
									        " is called from."			)			)
