






import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
		from .tokenization_nllb import NllbTokenizer
else:
		__magic_name__:					Dict							=     None


__magic_name__:					Optional[Any]							=     logging.get_logger(__name__)


__magic_name__:					List[Any]							=     {"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

__magic_name__:					Optional[Any]							=     {
    "vocab_file": {
        "facebook/nllb-200-distilled-600M": (
            "https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "facebook/nllb-200-distilled-600M": (
            "https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/tokenizer.json"
        ),
    },
}

__magic_name__:					Dict							=     {
    "facebook/nllb-large-en-ro": 1_024,
    "facebook/nllb-200-distilled-600M": 1_024,
}

# fmt: off
__magic_name__:					List[str]							=     ["ace_Arab", "ace_Latn", "acm_Arab", "acq_Arab", "aeb_Arab", "afr_Latn", "ajp_Arab", "aka_Latn", "amh_Ethi", "apc_Arab", "arb_Arab", "ars_Arab", "ary_Arab", "arz_Arab", "asm_Beng", "ast_Latn", "awa_Deva", "ayr_Latn", "azb_Arab", "azj_Latn", "bak_Cyrl", "bam_Latn", "ban_Latn", "bel_Cyrl", "bem_Latn", "ben_Beng", "bho_Deva", "bjn_Arab", "bjn_Latn", "bod_Tibt", "bos_Latn", "bug_Latn", "bul_Cyrl", "cat_Latn", "ceb_Latn", "ces_Latn", "cjk_Latn", "ckb_Arab", "crh_Latn", "cym_Latn", "dan_Latn", "deu_Latn", "dik_Latn", "dyu_Latn", "dzo_Tibt", "ell_Grek", "eng_Latn", "epo_Latn", "est_Latn", "eus_Latn", "ewe_Latn", "fao_Latn", "pes_Arab", "fij_Latn", "fin_Latn", "fon_Latn", "fra_Latn", "fur_Latn", "fuv_Latn", "gla_Latn", "gle_Latn", "glg_Latn", "grn_Latn", "guj_Gujr", "hat_Latn", "hau_Latn", "heb_Hebr", "hin_Deva", "hne_Deva", "hrv_Latn", "hun_Latn", "hye_Armn", "ibo_Latn", "ilo_Latn", "ind_Latn", "isl_Latn", "ita_Latn", "jav_Latn", "jpn_Jpan", "kab_Latn", "kac_Latn", "kam_Latn", "kan_Knda", "kas_Arab", "kas_Deva", "kat_Geor", "knc_Arab", "knc_Latn", "kaz_Cyrl", "kbp_Latn", "kea_Latn", "khm_Khmr", "kik_Latn", "kin_Latn", "kir_Cyrl", "kmb_Latn", "kon_Latn", "kor_Hang", "kmr_Latn", "lao_Laoo", "lvs_Latn", "lij_Latn", "lim_Latn", "lin_Latn", "lit_Latn", "lmo_Latn", "ltg_Latn", "ltz_Latn", "lua_Latn", "lug_Latn", "luo_Latn", "lus_Latn", "mag_Deva", "mai_Deva", "mal_Mlym", "mar_Deva", "min_Latn", "mkd_Cyrl", "plt_Latn", "mlt_Latn", "mni_Beng", "khk_Cyrl", "mos_Latn", "mri_Latn", "zsm_Latn", "mya_Mymr", "nld_Latn", "nno_Latn", "nob_Latn", "npi_Deva", "nso_Latn", "nus_Latn", "nya_Latn", "oci_Latn", "gaz_Latn", "ory_Orya", "pag_Latn", "pan_Guru", "pap_Latn", "pol_Latn", "por_Latn", "prs_Arab", "pbt_Arab", "quy_Latn", "ron_Latn", "run_Latn", "rus_Cyrl", "sag_Latn", "san_Deva", "sat_Beng", "scn_Latn", "shn_Mymr", "sin_Sinh", "slk_Latn", "slv_Latn", "smo_Latn", "sna_Latn", "snd_Arab", "som_Latn", "sot_Latn", "spa_Latn", "als_Latn", "srd_Latn", "srp_Cyrl", "ssw_Latn", "sun_Latn", "swe_Latn", "swh_Latn", "szl_Latn", "tam_Taml", "tat_Cyrl", "tel_Telu", "tgk_Cyrl", "tgl_Latn", "tha_Thai", "tir_Ethi", "taq_Latn", "taq_Tfng", "tpi_Latn", "tsn_Latn", "tso_Latn", "tuk_Latn", "tum_Latn", "tur_Latn", "twi_Latn", "tzm_Tfng", "uig_Arab", "ukr_Cyrl", "umb_Latn", "urd_Arab", "uzn_Latn", "vec_Latn", "vie_Latn", "war_Latn", "wol_Latn", "xho_Latn", "ydd_Hebr", "yor_Latn", "yue_Hant", "zho_Hans", "zho_Hant", "zul_Latn"]





class 	snake_case__ (							_lowerCAmelCase				):
	lowercase__						:					Optional[Any]											=   VOCAB_FILES_NAMES
	lowercase__						:					Dict											=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowercase__						:					int											=   PRETRAINED_VOCAB_FILES_MAP
	lowercase__						:					str											=   ['''input_ids''', '''attention_mask''']
	lowercase__						:					int											=   NllbTokenizer

	lowercase__						:					List[int]											=   []
	lowercase__						:					List[int]											=   []
	def __init__(    self      ,       lowerCAmelCase__=None      ,       lowerCAmelCase__=None      ,       lowerCAmelCase__="<s>"      ,       lowerCAmelCase__="</s>"      ,       lowerCAmelCase__="</s>"      ,       lowerCAmelCase__="<s>"      ,       lowerCAmelCase__="<unk>"      ,       lowerCAmelCase__="<pad>"      ,       lowerCAmelCase__="<mask>"      ,       lowerCAmelCase__=None      ,       lowerCAmelCase__=None      ,       lowerCAmelCase__=None      ,       lowerCAmelCase__=False      ,       **lowerCAmelCase__      ,       )							->				List[Any]:
							# Mask token behave like a normal word, i.e. include the space before it
							__magic_name__      :       Any       						= AddedToken(lowerCAmelCase__      ,       lstrip=lowerCAmelCase__      ,       rstrip=lowerCAmelCase__			) if isinstance(lowerCAmelCase__      ,       lowerCAmelCase__			) else mask_token
							__magic_name__      :       List[str]       						= legacy_behaviour
							super().__init__(
							    vocab_file=lowerCAmelCase__      ,       tokenizer_file=lowerCAmelCase__      ,       bos_token=lowerCAmelCase__      ,       eos_token=lowerCAmelCase__      ,       sep_token=lowerCAmelCase__      ,       cls_token=lowerCAmelCase__      ,       unk_token=lowerCAmelCase__      ,       pad_token=lowerCAmelCase__      ,       mask_token=lowerCAmelCase__      ,       src_lang=lowerCAmelCase__      ,       tgt_lang=lowerCAmelCase__      ,       additional_special_tokens=lowerCAmelCase__      ,       legacy_behaviour=lowerCAmelCase__      ,       **lowerCAmelCase__      ,       )

							__magic_name__      :       Optional[Any]       						= vocab_file
							__magic_name__      :       int       						= False if not self.vocab_file else True

							__magic_name__      :       Optional[int]       						= FAIRSEQ_LANGUAGE_CODES.copy()

							if additional_special_tokens is not None:
													# Only add those special tokens if they are not already there.
													_additional_special_tokens.extend(
													    [t for t in additional_special_tokens if t not in _additional_special_tokens]			)

							self.add_special_tokens({"""additional_special_tokens""": _additional_special_tokens}			)
							__magic_name__      :       Optional[int]       						= {
							    lang_code: self.convert_tokens_to_ids(lowerCAmelCase__			) for lang_code in FAIRSEQ_LANGUAGE_CODES
							}

							__magic_name__      :       Dict       						= src_lang if src_lang is not None else """eng_Latn"""
							__magic_name__      :       List[str]       						= self.convert_tokens_to_ids(self._src_lang			)
							__magic_name__      :       int       						= tgt_lang
							self.set_src_lang_special_tokens(self._src_lang			)
	@property
	def       __magic_name__    (    self			)							->				str:
							return self._src_lang
	@src_lang.setter
	def       __magic_name__    (    self      ,       lowerCAmelCase__			)							->				None:
							__magic_name__      :       str       						= new_src_lang
							self.set_src_lang_special_tokens(self._src_lang			)
	def       __magic_name__    (    self      ,       lowerCAmelCase__      ,       lowerCAmelCase__ = None			)							->				List[int]:
							if token_ids_a is None:
													return self.prefix_tokens + token_ids_a + self.suffix_tokens
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens
	def       __magic_name__    (    self      ,       lowerCAmelCase__      ,       lowerCAmelCase__ = None			)							->				List[int]:
							__magic_name__      :       List[Any]       						= [self.sep_token_id]
							__magic_name__      :       Any       						= [self.cls_token_id]

							if token_ids_a is None:
													return len(cls + token_ids_a + sep			) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]
	def       __magic_name__    (    self      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       **lowerCAmelCase__			)							->				List[Any]:
							if src_lang is None or tgt_lang is None:
													raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""			)
							__magic_name__      :       List[str]       						= src_lang
							__magic_name__      :       Optional[int]       						= self(lowerCAmelCase__      ,       add_special_tokens=lowerCAmelCase__      ,       return_tensors=lowerCAmelCase__      ,       **lowerCAmelCase__			)
							__magic_name__      :       Optional[Any]       						= self.convert_tokens_to_ids(lowerCAmelCase__			)
							__magic_name__      :       str       						= tgt_lang_id
							return inputs
	def       __magic_name__    (    self      ,       lowerCAmelCase__      ,       lowerCAmelCase__ = "eng_Latn"      ,       lowerCAmelCase__ = None      ,       lowerCAmelCase__ = "fra_Latn"      ,       **lowerCAmelCase__      ,       )							->				BatchEncoding:
							__magic_name__      :       Any       						= src_lang
							__magic_name__      :       Optional[int]       						= tgt_lang
							return super().prepare_seqaseq_batch(lowerCAmelCase__      ,       lowerCAmelCase__      ,       **lowerCAmelCase__			)
	def       __magic_name__    (    self			)							->				Dict:
							return self.set_src_lang_special_tokens(self.src_lang			)
	def       __magic_name__    (    self			)							->				List[str]:
							return self.set_tgt_lang_special_tokens(self.tgt_lang			)
	def       __magic_name__    (    self      ,       lowerCAmelCase__			)							->				None:
							__magic_name__      :       Optional[int]       						= self.convert_tokens_to_ids(lowerCAmelCase__			)

							if self.legacy_behaviour:
													__magic_name__      :       Tuple       						= []
													__magic_name__      :       str       						= [self.eos_token_id, self.cur_lang_code]
							else:
													__magic_name__      :       str       						= [self.cur_lang_code]
													__magic_name__      :       str       						= [self.eos_token_id]

							__magic_name__      :       Dict       						= self.convert_ids_to_tokens(self.prefix_tokens			)
							__magic_name__      :       List[str]       						= self.convert_ids_to_tokens(self.suffix_tokens			)

							__magic_name__      :       Dict       						= processors.TemplateProcessing(
							    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str      ,       pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str      ,       special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str      ,       self.prefix_tokens + self.suffix_tokens			)			)      ,       )
	def       __magic_name__    (    self      ,       lowerCAmelCase__			)							->				None:
							__magic_name__      :       List[Any]       						= self.convert_tokens_to_ids(lowerCAmelCase__			)
							if self.legacy_behaviour:
													__magic_name__      :       List[str]       						= []
													__magic_name__      :       Optional[Any]       						= [self.eos_token_id, self.cur_lang_code]
							else:
													__magic_name__      :       List[Any]       						= [self.cur_lang_code]
													__magic_name__      :       Union[str, Any]       						= [self.eos_token_id]

							__magic_name__      :       Dict       						= self.convert_ids_to_tokens(self.prefix_tokens			)
							__magic_name__      :       Union[str, Any]       						= self.convert_ids_to_tokens(self.suffix_tokens			)

							__magic_name__      :       Union[str, Any]       						= processors.TemplateProcessing(
							    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str      ,       pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str      ,       special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str      ,       self.prefix_tokens + self.suffix_tokens			)			)      ,       )







	def       __magic_name__    (    self      ,       lowerCAmelCase__      ,       lowerCAmelCase__ = None			)							->				Tuple[str]:
							if not self.can_save_slow_tokenizer:
													raise ValueError(
													    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
													    """tokenizer."""			)

							if not os.path.isdir(lowerCAmelCase__			):
													logger.error(F'Vocabulary path ({save_directory}) should be a directory.'			)
													return
							__magic_name__      :       int       						= os.path.join(
							    lowerCAmelCase__      ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]			)

							if os.path.abspath(self.vocab_file			) != os.path.abspath(lowerCAmelCase__			):
													copyfile(self.vocab_file      ,       lowerCAmelCase__			)

							return (out_vocab_file,)








from manim import *





class 	snake_case__ (							_lowerCAmelCase				):
	def       __magic_name__    (    self			)							->				Dict:
							__magic_name__      :       int       						= Rectangle(height=0.5      ,       width=0.5			)
							__magic_name__      :       Optional[int]       						= Rectangle(height=0.2_5      ,       width=0.2_5			)
							__magic_name__      :       str       						= Rectangle(height=0.4_6      ,       width=0.4_6			).set_stroke(width=0			)

							__magic_name__      :       List[Any]       						= [mem.copy() for i in range(6			)]
							__magic_name__      :       int       						= [mem.copy() for i in range(6			)]
							__magic_name__      :       Tuple       						= VGroup(*lowerCAmelCase__			).arrange(lowerCAmelCase__      ,       buff=0			)
							__magic_name__      :       List[str]       						= VGroup(*lowerCAmelCase__			).arrange(lowerCAmelCase__      ,       buff=0			)
							__magic_name__      :       str       						= VGroup(lowerCAmelCase__      ,       lowerCAmelCase__			).arrange(lowerCAmelCase__      ,       buff=0			)
							__magic_name__      :       Union[str, Any]       						= Text("""CPU"""      ,       font_size=24			)
							__magic_name__      :       Tuple       						= Group(lowerCAmelCase__      ,       lowerCAmelCase__			).arrange(lowerCAmelCase__      ,       buff=0.5      ,       aligned_edge=lowerCAmelCase__			)
							cpu.move_to([-2.5, -0.5, 0]			)
							self.add(lowerCAmelCase__			)

							__magic_name__      :       Any       						= [mem.copy() for i in range(4			)]
							__magic_name__      :       List[Any]       						= VGroup(*lowerCAmelCase__			).arrange(lowerCAmelCase__      ,       buff=0			)
							__magic_name__      :       Tuple       						= Text("""GPU"""      ,       font_size=24			)
							__magic_name__      :       Tuple       						= Group(lowerCAmelCase__      ,       lowerCAmelCase__			).arrange(lowerCAmelCase__      ,       buff=0.5      ,       aligned_edge=lowerCAmelCase__			)
							gpu.move_to([-1, -1, 0]			)
							self.add(lowerCAmelCase__			)

							__magic_name__      :       Union[str, Any]       						= [mem.copy() for i in range(6			)]
							__magic_name__      :       Union[str, Any]       						= VGroup(*lowerCAmelCase__			).arrange(lowerCAmelCase__      ,       buff=0			)

							__magic_name__      :       str       						= Text("""Model"""      ,       font_size=24			)
							__magic_name__      :       Optional[int]       						= Group(lowerCAmelCase__      ,       lowerCAmelCase__			).arrange(lowerCAmelCase__      ,       buff=0.5      ,       aligned_edge=lowerCAmelCase__			)
							model.move_to([3, -1.0, 0]			)
							self.add(lowerCAmelCase__			)

							__magic_name__      :       str       						= []
							__magic_name__      :       Tuple       						= []
							__magic_name__      :       Union[str, Any]       						= []

							for i, rect in enumerate(lowerCAmelCase__			):
													rect.set_stroke(lowerCAmelCase__			)

													__magic_name__      :       Optional[Any]       						= Rectangle(height=0.4_6 / 4      ,       width=0.4_6 / 3			).set_stroke(width=0.0			).set_fill(lowerCAmelCase__      ,       opacity=0.7			)

													if i == 0:
																			cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT			)      ,       buff=0.0_2      ,       direction=lowerCAmelCase__			)
																			cpu_target.set_x(cpu_target.get_x() + 0.1			)
													elif i == 3:
																			cpu_target.next_to(model_cpu_arr[0]      ,       direction=lowerCAmelCase__      ,       buff=0.0			)
													else:
																			cpu_target.next_to(model_cpu_arr[i - 1]      ,       direction=lowerCAmelCase__      ,       buff=0.0			)
													self.add(lowerCAmelCase__			)
													model_cpu_arr.append(lowerCAmelCase__			)

							self.add(*lowerCAmelCase__      ,       *lowerCAmelCase__      ,       *lowerCAmelCase__			)

							__magic_name__      :       Optional[Any]       						= [mem.copy() for i in range(6			)]
							__magic_name__      :       Optional[Any]       						= VGroup(*lowerCAmelCase__			).arrange(lowerCAmelCase__      ,       buff=0			)

							__magic_name__      :       Any       						= Text("""Loaded Checkpoint"""      ,       font_size=24			)
							__magic_name__      :       Optional[int]       						= Group(lowerCAmelCase__      ,       lowerCAmelCase__			).arrange(lowerCAmelCase__      ,       buff=0.5      ,       aligned_edge=lowerCAmelCase__			)
							checkpoint.move_to([3, 0.5, 0]			)

							self.add(lowerCAmelCase__			)

							__magic_name__      :       Optional[int]       						= []
							__magic_name__      :       Tuple       						= []

							for i, rect in enumerate(lowerCAmelCase__			):
													__magic_name__      :       Dict       						= fill.copy().set_fill(lowerCAmelCase__      ,       opacity=0.7			)
													target.move_to(lowerCAmelCase__			)
													ckpt_arr.append(lowerCAmelCase__			)

													__magic_name__      :       int       						= target.copy()
													if i < 5:
																			cpu_target.move_to(cpu_left_col_base[i + 1]			)
													else:
																			cpu_target.move_to(cpu_right_col_base[i - 5]			)
													ckpt_cpu_arr.append(lowerCAmelCase__			)
							self.add(*lowerCAmelCase__      ,       *lowerCAmelCase__			)

							__magic_name__      :       Tuple       						= Square(side_length=2.2			)
							key.move_to([-5, 2, 0]			)

							__magic_name__      :       str       						= MarkupText(
							    F'<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'      ,       font_size=18      ,       )

							key_text.move_to([-5, 2.4, 0]			)

							self.add(lowerCAmelCase__      ,       lowerCAmelCase__			)

							__magic_name__      :       Any       						= MarkupText(
							    F'<span fgcolor=\'{BLUE}\'>●</span> Checkpoint'      ,       font_size=18      ,       )

							blue_text.next_to(lowerCAmelCase__      ,       DOWN * 2.4      ,       aligned_edge=key_text.get_left()			)
							self.add(lowerCAmelCase__			)

							__magic_name__      :       Optional[Any]       						= MarkupText(
							    F'Based on the passed in configuration, weights are stored in\na variety of np.memmaps on disk or to a particular device.'      ,       font_size=24      ,       )
							step_a.move_to([2, 2, 0]			)

							__magic_name__      :       int       						= [meta_mem.copy() for i in range(6			)]
							__magic_name__      :       Union[str, Any]       						= [meta_mem.copy() for i in range(6			)]
							__magic_name__      :       Any       						= VGroup(*lowerCAmelCase__			).arrange(lowerCAmelCase__      ,       buff=0			)
							__magic_name__      :       str       						= VGroup(*lowerCAmelCase__			).arrange(lowerCAmelCase__      ,       buff=0			)
							__magic_name__      :       Tuple       						= VGroup(lowerCAmelCase__      ,       lowerCAmelCase__			).arrange(lowerCAmelCase__      ,       buff=0			)
							__magic_name__      :       int       						= Text("""Disk"""      ,       font_size=24			)
							__magic_name__      :       Union[str, Any]       						= Group(lowerCAmelCase__      ,       lowerCAmelCase__			).arrange(lowerCAmelCase__      ,       buff=0.5      ,       aligned_edge=lowerCAmelCase__			)
							disk.move_to([-4.0, -1.2_5, 0]			)
							self.play(Write(lowerCAmelCase__      ,       run_time=3			)      ,       Write(lowerCAmelCase__      ,       run_time=1			)      ,       Create(lowerCAmelCase__      ,       run_time=1			)			)

							__magic_name__      :       List[Any]       						= []
							for i, rect in enumerate(lowerCAmelCase__			):
													__magic_name__      :       Dict       						= rect.copy()
													target.generate_target()
													target.target.move_to(disk_left_col_base[i]			).scale(0.5			)
													animations.append(MoveToTarget(lowerCAmelCase__      ,       run_time=1.5			)			)
							self.play(*lowerCAmelCase__			)

							self.play(FadeOut(lowerCAmelCase__			)			)

							__magic_name__      :       str       						= MarkupText(F'Then, the checkpoint is removed from memory\nthrough garbage collection.'      ,       font_size=24			)
							step_a.move_to([2, 2, 0]			)

							self.play(Write(lowerCAmelCase__      ,       run_time=3			)			)

							self.play(
							    FadeOut(lowerCAmelCase__      ,       lowerCAmelCase__      ,       *lowerCAmelCase__      ,       *lowerCAmelCase__			)      ,       )

							self.wait()
