






import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


__magic_name__:					Any							=     datasets.utils.logging.get_logger(__name__)





@dataclass
class 	snake_case__ (							datasets.BuilderConfig				):
	lowercase__						:					int											=   10000
	lowercase__						:					Optional[List[str]]											=   None
	lowercase__						:					Optional[datasets.Features]											=   None





class 	snake_case__ (							datasets.ArrowBasedBuilder				):
	lowercase__						:					str											=   ParquetConfig
	def       __magic_name__    (    self			)							->				Optional[int]:
							return datasets.DatasetInfo(features=self.config.features			)
	def       __magic_name__    (    self      ,       lowerCAmelCase__			)							->				int:
							if not self.config.data_files:
													raise ValueError(F'At least one data file must be specified, but got data_files={self.config.data_files}'			)
							__magic_name__      :       int       						= dl_manager.download_and_extract(self.config.data_files			)
							if isinstance(lowerCAmelCase__      ,       (str, list, tuple)			):
													__magic_name__      :       Optional[Any]       						= data_files
													if isinstance(lowerCAmelCase__      ,       lowerCAmelCase__			):
																			__magic_name__      :       str       						= [files]
													# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
													__magic_name__      :       str       						= [dl_manager.iter_files(lowerCAmelCase__			) for file in files]
													return [datasets.SplitGenerator(name=datasets.Split.TRAIN      ,       gen_kwargs={"""files""": files}			)]
							__magic_name__      :       Dict       						= []
							for split_name, files in data_files.items():
													if isinstance(lowerCAmelCase__      ,       lowerCAmelCase__			):
																			__magic_name__      :       Union[str, Any]       						= [files]
													# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
													__magic_name__      :       Any       						= [dl_manager.iter_files(lowerCAmelCase__			) for file in files]
													# Infer features is they are stoed in the arrow schema
													if self.info.features is None:
																			for file in itertools.chain.from_iterable(lowerCAmelCase__			):
																									with open(lowerCAmelCase__      ,       """rb"""			) as f:
																															__magic_name__      :       List[Any]       						= datasets.Features.from_arrow_schema(pq.read_schema(lowerCAmelCase__			)			)
																									break
													splits.append(datasets.SplitGenerator(name=lowerCAmelCase__      ,       gen_kwargs={"""files""": files}			)			)
							return splits
	def       __magic_name__    (    self      ,       lowerCAmelCase__			)							->				pa.Table:
							if self.info.features is not None:
													# more expensive cast to support nested features with keys in a different order
													# allows str <-> int/float or str to Audio for example
													__magic_name__      :       Optional[int]       						= table_cast(lowerCAmelCase__      ,       self.info.features.arrow_schema			)
							return pa_table







	def       __magic_name__    (    self      ,       lowerCAmelCase__			)							->				Optional[int]:
							__magic_name__      :       int       						= self.info.features.arrow_schema if self.info.features is not None else None
							if self.info.features is not None and self.config.columns is not None:
													if sorted(field.name for field in schema			) != sorted(self.config.columns			):
																			raise ValueError(
																			    F'Tried to load parquet data with columns \'{self.config.columns}\' with mismatching features \'{self.info.features}\''			)
							for file_idx, file in enumerate(itertools.chain.from_iterable(lowerCAmelCase__			)			):
													with open(lowerCAmelCase__      ,       """rb"""			) as f:
																			__magic_name__      :       int       						= pq.ParquetFile(lowerCAmelCase__			)
																			try:
																									for batch_idx, record_batch in enumerate(
																									    parquet_file.iter_batches(batch_size=self.config.batch_size      ,       columns=self.config.columns			)			):
																															__magic_name__      :       Any       						= pa.Table.from_batches([record_batch]			)
																															# Uncomment for debugging (will print the Arrow table size and elements)
																															# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																															# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																															yield F'{file_idx}_{batch_idx}', self._cast_table(lowerCAmelCase__			)
																			except ValueError as e:
																									logger.error(F'Failed to read file \'{file}\' with error {type(lowerCAmelCase__			)}: {e}'			)
																									raise








from manim import *





class 	snake_case__ (							_lowerCAmelCase				):
	def       __magic_name__    (    self			)							->				Dict:
							__magic_name__      :       int       						= Rectangle(height=0.5      ,       width=0.5			)
							__magic_name__      :       Optional[int]       						= Rectangle(height=0.2_5      ,       width=0.2_5			)
							__magic_name__      :       str       						= Rectangle(height=0.4_6      ,       width=0.4_6			).set_stroke(width=0			)

							__magic_name__      :       List[Any]       						= [mem.copy() for i in range(6			)]
							__magic_name__      :       int       						= [mem.copy() for i in range(6			)]
							__magic_name__      :       Tuple       						= VGroup(*lowerCAmelCase__			).arrange(lowerCAmelCase__      ,       buff=0			)
							__magic_name__      :       List[str]       						= VGroup(*lowerCAmelCase__			).arrange(lowerCAmelCase__      ,       buff=0			)
							__magic_name__      :       str       						= VGroup(lowerCAmelCase__      ,       lowerCAmelCase__			).arrange(lowerCAmelCase__      ,       buff=0			)
							__magic_name__      :       Union[str, Any]       						= Text("""CPU"""      ,       font_size=24			)
							__magic_name__      :       Tuple       						= Group(lowerCAmelCase__      ,       lowerCAmelCase__			).arrange(lowerCAmelCase__      ,       buff=0.5      ,       aligned_edge=lowerCAmelCase__			)
							cpu.move_to([-2.5, -0.5, 0]			)
							self.add(lowerCAmelCase__			)

							__magic_name__      :       Any       						= [mem.copy() for i in range(4			)]
							__magic_name__      :       List[Any]       						= VGroup(*lowerCAmelCase__			).arrange(lowerCAmelCase__      ,       buff=0			)
							__magic_name__      :       Tuple       						= Text("""GPU"""      ,       font_size=24			)
							__magic_name__      :       Tuple       						= Group(lowerCAmelCase__      ,       lowerCAmelCase__			).arrange(lowerCAmelCase__      ,       buff=0.5      ,       aligned_edge=lowerCAmelCase__			)
							gpu.move_to([-1, -1, 0]			)
							self.add(lowerCAmelCase__			)

							__magic_name__      :       Union[str, Any]       						= [mem.copy() for i in range(6			)]
							__magic_name__      :       Union[str, Any]       						= VGroup(*lowerCAmelCase__			).arrange(lowerCAmelCase__      ,       buff=0			)

							__magic_name__      :       str       						= Text("""Model"""      ,       font_size=24			)
							__magic_name__      :       Optional[int]       						= Group(lowerCAmelCase__      ,       lowerCAmelCase__			).arrange(lowerCAmelCase__      ,       buff=0.5      ,       aligned_edge=lowerCAmelCase__			)
							model.move_to([3, -1.0, 0]			)
							self.add(lowerCAmelCase__			)

							__magic_name__      :       str       						= []
							__magic_name__      :       Tuple       						= []
							__magic_name__      :       Union[str, Any]       						= []

							for i, rect in enumerate(lowerCAmelCase__			):
													rect.set_stroke(lowerCAmelCase__			)

													__magic_name__      :       Optional[Any]       						= Rectangle(height=0.4_6 / 4      ,       width=0.4_6 / 3			).set_stroke(width=0.0			).set_fill(lowerCAmelCase__      ,       opacity=0.7			)

													if i == 0:
																			cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT			)      ,       buff=0.0_2      ,       direction=lowerCAmelCase__			)
																			cpu_target.set_x(cpu_target.get_x() + 0.1			)
													elif i == 3:
																			cpu_target.next_to(model_cpu_arr[0]      ,       direction=lowerCAmelCase__      ,       buff=0.0			)
													else:
																			cpu_target.next_to(model_cpu_arr[i - 1]      ,       direction=lowerCAmelCase__      ,       buff=0.0			)
													self.add(lowerCAmelCase__			)
													model_cpu_arr.append(lowerCAmelCase__			)

							self.add(*lowerCAmelCase__      ,       *lowerCAmelCase__      ,       *lowerCAmelCase__			)

							__magic_name__      :       Optional[Any]       						= [mem.copy() for i in range(6			)]
							__magic_name__      :       Optional[Any]       						= VGroup(*lowerCAmelCase__			).arrange(lowerCAmelCase__      ,       buff=0			)

							__magic_name__      :       Any       						= Text("""Loaded Checkpoint"""      ,       font_size=24			)
							__magic_name__      :       Optional[int]       						= Group(lowerCAmelCase__      ,       lowerCAmelCase__			).arrange(lowerCAmelCase__      ,       buff=0.5      ,       aligned_edge=lowerCAmelCase__			)
							checkpoint.move_to([3, 0.5, 0]			)

							self.add(lowerCAmelCase__			)

							__magic_name__      :       Optional[int]       						= []
							__magic_name__      :       Tuple       						= []

							for i, rect in enumerate(lowerCAmelCase__			):
													__magic_name__      :       Dict       						= fill.copy().set_fill(lowerCAmelCase__      ,       opacity=0.7			)
													target.move_to(lowerCAmelCase__			)
													ckpt_arr.append(lowerCAmelCase__			)

													__magic_name__      :       int       						= target.copy()
													if i < 5:
																			cpu_target.move_to(cpu_left_col_base[i + 1]			)
													else:
																			cpu_target.move_to(cpu_right_col_base[i - 5]			)
													ckpt_cpu_arr.append(lowerCAmelCase__			)
							self.add(*lowerCAmelCase__      ,       *lowerCAmelCase__			)

							__magic_name__      :       Tuple       						= Square(side_length=2.2			)
							key.move_to([-5, 2, 0]			)

							__magic_name__      :       str       						= MarkupText(
							    F'<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'      ,       font_size=18      ,       )

							key_text.move_to([-5, 2.4, 0]			)

							self.add(lowerCAmelCase__      ,       lowerCAmelCase__			)

							__magic_name__      :       Any       						= MarkupText(
							    F'<span fgcolor=\'{BLUE}\'>●</span> Checkpoint'      ,       font_size=18      ,       )

							blue_text.next_to(lowerCAmelCase__      ,       DOWN * 2.4      ,       aligned_edge=key_text.get_left()			)
							self.add(lowerCAmelCase__			)

							__magic_name__      :       Optional[Any]       						= MarkupText(
							    F'Based on the passed in configuration, weights are stored in\na variety of np.memmaps on disk or to a particular device.'      ,       font_size=24      ,       )
							step_a.move_to([2, 2, 0]			)

							__magic_name__      :       int       						= [meta_mem.copy() for i in range(6			)]
							__magic_name__      :       Union[str, Any]       						= [meta_mem.copy() for i in range(6			)]
							__magic_name__      :       Any       						= VGroup(*lowerCAmelCase__			).arrange(lowerCAmelCase__      ,       buff=0			)
							__magic_name__      :       str       						= VGroup(*lowerCAmelCase__			).arrange(lowerCAmelCase__      ,       buff=0			)
							__magic_name__      :       Tuple       						= VGroup(lowerCAmelCase__      ,       lowerCAmelCase__			).arrange(lowerCAmelCase__      ,       buff=0			)
							__magic_name__      :       int       						= Text("""Disk"""      ,       font_size=24			)
							__magic_name__      :       Union[str, Any]       						= Group(lowerCAmelCase__      ,       lowerCAmelCase__			).arrange(lowerCAmelCase__      ,       buff=0.5      ,       aligned_edge=lowerCAmelCase__			)
							disk.move_to([-4.0, -1.2_5, 0]			)
							self.play(Write(lowerCAmelCase__      ,       run_time=3			)      ,       Write(lowerCAmelCase__      ,       run_time=1			)      ,       Create(lowerCAmelCase__      ,       run_time=1			)			)

							__magic_name__      :       List[Any]       						= []
							for i, rect in enumerate(lowerCAmelCase__			):
													__magic_name__      :       Dict       						= rect.copy()
													target.generate_target()
													target.target.move_to(disk_left_col_base[i]			).scale(0.5			)
													animations.append(MoveToTarget(lowerCAmelCase__      ,       run_time=1.5			)			)
							self.play(*lowerCAmelCase__			)

							self.play(FadeOut(lowerCAmelCase__			)			)

							__magic_name__      :       str       						= MarkupText(F'Then, the checkpoint is removed from memory\nthrough garbage collection.'      ,       font_size=24			)
							step_a.move_to([2, 2, 0]			)

							self.play(Write(lowerCAmelCase__      ,       run_time=3			)			)

							self.play(
							    FadeOut(lowerCAmelCase__      ,       lowerCAmelCase__      ,       *lowerCAmelCase__      ,       *lowerCAmelCase__			)      ,       )

							self.wait()
