


import inspect
import re
from hashlib import shaaaa
from typing import Dict, List

from .arrow import arrow
from .audiofolder import audiofolder
from .csv import csv
from .imagefolder import imagefolder
from .json import json
from .pandas import pandas
from .parquet import parquet
from .sql import sql  # noqa F401
from .text import text
def       _a			(				SCREAMING_SNAKE_CASE					):


		"""simple docstring"""




		lowercase__						=				[]
		for line in lines:
				lowercase__						=				re.sub(R'''#.*'''						,    ''''''						,    _UpperCamelCase					)  # remove comments
				if line:
						filtered_lines.append(_UpperCamelCase					)
		lowercase__						=				'''\n'''.join(_UpperCamelCase					)

		# Make a hash from all this code
		lowercase__						=				full_str.encode('''utf-8'''					)
		return shaaaa(_UpperCamelCase					).hexdigest()


# get importable module names and hash for caching
lowerCAmelCase	=							{
    "csv": (csv.__name__, _hash_python_lines(inspect.getsource(csv).splitlines())),
    "json": (json.__name__, _hash_python_lines(inspect.getsource(json).splitlines())),
    "pandas": (pandas.__name__, _hash_python_lines(inspect.getsource(pandas).splitlines())),
    "parquet": (parquet.__name__, _hash_python_lines(inspect.getsource(parquet).splitlines())),
    "arrow": (arrow.__name__, _hash_python_lines(inspect.getsource(arrow).splitlines())),
    "text": (text.__name__, _hash_python_lines(inspect.getsource(text).splitlines())),
    "imagefolder": (imagefolder.__name__, _hash_python_lines(inspect.getsource(imagefolder).splitlines())),
    "audiofolder": (audiofolder.__name__, _hash_python_lines(inspect.getsource(audiofolder).splitlines())),
}

# Used to infer the module to use based on the data files extensions
lowerCAmelCase	=							{
    ".csv": ("csv", {}),
    ".tsv": ("csv", {"sep": "\t"}),
    ".json": ("json", {}),
    ".jsonl": ("json", {}),
    ".parquet": ("parquet", {}),
    ".arrow": ("arrow", {}),
    ".txt": ("text", {}),
}
_EXTENSION_TO_MODULE.update({ext: ('imagefolder', {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext.upper(): ('imagefolder', {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext: ('audiofolder', {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext.upper(): ('audiofolder', {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
lowerCAmelCase	=							{"imagefolder", "audiofolder"}

# Used to filter data files based on extensions given a module name
lowerCAmelCase	=							{}
for _ext, (_module, _) in _EXTENSION_TO_MODULE.items():
			_MODULE_TO_EXTENSIONS.setdefault(_module, []).append(_ext)

_MODULE_TO_EXTENSIONS["imagefolder"].append('.zip')
_MODULE_TO_EXTENSIONS["audiofolder"].append('.zip')






from abc import ABC, abstractmethod
from typing import List, Optional


class 	_a							(      UpperCamelCase__      ):

							def __init__(							self:     Optional[int]							)      ->       Union[str, Any]:





									"""simple docstring"""







									self.test()

							def 				lowerCamelCase_			(							self:     List[Any]							)      ->       List[Any]:





									"""simple docstring"""







									lowercase__						=				0
									lowercase__						=				False
									while not completed:
											if counter == 1:
													self.reset()
											lowercase__						=				self.advance()
											if not self.does_advance(UpperCamelCase_							):
													raise Exception(
													    '''Custom Constraint is not defined correctly. self.does_advance(self.advance()) must be true.'''							)

											lowercase__					,		lowercase__					,		lowercase__						=				self.update(UpperCamelCase_							)
											counter += 1

											if counter > 10_000:
													raise Exception('''update() does not fulfill the constraint.'''							)

									if self.remaining() != 0:
											raise Exception('''Custom Constraint is not defined correctly.'''							)

							@abstractmethod
							def 				lowerCamelCase_			(							self:     int							)      ->       Any:





									"""simple docstring"""







									raise NotImplementedError(
									    f'{self.__class__} is an abstract class. Only classes inheriting this class can be called.'							)

							@abstractmethod
							def 				lowerCamelCase_			(							self:     Tuple   ,						UpperCamelCase_:     int							)      ->       Optional[int]:





									"""simple docstring"""







									raise NotImplementedError(
									    f'{self.__class__} is an abstract class. Only classes inheriting this class can be called.'							)

							@abstractmethod
							def 				lowerCamelCase_			(							self:     Optional[int]   ,						UpperCamelCase_:     int							)      ->       str:





									"""simple docstring"""







									raise NotImplementedError(
									    f'{self.__class__} is an abstract class. Only classes inheriting this class can be called.'							)

							@abstractmethod
							def 				lowerCamelCase_			(							self:     Tuple							)      ->       List[Any]:





									"""simple docstring"""







									raise NotImplementedError(
									    f'{self.__class__} is an abstract class. Only classes inheriting this class can be called.'							)

							@abstractmethod
							def 				lowerCamelCase_			(							self:     Any							)      ->       Tuple:





									"""simple docstring"""







									raise NotImplementedError(
									    f'{self.__class__} is an abstract class. Only classes inheriting this class can be called.'							)


							@abstractmethod
							def 				lowerCamelCase_			(							self:     Tuple   ,						UpperCamelCase_:     int=False							)      ->       Any:





									"""simple docstring"""







									raise NotImplementedError(
									    f'{self.__class__} is an abstract class. Only classes inheriting this class can be called.'							)


class 	_a							(      UpperCamelCase__      ):

							def __init__(							self:     str   ,						UpperCamelCase_:     List[int]							)      ->       Tuple:





									"""simple docstring"""







									super(UpperCamelCase_   ,						self							).__init__()

									if not isinstance(UpperCamelCase_   ,						UpperCamelCase_							) or len(UpperCamelCase_							) == 0:
											raise ValueError(f'`token_ids` has to be a non-empty list, but is {token_ids}.'							)
									if any((not isinstance(UpperCamelCase_   ,						UpperCamelCase_							) or token_id < 0) for token_id in token_ids							):
											raise ValueError(f'Each list in `token_ids` has to be a list of positive integers, but is {token_ids}.'							)

									lowercase__						=				token_ids

									lowercase__						=				len(self.token_ids							)
									lowercase__						=				-1  # the index of the currently fulfilled step
									lowercase__						=				False

							def 				lowerCamelCase_			(							self:     Tuple							)      ->       Tuple:





									"""simple docstring"""







									if self.completed:
											return None
									return self.token_ids[self.fulfilled_idx + 1]

							def 				lowerCamelCase_			(							self:     Union[str, Any]   ,						UpperCamelCase_:     int							)      ->       Optional[Any]:





									"""simple docstring"""







									if not isinstance(UpperCamelCase_   ,						UpperCamelCase_							):
											raise ValueError(f'`token_id` has to be an `int`, but is {token_id} of type {type(UpperCamelCase_							)}'							)

									if self.completed:
											return False

									return token_id == self.token_ids[self.fulfilled_idx + 1]

							def 				lowerCamelCase_			(							self:     Optional[int]   ,						UpperCamelCase_:     int							)      ->       Dict:





									"""simple docstring"""







									if not isinstance(UpperCamelCase_   ,						UpperCamelCase_							):
											raise ValueError(f'`token_id` has to be an `int`, but is {token_id} of type {type(UpperCamelCase_							)}'							)

									lowercase__						=				False
									lowercase__						=				False
									lowercase__						=				False

									if self.does_advance(UpperCamelCase_							):
											self.fulfilled_idx += 1
											lowercase__						=				True
											if self.fulfilled_idx == (self.seqlen - 1):
													lowercase__						=				True
											lowercase__						=				completed
									else:
											# failed to make progress.
											lowercase__						=				True
											self.reset()
									return stepped, completed, reset

							def 				lowerCamelCase_			(							self:     Tuple							)      ->       int:





									"""simple docstring"""







									lowercase__						=				False
									lowercase__						=				0

							def 				lowerCamelCase_			(							self:     Any							)      ->       Dict:





									"""simple docstring"""







									return self.seqlen - (self.fulfilled_idx + 1)


							def 				lowerCamelCase_			(							self:     Any   ,						UpperCamelCase_:     Union[str, Any]=False							)      ->       Tuple:





									"""simple docstring"""







									lowercase__						=				PhrasalConstraint(self.token_ids							)

									if stateful:
											lowercase__						=				self.seqlen
											lowercase__						=				self.fulfilled_idx
											lowercase__						=				self.completed

									return new_constraint


class 	_a							:

							def __init__(							self:     Union[str, Any]   ,						UpperCamelCase_:     List[List[int]]   ,						UpperCamelCase_:     int=True							)      ->       int:





									"""simple docstring"""







									lowercase__						=				max([len(UpperCamelCase_							) for one in nested_token_ids]							)

									lowercase__						=				{}
									for token_ids in nested_token_ids:
											lowercase__						=				root
											for tidx, token_id in enumerate(UpperCamelCase_							):
													if token_id not in level:
															lowercase__						=				{}

													lowercase__						=				level[token_id]

									if no_subsets and self.has_subsets(UpperCamelCase_   ,						UpperCamelCase_							):
											raise ValueError(
											    '''Each list in `nested_token_ids` can\'t be a complete subset of another list, but is'''
											    f' {nested_token_ids}.'							)

									lowercase__						=				root

							def 				lowerCamelCase_			(							self:     int   ,						UpperCamelCase_:     str							)      ->       Any:





									"""simple docstring"""







									lowercase__						=				self.trie

									for current_token in current_seq:
											lowercase__						=				start[current_token]

									lowercase__						=				list(start.keys()							)

									return next_tokens

							def 				lowerCamelCase_			(							self:     Any   ,						UpperCamelCase_:     Tuple							)      ->       Optional[Any]:





									"""simple docstring"""







									lowercase__						=				self.next_tokens(UpperCamelCase_							)

									return len(UpperCamelCase_							) == 0

							def 				lowerCamelCase_			(							self:     List[str]   ,						UpperCamelCase_:     Optional[Any]							)      ->       Any:





									"""simple docstring"""







									lowercase__						=				list(root.values()							)
									if len(UpperCamelCase_							) == 0:
											return 1
									else:
											return sum([self.count_leaves(UpperCamelCase_							) for nn in next_nodes]							)


							def 				lowerCamelCase_			(							self:     Any   ,						UpperCamelCase_:     Union[str, Any]   ,						UpperCamelCase_:     int							)      ->       Tuple:





									"""simple docstring"""







									lowercase__						=				self.count_leaves(UpperCamelCase_							)
									return len(UpperCamelCase_							) != leaf_count


class 	_a							(      UpperCamelCase__      ):

							def __init__(							self:     Optional[int]   ,						UpperCamelCase_:     List[List[int]]							)      ->       List[Any]:





									"""simple docstring"""







									super(UpperCamelCase_   ,						self							).__init__()

									if not isinstance(UpperCamelCase_   ,						UpperCamelCase_							) or len(UpperCamelCase_							) == 0:
											raise ValueError(f'`nested_token_ids` has to be a non-empty list, but is {nested_token_ids}.'							)
									if any(not isinstance(UpperCamelCase_   ,						UpperCamelCase_							) for token_ids in nested_token_ids							):
											raise ValueError(f'`nested_token_ids` has to be a list of lists, but is {nested_token_ids}.'							)
									if any(
									    any((not isinstance(UpperCamelCase_   ,						UpperCamelCase_							) or token_id < 0) for token_id in token_ids							)
									    for token_ids in nested_token_ids							):
											raise ValueError(
											    f'Each list in `nested_token_ids` has to be a list of positive integers, but is {nested_token_ids}.'							)

									lowercase__						=				DisjunctiveTrie(UpperCamelCase_							)
									lowercase__						=				nested_token_ids

									lowercase__						=				self.trie.max_height
									lowercase__						=				[]
									lowercase__						=				False

							def 				lowerCamelCase_			(							self:     Union[str, Any]							)      ->       str:





									"""simple docstring"""







									lowercase__						=				self.trie.next_tokens(self.current_seq							)

									if len(UpperCamelCase_							) == 0:
											return None
									else:
											return token_list

							def 				lowerCamelCase_			(							self:     str   ,						UpperCamelCase_:     int							)      ->       Tuple:





									"""simple docstring"""







									if not isinstance(UpperCamelCase_   ,						UpperCamelCase_							):
											raise ValueError(f'`token_id` is supposed to be type `int`, but is {token_id} of type {type(UpperCamelCase_							)}'							)

									lowercase__						=				self.trie.next_tokens(self.current_seq							)

									return token_id in next_tokens

							def 				lowerCamelCase_			(							self:     int   ,						UpperCamelCase_:     int							)      ->       Dict:





									"""simple docstring"""







									if not isinstance(UpperCamelCase_   ,						UpperCamelCase_							):
											raise ValueError(f'`token_id` is supposed to be type `int`, but is {token_id} of type {type(UpperCamelCase_							)}'							)

									lowercase__						=				False
									lowercase__						=				False
									lowercase__						=				False

									if self.does_advance(UpperCamelCase_							):
											self.current_seq.append(UpperCamelCase_							)
											lowercase__						=				True
									else:
											lowercase__						=				True
											self.reset()

									lowercase__						=				self.trie.reached_leaf(self.current_seq							)
									lowercase__						=				completed

									return stepped, completed, reset

							def 				lowerCamelCase_			(							self:     List[str]							)      ->       List[Any]:





									"""simple docstring"""







									lowercase__						=				False
									lowercase__						=				[]

							def 				lowerCamelCase_			(							self:     List[str]							)      ->       Optional[Any]:





									"""simple docstring"""







									if self.completed:
											# since this can be completed without reaching max height
											return 0
									else:
											return self.seqlen - len(self.current_seq							)


							def 				lowerCamelCase_			(							self:     Any   ,						UpperCamelCase_:     Optional[Any]=False							)      ->       str:





									"""simple docstring"""







									lowercase__						=				DisjunctiveConstraint(self.token_ids							)

									if stateful:
											lowercase__						=				self.seqlen
											lowercase__						=				self.current_seq
											lowercase__						=				self.completed

									return new_constraint


class 	_a							:

							def __init__(							self:     int   ,						UpperCamelCase_:     List[Constraint]							)      ->       Dict:





									"""simple docstring"""







									lowercase__						=				constraints

									# max # of steps required to fulfill a given constraint
									lowercase__						=				max([c.seqlen for c in constraints]							)
									lowercase__						=				len(UpperCamelCase_							)
									lowercase__						=				False

									self.init_state()

							def 				lowerCamelCase_			(							self:     List[str]							)      ->       str:





									"""simple docstring"""







									lowercase__						=				[]
									lowercase__						=				None
									lowercase__						=				[constraint.copy(stateful=UpperCamelCase_							) for constraint in self.constraints]

							def 				lowerCamelCase_			(							self:     str							)      ->       Any:





									"""simple docstring"""







									lowercase__						=				0
									if self.inprogress_constraint:
											# extra points for having a constraint mid-fulfilled
											add += self.max_seqlen - self.inprogress_constraint.remaining()

									return (len(self.complete_constraints							) * self.max_seqlen) + add

							def 				lowerCamelCase_			(							self:     Union[str, Any]							)      ->       List[Any]:





									"""simple docstring"""







									lowercase__						=				[]
									if self.inprogress_constraint is None:
											for constraint in self.pending_constraints:  # "pending" == "unfulfilled yet"
													lowercase__						=				constraint.advance()
													if isinstance(UpperCamelCase_   ,						UpperCamelCase_							):
															token_list.append(UpperCamelCase_							)
													elif isinstance(UpperCamelCase_   ,						UpperCamelCase_							):
															token_list.extend(UpperCamelCase_							)
									else:
											lowercase__						=				self.inprogress_constraint.advance()
											if isinstance(UpperCamelCase_   ,						UpperCamelCase_							):
													token_list.append(UpperCamelCase_							)
											elif isinstance(UpperCamelCase_   ,						UpperCamelCase_							):
													token_list.extend(UpperCamelCase_							)

									if len(UpperCamelCase_							) == 0:
											return None
									else:
											return token_list

							def 				lowerCamelCase_			(							self:     Optional[Any]   ,						UpperCamelCase_:     Optional[List[int]]							)      ->       Optional[int]:





									"""simple docstring"""







									self.init_state()

									if token_ids is not None:
											for token in token_ids:
													# completes or steps **one** constraint
													lowercase__					,		lowercase__						=				self.add(UpperCamelCase_							)

													# the entire list of constraints are fulfilled
													if self.completed:
															break

							def 				lowerCamelCase_			(							self:     Optional[Any]   ,						UpperCamelCase_:     int							)      ->       int:





									"""simple docstring"""







									if not isinstance(UpperCamelCase_   ,						UpperCamelCase_							):
											raise ValueError(f'`token_id` should be an `int`, but is `{token_id}`.'							)

									lowercase__					,		lowercase__						=				False, False

									if self.completed:
											lowercase__						=				True
											lowercase__						=				False
											return complete, stepped

									if self.inprogress_constraint is not None:
											# In the middle of fulfilling a constraint. If the `token_id` *does* makes an incremental progress to current
											# job, simply update the state

											lowercase__					,		lowercase__					,		lowercase__						=				self.inprogress_constraint.update(UpperCamelCase_							)
											if reset:
													# 1. If the next token breaks the progress, then we must restart.
													#     e.g. constraint = "I love pies" and sequence so far is "I love" but `token_id` == "books".

													#     But that doesn't mean we self.init_state(), since we only reset the state for this particular
													#     constraint, not the full list of constraints.

													self.pending_constraints.append(self.inprogress_constraint.copy(stateful=UpperCamelCase_							)							)
													lowercase__						=				None

											if complete:
													# 2. If the next token completes the constraint, move it to completed list, set
													#     inprogress to None. If there are no pending constraints either, then this full list of constraints
													#     is complete.

													self.complete_constraints.append(self.inprogress_constraint							)
													lowercase__						=				None

													if len(self.pending_constraints							) == 0:
															# we're done!
															lowercase__						=				True

									else:
											# Not in the middle of fulfilling a constraint. So does this `token_id` helps us step towards any of our list
											# of constraints?

											for cidx, pending_constraint in enumerate(self.pending_constraints							):
													if pending_constraint.does_advance(UpperCamelCase_							):
															lowercase__					,		lowercase__					,		lowercase__						=				pending_constraint.update(UpperCamelCase_							)

															if not stepped:
																	raise Exception(
																	    '''`constraint.update(token_id)` is not yielding incremental progress, '''
																	    '''even though `constraint.does_advance(token_id)` is true.'''							)

															if complete:
																	self.complete_constraints.append(UpperCamelCase_							)
																	lowercase__						=				None

															if not complete and stepped:
																	lowercase__						=				pending_constraint

															if complete or stepped:
																	# If we made any progress at all, then it's at least not a "pending constraint".

																	lowercase__						=				(
																	    self.pending_constraints[:cidx] + self.pending_constraints[cidx + 1 :]
																	)

																	if len(self.pending_constraints							) == 0 and self.inprogress_constraint is None:
																			# If there's no longer any pending after this and no inprogress either, then we must be
																			# complete.

																			lowercase__						=				True

																	break  # prevent accidentally stepping through multiple constraints with just one token.

									return complete, stepped


							def 				lowerCamelCase_			(							self:     Any   ,						UpperCamelCase_:     Optional[Any]=True							)      ->       Dict:





									"""simple docstring"""







									lowercase__						=				ConstraintListState(self.constraints							)  # we actually never though self.constraints objects
									# throughout this process. So it's at initialization state.

									if stateful:
											lowercase__						=				[
											    constraint.copy(stateful=UpperCamelCase_							) for constraint in self.complete_constraints
											]
											if self.inprogress_constraint is not None:
													lowercase__						=				self.inprogress_constraint.copy(stateful=UpperCamelCase_							)
											lowercase__						=				[constraint.copy() for constraint in self.pending_constraints]

									return new_state


