


'''simple docstring'''






import contextlib
import os
import sqlitea

import pytest

from datasets import Dataset, Features, Value
from datasets.io.sql import SqlDatasetReader, SqlDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases, require_sqlalchemy







def     UpperCamelCase_      (   A__ :       int							,   A__ :       List[str] ):
  '''simple docstring'''





  assert isinstance(A__							,   A__ )
  assert dataset.num_rows == 4
  assert dataset.num_columns == 3
  assert dataset.column_names == ["col_1", "col_2", "col_3"]
  for feature, expected_dtype in expected_features.items():
    assert dataset.features[feature].dtype == expected_dtype







@require_sqlalchemy
@pytest.mark.parametrize("""keep_in_memory"""							,   [False, True] )
def     UpperCamelCase_      (   A__ :       int							,   A__ :       List[Any]							,   A__ :       str							,   A__ :       List[str] ):
  '''simple docstring'''





  lowerCAmelCase_      :				Optional[Any]      =       tmp_path / """cache"""
  lowerCAmelCase_      :				int      =       {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
  with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
    lowerCAmelCase_      :				Union[str, Any]      =       SqlDatasetReader(
        """dataset"""							,   """sqlite:///""" + sqlite_path							,   cache_dir=A__							,   keep_in_memory=A__ ).read()
  _check_sql_dataset(A__							,   A__ )







@require_sqlalchemy
@pytest.mark.parametrize(
    """features"""							,   [
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]							,   )
def     UpperCamelCase_      (   A__ :       Any							,   A__ :       Tuple							,   A__ :       int							,   A__ :       Union[str, Any] ):
  '''simple docstring'''





  lowerCAmelCase_      :				Tuple      =       tmp_path / """cache"""
  lowerCAmelCase_      :				Tuple      =       {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
  lowerCAmelCase_      :				List[str]      =       features.copy() if features else default_expected_features
  lowerCAmelCase_      :				List[Any]      =       (
      Features({feature: Value(A__ ) for feature, dtype in features.items()} ) if features is not None else None
  )
  lowerCAmelCase_      :				List[str]      =       SqlDatasetReader("""dataset"""							,   """sqlite:///""" + sqlite_path							,   features=A__							,   cache_dir=A__ ).read()
  _check_sql_dataset(A__							,   A__ )







def     UpperCamelCase_      (   A__ :       Dict ):
  '''simple docstring'''





  with contextlib.closing(sqlitea.connect(A__ ) ) as con:
    lowerCAmelCase_      :				List[Any]      =       con.cursor()
    cur.execute("""SELECT * FROM dataset""" )
    for row in cur:
      yield row







@require_sqlalchemy
def     UpperCamelCase_      (   A__ :       List[str]							,   A__ :       int							,   A__ :       int ):
  '''simple docstring'''





  lowerCAmelCase_      :				Optional[Any]      =       tmp_path / """cache"""
  lowerCAmelCase_      :				Optional[int]      =       os.path.join(A__							,   """tmp.sql""" )
  lowerCAmelCase_      :				Union[str, Any]      =       SqlDatasetReader("""dataset"""							,   """sqlite:///""" + sqlite_path							,   cache_dir=A__ ).read()
  SqlDatasetWriter(A__							,   """dataset"""							,   """sqlite:///""" + output_sqlite_path							,   num_proc=1 ).write()

  lowerCAmelCase_      :				List[str]      =       iter_sql_file(A__ )
  lowerCAmelCase_      :				List[str]      =       iter_sql_file(A__ )

  for rowa, rowa in zip(A__							,   A__ ):
    assert rowa == rowa







@require_sqlalchemy
def     UpperCamelCase_      (   A__ :       List[Any]							,   A__ :       Any							,   A__ :       Dict ):
  '''simple docstring'''





  lowerCAmelCase_      :				int      =       tmp_path / """cache"""
  lowerCAmelCase_      :				Any      =       os.path.join(A__							,   """tmp.sql""" )
  lowerCAmelCase_      :				Tuple      =       SqlDatasetReader("""dataset"""							,   """sqlite:///""" + sqlite_path							,   cache_dir=A__ ).read()
  SqlDatasetWriter(A__							,   """dataset"""							,   """sqlite:///""" + output_sqlite_path							,   num_proc=2 ).write()

  lowerCAmelCase_      :				Union[str, Any]      =       iter_sql_file(A__ )
  lowerCAmelCase_      :				str      =       iter_sql_file(A__ )

  for rowa, rowa in zip(A__							,   A__ ):
    assert rowa == rowa







@require_sqlalchemy
def     UpperCamelCase_      (   A__ :       Tuple							,   A__ :       Tuple							,   A__ :       List[str] ):
  '''simple docstring'''





  lowerCAmelCase_      :				int      =       tmp_path / """cache"""
  lowerCAmelCase_      :				str      =       os.path.join(A__							,   """tmp.sql""" )
  lowerCAmelCase_      :				List[str]      =       SqlDatasetReader("""dataset"""							,   """sqlite:///""" + sqlite_path							,   cache_dir=A__ ).read()
  with pytest.raises(A__ ):
    SqlDatasetWriter(A__							,   """dataset"""							,   """sqlite:///""" + output_sqlite_path							,   num_proc=0 ).write()







'''simple docstring'''






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_lxmert import LxmertTokenizer


__A :  Any    							=				{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__A :  int    							=				{
    "vocab_file": {
        "unc-nlp/lxmert-base-uncased": "https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/vocab.txt",
    },
    "tokenizer_file": {
        "unc-nlp/lxmert-base-uncased": (
            "https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/tokenizer.json"
        ),
    },
}

__A :  List[Any]    							=				{
    "unc-nlp/lxmert-base-uncased": 512,
}

__A :  Optional[Any]    							=				{
    "unc-nlp/lxmert-base-uncased": {"do_lower_case": True},
}






class        __snake_case	(       _SCREAMING_SNAKE_CASE):







    """simple docstring"""



    lowercase       			= VOCAB_FILES_NAMES
    lowercase       			= PRETRAINED_VOCAB_FILES_MAP
    lowercase       			= PRETRAINED_INIT_CONFIGURATION
    lowercase       			= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowercase       			= LxmertTokenizer

    def __init__(       self   :  Dict   ,						lowerCamelCase   :  Tuple=None   ,						lowerCamelCase   :  List[str]=None   ,						lowerCamelCase   :  Dict=True   ,						lowerCamelCase   :  List[Any]="[UNK]"   ,						lowerCamelCase   :  Tuple="[SEP]"   ,						lowerCamelCase   :  str="[PAD]"   ,						lowerCamelCase   :  Any="[CLS]"   ,						lowerCamelCase   :  str="[MASK]"   ,						lowerCamelCase   :  Dict=True   ,						lowerCamelCase   :  Union[str, Any]=None   ,						**lowerCamelCase   :  Optional[int]   ,						)       ->  Optional[Any]:
      super().__init__(
          lowerCamelCase   ,						tokenizer_file=lowerCamelCase   ,						do_lower_case=lowerCamelCase   ,						unk_token=lowerCamelCase   ,						sep_token=lowerCamelCase   ,						pad_token=lowerCamelCase   ,						cls_token=lowerCamelCase   ,						mask_token=lowerCamelCase   ,						tokenize_chinese_chars=lowerCamelCase   ,						strip_accents=lowerCamelCase   ,						**lowerCamelCase   ,						)

      lowerCAmelCase_      :				Optional[Any]      =       json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
      if (
          normalizer_state.get("""lowercase"""   ,						lowerCamelCase		) != do_lower_case
          or normalizer_state.get("""strip_accents"""   ,						lowerCamelCase		) != strip_accents
          or normalizer_state.get("""handle_chinese_chars"""   ,						lowerCamelCase		) != tokenize_chinese_chars
      ):
        lowerCAmelCase_      :				Tuple      =       getattr(lowerCamelCase   ,						normalizer_state.pop("""type"""		)		)
        lowerCAmelCase_      :				Optional[Any]      =       do_lower_case
        lowerCAmelCase_      :				Dict      =       strip_accents
        lowerCAmelCase_      :				List[Any]      =       tokenize_chinese_chars
        lowerCAmelCase_      :				List[Any]      =       normalizer_class(**lowerCamelCase		)

      lowerCAmelCase_      :				Tuple      =       do_lower_case

    def __lowercase       (       self   :  Tuple   ,						lowerCamelCase   :  str   ,						lowerCamelCase   :  Optional[Any]=None		)       ->  int:
      lowerCAmelCase_      :				Any      =       [self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
        output += token_ids_a + [self.sep_token_id]

      return output

    def __lowercase       (       self   :  str   ,						lowerCamelCase   :  List[int]   ,						lowerCamelCase   :  Optional[List[int]] = None		)       ->  List[int]:
      lowerCAmelCase_      :				str      =       [self.sep_token_id]
      lowerCAmelCase_      :				Optional[int]      =       [self.cls_token_id]
      if token_ids_a is None:
        return len(cls + token_ids_a + sep		) * [0]
      return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]





    def __lowercase       (       self   :  Tuple   ,						lowerCamelCase   :  str   ,						lowerCamelCase   :  Optional[str] = None		)       ->  Tuple[str]:
      lowerCAmelCase_      :				int      =       self._tokenizer.model.save(lowerCamelCase   ,						name=lowerCamelCase		)
      return tuple(lowerCamelCase		)



