


'''simple docstring'''






import re
import tempfile
from pathlib import Path

import pytest
import yaml

from datasets.utils.readme import ReadMe


# @pytest.fixture
# def example_yaml_structure():

a_   :      Dict				    =      yaml.safe_load(
    "\\nname: \"\"\nallow_empty: false\nallow_empty_text: true\nsubsections:\n  - name: \"Dataset Card for X\" # First-level markdown heading\n    allow_empty: false\n    allow_empty_text: true\n    subsections:\n      - name: \"Table of Contents\"\n        allow_empty: false\n        allow_empty_text: false\n        subsections: null\n      - name: \"Dataset Description\"\n        allow_empty: false\n        allow_empty_text: false\n        subsections:\n          - name: \"Dataset Summary\"\n            allow_empty: false\n            allow_empty_text: false\n            subsections: null\n          - name: \"Supported Tasks and Leaderboards\"\n            allow_empty: true\n            allow_empty_text: true\n            subsections: null\n          - name: Languages\n            allow_empty: false\n            allow_empty_text: true\n            subsections: null\n"
)


a_   :      Any				    =      {
    "name": "root",
    "text": "",
    "is_empty_text": True,
    "subsections": [
        {
            "name": "Dataset Card for My Dataset",
            "text": "",
            "is_empty_text": True,
            "subsections": [
                {"name": "Table of Contents", "text": "Some text here.", "is_empty_text": False, "subsections": []},
                {
                    "name": "Dataset Description",
                    "text": "Some text here.",
                    "is_empty_text": False,
                    "subsections": [
                        {
                            "name": "Dataset Summary",
                            "text": "Some text here.",
                            "is_empty_text": False,
                            "subsections": [],
                        },
                        {
                            "name": "Supported Tasks and Leaderboards",
                            "text": "",
                            "is_empty_text": True,
                            "subsections": [],
                        },
                        {"name": "Languages", "text": "Language Text", "is_empty_text": False, "subsections": []},
                    ],
                },
            ],
        }
    ],
}


a_   :      List[str]				    =      "\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n"


a_   :      List[Any]				    =      "\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n#### Extra Ignored Subsection\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n"

a_   :      int				    =      {
    "name": "root",
    "text": "",
    "is_empty_text": True,
    "subsections": [
        {
            "name": "Dataset Card for My Dataset",
            "text": "",
            "is_empty_text": True,
            "subsections": [
                {"name": "Table of Contents", "text": "Some text here.", "is_empty_text": False, "subsections": []},
                {
                    "name": "Dataset Description",
                    "text": "Some text here.",
                    "is_empty_text": False,
                    "subsections": [
                        {
                            "name": "Dataset Summary",
                            "text": "Some text here.",
                            "is_empty_text": False,
                            "subsections": [
                                {
                                    "name": "Extra Ignored Subsection",
                                    "text": "",
                                    "is_empty_text": True,
                                    "subsections": [],
                                }
                            ],
                        },
                        {
                            "name": "Supported Tasks and Leaderboards",
                            "text": "",
                            "is_empty_text": True,
                            "subsections": [],
                        },
                        {"name": "Languages", "text": "Language Text", "is_empty_text": False, "subsections": []},
                    ],
                },
            ],
        }
    ],
}

a_   :      Tuple				    =      "\\n---\n---\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n"

a_   :      int				    =      (
    "The following issues were found for the README at `{path}`:\n-\tEmpty YAML markers are present in the README."
)

a_   :      int				    =      "\\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n"

a_   :      str				    =      (
    "The following issues were found for the README at `{path}`:\n-\tNo YAML markers are present in the README."
)

a_   :      Optional[Any]				    =      "\\n---\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n"

a_   :      List[str]				    =      "The following issues were found for the README at `{path}`:\n-\tOnly the start of YAML tags present in the README."

a_   :      Tuple				    =      "\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n"
a_   :      Optional[Any]				    =      "The following issues were found for the README at `{path}`:\n-\tExpected some content in section `Dataset Summary` but it is empty.\n-\tExpected some text in section `Dataset Summary` but it is empty (text in subsections are ignored)."


a_   :      Optional[int]				    =      "\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n"
a_   :      Dict				    =      "The following issues were found for the README at `{path}`:\n-\tExpected some content in section `Dataset Card for My Dataset` but it is empty.\n-\tSection `Dataset Card for My Dataset` expected the following subsections: `Table of Contents`, `Dataset Description`. Found 'None'."

a_   :      Optional[int]				    =      "\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Languages\nLanguage Text\n"

a_   :      Tuple				    =      "The following issues were found for the README at `{path}`:\n-\tSection `Dataset Description` is missing subsection: `Supported Tasks and Leaderboards`."


a_   :      Tuple				    =      "\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\n"

a_   :      List[str]				    =      "The following issues were found for the README at `{path}`:\n-\tExpected some content in section `Languages` but it is empty."

a_   :      Optional[int]				    =      "\\n---\nlanguage:\n- zh\n- en\n---\n\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n"
a_   :      Optional[Any]				    =      "The following issues were found for the README at `{path}`:\n-\tThe README has no first-level headings. One heading is expected. Skipping further validation for this README."

a_   :      Optional[int]				    =      "\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n# Dataset Card My Dataset\n"

a_   :      int				    =      "The following issues were found for the README at `{path}`:\n-\tThe README has several first-level headings: `Dataset Card for My Dataset`, `Dataset Card My Dataset`. Only one heading is expected. Skipping further validation for this README."

a_   :      Optional[Any]				    =      "\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n"

a_   :      Tuple				    =      "The following issues were found for the README at `{path}`:\n-\tNo first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README."

a_   :      Any				    =      ""

a_   :      Optional[int]				    =      "The following issues were found for the README at `{path}`:\n-\tThe README has no first-level headings. One heading is expected. Skipping further validation for this README.\n-\tNo YAML markers are present in the README."

a_   :      Optional[int]				    =      "\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n"

a_   :      List[str]				    =      "The following issues were found while parsing the README at `{path}`:\n-\tMultiple sections with the same heading `Dataset Card for My Dataset` have been found. Please keep only one of these sections."






@pytest.mark.parametrize(
    'readme_md, expected_dict'			,  [
        (README_CORRECT, CORRECT_DICT),
        (README_CORRECT_FOUR_LEVEL, CORRECT_DICT_FOUR_LEVEL),
    ]			,  )
def 		_A			(lowerCAmelCase__					:Dict			,  lowerCAmelCase__					:List[str]						)				->					List[Any]:




     '''simple docstring'''



     assert ReadMe.from_string(lowerCAmelCase__			,  lowerCAmelCase__						).to_dict() == expected_dict






@pytest.mark.parametrize(
    'readme_md, expected_error'			,  [
        (README_NO_YAML, EXPECTED_ERROR_README_NO_YAML),
        (README_EMPTY_YAML, EXPECTED_ERROR_README_EMPTY_YAML),
        (README_INCORRECT_YAML, EXPECTED_ERROR_README_INCORRECT_YAML),
        (README_EMPTY, EXPECTED_ERROR_README_EMPTY),
        (README_NONE_SUBSECTION, EXPECTED_ERROR_README_NONE_SUBSECTION),
        (README_MISSING_FIRST_LEVEL, EXPECTED_ERROR_README_MISSING_FIRST_LEVEL),
        (README_MISSING_SUBSECTION, EXPECTED_ERROR_README_MISSING_SUBSECTION),
        (README_MISSING_TEXT, EXPECTED_ERROR_README_MISSING_TEXT),
        (README_WRONG_FIRST_LEVEL, EXPECTED_ERROR_README_WRONG_FIRST_LEVEL),
        (README_MULTIPLE_WRONG_FIRST_LEVEL, EXPECTED_ERROR_README_MULTIPLE_WRONG_FIRST_LEVEL),
        (README_MISSING_CONTENT, EXPECTED_ERROR_README_MISSING_CONTENT),
    ]			,  )
def 		_A			(lowerCAmelCase__					:List[str]			,  lowerCAmelCase__					:List[Any]						)				->					List[Any]:




     '''simple docstring'''



     with pytest.raises(lowerCAmelCase__			,  match=re.escape(expected_error.format(path='root'						)						)						):
          _a      =							ReadMe.from_string(lowerCAmelCase__			,  lowerCAmelCase__						)
          readme.validate()






@pytest.mark.parametrize(
    'readme_md, expected_error'			,  [
        (README_MULTIPLE_SAME_HEADING_1, EXPECTED_ERROR_README_MULTIPLE_SAME_HEADING_1),
    ]			,  )
def 		_A			(lowerCAmelCase__					:List[str]			,  lowerCAmelCase__					:str						)				->					str:




     '''simple docstring'''



     with pytest.raises(lowerCAmelCase__			,  match=re.escape(expected_error.format(path='root'						)						)						):
          ReadMe.from_string(lowerCAmelCase__			,  lowerCAmelCase__						)






@pytest.mark.parametrize(
    'readme_md,'			,  [
        (README_MULTIPLE_SAME_HEADING_1),
    ]			,  )
def 		_A			(lowerCAmelCase__					:Optional[Any]						)				->					int:




     '''simple docstring'''



     ReadMe.from_string(lowerCAmelCase__			,  lowerCAmelCase__			,  suppress_parsing_errors=lowerCAmelCase__						)






@pytest.mark.parametrize(
    'readme_md, expected_dict'			,  [
        (README_CORRECT, CORRECT_DICT),
        (README_CORRECT_FOUR_LEVEL, CORRECT_DICT_FOUR_LEVEL),
    ]			,  )
def 		_A			(lowerCAmelCase__					:Dict			,  lowerCAmelCase__					:Optional[Any]						)				->					Optional[int]:




     '''simple docstring'''



     with tempfile.TemporaryDirectory() as tmp_dir:
          _a      =							Path(lowerCAmelCase__						) / 'README.md'
          with open(lowerCAmelCase__			,  'w+'						) as readme_file:
               readme_file.write(lowerCAmelCase__						)
          _a      =							ReadMe.from_readme(lowerCAmelCase__			,  lowerCAmelCase__						).to_dict()
          assert out["name"] == path
          assert out["text"] == ""
          assert out["is_empty_text"]
          assert out["subsections"] == expected_dict["subsections"]






@pytest.mark.parametrize(
    'readme_md, expected_error'			,  [
        (README_NO_YAML, EXPECTED_ERROR_README_NO_YAML),
        (README_EMPTY_YAML, EXPECTED_ERROR_README_EMPTY_YAML),
        (README_INCORRECT_YAML, EXPECTED_ERROR_README_INCORRECT_YAML),
        (README_EMPTY, EXPECTED_ERROR_README_EMPTY),
        (README_NONE_SUBSECTION, EXPECTED_ERROR_README_NONE_SUBSECTION),
        (README_MISSING_FIRST_LEVEL, EXPECTED_ERROR_README_MISSING_FIRST_LEVEL),
        (README_MISSING_SUBSECTION, EXPECTED_ERROR_README_MISSING_SUBSECTION),
        (README_MISSING_TEXT, EXPECTED_ERROR_README_MISSING_TEXT),
        (README_WRONG_FIRST_LEVEL, EXPECTED_ERROR_README_WRONG_FIRST_LEVEL),
        (README_MULTIPLE_WRONG_FIRST_LEVEL, EXPECTED_ERROR_README_MULTIPLE_WRONG_FIRST_LEVEL),
        (README_MISSING_CONTENT, EXPECTED_ERROR_README_MISSING_CONTENT),
    ]			,  )
def 		_A			(lowerCAmelCase__					:Dict			,  lowerCAmelCase__					:List[str]						)				->					Optional[int]:




     '''simple docstring'''



     with tempfile.TemporaryDirectory() as tmp_dir:
          _a      =							Path(lowerCAmelCase__						) / 'README.md'
          with open(lowerCAmelCase__			,  'w+'						) as readme_file:
               readme_file.write(lowerCAmelCase__						)
          _a      =							expected_error.format(path=lowerCAmelCase__						)
          with pytest.raises(lowerCAmelCase__			,  match=re.escape(lowerCAmelCase__						)						):
               _a      =							ReadMe.from_readme(lowerCAmelCase__			,  lowerCAmelCase__						)
               readme.validate()






@pytest.mark.parametrize(
    'readme_md, expected_error'			,  [
        (README_MULTIPLE_SAME_HEADING_1, EXPECTED_ERROR_README_MULTIPLE_SAME_HEADING_1),
    ]			,  )
def 		_A			(lowerCAmelCase__					:List[str]			,  lowerCAmelCase__					:Optional[int]						)				->					List[str]:




     '''simple docstring'''



     with tempfile.TemporaryDirectory() as tmp_dir:
          _a      =							Path(lowerCAmelCase__						) / 'README.md'
          with open(lowerCAmelCase__			,  'w+'						) as readme_file:
               readme_file.write(lowerCAmelCase__						)
          _a      =							expected_error.format(path=lowerCAmelCase__						)
          with pytest.raises(lowerCAmelCase__			,  match=re.escape(lowerCAmelCase__						)						):
               ReadMe.from_readme(lowerCAmelCase__			,  lowerCAmelCase__						)






@pytest.mark.parametrize(
    'readme_md,'			,  [
        (README_MULTIPLE_SAME_HEADING_1),
    ]			,  )
def 		_A			(lowerCAmelCase__					:Union[str, Any]						)				->					Optional[Any]:




     '''simple docstring'''



     with tempfile.TemporaryDirectory() as tmp_dir:
          _a      =							Path(lowerCAmelCase__						) / 'README.md'
          with open(lowerCAmelCase__			,  'w+'						) as readme_file:
               readme_file.write(lowerCAmelCase__						)
          ReadMe.from_readme(lowerCAmelCase__			,  lowerCAmelCase__			,  suppress_parsing_errors=lowerCAmelCase__						)





'''simple docstring'''






from typing import TYPE_CHECKING

from ...file_utils import _LazyModule, is_torch_available
from ...utils import OptionalDependencyNotAvailable


a_   :      int				    =      {
    "configuration_gpt_neox_japanese": ["GPT_NEOX_JAPANESE_PRETRAINED_CONFIG_ARCHIVE_MAP", "GPTNeoXJapaneseConfig"],
    "tokenization_gpt_neox_japanese": ["GPTNeoXJapaneseTokenizer"],
}

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 a_   :      List[str]				    =      [
     "GPT_NEOX_JAPANESE_PRETRAINED_MODEL_ARCHIVE_LIST",
     "GPTNeoXJapaneseForCausalLM",
     "GPTNeoXJapaneseLayer",
     "GPTNeoXJapaneseModel",
     "GPTNeoXJapanesePreTrainedModel",
 ]


if TYPE_CHECKING:
 from .configuration_gpt_neox_japanese import GPT_NEOX_JAPANESE_PRETRAINED_CONFIG_ARCHIVE_MAP, GPTNeoXJapaneseConfig
 from .tokenization_gpt_neox_japanese import GPTNeoXJapaneseTokenizer

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_gpt_neox_japanese import (
      GPT_NEOX_JAPANESE_PRETRAINED_MODEL_ARCHIVE_LIST,
      GPTNeoXJapaneseForCausalLM,
      GPTNeoXJapaneseLayer,
      GPTNeoXJapaneseModel,
      GPTNeoXJapanesePreTrainedModel,
  )


else:
 import sys

 a_   :      int				    =      _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)

