

"""simple docstring"""





import re
import tempfile
from pathlib import Path

import pytest
import yaml

from datasets.utils.readme import ReadMe


# @pytest.fixture
# def example_yaml_structure():

_a							=							yaml.safe_load(
    '\\nname: ""\nallow_empty: false\nallow_empty_text: true\nsubsections:\n  - name: "Dataset Card for X" # First-level markdown heading\n    allow_empty: false\n    allow_empty_text: true\n    subsections:\n      - name: "Table of Contents"\n        allow_empty: false\n        allow_empty_text: false\n        subsections: null\n      - name: "Dataset Description"\n        allow_empty: false\n        allow_empty_text: false\n        subsections:\n          - name: "Dataset Summary"\n            allow_empty: false\n            allow_empty_text: false\n            subsections: null\n          - name: "Supported Tasks and Leaderboards"\n            allow_empty: true\n            allow_empty_text: true\n            subsections: null\n          - name: Languages\n            allow_empty: false\n            allow_empty_text: true\n            subsections: null\n'
)


_a							=							{
    'name': 'root',
    'text': '',
    'is_empty_text': True,
    'subsections': [
        {
            'name': 'Dataset Card for My Dataset',
            'text': '',
            'is_empty_text': True,
            'subsections': [
                {'name': 'Table of Contents', 'text': 'Some text here.', 'is_empty_text': False, 'subsections': []},
                {
                    'name': 'Dataset Description',
                    'text': 'Some text here.',
                    'is_empty_text': False,
                    'subsections': [
                        {
                            'name': 'Dataset Summary',
                            'text': 'Some text here.',
                            'is_empty_text': False,
                            'subsections': [],
                        },
                        {
                            'name': 'Supported Tasks and Leaderboards',
                            'text': '',
                            'is_empty_text': True,
                            'subsections': [],
                        },
                        {'name': 'Languages', 'text': 'Language Text', 'is_empty_text': False, 'subsections': []},
                    ],
                },
            ],
        }
    ],
}


_a							=							'\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'


_a							=							'\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n#### Extra Ignored Subsection\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'

_a							=							{
    'name': 'root',
    'text': '',
    'is_empty_text': True,
    'subsections': [
        {
            'name': 'Dataset Card for My Dataset',
            'text': '',
            'is_empty_text': True,
            'subsections': [
                {'name': 'Table of Contents', 'text': 'Some text here.', 'is_empty_text': False, 'subsections': []},
                {
                    'name': 'Dataset Description',
                    'text': 'Some text here.',
                    'is_empty_text': False,
                    'subsections': [
                        {
                            'name': 'Dataset Summary',
                            'text': 'Some text here.',
                            'is_empty_text': False,
                            'subsections': [
                                {
                                    'name': 'Extra Ignored Subsection',
                                    'text': '',
                                    'is_empty_text': True,
                                    'subsections': [],
                                }
                            ],
                        },
                        {
                            'name': 'Supported Tasks and Leaderboards',
                            'text': '',
                            'is_empty_text': True,
                            'subsections': [],
                        },
                        {'name': 'Languages', 'text': 'Language Text', 'is_empty_text': False, 'subsections': []},
                    ],
                },
            ],
        }
    ],
}

_a							=							'\\n---\n---\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'

_a							=							(
    'The following issues were found for the README at `{path}`:\n-\tEmpty YAML markers are present in the README.'
)

_a							=							'\\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'

_a							=							(
    'The following issues were found for the README at `{path}`:\n-\tNo YAML markers are present in the README.'
)

_a							=							'\\n---\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'

_a							=							'The following issues were found for the README at `{path}`:\n-\tOnly the start of YAML tags present in the README.'

_a							=							'\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'
_a							=							'The following issues were found for the README at `{path}`:\n-\tExpected some content in section `Dataset Summary` but it is empty.\n-\tExpected some text in section `Dataset Summary` but it is empty (text in subsections are ignored).'


_a							=							'\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n'
_a							=							'The following issues were found for the README at `{path}`:\n-\tExpected some content in section `Dataset Card for My Dataset` but it is empty.\n-\tSection `Dataset Card for My Dataset` expected the following subsections: `Table of Contents`, `Dataset Description`. Found \'None\'.'

_a							=							'\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Languages\nLanguage Text\n'

_a							=							'The following issues were found for the README at `{path}`:\n-\tSection `Dataset Description` is missing subsection: `Supported Tasks and Leaderboards`.'


_a							=							'\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\n'

_a							=							'The following issues were found for the README at `{path}`:\n-\tExpected some content in section `Languages` but it is empty.'

_a							=							'\\n---\nlanguage:\n- zh\n- en\n---\n\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'
_a							=							'The following issues were found for the README at `{path}`:\n-\tThe README has no first-level headings. One heading is expected. Skipping further validation for this README.'

_a							=							'\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n# Dataset Card My Dataset\n'

_a							=							'The following issues were found for the README at `{path}`:\n-\tThe README has several first-level headings: `Dataset Card for My Dataset`, `Dataset Card My Dataset`. Only one heading is expected. Skipping further validation for this README.'

_a							=							'\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'

_a							=							'The following issues were found for the README at `{path}`:\n-\tNo first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README.'

_a							=							''

_a							=							'The following issues were found for the README at `{path}`:\n-\tThe README has no first-level headings. One heading is expected. Skipping further validation for this README.\n-\tNo YAML markers are present in the README.'

_a							=							'\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'

_a							=							'The following issues were found while parsing the README at `{path}`:\n-\tMultiple sections with the same heading `Dataset Card for My Dataset` have been found. Please keep only one of these sections.'




@pytest.mark.parametrize(
    "readme_md, expected_dict",      [
        (README_CORRECT, CORRECT_DICT),
        (README_CORRECT_FOUR_LEVEL, CORRECT_DICT_FOUR_LEVEL),
    ],      )
def   _A  (  UpperCamelCase_					:	Union[str, Any],      UpperCamelCase_					:	List[str])     ->						Optional[int]:
   '''simple docstring'''


   assert ReadMe.from_string(__a,      __a).to_dict() == expected_dict




@pytest.mark.parametrize(
    "readme_md, expected_error",      [
        (README_NO_YAML, EXPECTED_ERROR_README_NO_YAML),
        (README_EMPTY_YAML, EXPECTED_ERROR_README_EMPTY_YAML),
        (README_INCORRECT_YAML, EXPECTED_ERROR_README_INCORRECT_YAML),
        (README_EMPTY, EXPECTED_ERROR_README_EMPTY),
        (README_NONE_SUBSECTION, EXPECTED_ERROR_README_NONE_SUBSECTION),
        (README_MISSING_FIRST_LEVEL, EXPECTED_ERROR_README_MISSING_FIRST_LEVEL),
        (README_MISSING_SUBSECTION, EXPECTED_ERROR_README_MISSING_SUBSECTION),
        (README_MISSING_TEXT, EXPECTED_ERROR_README_MISSING_TEXT),
        (README_WRONG_FIRST_LEVEL, EXPECTED_ERROR_README_WRONG_FIRST_LEVEL),
        (README_MULTIPLE_WRONG_FIRST_LEVEL, EXPECTED_ERROR_README_MULTIPLE_WRONG_FIRST_LEVEL),
        (README_MISSING_CONTENT, EXPECTED_ERROR_README_MISSING_CONTENT),
    ],      )
def   _A  (  UpperCamelCase_					:	List[str],      UpperCamelCase_					:	Optional[Any])     ->						Union[str, Any]:
   '''simple docstring'''


   with pytest.raises(__a,      match=re.escape(expected_error.format(path="root"))):
      __lowercase								=						ReadMe.from_string(__a,      __a)
      readme.validate()




@pytest.mark.parametrize(
    "readme_md, expected_error",      [
        (README_MULTIPLE_SAME_HEADING_1, EXPECTED_ERROR_README_MULTIPLE_SAME_HEADING_1),
    ],      )
def   _A  (  UpperCamelCase_					:	Dict,      UpperCamelCase_					:	Dict)     ->						Tuple:
   '''simple docstring'''


   with pytest.raises(__a,      match=re.escape(expected_error.format(path="root"))):
      ReadMe.from_string(__a,      __a)




@pytest.mark.parametrize(
    "readme_md,",      [
        (README_MULTIPLE_SAME_HEADING_1),
    ],      )
def   _A  (  UpperCamelCase_					:	Optional[Any])     ->						Tuple:
   '''simple docstring'''


   ReadMe.from_string(__a,      __a,      suppress_parsing_errors=__a)




@pytest.mark.parametrize(
    "readme_md, expected_dict",      [
        (README_CORRECT, CORRECT_DICT),
        (README_CORRECT_FOUR_LEVEL, CORRECT_DICT_FOUR_LEVEL),
    ],      )
def   _A  (  UpperCamelCase_					:	Union[str, Any],      UpperCamelCase_					:	Any)     ->						Optional[int]:
   '''simple docstring'''


   with tempfile.TemporaryDirectory() as tmp_dir:
      __lowercase								=						Path(__a) / '''README.md'''
      with open(__a,      "w+") as readme_file:
         readme_file.write(__a)
      __lowercase								=						ReadMe.from_readme(__a,      __a).to_dict()
      assert out["name"] == path
      assert out["text"] == ""
      assert out["is_empty_text"]
      assert out["subsections"] == expected_dict["subsections"]




@pytest.mark.parametrize(
    "readme_md, expected_error",      [
        (README_NO_YAML, EXPECTED_ERROR_README_NO_YAML),
        (README_EMPTY_YAML, EXPECTED_ERROR_README_EMPTY_YAML),
        (README_INCORRECT_YAML, EXPECTED_ERROR_README_INCORRECT_YAML),
        (README_EMPTY, EXPECTED_ERROR_README_EMPTY),
        (README_NONE_SUBSECTION, EXPECTED_ERROR_README_NONE_SUBSECTION),
        (README_MISSING_FIRST_LEVEL, EXPECTED_ERROR_README_MISSING_FIRST_LEVEL),
        (README_MISSING_SUBSECTION, EXPECTED_ERROR_README_MISSING_SUBSECTION),
        (README_MISSING_TEXT, EXPECTED_ERROR_README_MISSING_TEXT),
        (README_WRONG_FIRST_LEVEL, EXPECTED_ERROR_README_WRONG_FIRST_LEVEL),
        (README_MULTIPLE_WRONG_FIRST_LEVEL, EXPECTED_ERROR_README_MULTIPLE_WRONG_FIRST_LEVEL),
        (README_MISSING_CONTENT, EXPECTED_ERROR_README_MISSING_CONTENT),
    ],      )
def   _A  (  UpperCamelCase_					:	List[Any],      UpperCamelCase_					:	List[Any])     ->						int:
   '''simple docstring'''


   with tempfile.TemporaryDirectory() as tmp_dir:
      __lowercase								=						Path(__a) / '''README.md'''
      with open(__a,      "w+") as readme_file:
         readme_file.write(__a)
      __lowercase								=						expected_error.format(path=__a)
      with pytest.raises(__a,      match=re.escape(__a)):
         __lowercase								=						ReadMe.from_readme(__a,      __a)
         readme.validate()




@pytest.mark.parametrize(
    "readme_md, expected_error",      [
        (README_MULTIPLE_SAME_HEADING_1, EXPECTED_ERROR_README_MULTIPLE_SAME_HEADING_1),
    ],      )
def   _A  (  UpperCamelCase_					:	str,      UpperCamelCase_					:	Union[str, Any])     ->						Dict:
   '''simple docstring'''


   with tempfile.TemporaryDirectory() as tmp_dir:
      __lowercase								=						Path(__a) / '''README.md'''
      with open(__a,      "w+") as readme_file:
         readme_file.write(__a)
      __lowercase								=						expected_error.format(path=__a)
      with pytest.raises(__a,      match=re.escape(__a)):
         ReadMe.from_readme(__a,      __a)




@pytest.mark.parametrize(
    "readme_md,",      [
        (README_MULTIPLE_SAME_HEADING_1),
    ],      )
def   _A  (  UpperCamelCase_					:	Optional[Any])     ->						str:
   '''simple docstring'''


   with tempfile.TemporaryDirectory() as tmp_dir:
      __lowercase								=						Path(__a) / '''README.md'''
      with open(__a,      "w+") as readme_file:
         readme_file.write(__a)
      ReadMe.from_readme(__a,      __a,      suppress_parsing_errors=__a)







"""simple docstring"""





from unittest import TestCase

from datasets import Dataset
from minhash_deduplication import deduplicate_dataset, make_duplicate_clusters




def   _A  (  )     ->						Dict:
   '''simple docstring'''


   __lowercase								=						{
       "repo_name": ["test_repo1", "test_repo2", "test_repo3"],
       "path": ["test_1.py", "test_2.py", "unit_test.py"],
       "content": ["a " * 20, "a " * 30, "b " * 7],
   }
   __lowercase								=						Dataset.from_dict(UpperCamelCase_)
   return dataset




class  _lowerCAmelCase  (     lowercase       ):







 """simple docstring"""







 def  _lowercase			(      self							:      int     ):
    __lowercase								=						get_dataset()
    __lowercase								=						make_duplicate_clusters(UpperCAmelCase__, 0.85     )
    self.assertEqual(len(duplicate_clusters[0]     ), 2     )



 def  _lowercase			(      self							:      Any     ):
    __lowercase								=						get_dataset()
    __lowercase		,__lowercase								=						deduplicate_dataset(UpperCAmelCase__     )
    self.assertEqual(len(UpperCAmelCase__     ), 2     )
    print(UpperCAmelCase__     )
    self.assertEqual(duplicate_clusters[0][0]["copies"], 2     )
    self.assertEqual(duplicate_clusters[0][0]["is_extreme"], UpperCAmelCase__     )




