import os
from data_reading.basic_read import *


class MnliProcessor(DataProcessor):
    """Processor for the MultiNLI data set (GLUE version)."""

    def get_train_examples(self, data_dir,fname=None):
        """See base class."""
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "train.tsv")), "train")

    def get_dev_examples(self, data_dir,fname=None):
        """See base class."""
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "dev_matched.tsv")),
                "dev_matched")

    def get_labels(self):
        """See base class."""
        return ["contradiction", "entailment", "neutral"]

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for (i, line) in enumerate(lines):
            if i == 0:
                continue
            guid = "%s-%s" % (set_type, tokenization.convert_to_unicode(line[0]))
            text_a = tokenization.convert_to_unicode(line[8])
            text_b = tokenization.convert_to_unicode(line[9])
            label = tokenization.convert_to_unicode(line[-1])
            examples.append(
                    InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label))
        return examples

class QIMatch(DataProcessor):
    """Processor for the CoLA data set (GLUE version)."""

    def get_train_examples(self, data_dir,fname=None):
        """See base class."""
        if fname!=None:
            return self._create_examples(
                self._read_tsv(os.path.join(data_dir, fname)), "train")
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "train.txt.match")), "train")

    def get_dev_examples(self, data_dir,fname=None):
        """See base class."""
        if fname!=None:
            return self._create_examples(
                self._read_tsv(os.path.join(data_dir, fname)), "dev")
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "dev.txt.match")), "dev")

    def get_test_examples(self, data_dir,fname=None):
        """See base class."""
        if fname!=None:
            return self._create_examples(
                self._read_tsv(os.path.join(data_dir, fname)), "test")
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "test.txt.match")), "test")

    def get_labels(self):
        """See base class."""
        return ["0","1"]


    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for (i, line) in enumerate(lines):
            guid = "%s-%s" % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[1])
            text_b = tokenization.convert_to_unicode(line[2])
            label = tokenization.convert_to_unicode(line[3])
            examples.append(
                    InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label))
        return examples


class SimpleQMatch(DataProcessor):
    """Processor for the CoLA data set (GLUE version)."""

    def get_train_examples(self, data_dir,fname=None):
        """See base class."""
        if fname!=None:
            return self._create_examples(
                self._read_tsv(os.path.join(data_dir, fname)), "train")
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "train.txt.match")), "train")

    def get_dev_examples(self, data_dir,fname=None):
        """See base class."""
        if fname!=None:
            return self._create_examples(
                self._read_tsv(os.path.join(data_dir, fname)), "dev")
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "dev.txt.match")), "dev")

    def get_test_examples(self, data_dir,fname=None):
        """See base class."""
        if fname!=None:
            return self._create_examples(
                self._read_tsv(os.path.join(data_dir, fname)), "test")
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "test.txt.match")), "test")

    def get_labels(self):
        """See base class."""
        return ["0","1"]


    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for (i, line) in enumerate(lines):
            guid = "%s-%s" % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[0])
            text_b = tokenization.convert_to_unicode(line[1])
            label = tokenization.convert_to_unicode(line[2])
            examples.append(
                    InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label))
        return examples


class MrpcProcessor(DataProcessor):
    """Processor for the MRPC data set (GLUE version)."""

    def get_train_examples(self, data_dir,fname=None):
        """See base class."""
        if fname!=None:
            return self._create_examples(
                self._read_tsv(os.path.join(data_dir, fname)), "train")
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "train.tsv")), "train")

    def get_test_examples(self, data_dir,fname=None):
        """See base class."""
        if fname!=None:
            return self._create_examples(
                self._read_tsv(os.path.join(data_dir, fname)), "test")
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "test.tsv")), "test")

    def get_dev_examples(self, data_dir,fname=None):
        """See base class."""
        if fname!=None:
            return self._create_examples(
                self._read_tsv(os.path.join(data_dir, fname)), "dev")
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev")

    def get_labels(self):
        """See base class."""
        return ["0", "1"]

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for (i, line) in enumerate(lines):
            if i == 0:
                continue
            guid = "%s-%s" % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[3])
            text_b = tokenization.convert_to_unicode(line[4])
            label = tokenization.convert_to_unicode(line[0])
            examples.append(
                    InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label))
        return examples


class QICProcessorMovie(DataProcessor):
    """Processor for the CoLA data set (GLUE version)."""

    def get_train_examples(self, data_dir,fname=None):
        """See base class."""
        if fname!=None:
            return self._create_examples(
                self._read_tsv(os.path.join(data_dir, fname)), "train")
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "train.txt.format")), "train")

    def get_dev_examples(self, data_dir,fname=None):
        """See base class."""
        if fname!=None:
            return self._create_examples(
                self._read_tsv(os.path.join(data_dir, fname)), "dev")
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "dev.txt.format")), "dev")

    def get_test_examples(self, data_dir,fname=None):
        """See base class."""
        if fname!=None:
            return self._create_examples(
                self._read_tsv(os.path.join(data_dir, fname)), "test")
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "test.txt.format")), "test")

    def get_labels(self):
        """See base class."""
        return ['genre_film', 'film_review', 'film_writer', 'film_gross_revenue',
                        'actor_film', 'film_rating', 'film_director', 'film_cast',
                        'other', 'film_runtime', 'film_series', 'film_releasedate',
                        'topic_film', 'film_trailer', 'film_character_actor']


    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for (i, line) in enumerate(lines):
            guid = "%s-%s" % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[1])
            label = tokenization.convert_to_unicode(line[2])
            examples.append(
                    InputExample(guid=guid, text_a=text_a, text_b=None, label=label))
        return examples

class QICProcessor(DataProcessor):
    """Processor for the CoLA data set (GLUE version)."""

    def get_train_examples(self, data_dir,fname=None):
        """See base class."""
        if fname!=None:
            return self._create_examples(
                self._read_tsv(os.path.join(data_dir, fname)), "train")
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "train.txt.format")), "train")

    def get_dev_examples(self, data_dir,fname=None):
        """See base class."""
        if fname!=None:
            return self._create_examples(
                self._read_tsv(os.path.join(data_dir, fname)), "dev")
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "dev.txt.format")), "dev")

    def get_test_examples(self, data_dir,fname=None):
        """See base class."""
        if fname!=None:
            return self._create_examples(
                self._read_tsv(os.path.join(data_dir, fname)), "test")
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "new-test.tsv.format")), "test")

    def get_labels(self):
        """See base class."""
        return ['organization_date_founded',
                             'organization_founder',
                             'organization_parent_org',
                             'organization_headquarter',
                             'organization_president',
                             'organization_phone',
                             'organization_ceo',
                             'organization_place_founded',
                             'other']

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for (i, line) in enumerate(lines):
            guid = "%s-%s" % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[1])
            label = tokenization.convert_to_unicode(line[2])
            examples.append(
                    InputExample(guid=guid, text_a=text_a, text_b=None, label=label))
        return examples


class ColaProcessor(DataProcessor):
    """Processor for the CoLA data set (GLUE version)."""

    def get_train_examples(self, data_dir,fname=None):
        """See base class."""
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "train.tsv")), "train")

    def get_dev_examples(self, data_dir,fname=None):
        """See base class."""
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev")

    def get_labels(self):
        """See base class."""
        return ["0", "1"]

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for (i, line) in enumerate(lines):
            guid = "%s-%s" % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[3])
            label = tokenization.convert_to_unicode(line[1])
            examples.append(
                    InputExample(guid=guid, text_a=text_a, text_b=None, label=label))
        return examples



a8e2d44a945001f49fdcafd26031325ee4851d6a
<mask token>


class MrpcProcessor(DataProcessor):
    <mask token>

    def get_train_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'train')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'train.tsv')), 'train')
    <mask token>

    def get_dev_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'dev')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'dev.tsv')), 'dev')
    <mask token>

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for i, line in enumerate(lines):
            if i == 0:
                continue
            guid = '%s-%s' % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[3])
            text_b = tokenization.convert_to_unicode(line[4])
            label = tokenization.convert_to_unicode(line[0])
            examples.append(InputExample(guid=guid, text_a=text_a, text_b=
                text_b, label=label))
        return examples


class QICProcessorMovie(DataProcessor):
    """Processor for the CoLA data set (GLUE version)."""

    def get_train_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'train')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'train.txt.format')), 'train')

    def get_dev_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'dev')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'dev.txt.format')), 'dev')

    def get_test_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'test')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'test.txt.format')), 'test')

    def get_labels(self):
        """See base class."""
        return ['genre_film', 'film_review', 'film_writer',
            'film_gross_revenue', 'actor_film', 'film_rating',
            'film_director', 'film_cast', 'other', 'film_runtime',
            'film_series', 'film_releasedate', 'topic_film', 'film_trailer',
            'film_character_actor']

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for i, line in enumerate(lines):
            guid = '%s-%s' % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[1])
            label = tokenization.convert_to_unicode(line[2])
            examples.append(InputExample(guid=guid, text_a=text_a, text_b=
                None, label=label))
        return examples


class QICProcessor(DataProcessor):
    """Processor for the CoLA data set (GLUE version)."""

    def get_train_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'train')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'train.txt.format')), 'train')

    def get_dev_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'dev')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'dev.txt.format')), 'dev')

    def get_test_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'test')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'new-test.tsv.format')), 'test')

    def get_labels(self):
        """See base class."""
        return ['organization_date_founded', 'organization_founder',
            'organization_parent_org', 'organization_headquarter',
            'organization_president', 'organization_phone',
            'organization_ceo', 'organization_place_founded', 'other']

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for i, line in enumerate(lines):
            guid = '%s-%s' % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[1])
            label = tokenization.convert_to_unicode(line[2])
            examples.append(InputExample(guid=guid, text_a=text_a, text_b=
                None, label=label))
        return examples


class ColaProcessor(DataProcessor):
    """Processor for the CoLA data set (GLUE version)."""

    def get_train_examples(self, data_dir, fname=None):
        """See base class."""
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'train.tsv')), 'train')

    def get_dev_examples(self, data_dir, fname=None):
        """See base class."""
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'dev.tsv')), 'dev')

    def get_labels(self):
        """See base class."""
        return ['0', '1']

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for i, line in enumerate(lines):
            guid = '%s-%s' % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[3])
            label = tokenization.convert_to_unicode(line[1])
            examples.append(InputExample(guid=guid, text_a=text_a, text_b=
                None, label=label))
        return examples

<mask token>


class SimpleQMatch(DataProcessor):
    <mask token>

    def get_train_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'train')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'train.txt.match')), 'train')

    def get_dev_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'dev')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'dev.txt.match')), 'dev')

    def get_test_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'test')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'test.txt.match')), 'test')
    <mask token>
    <mask token>


class MrpcProcessor(DataProcessor):
    """Processor for the MRPC data set (GLUE version)."""

    def get_train_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'train')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'train.tsv')), 'train')

    def get_test_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'test')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'test.tsv')), 'test')

    def get_dev_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'dev')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'dev.tsv')), 'dev')

    def get_labels(self):
        """See base class."""
        return ['0', '1']

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for i, line in enumerate(lines):
            if i == 0:
                continue
            guid = '%s-%s' % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[3])
            text_b = tokenization.convert_to_unicode(line[4])
            label = tokenization.convert_to_unicode(line[0])
            examples.append(InputExample(guid=guid, text_a=text_a, text_b=
                text_b, label=label))
        return examples


class QICProcessorMovie(DataProcessor):
    """Processor for the CoLA data set (GLUE version)."""

    def get_train_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'train')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'train.txt.format')), 'train')

    def get_dev_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'dev')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'dev.txt.format')), 'dev')

    def get_test_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'test')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'test.txt.format')), 'test')

    def get_labels(self):
        """See base class."""
        return ['genre_film', 'film_review', 'film_writer',
            'film_gross_revenue', 'actor_film', 'film_rating',
            'film_director', 'film_cast', 'other', 'film_runtime',
            'film_series', 'film_releasedate', 'topic_film', 'film_trailer',
            'film_character_actor']

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for i, line in enumerate(lines):
            guid = '%s-%s' % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[1])
            label = tokenization.convert_to_unicode(line[2])
            examples.append(InputExample(guid=guid, text_a=text_a, text_b=
                None, label=label))
        return examples


class QICProcessor(DataProcessor):
    """Processor for the CoLA data set (GLUE version)."""

    def get_train_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'train')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'train.txt.format')), 'train')

    def get_dev_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'dev')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'dev.txt.format')), 'dev')

    def get_test_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'test')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'new-test.tsv.format')), 'test')

    def get_labels(self):
        """See base class."""
        return ['organization_date_founded', 'organization_founder',
            'organization_parent_org', 'organization_headquarter',
            'organization_president', 'organization_phone',
            'organization_ceo', 'organization_place_founded', 'other']

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for i, line in enumerate(lines):
            guid = '%s-%s' % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[1])
            label = tokenization.convert_to_unicode(line[2])
            examples.append(InputExample(guid=guid, text_a=text_a, text_b=
                None, label=label))
        return examples


class ColaProcessor(DataProcessor):
    """Processor for the CoLA data set (GLUE version)."""

    def get_train_examples(self, data_dir, fname=None):
        """See base class."""
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'train.tsv')), 'train')

    def get_dev_examples(self, data_dir, fname=None):
        """See base class."""
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'dev.tsv')), 'dev')

    def get_labels(self):
        """See base class."""
        return ['0', '1']

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for i, line in enumerate(lines):
            guid = '%s-%s' % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[3])
            label = tokenization.convert_to_unicode(line[1])
            examples.append(InputExample(guid=guid, text_a=text_a, text_b=
                None, label=label))
        return examples

<mask token>


class SimpleQMatch(DataProcessor):
    <mask token>

    def get_train_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'train')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'train.txt.match')), 'train')

    def get_dev_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'dev')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'dev.txt.match')), 'dev')

    def get_test_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'test')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'test.txt.match')), 'test')
    <mask token>

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for i, line in enumerate(lines):
            guid = '%s-%s' % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[0])
            text_b = tokenization.convert_to_unicode(line[1])
            label = tokenization.convert_to_unicode(line[2])
            examples.append(InputExample(guid=guid, text_a=text_a, text_b=
                text_b, label=label))
        return examples


class MrpcProcessor(DataProcessor):
    """Processor for the MRPC data set (GLUE version)."""

    def get_train_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'train')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'train.tsv')), 'train')

    def get_test_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'test')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'test.tsv')), 'test')

    def get_dev_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'dev')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'dev.tsv')), 'dev')

    def get_labels(self):
        """See base class."""
        return ['0', '1']

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for i, line in enumerate(lines):
            if i == 0:
                continue
            guid = '%s-%s' % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[3])
            text_b = tokenization.convert_to_unicode(line[4])
            label = tokenization.convert_to_unicode(line[0])
            examples.append(InputExample(guid=guid, text_a=text_a, text_b=
                text_b, label=label))
        return examples


class QICProcessorMovie(DataProcessor):
    """Processor for the CoLA data set (GLUE version)."""

    def get_train_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'train')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'train.txt.format')), 'train')

    def get_dev_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'dev')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'dev.txt.format')), 'dev')

    def get_test_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'test')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'test.txt.format')), 'test')

    def get_labels(self):
        """See base class."""
        return ['genre_film', 'film_review', 'film_writer',
            'film_gross_revenue', 'actor_film', 'film_rating',
            'film_director', 'film_cast', 'other', 'film_runtime',
            'film_series', 'film_releasedate', 'topic_film', 'film_trailer',
            'film_character_actor']

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for i, line in enumerate(lines):
            guid = '%s-%s' % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[1])
            label = tokenization.convert_to_unicode(line[2])
            examples.append(InputExample(guid=guid, text_a=text_a, text_b=
                None, label=label))
        return examples


class QICProcessor(DataProcessor):
    """Processor for the CoLA data set (GLUE version)."""

    def get_train_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'train')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'train.txt.format')), 'train')

    def get_dev_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'dev')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'dev.txt.format')), 'dev')

    def get_test_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'test')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'new-test.tsv.format')), 'test')

    def get_labels(self):
        """See base class."""
        return ['organization_date_founded', 'organization_founder',
            'organization_parent_org', 'organization_headquarter',
            'organization_president', 'organization_phone',
            'organization_ceo', 'organization_place_founded', 'other']

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for i, line in enumerate(lines):
            guid = '%s-%s' % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[1])
            label = tokenization.convert_to_unicode(line[2])
            examples.append(InputExample(guid=guid, text_a=text_a, text_b=
                None, label=label))
        return examples


class ColaProcessor(DataProcessor):
    """Processor for the CoLA data set (GLUE version)."""

    def get_train_examples(self, data_dir, fname=None):
        """See base class."""
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'train.tsv')), 'train')

    def get_dev_examples(self, data_dir, fname=None):
        """See base class."""
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'dev.tsv')), 'dev')

    def get_labels(self):
        """See base class."""
        return ['0', '1']

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for i, line in enumerate(lines):
            guid = '%s-%s' % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[3])
            label = tokenization.convert_to_unicode(line[1])
            examples.append(InputExample(guid=guid, text_a=text_a, text_b=
                None, label=label))
        return examples

<mask token>


class QIMatch(DataProcessor):
    <mask token>
    <mask token>
    <mask token>

    def get_test_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'test')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'test.txt.match')), 'test')
    <mask token>

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for i, line in enumerate(lines):
            guid = '%s-%s' % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[1])
            text_b = tokenization.convert_to_unicode(line[2])
            label = tokenization.convert_to_unicode(line[3])
            examples.append(InputExample(guid=guid, text_a=text_a, text_b=
                text_b, label=label))
        return examples


class SimpleQMatch(DataProcessor):
    """Processor for the CoLA data set (GLUE version)."""

    def get_train_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'train')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'train.txt.match')), 'train')

    def get_dev_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'dev')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'dev.txt.match')), 'dev')

    def get_test_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'test')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'test.txt.match')), 'test')

    def get_labels(self):
        """See base class."""
        return ['0', '1']

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for i, line in enumerate(lines):
            guid = '%s-%s' % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[0])
            text_b = tokenization.convert_to_unicode(line[1])
            label = tokenization.convert_to_unicode(line[2])
            examples.append(InputExample(guid=guid, text_a=text_a, text_b=
                text_b, label=label))
        return examples


class MrpcProcessor(DataProcessor):
    """Processor for the MRPC data set (GLUE version)."""

    def get_train_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'train')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'train.tsv')), 'train')

    def get_test_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'test')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'test.tsv')), 'test')

    def get_dev_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'dev')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'dev.tsv')), 'dev')

    def get_labels(self):
        """See base class."""
        return ['0', '1']

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for i, line in enumerate(lines):
            if i == 0:
                continue
            guid = '%s-%s' % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[3])
            text_b = tokenization.convert_to_unicode(line[4])
            label = tokenization.convert_to_unicode(line[0])
            examples.append(InputExample(guid=guid, text_a=text_a, text_b=
                text_b, label=label))
        return examples


class QICProcessorMovie(DataProcessor):
    """Processor for the CoLA data set (GLUE version)."""

    def get_train_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'train')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'train.txt.format')), 'train')

    def get_dev_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'dev')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'dev.txt.format')), 'dev')

    def get_test_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'test')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'test.txt.format')), 'test')

    def get_labels(self):
        """See base class."""
        return ['genre_film', 'film_review', 'film_writer',
            'film_gross_revenue', 'actor_film', 'film_rating',
            'film_director', 'film_cast', 'other', 'film_runtime',
            'film_series', 'film_releasedate', 'topic_film', 'film_trailer',
            'film_character_actor']

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for i, line in enumerate(lines):
            guid = '%s-%s' % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[1])
            label = tokenization.convert_to_unicode(line[2])
            examples.append(InputExample(guid=guid, text_a=text_a, text_b=
                None, label=label))
        return examples


class QICProcessor(DataProcessor):
    """Processor for the CoLA data set (GLUE version)."""

    def get_train_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'train')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'train.txt.format')), 'train')

    def get_dev_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'dev')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'dev.txt.format')), 'dev')

    def get_test_examples(self, data_dir, fname=None):
        """See base class."""
        if fname != None:
            return self._create_examples(self._read_tsv(os.path.join(
                data_dir, fname)), 'test')
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'new-test.tsv.format')), 'test')

    def get_labels(self):
        """See base class."""
        return ['organization_date_founded', 'organization_founder',
            'organization_parent_org', 'organization_headquarter',
            'organization_president', 'organization_phone',
            'organization_ceo', 'organization_place_founded', 'other']

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for i, line in enumerate(lines):
            guid = '%s-%s' % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[1])
            label = tokenization.convert_to_unicode(line[2])
            examples.append(InputExample(guid=guid, text_a=text_a, text_b=
                None, label=label))
        return examples


class ColaProcessor(DataProcessor):
    """Processor for the CoLA data set (GLUE version)."""

    def get_train_examples(self, data_dir, fname=None):
        """See base class."""
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'train.tsv')), 'train')

    def get_dev_examples(self, data_dir, fname=None):
        """See base class."""
        return self._create_examples(self._read_tsv(os.path.join(data_dir,
            'dev.tsv')), 'dev')

    def get_labels(self):
        """See base class."""
        return ['0', '1']

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for i, line in enumerate(lines):
            guid = '%s-%s' % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[3])
            label = tokenization.convert_to_unicode(line[1])
            examples.append(InputExample(guid=guid, text_a=text_a, text_b=
                None, label=label))
        return examples

import os
from data_reading.basic_read import *


class MnliProcessor(DataProcessor):
    """Processor for the MultiNLI data set (GLUE version)."""

    def get_train_examples(self, data_dir,fname=None):
        """See base class."""
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "train.tsv")), "train")

    def get_dev_examples(self, data_dir,fname=None):
        """See base class."""
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "dev_matched.tsv")),
                "dev_matched")

    def get_labels(self):
        """See base class."""
        return ["contradiction", "entailment", "neutral"]

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for (i, line) in enumerate(lines):
            if i == 0:
                continue
            guid = "%s-%s" % (set_type, tokenization.convert_to_unicode(line[0]))
            text_a = tokenization.convert_to_unicode(line[8])
            text_b = tokenization.convert_to_unicode(line[9])
            label = tokenization.convert_to_unicode(line[-1])
            examples.append(
                    InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label))
        return examples

class QIMatch(DataProcessor):
    """Processor for the CoLA data set (GLUE version)."""

    def get_train_examples(self, data_dir,fname=None):
        """See base class."""
        if fname!=None:
            return self._create_examples(
                self._read_tsv(os.path.join(data_dir, fname)), "train")
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "train.txt.match")), "train")

    def get_dev_examples(self, data_dir,fname=None):
        """See base class."""
        if fname!=None:
            return self._create_examples(
                self._read_tsv(os.path.join(data_dir, fname)), "dev")
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "dev.txt.match")), "dev")

    def get_test_examples(self, data_dir,fname=None):
        """See base class."""
        if fname!=None:
            return self._create_examples(
                self._read_tsv(os.path.join(data_dir, fname)), "test")
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "test.txt.match")), "test")

    def get_labels(self):
        """See base class."""
        return ["0","1"]


    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for (i, line) in enumerate(lines):
            guid = "%s-%s" % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[1])
            text_b = tokenization.convert_to_unicode(line[2])
            label = tokenization.convert_to_unicode(line[3])
            examples.append(
                    InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label))
        return examples


class SimpleQMatch(DataProcessor):
    """Processor for the CoLA data set (GLUE version)."""

    def get_train_examples(self, data_dir,fname=None):
        """See base class."""
        if fname!=None:
            return self._create_examples(
                self._read_tsv(os.path.join(data_dir, fname)), "train")
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "train.txt.match")), "train")

    def get_dev_examples(self, data_dir,fname=None):
        """See base class."""
        if fname!=None:
            return self._create_examples(
                self._read_tsv(os.path.join(data_dir, fname)), "dev")
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "dev.txt.match")), "dev")

    def get_test_examples(self, data_dir,fname=None):
        """See base class."""
        if fname!=None:
            return self._create_examples(
                self._read_tsv(os.path.join(data_dir, fname)), "test")
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "test.txt.match")), "test")

    def get_labels(self):
        """See base class."""
        return ["0","1"]


    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for (i, line) in enumerate(lines):
            guid = "%s-%s" % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[0])
            text_b = tokenization.convert_to_unicode(line[1])
            label = tokenization.convert_to_unicode(line[2])
            examples.append(
                    InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label))
        return examples


class MrpcProcessor(DataProcessor):
    """Processor for the MRPC data set (GLUE version)."""

    def get_train_examples(self, data_dir,fname=None):
        """See base class."""
        if fname!=None:
            return self._create_examples(
                self._read_tsv(os.path.join(data_dir, fname)), "train")
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "train.tsv")), "train")

    def get_test_examples(self, data_dir,fname=None):
        """See base class."""
        if fname!=None:
            return self._create_examples(
                self._read_tsv(os.path.join(data_dir, fname)), "test")
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "test.tsv")), "test")

    def get_dev_examples(self, data_dir,fname=None):
        """See base class."""
        if fname!=None:
            return self._create_examples(
                self._read_tsv(os.path.join(data_dir, fname)), "dev")
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev")

    def get_labels(self):
        """See base class."""
        return ["0", "1"]

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for (i, line) in enumerate(lines):
            if i == 0:
                continue
            guid = "%s-%s" % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[3])
            text_b = tokenization.convert_to_unicode(line[4])
            label = tokenization.convert_to_unicode(line[0])
            examples.append(
                    InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label))
        return examples


class QICProcessorMovie(DataProcessor):
    """Processor for the CoLA data set (GLUE version)."""

    def get_train_examples(self, data_dir,fname=None):
        """See base class."""
        if fname!=None:
            return self._create_examples(
                self._read_tsv(os.path.join(data_dir, fname)), "train")
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "train.txt.format")), "train")

    def get_dev_examples(self, data_dir,fname=None):
        """See base class."""
        if fname!=None:
            return self._create_examples(
                self._read_tsv(os.path.join(data_dir, fname)), "dev")
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "dev.txt.format")), "dev")

    def get_test_examples(self, data_dir,fname=None):
        """See base class."""
        if fname!=None:
            return self._create_examples(
                self._read_tsv(os.path.join(data_dir, fname)), "test")
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "test.txt.format")), "test")

    def get_labels(self):
        """See base class."""
        return ['genre_film', 'film_review', 'film_writer', 'film_gross_revenue',
                        'actor_film', 'film_rating', 'film_director', 'film_cast',
                        'other', 'film_runtime', 'film_series', 'film_releasedate',
                        'topic_film', 'film_trailer', 'film_character_actor']


    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for (i, line) in enumerate(lines):
            guid = "%s-%s" % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[1])
            label = tokenization.convert_to_unicode(line[2])
            examples.append(
                    InputExample(guid=guid, text_a=text_a, text_b=None, label=label))
        return examples

class QICProcessor(DataProcessor):
    """Processor for the CoLA data set (GLUE version)."""

    def get_train_examples(self, data_dir,fname=None):
        """See base class."""
        if fname!=None:
            return self._create_examples(
                self._read_tsv(os.path.join(data_dir, fname)), "train")
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "train.txt.format")), "train")

    def get_dev_examples(self, data_dir,fname=None):
        """See base class."""
        if fname!=None:
            return self._create_examples(
                self._read_tsv(os.path.join(data_dir, fname)), "dev")
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "dev.txt.format")), "dev")

    def get_test_examples(self, data_dir,fname=None):
        """See base class."""
        if fname!=None:
            return self._create_examples(
                self._read_tsv(os.path.join(data_dir, fname)), "test")
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "new-test.tsv.format")), "test")

    def get_labels(self):
        """See base class."""
        return ['organization_date_founded',
                             'organization_founder',
                             'organization_parent_org',
                             'organization_headquarter',
                             'organization_president',
                             'organization_phone',
                             'organization_ceo',
                             'organization_place_founded',
                             'other']

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for (i, line) in enumerate(lines):
            guid = "%s-%s" % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[1])
            label = tokenization.convert_to_unicode(line[2])
            examples.append(
                    InputExample(guid=guid, text_a=text_a, text_b=None, label=label))
        return examples


class ColaProcessor(DataProcessor):
    """Processor for the CoLA data set (GLUE version)."""

    def get_train_examples(self, data_dir,fname=None):
        """See base class."""
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "train.tsv")), "train")

    def get_dev_examples(self, data_dir,fname=None):
        """See base class."""
        return self._create_examples(
                self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev")

    def get_labels(self):
        """See base class."""
        return ["0", "1"]

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for (i, line) in enumerate(lines):
            guid = "%s-%s" % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[3])
            label = tokenization.convert_to_unicode(line[1])
            examples.append(
                    InputExample(guid=guid, text_a=text_a, text_b=None, label=label))
        return examples


