/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.internal.filter2.columnindex;

import java.util.PrimitiveIterator;
import java.util.Set;
import java.util.function.Function;

import org.apache.parquet.filter2.compat.FilterCompat;
import org.apache.parquet.filter2.compat.FilterCompat.FilterPredicateCompat;
import org.apache.parquet.filter2.compat.FilterCompat.NoOpFilter;
import org.apache.parquet.filter2.compat.FilterCompat.UnboundRecordFilterCompat;
import org.apache.parquet.filter2.predicate.FilterPredicate.Visitor;
import org.apache.parquet.filter2.predicate.Operators.And;
import org.apache.parquet.filter2.predicate.Operators.Column;
import org.apache.parquet.filter2.predicate.Operators.Eq;
import org.apache.parquet.filter2.predicate.Operators.Gt;
import org.apache.parquet.filter2.predicate.Operators.GtEq;
import org.apache.parquet.filter2.predicate.Operators.LogicalNotUserDefined;
import org.apache.parquet.filter2.predicate.Operators.Lt;
import org.apache.parquet.filter2.predicate.Operators.LtEq;
import org.apache.parquet.filter2.predicate.Operators.Not;
import org.apache.parquet.filter2.predicate.Operators.NotEq;
import org.apache.parquet.filter2.predicate.Operators.Or;
import org.apache.parquet.filter2.predicate.Operators.UserDefined;
import org.apache.parquet.filter2.predicate.UserDefinedPredicate;
import org.apache.parquet.hadoop.metadata.ColumnPath;
import org.apache.parquet.internal.column.columnindex.ColumnIndex;
import org.apache.parquet.internal.column.columnindex.OffsetIndex;
import org.apache.parquet.internal.filter2.columnindex.ColumnIndexStore.MissingOffsetIndexException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Filter implementation based on column indexes.
 * No filtering will be applied for columns where no column index is available.
 * Offset index is required for all the columns in the projection, therefore a {@link MissingOffsetIndexException} will
 * be thrown from any {@code visit} methods if any of the required offset indexes is missing.
 */
public class ColumnIndexFilter implements Visitor<RowRanges> {

  private static final Logger LOGGER = LoggerFactory.getLogger(ColumnIndexFilter.class);
  private final ColumnIndexStore columnIndexStore;
  private final Set<ColumnPath> columns;
  private final long rowCount;
  private RowRanges allRows;

  /**
   * Calculates the row ranges containing the indexes of the rows might match the specified filter.
   *
   * @param filter
   *          to be used for filtering the rows
   * @param columnIndexStore
   *          the store for providing column/offset indexes
   * @param paths
   *          the paths of the columns used in the actual projection; a column not being part of the projection will be
   *          handled as containing {@code null} values only even if the column has values written in the file
   * @param rowCount
   *          the total number of rows in the row-group
   * @return the ranges of the possible matching row indexes; the returned ranges will contain all the rows if any of
   *         the required offset index is missing
   */
  public static RowRanges calculateRowRanges(FilterCompat.Filter filter, ColumnIndexStore columnIndexStore,
      Set<ColumnPath> paths, long rowCount) {
    return filter.accept(new FilterCompat.Visitor<RowRanges>() {
      @Override
      public RowRanges visit(FilterPredicateCompat filterPredicateCompat) {
        try {
          return filterPredicateCompat.getFilterPredicate()
              .accept(new ColumnIndexFilter(columnIndexStore, paths, rowCount));
        } catch (MissingOffsetIndexException e) {
          LOGGER.info(e.getMessage());
          return RowRanges.createSingle(rowCount);
        }
      }

      @Override
      public RowRanges visit(UnboundRecordFilterCompat unboundRecordFilterCompat) {
        return RowRanges.createSingle(rowCount);
      }

      @Override
      public RowRanges visit(NoOpFilter noOpFilter) {
        return RowRanges.createSingle(rowCount);
      }
    });
  }

  private ColumnIndexFilter(ColumnIndexStore columnIndexStore, Set<ColumnPath> paths, long rowCount) {
    this.columnIndexStore = columnIndexStore;
    this.columns = paths;
    this.rowCount = rowCount;
  }

  private RowRanges allRows() {
    if (allRows == null) {
      allRows = RowRanges.createSingle(rowCount);
    }
    return allRows;
  }

  @Override
  public <T extends Comparable<T>> RowRanges visit(Eq<T> eq) {
    return applyPredicate(eq.getColumn(), ci -> ci.visit(eq), eq.getValue() == null ? allRows() : RowRanges.EMPTY);
  }

  @Override
  public <T extends Comparable<T>> RowRanges visit(NotEq<T> notEq) {
    return applyPredicate(notEq.getColumn(), ci -> ci.visit(notEq),
        notEq.getValue() == null ? RowRanges.EMPTY : allRows());
  }

  @Override
  public <T extends Comparable<T>> RowRanges visit(Lt<T> lt) {
    return applyPredicate(lt.getColumn(), ci -> ci.visit(lt), RowRanges.EMPTY);
  }

  @Override
  public <T extends Comparable<T>> RowRanges visit(LtEq<T> ltEq) {
    return applyPredicate(ltEq.getColumn(), ci -> ci.visit(ltEq), RowRanges.EMPTY);
  }

  @Override
  public <T extends Comparable<T>> RowRanges visit(Gt<T> gt) {
    return applyPredicate(gt.getColumn(), ci -> ci.visit(gt), RowRanges.EMPTY);
  }

  @Override
  public <T extends Comparable<T>> RowRanges visit(GtEq<T> gtEq) {
    return applyPredicate(gtEq.getColumn(), ci -> ci.visit(gtEq), RowRanges.EMPTY);
  }

  @Override
  public <T extends Comparable<T>, U extends UserDefinedPredicate<T>> RowRanges visit(UserDefined<T, U> udp) {
    return applyPredicate(udp.getColumn(), ci -> ci.visit(udp),
        udp.getUserDefinedPredicate().keep(null) ? allRows() : RowRanges.EMPTY);
  }

  @Override
  public <T extends Comparable<T>, U extends UserDefinedPredicate<T>> RowRanges visit(
      LogicalNotUserDefined<T, U> udp) {
    return applyPredicate(udp.getUserDefined().getColumn(), ci -> ci.visit(udp),
        udp.getUserDefined().getUserDefinedPredicate().keep(null) ? RowRanges.EMPTY : allRows());
  }

  private RowRanges applyPredicate(Column<?> column, Function<ColumnIndex, PrimitiveIterator.OfInt> func,
      RowRanges rangesForMissingColumns) {
    ColumnPath columnPath = column.getColumnPath();
    if (!columns.contains(columnPath)) {
      return rangesForMissingColumns;
    }

    OffsetIndex oi = columnIndexStore.getOffsetIndex(columnPath);
    ColumnIndex ci = columnIndexStore.getColumnIndex(columnPath);
    if (ci == null) {
      LOGGER.info("No column index for column {} is available; Unable to filter on this column", columnPath);
      return allRows();
    }

    return RowRanges.create(rowCount, func.apply(ci), oi);
  }

  @Override
  public RowRanges visit(And and) {
    return RowRanges.intersection(and.getLeft().accept(this), and.getRight().accept(this));
  }

  @Override
  public RowRanges visit(Or or) {
    return RowRanges.union(or.getLeft().accept(this), or.getRight().accept(this));
  }

  @Override
  public RowRanges visit(Not not) {
    throw new IllegalArgumentException(
        "Predicates containing a NOT must be run through LogicalInverseRewriter. " + not);
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.hadoop.thrift;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.JobConf;

import org.apache.parquet.hadoop.ParquetInputFormat;

public class ParquetThriftInputFormat<T> extends ParquetInputFormat<T> {

  @SuppressWarnings("unchecked")
  public ParquetThriftInputFormat() {
    this(ThriftReadSupport.class);
  }

  /**
   * ScroogeReadSupport can be used when reading scrooge records out of parquet file
   * @param readSupportClass a read support class
   */
  protected ParquetThriftInputFormat(Class readSupportClass) {
    super(readSupportClass);
  }

  /**
   * Call this method when setting up your Hadoop job if reading into a Thrift object
   * that is not encoded into the parquet-serialized thrift metadata (for example,
   * writing with Apache Thrift, but reading back into Twitter Scrooge version of
   * the same thrift definition, or a different but compatible Apache Thrift class).
   * @param conf a mapred jobconf
   * @param klass a thrift class
   * @param <T> the Java type of records the configured ReadSupport will produce
   */
  public static <T> void setThriftClass(JobConf conf, Class<T> klass) {
    conf.set(ThriftReadSupport.THRIFT_READ_CLASS_KEY, klass.getName());
  }

  /**
   * Call this method when setting up your Hadoop job if reading into a Thrift object
   * that is not encoded into the parquet-serialized thrift metadata (for example,
   * writing with Apache Thrift, but reading back into Twitter Scrooge version of
   * the same thrift definition, or a different but compatible Apache Thrift class).
   * @param conf a configuration
   * @param klass a thrift class
   * @param <T> the Java type of records the configured ReadSupport will produce
   */
  public static  <T> void setThriftClass(Configuration conf, Class<T> klass) {
    conf.set(ThriftReadSupport.THRIFT_READ_CLASS_KEY, klass.getName());
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.hadoop.hive.ql.io.parquet.convert;

import java.util.List;

import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;

import org.apache.parquet.schema.ConversionPatterns;
import org.apache.parquet.schema.GroupType;
import org.apache.parquet.schema.LogicalTypeAnnotation;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.PrimitiveType;
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
import org.apache.parquet.schema.Type;
import org.apache.parquet.schema.Type.Repetition;
import org.apache.parquet.schema.Types;

import static org.apache.parquet.schema.LogicalTypeAnnotation.listType;

public class HiveSchemaConverter {

  public static MessageType convert(final List<String> columnNames, final List<TypeInfo> columnTypes) {
    final MessageType schema = new MessageType("hive_schema", convertTypes(columnNames, columnTypes));
    return schema;
  }

  private static Type[] convertTypes(final List<String> columnNames, final List<TypeInfo> columnTypes) {
    if (columnNames.size() != columnTypes.size()) {
      throw new IllegalStateException("Mismatched Hive columns and types. Hive columns names" +
        " found : " + columnNames + " . And Hive types found : " + columnTypes);
    }
    final Type[] types = new Type[columnNames.size()];
    for (int i = 0; i < columnNames.size(); ++i) {
      types[i] = convertType(columnNames.get(i), columnTypes.get(i));
    }
    return types;
  }

  private static Type convertType(final String name, final TypeInfo typeInfo) {
    return convertType(name, typeInfo, Repetition.OPTIONAL);
  }

  private static Type convertType(final String name, final TypeInfo typeInfo, final Repetition repetition) {
    if (typeInfo.getCategory().equals(Category.PRIMITIVE)) {
      if (typeInfo.equals(TypeInfoFactory.stringTypeInfo)) {
        return new PrimitiveType(repetition, PrimitiveTypeName.BINARY, name);
      } else if (typeInfo.equals(TypeInfoFactory.intTypeInfo) ||
          typeInfo.equals(TypeInfoFactory.shortTypeInfo) ||
          typeInfo.equals(TypeInfoFactory.byteTypeInfo)) {
        return new PrimitiveType(repetition, PrimitiveTypeName.INT32, name);
      } else if (typeInfo.equals(TypeInfoFactory.longTypeInfo)) {
        return new PrimitiveType(repetition, PrimitiveTypeName.INT64, name);
      } else if (typeInfo.equals(TypeInfoFactory.doubleTypeInfo)) {
        return new PrimitiveType(repetition, PrimitiveTypeName.DOUBLE, name);
      } else if (typeInfo.equals(TypeInfoFactory.floatTypeInfo)) {
        return new PrimitiveType(repetition, PrimitiveTypeName.FLOAT, name);
      } else if (typeInfo.equals(TypeInfoFactory.booleanTypeInfo)) {
        return new PrimitiveType(repetition, PrimitiveTypeName.BOOLEAN, name);
      } else if (typeInfo.equals(TypeInfoFactory.binaryTypeInfo)) {
        // TODO : binaryTypeInfo is a byte array. Need to map it
        throw new UnsupportedOperationException("Binary type not implemented");
      } else if (typeInfo.equals(TypeInfoFactory.timestampTypeInfo)) {
        throw new UnsupportedOperationException("Timestamp type not implemented");
      } else if (typeInfo.equals(TypeInfoFactory.voidTypeInfo)) {
        throw new UnsupportedOperationException("Void type not implemented");
      } else if (typeInfo.equals(TypeInfoFactory.unknownTypeInfo)) {
        throw new UnsupportedOperationException("Unknown type not implemented");
      } else {
        throw new IllegalArgumentException("Unknown type: " + typeInfo);
      }
    } else if (typeInfo.getCategory().equals(Category.LIST)) {
      return convertArrayType(name, (ListTypeInfo) typeInfo);
    } else if (typeInfo.getCategory().equals(Category.STRUCT)) {
      return convertStructType(name, (StructTypeInfo) typeInfo);
    } else if (typeInfo.getCategory().equals(Category.MAP)) {
      return convertMapType(name, (MapTypeInfo) typeInfo);
    } else if (typeInfo.getCategory().equals(Category.UNION)) {
      throw new UnsupportedOperationException("Union type not implemented");
    } else {
      throw new IllegalArgumentException("Unknown type: " + typeInfo);
    }
  }

  // An optional group containing a repeated anonymous group "bag", containing
  // 1 anonymous element "array_element"
  private static GroupType convertArrayType(final String name, final ListTypeInfo typeInfo) {
    final TypeInfo subType = typeInfo.getListElementTypeInfo();
    return listWrapper(name, listType(), new GroupType(Repetition.REPEATED,
        ParquetHiveSerDe.ARRAY.toString(), convertType("array_element", subType)));
  }

  // An optional group containing multiple elements
  private static GroupType convertStructType(final String name, final StructTypeInfo typeInfo) {
    final List<String> columnNames = typeInfo.getAllStructFieldNames();
    final List<TypeInfo> columnTypes = typeInfo.getAllStructFieldTypeInfos();
    return new GroupType(Repetition.OPTIONAL, name, convertTypes(columnNames, columnTypes));

  }

  // An optional group containing a repeated anonymous group "map", containing
  // 2 elements: "key", "value"
  private static GroupType convertMapType(final String name, final MapTypeInfo typeInfo) {
    final Type keyType = convertType(ParquetHiveSerDe.MAP_KEY.toString(),
        typeInfo.getMapKeyTypeInfo(), Repetition.REQUIRED);
    final Type valueType = convertType(ParquetHiveSerDe.MAP_VALUE.toString(),
        typeInfo.getMapValueTypeInfo());
    return ConversionPatterns.mapType(Repetition.OPTIONAL, name, keyType, valueType);
  }

  private static GroupType listWrapper(final String name, final LogicalTypeAnnotation logicalTypeAnnotation,
      final GroupType groupType) {
    return Types.optionalGroup().addField(groupType).as(logicalTypeAnnotation).named(name);
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.glob;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

import org.apache.parquet.glob.GlobNode.Atom;
import org.apache.parquet.glob.GlobNode.GlobNodeSequence;
import org.apache.parquet.glob.GlobNode.OneOf;

/**
 * Implementation of {@link org.apache.parquet.Strings#expandGlob(String)}
 */
public final class GlobExpander {
  private GlobExpander()  { }

  /**
   * Expands a string with braces ("{}") into all of its possible permutations.
   * We call anything inside of {} braces a "one-of" group.
   *
   * The only special characters in this glob syntax are '}', '{' and ','
   *
   * The top-level pattern must not contain any commas, but a "one-of" group separates
   * its elements with commas, and a one-of group may contain sub one-of groups.
   *
   * For example:
   * start{a,b,c}end -> startaend, startbend, startcend
   * start{a,{b,c},d} -> startaend, startbend, startcend, startdend
   * {a,b,c} -> a, b, c
   * start{a, b{x,y}} -> starta, startbx, startby
   *
   * @param globPattern a string in the format described above
   * @return a list of all the strings that would satisfy globPattern, including duplicates
   */
  public static List<String> expand(String globPattern) {
    return GlobExpanderImpl.expand(GlobParser.parse(globPattern));
  }

  /**
   * Transforms a tree of {@link GlobNode} into a list of all the strings that satisfy
   * this tree.
   */
  private final static class GlobExpanderImpl implements GlobNode.Visitor<List<String>> {
    private static final GlobExpanderImpl INSTANCE = new GlobExpanderImpl();

    private GlobExpanderImpl() {}

    public static List<String> expand(GlobNode node) {
      return node.accept(INSTANCE);
    }

    @Override
    public List<String> visit(Atom atom) {
      // atoms are the base case, just return a singleton list
      return Arrays.asList(atom.get());
    }

    @Override
    public List<String> visit(OneOf oneOf) {
      // in the case of OneOf, we just need to take all of
      // the possible values the OneOf represents and
      // union them together
      List<String> results = new ArrayList<String>();
      for (GlobNode n : oneOf.getChildren()) {
        results.addAll(n.accept(this));
      }
      return results;
    }

    @Override
    public List<String> visit(GlobNodeSequence seq) {
      // in the case of a sequence, for each child
      // we need to expand the child into all of its
      // possibilities, then do a cross product of
      // all the children, in order.

      List<String> results = new ArrayList<String>();
      for (GlobNode n : seq.getChildren()) {
        results = crossOrTakeNonEmpty(results, n.accept(this));
      }
      return results;
    }

    /**
     * Computes the cross product of two lists by adding each string in list1 to each string in list2.
     * If one of the lists is empty, a copy of the other list is returned.
     * If both are empty, an empty list is returned.
     */
    public static List<String> crossOrTakeNonEmpty(List<String> list1, List<String> list2) {
      if (list1.isEmpty()) {
        ArrayList<String> result = new ArrayList<String>(list2.size());
        result.addAll(list2);
        return result;
      }

      if (list2.isEmpty()) {
        ArrayList<String> result = new ArrayList<String>(list1.size());
        result.addAll(list1);
        return result;
      }

      List<String> result = new ArrayList<String>(list1.size() * list2.size());
      for (String s1 : list1) {
        for (String s2 : list2) {
          result.add(s1 + s2);
        }
      }
      return result;
    }
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.internal.filter2.columnindex;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.PrimitiveIterator;
import java.util.Set;

import org.apache.parquet.filter2.compat.FilterCompat.Filter;
import org.apache.parquet.internal.column.columnindex.OffsetIndex;

/**
 * Class representing row ranges in a row-group. These row ranges are calculated as a result of the column index based
 * filtering. To be used iterate over the matching row indexes to be read from a row-group, retrieve the count of the
 * matching rows or check overlapping of a row index range.
 *
 * @see ColumnIndexFilter#calculateRowRanges(Filter, ColumnIndexStore, Set, long)
 */
public class RowRanges {
  private static class Range {

    // Returns the union of the two ranges or null if there are elements between them.
    private static Range union(Range left, Range right) {
      if (left.from <= right.from) {
        if (left.to + 1 >= right.from) {
          return new Range(left.from, Math.max(left.to, right.to));
        }
      } else if (right.to + 1 >= left.from) {
        return new Range(right.from, Math.max(left.to, right.to));
      }
      return null;
    }

    // Returns the intersection of the two ranges of null if they are not overlapped.
    private static Range intersection(Range left, Range right) {
      if (left.from <= right.from) {
        if (left.to >= right.from) {
          return new Range(right.from, Math.min(left.to, right.to));
        }
      } else if (right.to >= left.from) {
        return new Range(left.from, Math.min(left.to, right.to));
      }
      return null;
    }

    final long from;
    final long to;

    // Creates a range of [from, to] (from and to are inclusive; empty ranges are not valid)
    Range(long from, long to) {
      assert from <= to;
      this.from = from;
      this.to = to;
    }

    long count() {
      return to - from + 1;
    }

    boolean isBefore(Range other) {
      return to < other.from;
    }

    boolean isAfter(Range other) {
      return from > other.to;
    }

    @Override
    public String toString() {
      return "[" + from + ", " + to + ']';
    }
  }

  static final RowRanges EMPTY = new RowRanges();

  /*
   * Creates a new RowRanges object with the single range [0, rowCount - 1].
   */
  static RowRanges createSingle(long rowCount) {
    RowRanges ranges = new RowRanges();
    ranges.add(new Range(0, rowCount - 1));
    return ranges;
  }

  /*
   * Creates a new RowRanges object with the following ranges.
   * [firstRowIndex[0], lastRowIndex[0]],
   * [firstRowIndex[1], lastRowIndex[1]],
   * ...,
   * [firstRowIndex[n], lastRowIndex[n]]
   * (See OffsetIndex.getFirstRowIndex and OffsetIndex.getLastRowIndex for details.)
   *
   * The union of the ranges are calculated so the result ranges always contain the disjunct ranges. See union for
   * details.
   */
  static RowRanges create(long rowCount, PrimitiveIterator.OfInt pageIndexes, OffsetIndex offsetIndex) {
    RowRanges ranges = new RowRanges();
    while (pageIndexes.hasNext()) {
      int pageIndex = pageIndexes.nextInt();
      ranges.add(new Range(offsetIndex.getFirstRowIndex(pageIndex), offsetIndex.getLastRowIndex(pageIndex, rowCount)));
    }
    return ranges;
  }

  /*
   * Calculates the union of the two specified RowRanges object. The union of two range is calculated if there are no
   * elements between them. Otherwise, the two disjunct ranges are stored separately.
   * For example:
   * [113, 241] ∪ [221, 340] = [113, 330]
   * [113, 230] ∪ [231, 340] = [113, 340]
   * while
   * [113, 230] ∪ [232, 340] = [113, 230], [232, 340]
   *
   * The result RowRanges object will contain all the row indexes that were contained in one of the specified objects.
   */
  static RowRanges union(RowRanges left, RowRanges right) {
    RowRanges result = new RowRanges();
    Iterator<Range> it1 = left.ranges.iterator();
    Iterator<Range> it2 = right.ranges.iterator();
    if (it2.hasNext()) {
      Range range2 = it2.next();
      while (it1.hasNext()) {
        Range range1 = it1.next();
        if (range1.isAfter(range2)) {
          result.add(range2);
          range2 = range1;
          Iterator<Range> tmp = it1;
          it1 = it2;
          it2 = tmp;
        } else {
          result.add(range1);
        }
      }
      result.add(range2);
    } else {
      it2 = it1;
    }
    while (it2.hasNext()) {
      result.add(it2.next());
    }

    return result;
  }

  /*
   * Calculates the intersection of the two specified RowRanges object. Two ranges intersect if they have common
   * elements otherwise the result is empty.
   * For example:
   * [113, 241] ∩ [221, 340] = [221, 241]
   * while
   * [113, 230] ∩ [231, 340] = <EMPTY>
   *
   * The result RowRanges object will contain all the row indexes there were contained in both of the specified objects
   */
  static RowRanges intersection(RowRanges left, RowRanges right) {
    RowRanges result = new RowRanges();

    int rightIndex = 0;
    for (Range l : left.ranges) {
      for (int i = rightIndex, n = right.ranges.size(); i < n; ++i) {
        Range r = right.ranges.get(i);
        if (l.isBefore(r)) {
          break;
        } else if (l.isAfter(r)) {
          rightIndex = i + 1;
          continue;
        }
        result.add(Range.intersection(l, r));
      }
    }

    return result;
  }

  private final List<Range> ranges = new ArrayList<>();

  private RowRanges() {
  }

  /*
   * Adds a range to the end of the list of ranges. It maintains the disjunct ascending order(*) of the ranges by
   * trying to union the specified range to the last ranges in the list. The specified range shall be larger(*) than
   * the last one or might be overlapped with some of the last ones.
   * (*) [a, b] < [c, d] if b < c
   */
  private void add(Range range) {
    Range rangeToAdd = range;
    for (int i = ranges.size() - 1; i >= 0; --i) {
      Range last = ranges.get(i);
      assert !last.isAfter(range);
      Range u = Range.union(last, rangeToAdd);
      if (u == null) {
        break;
      }
      rangeToAdd = u;
      ranges.remove(i);
    }
    ranges.add(rangeToAdd);
  }

  /**
   * @return the number of rows in the ranges
   */
  public long rowCount() {
    long cnt = 0;
    for (Range range : ranges) {
      cnt += range.count();
    }
    return cnt;
  }

  /**
   * @return the ascending iterator of the row indexes contained in the ranges
   */
  public PrimitiveIterator.OfLong iterator() {
    return new PrimitiveIterator.OfLong() {
      private int currentRangeIndex = -1;
      private Range currentRange;
      private long next = findNext();

      private long findNext() {
        if (currentRange == null || next + 1 > currentRange.to) {
          if (currentRangeIndex + 1 < ranges.size()) {
            currentRange = ranges.get(++currentRangeIndex);
            next = currentRange.from;
          } else {
            return -1;
          }
        } else {
          ++next;
        }
        return next;
      }

      @Override
      public boolean hasNext() {
        return next >= 0;
      }

      @Override
      public long nextLong() {
        long ret = next;
        if (ret < 0) {
          throw new NoSuchElementException();
        }
        next = findNext();
        return ret;
      }
    };
  }

  /**
   * @param from
   *          the first row of the range to be checked for connection
   * @param to
   *          the last row of the range to be checked for connection
   * @return {@code true} if the specified range is overlapping (have common elements) with one of the ranges
   */
  public boolean isOverlapping(long from, long to) {
    return Collections.binarySearch(ranges, new Range(from, to),
        (r1, r2) -> r1.isBefore(r2) ? -1 : r1.isAfter(r2) ? 1 : 0) >= 0;
  }

  @Override
  public String toString() {
    return ranges.toString();
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.hadoop.metadata;

import java.util.Set;

import org.apache.parquet.column.Encoding;
import org.apache.parquet.column.EncodingStats;
import org.apache.parquet.column.statistics.BooleanStatistics;
import org.apache.parquet.column.statistics.Statistics;
import org.apache.parquet.internal.hadoop.metadata.IndexReference;
import org.apache.parquet.schema.PrimitiveType;
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
import org.apache.parquet.schema.Types;
import org.apache.yetus.audience.InterfaceAudience.Private;

/**
 * Column meta data for a block stored in the file footer and passed in the InputSplit
 */
abstract public class ColumnChunkMetaData {

  @Deprecated
  public static ColumnChunkMetaData get(
      ColumnPath path,
      PrimitiveTypeName type,
      CompressionCodecName codec,
      Set<Encoding> encodings,
      long firstDataPage,
      long dictionaryPageOffset,
      long valueCount,
      long totalSize,
      long totalUncompressedSize) {
    return get(
        path, type, codec, null, encodings, new BooleanStatistics(), firstDataPage,
        dictionaryPageOffset, valueCount, totalSize, totalUncompressedSize);
  }

  @Deprecated
  public static ColumnChunkMetaData get(
      ColumnPath path,
      PrimitiveTypeName type,
      CompressionCodecName codec,
      Set<Encoding> encodings,
      Statistics statistics,
      long firstDataPage,
      long dictionaryPageOffset,
      long valueCount,
      long totalSize,
      long totalUncompressedSize) {
    return get(
        path, type, codec, null, encodings, statistics, firstDataPage, dictionaryPageOffset,
        valueCount, totalSize, totalUncompressedSize);
  }

  /**
   * @param path the path of this column in the write schema
   * @param type primitive type for this column
   * @param codec the compression codec used to compress
   * @param encodingStats EncodingStats for the encodings used in this column
   * @param encodings a set of encoding used in this column
   * @param statistics statistics for the data in this column
   * @param firstDataPage offset of the first non-dictionary page
   * @param dictionaryPageOffset offset of the the dictionary page
   * @param valueCount number of values
   * @param totalSize total compressed size
   * @param totalUncompressedSize uncompressed data size
   * @return a column chunk metadata instance
   * @deprecated will be removed in 2.0.0. Use
   *             {@link #get(ColumnPath, PrimitiveType, CompressionCodecName, EncodingStats, Set, Statistics, long, long, long, long, long)}
   *             instead.
   */
  @Deprecated
  public static ColumnChunkMetaData get(
      ColumnPath path,
      PrimitiveTypeName type,
      CompressionCodecName codec,
      EncodingStats encodingStats,
      Set<Encoding> encodings,
      Statistics statistics,
      long firstDataPage,
      long dictionaryPageOffset,
      long valueCount,
      long totalSize,
      long totalUncompressedSize) {
    return get(path, Types.optional(type).named("fake_type"), codec, encodingStats, encodings, statistics,
        firstDataPage, dictionaryPageOffset, valueCount, totalSize, totalUncompressedSize);
  }

  public static ColumnChunkMetaData get(
      ColumnPath path,
      PrimitiveType type,
      CompressionCodecName codec,
      EncodingStats encodingStats,
      Set<Encoding> encodings,
      Statistics statistics,
      long firstDataPage,
      long dictionaryPageOffset,
      long valueCount,
      long totalSize,
      long totalUncompressedSize) {
    // to save space we store those always positive longs in ints when they fit.
    if (positiveLongFitsInAnInt(firstDataPage)
        && positiveLongFitsInAnInt(dictionaryPageOffset)
        && positiveLongFitsInAnInt(valueCount)
        && positiveLongFitsInAnInt(totalSize)
        && positiveLongFitsInAnInt(totalUncompressedSize)) {
      return new IntColumnChunkMetaData(
          path, type, codec,
          encodingStats, encodings,
          statistics,
          firstDataPage,
          dictionaryPageOffset,
          valueCount,
          totalSize,
          totalUncompressedSize);
    } else {
      return new LongColumnChunkMetaData(
          path, type, codec,
          encodingStats, encodings,
          statistics,
          firstDataPage,
          dictionaryPageOffset,
          valueCount,
          totalSize,
          totalUncompressedSize);
    }
  }

  /**
   * @return the offset of the first byte in the chunk
   */
  public long getStartingPos() {
    long dictionaryPageOffset = getDictionaryPageOffset();
    long firstDataPageOffset = getFirstDataPageOffset();
    if (dictionaryPageOffset > 0 && dictionaryPageOffset < firstDataPageOffset) {
      // if there's a dictionary and it's before the first data page, start from there
      return dictionaryPageOffset;
    }
    return firstDataPageOffset;
  }

  /**
   * checks that a positive long value fits in an int.
   * (reindexed on Integer.MIN_VALUE)
   * @param value a long value
   * @return whether it fits
   */
  protected static boolean positiveLongFitsInAnInt(long value) {
    return (value >= 0) && (value + Integer.MIN_VALUE <= Integer.MAX_VALUE);
  }

  private final EncodingStats encodingStats;

  // we save 3 references by storing together the column properties that have few distinct values
  private final ColumnChunkProperties properties;

  private IndexReference columnIndexReference;
  private IndexReference offsetIndexReference;

  protected ColumnChunkMetaData(ColumnChunkProperties columnChunkProperties) {
    this(null, columnChunkProperties);
  }

  protected ColumnChunkMetaData(EncodingStats encodingStats, ColumnChunkProperties columnChunkProperties) {
    this.encodingStats = encodingStats;
    this.properties = columnChunkProperties;
  }

  public CompressionCodecName getCodec() {
    return properties.getCodec();
  }

  /**
   *
   * @return column identifier
   */
  public ColumnPath getPath() {
    return properties.getPath();
  }

  /**
   * @return type of the column
   * @deprecated will be removed in 2.0.0. Use {@link #getPrimitiveType()} instead.
   */
  @Deprecated
  public PrimitiveTypeName getType() {
    return properties.getType();
  }

  /**
   * @return the primitive type object of the column
   */
  public PrimitiveType getPrimitiveType() {
    return properties.getPrimitiveType();
  }

  /**
   * @return start of the column data offset
   */
  abstract public long getFirstDataPageOffset();

  /**
   * @return the location of the dictionary page if any
   */
  abstract public long getDictionaryPageOffset();

  /**
   * @return count of values in this block of the column
   */
  abstract public long getValueCount();

  /**
   * @return the totalUncompressedSize
   */
  abstract public long getTotalUncompressedSize();

  /**
   * @return the totalSize
   */
  abstract public long getTotalSize();

  /**
   * @return the stats for this column
   */
  abstract public Statistics getStatistics();

  /**
   * @return the reference to the column index
   */
  @Private
  public IndexReference getColumnIndexReference() {
    return columnIndexReference;
  }

  /**
   * @param indexReference
   *          the reference to the column index
   */
  @Private
  public void setColumnIndexReference(IndexReference indexReference) {
    this.columnIndexReference = indexReference;
  }

  /**
   * @return the reference to the offset index
   */
  @Private
  public IndexReference getOffsetIndexReference() {
    return offsetIndexReference;
  }

  /**
   * @param offsetIndexReference
   *          the reference to the offset index
   */
  @Private
  public void setOffsetIndexReference(IndexReference offsetIndexReference) {
    this.offsetIndexReference = offsetIndexReference;
  }

  /**
   * @return all the encodings used in this column
   */
  public Set<Encoding> getEncodings() {
    return properties.getEncodings();
  }

  public EncodingStats getEncodingStats() {
    return encodingStats;
  }

  @Override
  public String toString() {
    return "ColumnMetaData{" + properties.toString() + ", " + getFirstDataPageOffset() + "}";
  }
}

class IntColumnChunkMetaData extends ColumnChunkMetaData {

  private final int firstDataPage;
  private final int dictionaryPageOffset;
  private final int valueCount;
  private final int totalSize;
  private final int totalUncompressedSize;
  private final Statistics statistics;

  /**
   * @param path column identifier
   * @param type type of the column
   * @param codec
   * @param encodings
   * @param statistics
   * @param firstDataPage
   * @param dictionaryPageOffset
   * @param valueCount
   * @param totalSize
   * @param totalUncompressedSize
   */
  IntColumnChunkMetaData(
      ColumnPath path,
      PrimitiveType type,
      CompressionCodecName codec,
      EncodingStats encodingStats,
      Set<Encoding> encodings,
      Statistics statistics,
      long firstDataPage,
      long dictionaryPageOffset,
      long valueCount,
      long totalSize,
      long totalUncompressedSize) {
    super(encodingStats, ColumnChunkProperties.get(path, type, codec, encodings));
    this.firstDataPage = positiveLongToInt(firstDataPage);
    this.dictionaryPageOffset = positiveLongToInt(dictionaryPageOffset);
    this.valueCount = positiveLongToInt(valueCount);
    this.totalSize = positiveLongToInt(totalSize);
    this.totalUncompressedSize = positiveLongToInt(totalUncompressedSize);
    this.statistics = statistics;
  }

  /**
   * stores a positive long into an int (assuming it fits)
   * @param value
   * @return
   */
  private int positiveLongToInt(long value) {
    if (!ColumnChunkMetaData.positiveLongFitsInAnInt(value)) {
      throw new IllegalArgumentException("value should be positive and fit in an int: " + value);
    }
    return (int)(value + Integer.MIN_VALUE);
  }

  /**
   * turns the int back into a positive long
   * @param value
   * @return
   */
  private long intToPositiveLong(int value) {
    return (long)value - Integer.MIN_VALUE;
  }

  /**
   * @return start of the column data offset
   */
  public long getFirstDataPageOffset() {
    return intToPositiveLong(firstDataPage);
  }

  /**
   * @return the location of the dictionary page if any
   */
  public long getDictionaryPageOffset() {
    return intToPositiveLong(dictionaryPageOffset);
  }

  /**
   * @return count of values in this block of the column
   */
  public long getValueCount() {
    return intToPositiveLong(valueCount);
  }

  /**
   * @return the totalUncompressedSize
   */
  public long getTotalUncompressedSize() {
    return intToPositiveLong(totalUncompressedSize);
  }

  /**
   * @return the totalSize
   */
  public long getTotalSize() {
    return intToPositiveLong(totalSize);
  }

  /**
   * @return the stats for this column
   */
  public Statistics getStatistics() {
   return statistics;
  }
}
class LongColumnChunkMetaData extends ColumnChunkMetaData {

  private final long firstDataPageOffset;
  private final long dictionaryPageOffset;
  private final long valueCount;
  private final long totalSize;
  private final long totalUncompressedSize;
  private final Statistics statistics;

  /**
   * @param path column identifier
   * @param type type of the column
   * @param codec
   * @param encodings
   * @param statistics
   * @param firstDataPageOffset
   * @param dictionaryPageOffset
   * @param valueCount
   * @param totalSize
   * @param totalUncompressedSize
   */
  LongColumnChunkMetaData(
      ColumnPath path,
      PrimitiveType type,
      CompressionCodecName codec,
      EncodingStats encodingStats,
      Set<Encoding> encodings,
      Statistics statistics,
      long firstDataPageOffset,
      long dictionaryPageOffset,
      long valueCount,
      long totalSize,
      long totalUncompressedSize) {
    super(encodingStats, ColumnChunkProperties.get(path, type, codec, encodings));
    this.firstDataPageOffset = firstDataPageOffset;
    this.dictionaryPageOffset = dictionaryPageOffset;
    this.valueCount = valueCount;
    this.totalSize = totalSize;
    this.totalUncompressedSize = totalUncompressedSize;
    this.statistics = statistics;
  }

  /**
   * @return start of the column data offset
   */
  public long getFirstDataPageOffset() {
    return firstDataPageOffset;
  }

  /**
   * @return the location of the dictionary page if any
   */
  public long getDictionaryPageOffset() {
    return dictionaryPageOffset;
  }

  /**
   * @return count of values in this block of the column
   */
  public long getValueCount() {
    return valueCount;
  }

  /**
   * @return the totalUncompressedSize
   */
  public long getTotalUncompressedSize() {
    return totalUncompressedSize;
  }

  /**
   * @return the totalSize
   */
  public long getTotalSize() {
    return totalSize;
  }

  /**
   * @return the stats for this column
   */
  public Statistics getStatistics() {
   return statistics;
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.glob;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

import org.apache.parquet.glob.GlobNode.Atom;
import org.apache.parquet.glob.GlobNode.GlobNodeSequence;
import org.apache.parquet.glob.GlobNode.OneOf;

final class GlobParser {
  private GlobParser() { }

  /**
   * Parse a String into a {@link GlobNodeSequence}
   *
   * See {@link org.apache.parquet.Strings#expandGlob(String)}
   */
  public static GlobNodeSequence parse(String pattern) {
    /*
     * The parse algorithm works as follows, assuming we are parsing:
     * "apache{one,pre{x,y}post,two}parquet{a,b}"
     *
     * 1) Begin scanning the string until we find the first {
     *
     * 2) Now that we've found the beginning of a glob group, scan forwards
     *    until the end of this glob group (by counting { and } we see until we find
     *    the closing } for the group we found in step 1).
     *
     * 3) Once the matching closing } is found we need to do two things. First, everything
     *    from the end of the last group up to start of this group is an Atom, so in the example
     *    above, once we've found that "{one,pre{x,y}post,two}" is the first group, we need to grab
     *    "apache" and treat it as an atom and add it to our sequence.
     *    Then, we parse "{one,pre{x,y}post,two}" using a similar but slightly different function (parseOneOf)
     *    and add the result from that to our sequence.
     *
     * 4) Repeat until the end of the string -- so next we find {a,b} and add "parquet" as an Atom and parse
     *    {a,b} using parseOneOf.
     */

    if (pattern.isEmpty() || pattern.equals("{}")) {
      return new GlobNodeSequence(Arrays.<GlobNode>asList(new Atom("")));
    }

    // the outer parse method needs to parse the pattern into a
    // GlobNodeSequence, though it may end up being a singleton sequence
    List<GlobNode> children = new ArrayList<GlobNode>();

    int unmatchedBraces = 0; // count of unmatched braces
    int firstBrace = 0; // open brace of current group being processsed
    int anchor = 0; // first un-parsed character position

    for (int i = 0; i < pattern.length(); i++) {
      char c = pattern.charAt(i);

      switch (c) {
        case ',':
          if (unmatchedBraces == 0) {
            // commas not allowed in the top level expression
            // TODO: maybe turn this check off?
            throw new GlobParseException("Unexpected comma outside of a {} group:\n"
                + annotateMessage(pattern, i));
          }
          break;
        case '{':
          if (unmatchedBraces == 0) {
            // this is the first brace of an outermost {} group
            firstBrace = i;
          }
          unmatchedBraces++;
          break;
        case '}':
          unmatchedBraces--;
          if (unmatchedBraces < 0) {
            throw new GlobParseException("Unexpected closing }:\n"
                + annotateMessage(pattern, i));
          }
          if (unmatchedBraces == 0) {
            // grab everything from the end of the last group up to here,
            // not including the close brace, it is an Atom in our sequence
            // (assuming it's not empty)
            if (anchor != firstBrace) {
              // not empty!
              // (substring's end param is exclusive)
              children.add(new Atom(pattern.substring(anchor, firstBrace)));
            }

            // grab the group, parse it, add it to our sequence, and then continue
            // note that we skip the braces on both sides (substring's end param is exclusive)
            children.add(parseOneOf(pattern.substring(firstBrace + 1, i)));

            // we have now parsed all the way up to here, the next un-parsed char is i + 1
            anchor = i + 1;
          }
          break;
      }
    }

    if (unmatchedBraces > 0) {
      throw new GlobParseException("Not enough close braces in: " + pattern);
    }

    if (anchor != pattern.length()) {
      // either there were no {} groups, or there were some characters after the
      // last }, either way whatever is left (could be the entire input) is an Atom
      // in our sequence
      children.add(new Atom(pattern.substring(anchor, pattern.length())));
    }

    return new GlobNodeSequence(children);
  }

  private static OneOf parseOneOf(String pattern) {
    /*
     * This method is only called when parsing the inside of a {} expression.
     * So in the example above, of calling parse("apache{one,pre{x,y}post,two}parquet{a,b}")
     * this method will get called on first "one,pre{x,y}post,two", then on "x,y" and then on "a,b"
     *
     * The inside of a {} expression essentially means "one of these comma separated expressions".
     * So this gets parsed slightly differently than the top level string passed to parse().
     *
     * The algorithm works as follows:
     * 1) Split the string on ',' -- but only commas that are not inside of {} expressions
     * 2) Each of the splits can be parsed via the parse() method above
     * 3) Add all parsed splits to a single parent OneOf.
     */

    // this inner parse method needs to parse the pattern into a
    // OneOf, though it may end up being a singleton OneOf
    List<GlobNode> children = new ArrayList<GlobNode>();

    int unmatchedBraces = 0; // count of unmatched braces
    int anchor = 0; // first un-parsed character position

    for (int i = 0; i < pattern.length(); i++) {
      char c = pattern.charAt(i);

      switch (c) {
        case ',':
          // only "split" on commas not nested inside of {}
          if (unmatchedBraces == 0) {
            // ok, this comma is not inside of a {}, so
            // grab everything from anchor to here, parse it, and add it
            // as one of the options in this OneOf
            children.add(parse(pattern.substring(anchor, i)));

            // we have now parsed up to this comma, the next un-parsed char is i + 1
            anchor = i + 1;
          }
          break;
        case '{':
          unmatchedBraces++;
          break;
        case '}':
          unmatchedBraces--;
          if (unmatchedBraces < 0) {
            throw new GlobParseException("Unexpected closing }:\n"
                + annotateMessage(pattern, i));
          }
          break;
      }
    }

    if (unmatchedBraces > 0) {
      throw new GlobParseException("Not enough close braces in: " + pattern);
    }

    if (anchor != pattern.length()) {
      // either there were no commas outside of {} groups, or there were some characters after the
      // last comma, either way whatever is left (could be the entire input) is an Atom
      // in our sequence
      children.add(parse(pattern.substring(anchor, pattern.length())));
    }

    if (pattern.length() > 0 && pattern.charAt(pattern.length() - 1) == ',') {
      // the above loop won't handle a trailing comma
      children.add(parse(""));
    }

    return new OneOf(children);
  }

  // for pretty printing which character had the error
  private static String annotateMessage(String message, int pos) {
    StringBuilder sb = new StringBuilder(message);
    sb.append('\n');
    for (int i = 0; i < pos; i++) {
      sb.append('-');
    }
    sb.append('^');
    return sb.toString();
  }

  public static class GlobParseException extends RuntimeException {
    public GlobParseException() {
    }

    public GlobParseException(String message) {
      super(message);
    }

    public GlobParseException(String message, Throwable cause) {
      super(message, cause);
    }
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.column.values.rle;

import java.io.IOException;

import org.apache.parquet.bytes.ByteBufferAllocator;
import org.apache.parquet.Preconditions;
import org.apache.parquet.bytes.BytesInput;
import org.apache.parquet.bytes.BytesUtils;
import org.apache.parquet.bytes.CapacityByteArrayOutputStream;
import org.apache.parquet.column.values.bitpacking.BytePacker;
import org.apache.parquet.column.values.bitpacking.Packer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Encodes values using a combination of run length encoding and bit packing,
 * according to the following grammar:
 *
 *

 * {@code
 * rle-bit-packed-hybrid: <length> <encoded-data>
 * length := length of the <encoded-data> in bytes stored as 4 bytes little endian
 * encoded-data := <run>*
 * run := <bit-packed-run> | <rle-run>
 * bit-packed-run := <bit-packed-header> <bit-packed-values>
 * bit-packed-header := varint-encode(<bit-pack-count> << 1 | 1)
 * // we always bit-pack a multiple of 8 values at a time, so we only store the number of values / 8
 * bit-pack-count := (number of values in this run) / 8
 * bit-packed-values :=  bit packed back to back, from LSB to MSB
 * rle-run := <rle-header> <repeated-value>
 * rle-header := varint-encode( (number of times repeated) << 1)
 * repeated-value := value that is repeated, using a fixed-width of round-up-to-next-byte(bit-width)
 * }
 *

 * NOTE: this class is only responsible for creating and returning the {@code <encoded-data>}
 *       portion of the above grammar. The {@code <length>} portion is done by
 *       {@link RunLengthBitPackingHybridValuesWriter}
 *

 * Only supports positive values (including 0) // TODO: is that ok? Should we make a signed version?
 */
public class RunLengthBitPackingHybridEncoder {
  private static final Logger LOG = LoggerFactory.getLogger(RunLengthBitPackingHybridEncoder.class);

  private final BytePacker packer;

  private final CapacityByteArrayOutputStream baos;

  /**
   * The bit width used for bit-packing and for writing
   * the repeated-value
   */
  private final int bitWidth;

  /**
   * Values that are bit packed 8 at at a time are packed into this
   * buffer, which is then written to baos
   */
  private final byte[] packBuffer;

  /**
   * Previous value written, used to detect repeated values
   */
  private int previousValue;

  /**
   * We buffer 8 values at a time, and either bit pack them
   * or discard them after writing a rle-run
   */
  private final int[] bufferedValues;
  private int numBufferedValues;

  /**
   * How many times a value has been repeated
   */
  private int repeatCount;

  /**
   * How many groups of 8 values have been written
   * to the current bit-packed-run
   */
  private int bitPackedGroupCount;

  /**
   * A "pointer" to a single byte in baos,
   * which we use as our bit-packed-header. It's really
   * the logical index of the byte in baos.
   *
   * We are only using one byte for this header,
   * which limits us to writing 504 values per bit-packed-run.
   *
   * MSB must be 0 for varint encoding, LSB must be 1 to signify
   * that this is a bit-packed-header leaves 6 bits to write the
   * number of 8-groups -> (2^6 - 1) * 8 = 504
   */
  private long bitPackedRunHeaderPointer;

  private boolean toBytesCalled;

  public RunLengthBitPackingHybridEncoder(int bitWidth, int initialCapacity, int pageSize, ByteBufferAllocator allocator) {
    LOG.debug("Encoding: RunLengthBitPackingHybridEncoder with "
      + "bithWidth: {} initialCapacity {}", bitWidth, initialCapacity);

    Preconditions.checkArgument(bitWidth >= 0 && bitWidth <= 32, "bitWidth must be >= 0 and <= 32");

    this.bitWidth = bitWidth;
    this.baos = new CapacityByteArrayOutputStream(initialCapacity, pageSize, allocator);
    this.packBuffer = new byte[bitWidth];
    this.bufferedValues = new int[8];
    this.packer = Packer.LITTLE_ENDIAN.newBytePacker(bitWidth);
    reset(false);
  }

  private void reset(boolean resetBaos) {
    if (resetBaos) {
      this.baos.reset();
    }
    this.previousValue = 0;
    this.numBufferedValues = 0;
    this.repeatCount = 0;
    this.bitPackedGroupCount = 0;
    this.bitPackedRunHeaderPointer = -1;
    this.toBytesCalled = false;
  }

  public void writeInt(int value) throws IOException {
    if (value == previousValue) {
      // keep track of how many times we've seen this value
      // consecutively
      ++repeatCount;

      if (repeatCount >= 8) {
        // we've seen this at least 8 times, we're
        // certainly going to write an rle-run,
        // so just keep on counting repeats for now
        return;
      }
    } else {
      // This is a new value, check if it signals the end of
      // an rle-run
      if (repeatCount >= 8) {
        // it does! write an rle-run
        writeRleRun();
      }

      // this is a new value so we've only seen it once
      repeatCount = 1;
      // start tracking this value for repeats
      previousValue = value;
    }

    // We have not seen enough repeats to justify an rle-run yet,
    // so buffer this value in case we decide to write a bit-packed-run
    bufferedValues[numBufferedValues] = value;
    ++numBufferedValues;

    if (numBufferedValues == 8) {
      // we've encountered less than 8 repeated values, so
      // either start a new bit-packed-run or append to the
      // current bit-packed-run
      writeOrAppendBitPackedRun();
    }
  }

  private void writeOrAppendBitPackedRun() throws IOException {
    if (bitPackedGroupCount >= 63) {
      // we've packed as many values as we can for this run,
      // end it and start a new one
      endPreviousBitPackedRun();
    }

    if (bitPackedRunHeaderPointer == -1) {
      // this is a new bit-packed-run, allocate a byte for the header
      // and keep a "pointer" to it so that it can be mutated later
      baos.write(0); // write a sentinel value
      bitPackedRunHeaderPointer = baos.getCurrentIndex();
    }

    packer.pack8Values(bufferedValues, 0, packBuffer, 0);
    baos.write(packBuffer);

    // empty the buffer, they've all been written
    numBufferedValues = 0;

    // clear the repeat count, as some repeated values
    // may have just been bit packed into this run
    repeatCount = 0;

    ++bitPackedGroupCount;
  }

  /**
   * If we are currently writing a bit-packed-run, update the
   * bit-packed-header and consider this run to be over
   *
   * does nothing if we're not currently writing a bit-packed run
   */
  private void endPreviousBitPackedRun() {
    if (bitPackedRunHeaderPointer == -1) {
      // we're not currently in a bit-packed-run
      return;
    }

    // create bit-packed-header, which needs to fit in 1 byte
    byte bitPackHeader = (byte) ((bitPackedGroupCount << 1) | 1);

    // update this byte
    baos.setByte(bitPackedRunHeaderPointer, bitPackHeader);

    // mark that this run is over
    bitPackedRunHeaderPointer = -1;

    // reset the number of groups
    bitPackedGroupCount = 0;
  }

  private void writeRleRun() throws IOException {
    // we may have been working on a bit-packed-run
    // so close that run if it exists before writing this
    // rle-run
    endPreviousBitPackedRun();

    // write the rle-header (lsb of 0 signifies a rle run)
    BytesUtils.writeUnsignedVarInt(repeatCount << 1, baos);
    // write the repeated-value
    BytesUtils.writeIntLittleEndianPaddedOnBitWidth(baos, previousValue, bitWidth);

    // reset the repeat count
    repeatCount = 0;

    // throw away all the buffered values, they were just repeats and they've been written
    numBufferedValues = 0;
  }

  public BytesInput toBytes() throws IOException {
    Preconditions.checkArgument(!toBytesCalled,
        "You cannot call toBytes() more than once without calling reset()");

    // write anything that is buffered / queued up for an rle-run
    if (repeatCount >= 8) {
      writeRleRun();
    } else if(numBufferedValues > 0) {
      for (int i = numBufferedValues; i < 8; i++) {
        bufferedValues[i] = 0;
      }
      writeOrAppendBitPackedRun();
      endPreviousBitPackedRun();
    } else {
      endPreviousBitPackedRun();
    }

    toBytesCalled = true;
    return BytesInput.from(baos);
  }

  /**
   * Reset this encoder for re-use
   */
  public void reset() {
    reset(true);
  }

  public void close() {
    reset(false);
    baos.close();
  }

  public long getBufferedSize() {
    return baos.size();
  }

  public long getAllocatedSize() {
    return baos.getCapacity();
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.thrift;

import org.apache.thrift.TException;
import org.apache.thrift.protocol.TField;
import org.apache.thrift.protocol.TList;
import org.apache.thrift.protocol.TMap;
import org.apache.thrift.protocol.TProtocol;
import org.apache.thrift.protocol.TSet;
import org.apache.thrift.protocol.TStruct;
import org.apache.thrift.protocol.TType;

/**
 * Class to read from one protocol and write to another one
 */
public class ProtocolReadToWrite implements ProtocolPipe {

  /**
   * reads one record from in and writes it to out
   * exceptions are not recoverable as record might be halfway written
   * @param in input protocol
   * @param out output protocol
   * @throws TException if there is an error while reading or writing
   */
  @Override
  public void readOne(TProtocol in, TProtocol out) throws TException {
    readOneStruct(in, out);
  }

  void readOneValue(TProtocol in, TProtocol out, byte type)
      throws TException {
    switch (type) {
    case TType.LIST:
      readOneList(in, out);
      break;
    case TType.MAP:
      readOneMap(in, out);
      break;
    case TType.SET:
      readOneSet(in, out);
      break;
    case TType.STRUCT:
      readOneStruct(in, out);
      break;
    case TType.STOP:
      break;
    case TType.BOOL:
      out.writeBool(in.readBool());
      break;
    case TType.BYTE:
      out.writeByte(in.readByte());
      break;
    case TType.DOUBLE:
      out.writeDouble(in.readDouble());
      break;
    case TType.I16:
      out.writeI16(in.readI16());
      break;
    case TType.ENUM: // same as i32 => actually never seen in the protocol layer as enums are written as a i32 field
    case TType.I32:
      out.writeI32(in.readI32());
      break;
    case TType.I64:
      out.writeI64(in.readI64());
      break;
    case TType.STRING:
      out.writeBinary(in.readBinary());
      break;
    case TType.VOID:
      break;
    default:
      throw new TException("Unknown type: " + type);
    }
  }

  private void readOneStruct(TProtocol in, TProtocol out) throws TException {
    final TStruct struct = in.readStructBegin();
    out.writeStructBegin(struct);
    TField field;
    while ((field = in.readFieldBegin()).type != TType.STOP) {
      out.writeFieldBegin(field);
      readOneValue(in, out, field.type);
      in.readFieldEnd();
      out.writeFieldEnd();
    }
    out.writeFieldStop();
    in.readStructEnd();
    out.writeStructEnd();
  }

  private void readOneMap(TProtocol in, TProtocol out) throws TException {
    final TMap map = in.readMapBegin();
    out.writeMapBegin(map);
    for (int i = 0; i < map.size; i++) {
      readOneValue(in, out, map.keyType);
      readOneValue(in, out, map.valueType);
    }
    in.readMapEnd();
    out.writeMapEnd();
  }

  private void readOneSet(TProtocol in, TProtocol out) throws TException {
    final TSet set = in.readSetBegin();
    out.writeSetBegin(set);
    readCollectionElements(in, out, set.size, set.elemType);
    in.readSetEnd();
    out.writeSetEnd();
  }

  private void readOneList(TProtocol in, TProtocol out) throws TException {
    final TList list = in.readListBegin();
    out.writeListBegin(list);
    readCollectionElements(in, out, list.size, list.elemType);
    in.readListEnd();
    out.writeListEnd();
  }

  private void readCollectionElements(TProtocol in, TProtocol out,
      final int size, final byte elemType) throws TException {
    for (int i = 0; i < size; i++) {
      readOneValue(in, out, elemType);
    }
  }

}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.filter2.compat;

import org.apache.parquet.filter.UnboundRecordFilter;
import org.apache.parquet.filter2.predicate.FilterPredicate;
import org.apache.parquet.filter2.predicate.LogicalInverseRewriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static org.apache.parquet.Preconditions.checkArgument;
import static org.apache.parquet.Preconditions.checkNotNull;

/**
 * Parquet currently has two ways to specify a filter for dropping records at read time.
 * The first way, that only supports filtering records during record assembly, is found
 * in {@link org.apache.parquet.filter}. The new API (found in {@link org.apache.parquet.filter2}) supports
 * also filtering entire rowgroups of records without reading them at all.
 *

 * This class defines a common interface that both of these filters share,
 * {@link Filter}. A Filter can be either an {@link UnboundRecordFilter} from the old API, or
 * a {@link FilterPredicate} from the new API, or a sentinel no-op filter.
 *

 * Having this common interface simplifies passing a filter through the read path of parquet's
 * codebase.
 */
public class FilterCompat {
  private static final Logger LOG = LoggerFactory.getLogger(FilterCompat.class);

  /**
   * Anyone wanting to use a {@link Filter} need only implement this interface,
   * per the visitor pattern.
   */
  public static interface Visitor<T> {
    T visit(FilterPredicateCompat filterPredicateCompat);
    T visit(UnboundRecordFilterCompat unboundRecordFilterCompat);
    T visit(NoOpFilter noOpFilter);
  }

  public static interface Filter {
    <R> R accept(Visitor<R> visitor);
  }

  // sentinel no op filter that signals "do no filtering"
  public static final Filter NOOP = new NoOpFilter();

  /**
   * Given a FilterPredicate, return a Filter that wraps it.
   * This method also logs the filter being used and rewrites
   * the predicate to not include the not() operator.
   *
   * @param filterPredicate a filter predicate
   * @return a filter for the given predicate
   */
  public static Filter get(FilterPredicate filterPredicate) {
    checkNotNull(filterPredicate, "filterPredicate");

    LOG.info("Filtering using predicate: {}", filterPredicate);

    // rewrite the predicate to not include the not() operator
    FilterPredicate collapsedPredicate = LogicalInverseRewriter.rewrite(filterPredicate);

    if (!filterPredicate.equals(collapsedPredicate)) {
      LOG.info("Predicate has been collapsed to: {}", collapsedPredicate);
    }

    return new FilterPredicateCompat(collapsedPredicate);
  }

  /**
   * Given an UnboundRecordFilter, return a Filter that wraps it.
   *
   * @param unboundRecordFilter an unbound record filter
   * @return a Filter for the given record filter (from the old API)
   */
  public static Filter get(UnboundRecordFilter unboundRecordFilter) {
    return new UnboundRecordFilterCompat(unboundRecordFilter);
  }

  /**
   * Given either a FilterPredicate or the class of an UnboundRecordFilter, or neither (but not both)
   * return a Filter that wraps whichever was provided.
   *

   * Either filterPredicate or unboundRecordFilterClass must be null, or an exception is thrown.
   *

   * If both are null, the no op filter will be returned.
   *
   * @param filterPredicate a filter predicate, or null
   * @param unboundRecordFilter an unbound record filter, or null
   * @return a Filter wrapping either the predicate or the unbound record filter (from the old API)
   */
  public static Filter get(FilterPredicate filterPredicate, UnboundRecordFilter unboundRecordFilter) {
    checkArgument(filterPredicate == null || unboundRecordFilter == null,
        "Cannot provide both a FilterPredicate and an UnboundRecordFilter");

    if (filterPredicate != null) {
      return get(filterPredicate);
    }

    if (unboundRecordFilter != null) {
      return get(unboundRecordFilter);
    }

    return NOOP;
  }

  // wraps a FilterPredicate
  public static final class FilterPredicateCompat implements Filter {
    private final FilterPredicate filterPredicate;

    private FilterPredicateCompat(FilterPredicate filterPredicate) {
      this.filterPredicate = checkNotNull(filterPredicate, "filterPredicate");
    }

    public FilterPredicate getFilterPredicate() {
      return filterPredicate;
    }

    @Override
    public <R> R accept(Visitor<R> visitor) {
      return visitor.visit(this);
    }
  }

  // wraps an UnboundRecordFilter
  public static final class UnboundRecordFilterCompat implements Filter {
    private final UnboundRecordFilter unboundRecordFilter;

    private UnboundRecordFilterCompat(UnboundRecordFilter unboundRecordFilter) {
      this.unboundRecordFilter = checkNotNull(unboundRecordFilter, "unboundRecordFilter");
    }

    public UnboundRecordFilter getUnboundRecordFilter() {
      return unboundRecordFilter;
    }

    @Override
    public <R> R accept(Visitor<R> visitor) {
      return visitor.visit(this);
    }
  }

  // sentinel no op filter
  public static final class NoOpFilter implements Filter {
    private NoOpFilter() {}

    @Override
    public <R> R accept(Visitor<R> visitor) {
      return visitor.visit(this);
    }
  }

}