/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.filter2.predicate;

import java.io.Serializable;

import org.apache.parquet.hadoop.metadata.ColumnPath;
import org.apache.parquet.filter2.predicate.Operators.And;
import org.apache.parquet.filter2.predicate.Operators.BinaryColumn;
import org.apache.parquet.filter2.predicate.Operators.BooleanColumn;
import org.apache.parquet.filter2.predicate.Operators.Column;
import org.apache.parquet.filter2.predicate.Operators.DoubleColumn;
import org.apache.parquet.filter2.predicate.Operators.Eq;
import org.apache.parquet.filter2.predicate.Operators.FloatColumn;
import org.apache.parquet.filter2.predicate.Operators.Gt;
import org.apache.parquet.filter2.predicate.Operators.GtEq;
import org.apache.parquet.filter2.predicate.Operators.IntColumn;
import org.apache.parquet.filter2.predicate.Operators.LongColumn;
import org.apache.parquet.filter2.predicate.Operators.Lt;
import org.apache.parquet.filter2.predicate.Operators.LtEq;
import org.apache.parquet.filter2.predicate.Operators.Not;
import org.apache.parquet.filter2.predicate.Operators.NotEq;
import org.apache.parquet.filter2.predicate.Operators.Or;
import org.apache.parquet.filter2.predicate.Operators.SupportsEqNotEq;
import org.apache.parquet.filter2.predicate.Operators.SupportsLtGt;
import org.apache.parquet.filter2.predicate.Operators.UserDefined;
import org.apache.parquet.filter2.predicate.Operators.UserDefinedByClass;
import org.apache.parquet.filter2.predicate.Operators.UserDefinedByInstance;

/**
 * The Filter API is expressed through these static methods.
 *

 * Example usage:
 *

 *   IntColumn foo = intColumn("foo");
 *   DoubleColumn bar = doubleColumn("x.y.bar");
 *
 *   // foo == 10 || bar <= 17.0
 *   FilterPredicate pred = or(eq(foo, 10), ltEq(bar, 17.0));
 *

 */
// TODO: Support repeated columns (https://issues.apache.org/jira/browse/PARQUET-34)
//
// TODO: Support filtering on groups (eg, filter where this group is / isn't null)
// TODO: (https://issues.apache.org/jira/browse/PARQUET-43)

// TODO: Consider adding support for more column types that aren't coupled with parquet types, eg Column<String>
// TODO: (https://issues.apache.org/jira/browse/PARQUET-35)
public final class FilterApi {
  private FilterApi() { }

  public static IntColumn intColumn(String columnPath) {
    return new IntColumn(ColumnPath.fromDotString(columnPath));
  }

  public static LongColumn longColumn(String columnPath) {
    return new LongColumn(ColumnPath.fromDotString(columnPath));
  }

  public static FloatColumn floatColumn(String columnPath) {
    return new FloatColumn(ColumnPath.fromDotString(columnPath));
  }

  public static DoubleColumn doubleColumn(String columnPath) {
    return new DoubleColumn(ColumnPath.fromDotString(columnPath));
  }

  public static BooleanColumn booleanColumn(String columnPath) {
    return new BooleanColumn(ColumnPath.fromDotString(columnPath));
  }

  public static BinaryColumn binaryColumn(String columnPath) {
    return new BinaryColumn(ColumnPath.fromDotString(columnPath));
  }

  /**
   * Keeps records if their value is equal to the provided value.
   * Nulls are treated the same way the java programming language does.
   *

   * For example:
   *   eq(column, null) will keep all records whose value is null.
   *   eq(column, 7) will keep all records whose value is 7, and will drop records whose value is null
   *
   * @param column a column reference created by FilterApi
   * @param value a value that matches the column's type
   * @param <T> the Java type of values in the column
   * @param <C> the column type that corresponds to values of type T
   * @return an equals predicate for the given column and value
   */
  public static <T extends Comparable<T>, C extends Column<T> & SupportsEqNotEq> Eq<T> eq(C column, T value) {
    return new Eq<T>(column, value);
  }

  /**
   * Keeps records if their value is not equal to the provided value.
   * Nulls are treated the same way the java programming language does.
   *

   * For example:
   *   notEq(column, null) will keep all records whose value is not null.
   *   notEq(column, 7) will keep all records whose value is not 7, including records whose value is null.
   *
   *   NOTE: this is different from how some query languages handle null. For example, SQL and pig will drop
   *   nulls when you filter by not equal to 7. To achieve similar behavior in this api, do:
   *   and(notEq(column, 7), notEq(column, null))
   *
   *   NOTE: be sure to read the {@link #lt}, {@link #ltEq}, {@link #gt}, {@link #gtEq} operator's docs
   *         for how they handle nulls
   *
   * @param column a column reference created by FilterApi
   * @param value a value that matches the column's type
   * @param <T> the Java type of values in the column
   * @param <C> the column type that corresponds to values of type T
   * @return a not-equals predicate for the given column and value
   */
  public static <T extends Comparable<T>, C extends Column<T> & SupportsEqNotEq> NotEq<T> notEq(C column, T value) {
    return new NotEq<T>(column, value);
  }

  /**
   * Keeps records if their value is less than (but not equal to) the provided value.
   * The provided value cannot be null, as less than null has no meaning.
   * Records with null values will be dropped.
   *

   * For example:
   *   lt(column, 7) will keep all records whose value is less than (but not equal to) 7, and not null.
   *
   * @param column a column reference created by FilterApi
   * @param value a value that matches the column's type
   * @param <T> the Java type of values in the column
   * @param <C> the column type that corresponds to values of type T
   * @return a less-than predicate for the given column and value
   */
  public static <T extends Comparable<T>, C extends Column<T> & SupportsLtGt> Lt<T> lt(C column, T value) {
    return new Lt<T>(column, value);
  }

  /**
   * Keeps records if their value is less than or equal to the provided value.
   * The provided value cannot be null, as less than null has no meaning.
   * Records with null values will be dropped.
   *

   * For example:
   *   ltEq(column, 7) will keep all records whose value is less than or equal to 7, and not null.
   *
   * @param column a column reference created by FilterApi
   * @param value a value that matches the column's type
   * @param <T> the Java type of values in the column
   * @param <C> the column type that corresponds to values of type T
   * @return a less-than-or-equal predicate for the given column and value
   */
  public static <T extends Comparable<T>, C extends Column<T> & SupportsLtGt> LtEq<T> ltEq(C column, T value) {
    return new LtEq<T>(column, value);
  }

  /**
   * Keeps records if their value is greater than (but not equal to) the provided value.
   * The provided value cannot be null, as less than null has no meaning.
   * Records with null values will be dropped.
   *

   * For example:
   *   gt(column, 7) will keep all records whose value is greater than (but not equal to) 7, and not null.
   *
   * @param column a column reference created by FilterApi
   * @param value a value that matches the column's type
   * @param <T> the Java type of values in the column
   * @param <C> the column type that corresponds to values of type T
   * @return a greater-than predicate for the given column and value
   */
  public static <T extends Comparable<T>, C extends Column<T> & SupportsLtGt> Gt<T> gt(C column, T value) {
    return new Gt<T>(column, value);
  }

  /**
   * Keeps records if their value is greater than or equal to the provided value.
   * The provided value cannot be null, as less than null has no meaning.
   * Records with null values will be dropped.
   *

   * For example:
   *   gtEq(column, 7) will keep all records whose value is greater than or equal to 7, and not null.
   *
   * @param column a column reference created by FilterApi
   * @param value a value that matches the column's type
   * @param <T> the Java type of values in the column
   * @param <C> the column type that corresponds to values of type T
   * @return a greater-than-or-equal predicate for the given column and value
   */
  public static <T extends Comparable<T>, C extends Column<T> & SupportsLtGt> GtEq<T> gtEq(C column, T value) {
    return new GtEq<T>(column, value);
  }

  /**
   * Keeps records that pass the provided {@link UserDefinedPredicate}
   *

   * The provided class must have a default constructor. To use an instance
   * of a UserDefinedPredicate instead, see userDefined below.
   *
   * @param column a column reference created by FilterApi
   * @param clazz a user-defined predicate class
   * @param <T> the Java type of values in the column
   * @param  a user-defined predicate for values of type T
   * @return a user-defined predicate for the given column
   */
  public static <T extends Comparable<T>, U extends UserDefinedPredicate<T>>
    UserDefined<T, U> userDefined(Column<T> column, Class clazz) {
    return new UserDefinedByClass<T, U>(column, clazz);
  }

  /**
   * Keeps records that pass the provided {@link UserDefinedPredicate}
   *

   * The provided instance of UserDefinedPredicate must be serializable.
   *
   * @param column a column reference created by FilterApi
   * @param udp a user-defined predicate instance
   * @param <T> the Java type of values in the column
   * @param  a user-defined predicate for values of type T
   * @return a user-defined predicate for the given column
   */
  public static <T extends Comparable<T>, U extends UserDefinedPredicate<T> & Serializable>
    UserDefined<T, U> userDefined(Column<T> column, U udp) {
    return new UserDefinedByInstance<T, U>(column, udp);
  }

  /**
   * Constructs the logical and of two predicates. Records will be kept if both the left and right predicate agree
   * that the record should be kept.
   *
   * @param left a predicate
   * @param right a predicate
   * @return an and predicate from the result of the left and right predicates
   */
  public static FilterPredicate and(FilterPredicate left, FilterPredicate right) {
    return new And(left, right);
  }

  /**
   * Constructs the logical or of two predicates. Records will be kept if either the left or right predicate
   * is satisfied (or both).
   *
   * @param left a predicate
   * @param right a predicate
   * @return an or predicate from the result of the left and right predicates
   */
  public static FilterPredicate or(FilterPredicate left, FilterPredicate right) {
    return new Or(left, right);
  }

  /**
   * Constructs the logical not (or inverse) of a predicate.
   * Records will be kept if the provided predicate is not satisfied.
   *
   * @param predicate a predicate
   * @return a not predicate wrapping the result of the given predicate
   */
  public static FilterPredicate not(FilterPredicate predicate) {
    return new Not(predicate);
  }

}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.logging.Level;

/**
 * Simple wrapper around java.util.logging
 * Adds compile time log level.
 * The compiler removes completely if statements that reference to a false constant
 *
 * <code>
 *   if (DEBUG) LOG.debug("removed by the compiler if DEBUG is a false constant")
 * </code>
 */
public class Log {

  /**
   * this is the compile time log level
   */
  public static final Level LEVEL = Level.INFO; // should be INFO unless for debugging

  public static final boolean DEBUG = (LEVEL.intValue() <= Level.FINE.intValue());
  public static final boolean INFO = (LEVEL.intValue() <= Level.INFO.intValue());
  public static final boolean WARN = (LEVEL.intValue() <= Level.WARNING.intValue());
  public static final boolean ERROR = (LEVEL.intValue() <= Level.SEVERE.intValue());

  /**
   *
   * @param c the current class
   * @return the corresponding logger
   * @deprecated will be removed in 2.0.0; use org.slf4j.LoggerFactory instead.
   */
  public static Log getLog(Class<?> c) {
    return new Log(c);
  }

  private Logger logger;

  public Log(Class<?> c) {
    this.logger = LoggerFactory.getLogger(c);
  }

  /**
   * prints a debug message
   * @param m a log message
   */
  public void debug(Object m) {
    if (m instanceof Throwable) {
      logger.debug("", (Throwable) m);
    } else {
      logger.debug(String.valueOf(m));
    }
  }

  /**
   * prints a debug message
   * @param m a log message
   * @param t a throwable error
   */
  public void debug(Object m, Throwable t) {
    logger.debug(String.valueOf(m), t);
  }

  /**
   * prints an info message
   * @param m a log message
   */
  public void info(Object m) {
    if (m instanceof Throwable) {
      logger.info("", (Throwable) m);
    } else {
      logger.info(String.valueOf(m));
    }
  }

  /**
   * prints an info message
   * @param m a log message
   * @param t a throwable error
   */
  public void info(Object m, Throwable t) {
    logger.info(String.valueOf(m), t);
  }

  /**
   * prints a warn message
   * @param m a log message
   */
  public void warn(Object m) {
    if (m instanceof Throwable) {
      logger.warn("", (Throwable) m);
    } else {
      logger.warn(String.valueOf(m));
    }
  }

  /**
   * prints a warn message
   * @param m a log message
   * @param t a throwable error
   */
  public void warn(Object m, Throwable t) {
    logger.warn(String.valueOf(m), t);
  }

  /**
   * prints an error message
   * @param m a log message
   */
  public void error(Object m) {
    if (m instanceof Throwable) {
      logger.error("", (Throwable) m);
    } else {
      logger.error(String.valueOf(m));
    }
  }

  /**
   * prints an error message
   * @param m a log message
   * @param t a throwable error
   */
  public void error(Object m, Throwable t) {
    logger.error(String.valueOf(m), t);
  }

}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.filter;

import org.apache.parquet.Preconditions;
import org.apache.parquet.column.ColumnReader;

/**
 * Provides ability to chain two filters together.
 */
public final class OrRecordFilter implements RecordFilter {

  private final RecordFilter boundFilter1;
  private final RecordFilter boundFilter2;

  /**
   * Returns builder for creating an and filter.
   * @param filter1 The first filter to check.
   * @param filter2 The second filter to check.
   * @return an or record filter
   */
  public static final UnboundRecordFilter or( final UnboundRecordFilter filter1, final UnboundRecordFilter filter2 ) {
    Preconditions.checkNotNull( filter1, "filter1" );
    Preconditions.checkNotNull( filter2, "filter2" );
    return new UnboundRecordFilter() {
      @Override
      public RecordFilter bind(Iterable<ColumnReader> readers) {
        return new OrRecordFilter( filter1.bind(readers), filter2.bind( readers) );
      }
    };
  }

  /**
   * Private constructor, use OrRecordFilter.or() instead.
   */
  private OrRecordFilter( RecordFilter boundFilter1, RecordFilter boundFilter2 ) {
    this.boundFilter1 = boundFilter1;
    this.boundFilter2 = boundFilter2;
  }

  @Override
  public boolean isMatch() {
    return boundFilter1.isMatch() || boundFilter2.isMatch();
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.proto;

import com.google.protobuf.*;
import com.google.protobuf.Descriptors.Descriptor;
import com.google.protobuf.Descriptors.FieldDescriptor;
import com.twitter.elephantbird.util.Protobufs;
import org.apache.hadoop.conf.Configuration;
import org.apache.parquet.hadoop.BadConfigurationException;
import org.apache.parquet.hadoop.api.WriteSupport;
import org.apache.parquet.io.InvalidRecordException;
import org.apache.parquet.io.api.Binary;
import org.apache.parquet.io.api.RecordConsumer;
import org.apache.parquet.schema.*;
import org.apache.parquet.schema.Type;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.lang.reflect.Array;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;

import static java.util.Optional.ofNullable;

/**
 * Implementation of {@link WriteSupport} for writing Protocol Buffers.
 */
public class ProtoWriteSupport<T extends MessageOrBuilder> extends WriteSupport<T> {

  private static final Logger LOG = LoggerFactory.getLogger(ProtoWriteSupport.class);
  public static final String PB_CLASS_WRITE = "parquet.proto.writeClass";
  // PARQUET-968 introduces changes to allow writing specs compliant schemas with parquet-protobuf.
  // In the past, collection were not written using the LIST and MAP wrappers and thus were not compliant
  // with the parquet specs. This flag, is set to true, allows to write using spec compliant schemas
  // but is set to false by default to keep backward compatibility
  public static final String PB_SPECS_COMPLIANT_WRITE = "parquet.proto.writeSpecsCompliant";

  private boolean writeSpecsCompliant = false;
  private RecordConsumer recordConsumer;
  private Class<? extends Message> protoMessage;
  private MessageWriter messageWriter;

  public ProtoWriteSupport() {
  }

  public ProtoWriteSupport(Class<? extends Message> protobufClass) {
    this.protoMessage = protobufClass;
  }

  @Override
  public String getName() {
    return "protobuf";
  }

  public static void setSchema(Configuration configuration, Class<? extends Message> protoClass) {
    configuration.setClass(PB_CLASS_WRITE, protoClass, Message.class);
  }

  /**
   * Make parquet-protobuf use the LIST and MAP wrappers for collections. Set to false if you need backward
   * compatibility with parquet before PARQUET-968 (1.9.0 and older).
   * @param configuration           The hadoop configuration
   * @param writeSpecsCompliant     If set to true, the old schema style will be used (without wrappers).
   */
  public static void setWriteSpecsCompliant(Configuration configuration, boolean writeSpecsCompliant) {
    configuration.setBoolean(PB_SPECS_COMPLIANT_WRITE, writeSpecsCompliant);
  }

  /**
   * Writes Protocol buffer to parquet file.
   * @param record instance of Message.Builder or Message.
   * */
  @Override
  public void write(T record) {
    recordConsumer.startMessage();
    try {
      messageWriter.writeTopLevelMessage(record);
    } catch (RuntimeException e) {
      Message m = (record instanceof Message.Builder) ? ((Message.Builder) record).build() : (Message) record;
      LOG.error("Cannot write message " + e.getMessage() + " : " + m);
      throw e;
    }
    recordConsumer.endMessage();
  }

  @Override
  public void prepareForWrite(RecordConsumer recordConsumer) {
    this.recordConsumer = recordConsumer;
  }

  @Override
  public WriteContext init(Configuration configuration) {

    // if no protobuf descriptor was given in constructor, load descriptor from configuration (set with setProtobufClass)
    if (protoMessage == null) {
      Class<? extends Message> pbClass = configuration.getClass(PB_CLASS_WRITE, null, Message.class);
      if (pbClass != null) {
        protoMessage = pbClass;
      } else {
        String msg = "Protocol buffer class not specified.";
        String hint = " Please use method ProtoParquetOutputFormat.setProtobufClass(...) or other similar method.";
        throw new BadConfigurationException(msg + hint);
      }
    }

    writeSpecsCompliant = configuration.getBoolean(PB_SPECS_COMPLIANT_WRITE, writeSpecsCompliant);
    MessageType rootSchema = new ProtoSchemaConverter(writeSpecsCompliant).convert(protoMessage);
    Descriptor messageDescriptor = Protobufs.getMessageDescriptor(protoMessage);
    validatedMapping(messageDescriptor, rootSchema);

    this.messageWriter = new MessageWriter(messageDescriptor, rootSchema);

    Map<String, String> extraMetaData = new HashMap<String, String>();
    extraMetaData.put(ProtoReadSupport.PB_CLASS, protoMessage.getName());
    extraMetaData.put(ProtoReadSupport.PB_DESCRIPTOR, serializeDescriptor(protoMessage));
    extraMetaData.put(PB_SPECS_COMPLIANT_WRITE, String.valueOf(writeSpecsCompliant));
    return new WriteContext(rootSchema, extraMetaData);
  }

  class FieldWriter {
    String fieldName;
    int index = -1;

     void setFieldName(String fieldName) {
      this.fieldName = fieldName;
    }

    /** sets index of field inside parquet message.*/
     void setIndex(int index) {
      this.index = index;
    }

    /** Used for writing repeated fields*/
     void writeRawValue(Object value) {

    }

    /** Used for writing nonrepeated (optional, required) fields*/
    void writeField(Object value) {
      recordConsumer.startField(fieldName, index);
      writeRawValue(value);
      recordConsumer.endField(fieldName, index);
    }
  }

  class MessageWriter extends FieldWriter {

    final FieldWriter[] fieldWriters;

    @SuppressWarnings("unchecked")
    MessageWriter(Descriptor descriptor, GroupType schema) {
      List<FieldDescriptor> fields = descriptor.getFields();
      fieldWriters = (FieldWriter[]) Array.newInstance(FieldWriter.class, fields.size());

      for (FieldDescriptor fieldDescriptor: fields) {
        String name = fieldDescriptor.getName();
        Type type = schema.getType(name);
        FieldWriter writer = createWriter(fieldDescriptor, type);

        if(writeSpecsCompliant && fieldDescriptor.isRepeated() && !fieldDescriptor.isMapField()) {
          writer = new ArrayWriter(writer);
        }
        else if (!writeSpecsCompliant && fieldDescriptor.isRepeated()) {
          // the old schemas style used to write maps as repeated fields instead of wrapping them in a LIST
          writer = new RepeatedWriter(writer);
        }

        writer.setFieldName(name);
        writer.setIndex(schema.getFieldIndex(name));

        fieldWriters[fieldDescriptor.getIndex()] = writer;
      }
    }

    private FieldWriter createWriter(FieldDescriptor fieldDescriptor, Type type) {

      switch (fieldDescriptor.getJavaType()) {
        case STRING: return new StringWriter() ;
        case MESSAGE: return createMessageWriter(fieldDescriptor, type);
        case INT: return new IntWriter();
        case LONG: return new LongWriter();
        case FLOAT: return new FloatWriter();
        case DOUBLE: return new DoubleWriter();
        case ENUM: return new EnumWriter();
        case BOOLEAN: return new BooleanWriter();
        case BYTE_STRING: return new BinaryWriter();
      }

      return unknownType(fieldDescriptor);//should not be executed, always throws exception.
    }

    private FieldWriter createMessageWriter(FieldDescriptor fieldDescriptor, Type type) {
      if (fieldDescriptor.isMapField() && writeSpecsCompliant) {
        return createMapWriter(fieldDescriptor, type);
      }

      return new MessageWriter(fieldDescriptor.getMessageType(), getGroupType(type));
    }

    private GroupType getGroupType(Type type) {
      LogicalTypeAnnotation logicalTypeAnnotation = type.getLogicalTypeAnnotation();
      if (logicalTypeAnnotation == null) {
        return type.asGroupType();
      }
      return logicalTypeAnnotation.accept(new LogicalTypeAnnotation.LogicalTypeAnnotationVisitor<GroupType>() {
        @Override
        public Optional<GroupType> visit(LogicalTypeAnnotation.ListLogicalTypeAnnotation listLogicalType) {
          return ofNullable(type.asGroupType().getType("list").asGroupType().getType("element").asGroupType());
        }

        @Override
        public Optional<GroupType> visit(LogicalTypeAnnotation.MapLogicalTypeAnnotation mapLogicalType) {
          return ofNullable(type.asGroupType().getType("key_value").asGroupType().getType("value").asGroupType());
        }
      }).orElse(type.asGroupType());
    }

    private MapWriter createMapWriter(FieldDescriptor fieldDescriptor, Type type) {
      List<FieldDescriptor> fields = fieldDescriptor.getMessageType().getFields();
      if (fields.size() != 2) {
        throw new UnsupportedOperationException("Expected two fields for the map (key/value), but got: " + fields);
      }

      // KeyFieldWriter
      FieldDescriptor keyProtoField = fields.get(0);
      FieldWriter keyWriter = createWriter(keyProtoField, type);
      keyWriter.setFieldName(keyProtoField.getName());
      keyWriter.setIndex(0);

      // ValueFieldWriter
      FieldDescriptor valueProtoField = fields.get(1);
      FieldWriter valueWriter = createWriter(valueProtoField, type);
      valueWriter.setFieldName(valueProtoField.getName());
      valueWriter.setIndex(1);

      return new MapWriter(keyWriter, valueWriter);
    }

    /** Writes top level message. It cannot call startGroup() */
    void writeTopLevelMessage(Object value) {
      writeAllFields((MessageOrBuilder) value);
    }

    /** Writes message as part of repeated field. It cannot start field*/
    @Override
    final void writeRawValue(Object value) {
      recordConsumer.startGroup();
      writeAllFields((MessageOrBuilder) value);
      recordConsumer.endGroup();
    }

    /** Used for writing nonrepeated (optional, required) fields*/
    @Override
    final void writeField(Object value) {
      recordConsumer.startField(fieldName, index);
      writeRawValue(value);
      recordConsumer.endField(fieldName, index);
    }

    private void writeAllFields(MessageOrBuilder pb) {
      //returns changed fields with values. Map is ordered by id.
      Map<FieldDescriptor, Object> changedPbFields = pb.getAllFields();

      for (Map.Entry<FieldDescriptor, Object> entry : changedPbFields.entrySet()) {
        FieldDescriptor fieldDescriptor = entry.getKey();

        if(fieldDescriptor.isExtension()) {
          // Field index of an extension field might overlap with a base field.
          throw new UnsupportedOperationException(
                  "Cannot convert Protobuf message with extension field(s)");
        }

        int fieldIndex = fieldDescriptor.getIndex();
        fieldWriters[fieldIndex].writeField(entry.getValue());
      }
    }
  }

  class ArrayWriter extends FieldWriter {
    final FieldWriter fieldWriter;

    ArrayWriter(FieldWriter fieldWriter) {
      this.fieldWriter = fieldWriter;
    }

    @Override
    final void writeRawValue(Object value) {
      throw new UnsupportedOperationException("Array has no raw value");
    }

    @Override
    final void writeField(Object value) {
      recordConsumer.startField(fieldName, index);
      recordConsumer.startGroup();
      List<?> list = (List<?>) value;

      recordConsumer.startField("list", 0); // This is the wrapper group for the array field
      for (Object listEntry: list) {
        recordConsumer.startGroup();
        recordConsumer.startField("element", 0); // This is the mandatory inner field

        fieldWriter.writeRawValue(listEntry);

        recordConsumer.endField("element", 0);
        recordConsumer.endGroup();
      }
      recordConsumer.endField("list", 0);

      recordConsumer.endGroup();
      recordConsumer.endField(fieldName, index);
    }
  }

  /**
   * The RepeatedWriter is used to write collections (lists and maps) using the old style (without LIST and MAP
   * wrappers).
   */
  class RepeatedWriter extends FieldWriter {
    final FieldWriter fieldWriter;

    RepeatedWriter(FieldWriter fieldWriter) {
      this.fieldWriter = fieldWriter;
    }

    @Override
    final void writeRawValue(Object value) {
      throw new UnsupportedOperationException("Array has no raw value");
    }

    @Override
    final void writeField(Object value) {
      recordConsumer.startField(fieldName, index);
      List<?> list = (List<?>) value;

      for (Object listEntry: list) {
        fieldWriter.writeRawValue(listEntry);
      }

      recordConsumer.endField(fieldName, index);
    }
  }

  /** validates mapping between protobuffer fields and parquet fields.*/
  private void validatedMapping(Descriptor descriptor, GroupType parquetSchema) {
    List<FieldDescriptor> allFields = descriptor.getFields();

    for (FieldDescriptor fieldDescriptor: allFields) {
      String fieldName = fieldDescriptor.getName();
      int fieldIndex = fieldDescriptor.getIndex();
      int parquetIndex = parquetSchema.getFieldIndex(fieldName);
      if (fieldIndex != parquetIndex) {
        String message = "FieldIndex mismatch name=" + fieldName + ": " + fieldIndex + " != " + parquetIndex;
        throw new IncompatibleSchemaModificationException(message);
      }
    }
  }

  class StringWriter extends FieldWriter {
    @Override
    final void writeRawValue(Object value) {
      Binary binaryString = Binary.fromString((String) value);
      recordConsumer.addBinary(binaryString);
    }
  }

  class IntWriter extends FieldWriter {
    @Override
    final void writeRawValue(Object value) {
      recordConsumer.addInteger((Integer) value);
    }
  }

  class LongWriter extends FieldWriter {

    @Override
    final void writeRawValue(Object value) {
      recordConsumer.addLong((Long) value);
    }
  }

  class MapWriter extends FieldWriter {

    private final FieldWriter keyWriter;
    private final FieldWriter valueWriter;

    public MapWriter(FieldWriter keyWriter, FieldWriter valueWriter) {
      super();
      this.keyWriter = keyWriter;
      this.valueWriter = valueWriter;
    }

    @Override
    final void writeRawValue(Object value) {
      recordConsumer.startGroup();

      recordConsumer.startField("key_value", 0); // This is the wrapper group for the map field
      for (Message msg : (Collection<Message>) value) {
        recordConsumer.startGroup();

        final Descriptor descriptorForType = msg.getDescriptorForType();
        final FieldDescriptor keyDesc = descriptorForType.findFieldByName("key");
        final FieldDescriptor valueDesc = descriptorForType.findFieldByName("value");

        keyWriter.writeField(msg.getField(keyDesc));
        valueWriter.writeField(msg.getField(valueDesc));

        recordConsumer.endGroup();
      }

      recordConsumer.endField("key_value", 0);

      recordConsumer.endGroup();
    }
  }

  class FloatWriter extends FieldWriter {
    @Override
    final void writeRawValue(Object value) {
      recordConsumer.addFloat((Float) value);
    }
  }

  class DoubleWriter extends FieldWriter {
    @Override
    final void writeRawValue(Object value) {
      recordConsumer.addDouble((Double) value);
    }
  }

  class EnumWriter extends FieldWriter {
    @Override
    final void writeRawValue(Object value) {
      Binary binary = Binary.fromString(((Descriptors.EnumValueDescriptor) value).getName());
      recordConsumer.addBinary(binary);
    }
  }

  class BooleanWriter extends FieldWriter {
    @Override
    final void writeRawValue(Object value) {
      recordConsumer.addBoolean((Boolean) value);
    }
  }

  class BinaryWriter extends FieldWriter {
    @Override
    final void writeRawValue(Object value) {
      ByteString byteString = (ByteString) value;
      Binary binary = Binary.fromConstantByteArray(byteString.toByteArray());
      recordConsumer.addBinary(binary);
    }
  }

  private FieldWriter unknownType(FieldDescriptor fieldDescriptor) {
    String exceptionMsg = "Unknown type with descriptor \"" + fieldDescriptor
            + "\" and type \"" + fieldDescriptor.getJavaType() + "\".";
    throw new InvalidRecordException(exceptionMsg);
  }

  /** Returns message descriptor as JSON String*/
  private String serializeDescriptor(Class<? extends Message> protoClass) {
    Descriptor descriptor = Protobufs.getMessageDescriptor(protoClass);
    DescriptorProtos.DescriptorProto asProto = descriptor.toProto();
    return TextFormat.printToString(asProto);
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.hadoop;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.compress.CodecPool;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionOutputStream;
import org.apache.hadoop.io.compress.Compressor;
import org.apache.hadoop.io.compress.Decompressor;
import org.apache.hadoop.util.ReflectionUtils;

import org.apache.parquet.bytes.ByteBufferAllocator;
import org.apache.parquet.bytes.BytesInput;
import org.apache.parquet.compression.CompressionCodecFactory;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;

public class CodecFactory implements CompressionCodecFactory {

  protected static final Map<String, CompressionCodec> CODEC_BY_NAME = Collections
      .synchronizedMap(new HashMap<String, CompressionCodec>());

  private final Map<CompressionCodecName, BytesCompressor> compressors = new HashMap<CompressionCodecName, BytesCompressor>();
  private final Map<CompressionCodecName, BytesDecompressor> decompressors = new HashMap<CompressionCodecName, BytesDecompressor>();

  protected final Configuration configuration;
  protected final int pageSize;

  /**
   * Create a new codec factory.
   *
   * @param configuration used to pass compression codec configuration information
   * @param pageSize the expected page size, does not set a hard limit, currently just
   *                 used to set the initial size of the output stream used when
   *                 compressing a buffer. If this factory is only used to construct
   *                 decompressors this parameter has no impact on the function of the factory
   */
  public CodecFactory(Configuration configuration, int pageSize) {
    this.configuration = configuration;
    this.pageSize = pageSize;
  }

  /**
   * Create a codec factory that will provide compressors and decompressors
   * that will work natively with ByteBuffers backed by direct memory.
   *
   * @param config configuration options for different compression codecs
   * @param allocator an allocator for creating result buffers during compression
   *                  and decompression, must provide buffers backed by Direct
   *                  memory and return true for the isDirect() method
   *                  on the ByteBufferAllocator interface
   * @param pageSize the default page size. This does not set a hard limit on the
   *                 size of buffers that can be compressed, but performance may
   *                 be improved by setting it close to the expected size of buffers
   *                 (in the case of parquet, pages) that will be compressed. This
   *                 setting is unused in the case of decompressing data, as parquet
   *                 always records the uncompressed size of a buffer. If this
   *                 CodecFactory is only going to be used for decompressors, this
   *                 parameter will not impact the function of the factory.
   * @return a configured direct codec factory
   */
  public static CodecFactory createDirectCodecFactory(Configuration config, ByteBufferAllocator allocator, int pageSize) {
    return new DirectCodecFactory(config, allocator, pageSize);
  }

  class HeapBytesDecompressor extends BytesDecompressor {

    private final CompressionCodec codec;
    private final Decompressor decompressor;

    HeapBytesDecompressor(CompressionCodecName codecName) {
      this.codec = getCodec(codecName);
      if (codec != null) {
        decompressor = CodecPool.getDecompressor(codec);
      } else {
        decompressor = null;
      }
    }

    @Override
    public BytesInput decompress(BytesInput bytes, int uncompressedSize) throws IOException {
      final BytesInput decompressed;
      if (codec != null) {
        decompressor.reset();
        InputStream is = codec.createInputStream(bytes.toInputStream(), decompressor);
        decompressed = BytesInput.from(is, uncompressedSize);
      } else {
        decompressed = bytes;
      }
      return decompressed;
    }

    @Override
    public void decompress(ByteBuffer input, int compressedSize, ByteBuffer output, int uncompressedSize) throws IOException {
      ByteBuffer decompressed = decompress(BytesInput.from(input), uncompressedSize).toByteBuffer();
      output.put(decompressed);
    }

    public void release() {
      if (decompressor != null) {
        CodecPool.returnDecompressor(decompressor);
      }
    }
  }

  /**
   * Encapsulates the logic around hadoop compression
   */
  class HeapBytesCompressor extends BytesCompressor {

    private final CompressionCodec codec;
    private final Compressor compressor;
    private final ByteArrayOutputStream compressedOutBuffer;
    private final CompressionCodecName codecName;

    HeapBytesCompressor(CompressionCodecName codecName) {
      this.codecName = codecName;
      this.codec = getCodec(codecName);
      if (codec != null) {
        this.compressor = CodecPool.getCompressor(codec);
        this.compressedOutBuffer = new ByteArrayOutputStream(pageSize);
      } else {
        this.compressor = null;
        this.compressedOutBuffer = null;
      }
    }

    @Override
    public BytesInput compress(BytesInput bytes) throws IOException {
      final BytesInput compressedBytes;
      if (codec == null) {
        compressedBytes = bytes;
      } else {
        compressedOutBuffer.reset();
        if (compressor != null) {
          // null compressor for non-native gzip
          compressor.reset();
        }
        CompressionOutputStream cos = codec.createOutputStream(compressedOutBuffer, compressor);
        bytes.writeAllTo(cos);
        cos.finish();
        cos.close();
        compressedBytes = BytesInput.from(compressedOutBuffer);
      }
      return compressedBytes;
    }

    @Override
    public void release() {
      if (compressor != null) {
        CodecPool.returnCompressor(compressor);
      }
    }

    public CompressionCodecName getCodecName() {
      return codecName;
    }

  }

  @Override
  public BytesCompressor getCompressor(CompressionCodecName codecName) {
    BytesCompressor comp = compressors.get(codecName);
    if (comp == null) {
      comp = createCompressor(codecName);
      compressors.put(codecName, comp);
    }
    return comp;
  }

  @Override
  public BytesDecompressor getDecompressor(CompressionCodecName codecName) {
    BytesDecompressor decomp = decompressors.get(codecName);
    if (decomp == null) {
      decomp = createDecompressor(codecName);
      decompressors.put(codecName, decomp);
    }
    return decomp;
  }

  protected BytesCompressor createCompressor(CompressionCodecName codecName) {
    return new HeapBytesCompressor(codecName);
  }

  protected BytesDecompressor createDecompressor(CompressionCodecName codecName) {
    return new HeapBytesDecompressor(codecName);
  }

  /**
   *
   * @param codecName
   *          the requested codec
   * @return the corresponding hadoop codec. null if UNCOMPRESSED
   */
  protected CompressionCodec getCodec(CompressionCodecName codecName) {
    String codecClassName = codecName.getHadoopCompressionCodecClassName();
    if (codecClassName == null) {
      return null;
    }
    CompressionCodec codec = CODEC_BY_NAME.get(codecClassName);
    if (codec != null) {
      return codec;
    }

    try {
      Class<?> codecClass = Class.forName(codecClassName);
      codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, configuration);
      CODEC_BY_NAME.put(codecClassName, codec);
      return codec;
    } catch (ClassNotFoundException e) {
      throw new BadConfigurationException("Class " + codecClassName + " was not found", e);
    }
  }

  @Override
  public void release() {
    for (BytesCompressor compressor : compressors.values()) {
      compressor.release();
    }
    compressors.clear();
    for (BytesDecompressor decompressor : decompressors.values()) {
      decompressor.release();
    }
    decompressors.clear();
  }

  /**
   * @deprecated will be removed in 2.0.0; use CompressionCodecFactory.BytesInputCompressor instead.
   */
  @Deprecated
  public static abstract class BytesCompressor implements CompressionCodecFactory.BytesInputCompressor {
    public abstract BytesInput compress(BytesInput bytes) throws IOException;
    public abstract CompressionCodecName getCodecName();
    public abstract void release();
  }

  /**
   * @deprecated will be removed in 2.0.0; use CompressionCodecFactory.BytesInputDecompressor instead.
   */
  @Deprecated
  public static abstract class BytesDecompressor implements CompressionCodecFactory.BytesInputDecompressor {
    public abstract BytesInput decompress(BytesInput bytes, int uncompressedSize) throws IOException;
    public abstract void decompress(ByteBuffer input, int compressedSize, ByteBuffer output, int uncompressedSize) throws IOException;
    public abstract void release();
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.parquet.cli.util;

import com.google.common.base.MoreObjects;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.util.Utf8;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;

public class Expressions {
  private static final Pattern NUMERIC_RE = Pattern.compile("^\\d+$");

  public static Object select(Schema schema, Object datum, String path) {
    return select(schema, datum, Lists.newArrayList(parse(path)));
  }

  @SuppressWarnings("unchecked")
  private static Object select(Schema schema, Object datum, List<PathExpr> tokens) {
    if (tokens.isEmpty()) {
      return datum;
    }

    Preconditions.checkArgument(tokens.size() == 1, "Cannot return multiple values");
    PathExpr token = tokens.get(0);

    switch (schema.getType()) {
      case RECORD:
        if (!(datum instanceof GenericRecord) && "json".equals(schema.getName())) {
          // skip the placeholder record schema
          return select(schema.getField("value").schema(), datum, tokens);
        }
        Preconditions.checkArgument(token.type == PathExpr.Type.FIELD,
            "Cannot dereference records");
        Preconditions.checkArgument(datum instanceof GenericRecord,
            "Not a record: %s", datum);
        GenericRecord record = (GenericRecord) datum;
        Schema.Field field = schema.getField(token.value);
        Preconditions.checkArgument(field != null,
            "No such field '%s' in schema: %s", token.value, schema);
        return select(field.schema(), record.get(token.value), token.children);

      case MAP:
        Preconditions.checkArgument(datum instanceof Map,
            "Not a map: %s", datum);
        Map<Object, Object> map = (Map<Object, Object>) datum;
        Object value = map.get(token.value);
        if (value == null) {
          // try with a Utf8
          value = map.get(new Utf8(token.value));
        }
        return select(schema.getValueType(), value, token.children);

      case ARRAY:
        Preconditions.checkArgument(token.type == PathExpr.Type.DEREF,
            "Cannot access fields of an array");
        Preconditions.checkArgument(datum instanceof Collection,
            "Not an array: %s", datum);
        Preconditions.checkArgument(NUMERIC_RE.matcher(token.value).matches(),
            "Not an array index: %s", token.value);
        List<Object> list = (List<Object>) datum;
        return select(schema.getElementType(), list.get(Integer.parseInt(token.value)),
            token.children);

      case UNION:
        int branch = GenericData.get().resolveUnion(schema, datum);
        return select(schema.getTypes().get(branch), datum, tokens);

      default:
        throw new IllegalArgumentException("Cannot access child of primitive value: " + datum);
    }
  }

  /**
   * a.2.b[3]["key"]
   * * optional (union with null) should be ignored
   * * unions should match by position number or short name (e.g. 2, user)
   * * fields should match by name
   * * arrays are dereferenced by position [n] => schema is the element schema
   * * maps are dereferenced by key => schema is the value schema
   *
   * @param schema an Avro schema
   * @param fieldPaths selected field paths
   * @return a filtered schema
   */
  public static Schema filterSchema(Schema schema, String... fieldPaths) {
    return filterSchema(schema, Lists.newArrayList(fieldPaths));
  }

  public static Schema filterSchema(Schema schema, List<String> fieldPaths) {
    if (fieldPaths == null) {
      return schema;
    }
    List<PathExpr> paths = merge(Lists.newArrayList(fieldPaths));
    return filter(schema, paths);
  }

  private static PathExpr parse(String path) {
    PathExpr expr = null;
    PathExpr last = null;
    boolean inDeref = false;
    boolean afterDeref = false;
    int valueStart = 0;
    for (int i = 0; i < path.length(); i += 1) {
      switch (path.charAt(i)) {
        case '.':
          Preconditions.checkState(valueStart != i || afterDeref, "Empty reference: ''");
          if (!inDeref) {
            if (valueStart != i) {
              PathExpr current = PathExpr.field(path.substring(valueStart, i));
              if (last != null) {
                last.children.add(current);
              } else {
                expr = current;
              }
              last = current;
            }
            valueStart = i + 1;
            afterDeref = false;
          }
          break;
        case '[':
          Preconditions.checkState(!inDeref, "Cannot nest [ within []");
          Preconditions.checkState(valueStart != i || afterDeref, "Empty reference: ''");
          if (valueStart != i) {
            PathExpr current = PathExpr.field(path.substring(valueStart, i));
            if (last != null) {
              last.children.add(current);
            } else {
              expr = current;
            }
            last = current;
          }
          valueStart = i + 1;
          inDeref = true;
          afterDeref = false;
          break;
        case ']':
          Preconditions.checkState(inDeref, "Cannot use ] without a starting [");
          Preconditions.checkState(valueStart != i, "Empty reference: ''");
          PathExpr current = PathExpr.deref(path.substring(valueStart, i));
          if (last != null) {
            last.children.add(current);
          } else {
            expr = current;
          }
          last = current;
          valueStart = i + 1;
          inDeref = false;
          afterDeref = true;
          break;
        default:
          Preconditions.checkState(!afterDeref, "Fields after [] must start with .");
      }
    }
    Preconditions.checkState(!inDeref, "Fields after [ must end with ]");
    if (valueStart < path.length()) {
      PathExpr current = PathExpr.field(path.substring(valueStart, path.length()));
      if (last != null) {
        last.children.add(current);
      } else {
        expr = current;
      }
    }
    return expr;
  }

  private static List<PathExpr> merge(List<String> fields) {
    List<PathExpr> paths = Lists.newArrayList();
    for (String field : fields) {
      merge(paths, parse(field));
    }
    return paths;
  }

  private static List<PathExpr> merge(List<PathExpr> tokens, PathExpr toAdd) {
    boolean merged = false;
    for (PathExpr token : tokens) {
      if ((token.type == toAdd.type) &&
          (token.type == PathExpr.Type.DEREF || token.value.equals(toAdd.value))) {
        for (PathExpr child : toAdd.children) {
          merge(token.children, child);
        }
        merged = true;
      }
    }
    if (!merged) {
      tokens.add(toAdd);
    }
    return tokens;
  }

  private static Schema filter(Schema schema, List<PathExpr> exprs) {
    if (exprs.isEmpty()) {
      return schema;
    }

    switch (schema.getType()) {
      case RECORD:
        List<Schema.Field> fields = Lists.newArrayList();
        for (PathExpr expr : exprs) {
          Schema.Field field = schema.getField(expr.value);
          Preconditions.checkArgument(field != null,
              "Cannot find field '%s' in schema: %s", expr.value, schema);
          fields.add(new Schema.Field(expr.value, filter(field.schema(), expr.children),
              field.doc(), field.defaultVal(), field.order()));
        }
        return Schema.createRecord(schema.getName(),
            schema.getDoc(), schema.getNamespace(), schema.isError(), fields);

      case UNION:
        // Ignore schemas that are a union with null because there is another token
        if (schema.getTypes().size() == 2) {
          if (schema.getTypes().get(0).getType() == Schema.Type.NULL) {
            return filter(schema.getTypes().get(1), exprs);
          } else if (schema.getTypes().get(1).getType() == Schema.Type.NULL) {
            return filter(schema.getTypes().get(0), exprs);
          }
        }

        List<Schema> schemas = Lists.newArrayList();
        for (PathExpr expr : exprs) {
          schemas.add(filter(schema, expr));
        }

        if (schemas.size() > 1) {
          return Schema.createUnion(schemas);
        } else {
          return schemas.get(0);
        }

      case MAP:
        Preconditions.checkArgument(exprs.size() == 1,
            "Cannot find multiple children of map schema: %s", schema);
        return filter(schema, exprs.get(0));

      case ARRAY:
        Preconditions.checkArgument(exprs.size() == 1,
            "Cannot find multiple children of array schema: %s", schema);
        return filter(schema, exprs.get(0));

      default:
        throw new IllegalArgumentException(String.format(
            "Cannot find child of primitive schema: %s", schema));
    }
  }

  private static Schema filter(Schema schema, PathExpr expr) {
    if (expr == null) {
      return schema;
    }

    switch (schema.getType()) {
      case RECORD:
        Preconditions.checkArgument(expr.type == PathExpr.Type.FIELD,
            "Cannot index a record: [%s]", expr.value);
        Schema.Field field = schema.getField(expr.value);
        if (field != null) {
          return filter(field.schema(), expr.children);
        } else {
          throw new IllegalArgumentException(String.format(
              "Cannot find field '%s' in schema: %s", expr.value, schema.toString(true)));
        }

      case MAP:
        return Schema.createMap(filter(schema.getValueType(), expr.children));

      case ARRAY:
        Preconditions.checkArgument(expr.type == PathExpr.Type.DEREF,
            "Cannot find field '%s' in an array", expr.value);
        Preconditions.checkArgument(NUMERIC_RE.matcher(expr.value).matches(),
            "Cannot index array by non-numeric value '%s'", expr.value);
        return Schema.createArray(filter(schema.getElementType(), expr.children));

      case UNION:
        // TODO: this should only return something if the type can match rather than explicitly
        // accessing parts of a union. when selecting data, unions are ignored.
        Preconditions.checkArgument(expr.type == PathExpr.Type.DEREF,
            "Cannot find field '%s' in a union", expr.value);
        List<Schema> options = schema.getTypes();
        if (NUMERIC_RE.matcher(expr.value).matches()) {
          // look up the option by position
          int i = Integer.parseInt(expr.value);
          if (i < options.size()) {
            return filter(options.get(i), expr.children);
          }
        } else {
          // look up the option by name
          for (Schema option : options) {
            if (expr.value.equalsIgnoreCase(option.getName())) {
              return filter(option, expr.children);
            }
          }
        }
        throw new IllegalArgumentException(String.format(
            "Invalid union index '%s' for schema: %s", expr.value, schema));

      default:
        throw new IllegalArgumentException(String.format(
            "Cannot find '%s' in primitive schema: %s", expr.value, schema));
    }
  }

  private static class PathExpr {
    enum Type {
      DEREF,
      FIELD
    }

    static PathExpr deref(String value) {
      return new PathExpr(Type.DEREF, value);
    }

    static PathExpr deref(String value, PathExpr child) {
      return new PathExpr(Type.DEREF, value, Lists.newArrayList(child));
    }

    static PathExpr field(String value) {
      return new PathExpr(Type.FIELD, value);
    }

    static PathExpr field(String value, PathExpr child) {
      return new PathExpr(Type.FIELD, value, Lists.newArrayList(child));
    }

    private final Type type;
    private final String value;
    private final List<PathExpr> children;

    PathExpr(Type type, String value) {
      this.type = type;
      this.value = value;
      this.children = Lists.newArrayList();
    }

    PathExpr(Type type, String value, List<PathExpr> children) {
      this.type = type;
      this.value = value;
      this.children = children;
    }

    @Override
    public boolean equals(Object o) {
      if (this == o) return true;
      if (o == null || getClass() != o.getClass()) return false;

      PathExpr pathExpr = (PathExpr) o;

      if (type != pathExpr.type) return false;
      if (value != null ? !value.equals(pathExpr.value) : pathExpr.value != null) return false;
      return children != null ? children.equals(pathExpr.children) : pathExpr.children == null;
    }

    @Override
    public int hashCode() {
      int result = type != null ? type.hashCode() : 0;
      result = 31 * result + (value != null ? value.hashCode() : 0);
      result = 31 * result + (children != null ? children.hashCode() : 0);
      return result;
    }

    @Override
    public String toString() {
      return MoreObjects.toStringHelper(this)
          .add("type", type)
          .add("value", value)
          .add("children", children)
          .toString();
    }
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.thrift.projection.amend;

import org.apache.thrift.TException;
import org.apache.thrift.protocol.*;
import org.apache.parquet.thrift.struct.ThriftField;
import org.apache.parquet.thrift.struct.ThriftType;
import org.apache.parquet.thrift.struct.ThriftType.StructType;

import java.util.*;

/**
 * fill in default value for required fields in TProtocols after projection is specified.
 */
public class ProtocolEventsAmender {
  List<TProtocol> rootEvents;
  List<TProtocol> fixedEvents = new ArrayList<TProtocol>();

  public ProtocolEventsAmender(List<TProtocol> rootEvents) {
    this.rootEvents = rootEvents;
  }

  /**
   * Given a thrift definition, protocols events, it checks all the required fields,
   * and create default value if a required field is missing
   *
   * @param recordThriftType the Thrift Struct definition for events
   * @return a list of events
   * @throws TException if there is an error while amending events
   */
  public List<TProtocol> amendMissingRequiredFields(StructType recordThriftType) throws TException {
    Iterator<TProtocol> protocolIter = rootEvents.iterator();
    checkStruct(protocolIter, recordThriftType);
    return fixedEvents;
  }

  private TProtocol acceptProtocol(TProtocol p) {
    this.fixedEvents.add(p);
    return p;
  }

  private void checkStruct(Iterator<TProtocol> eventIter, ThriftType.StructType thriftStructType) throws TException {
    TStruct tStruct = acceptProtocol(eventIter.next()).readStructBegin();

    List<ThriftField> childrenFields = thriftStructType.getChildren();

    Set<Short> includedFieldsIds = new HashSet<Short>();

    while (true) {
      TProtocol next = eventIter.next();
      TField field = next.readFieldBegin();

      if (isStopField(field))
        break;
      acceptProtocol(next);
      includedFieldsIds.add(field.id);
      ThriftField fieldDefinition = thriftStructType.getChildById(field.id);
      checkField(field.type, eventIter, fieldDefinition);
      acceptProtocol(eventIter.next()).readFieldEnd();
    }

    for (ThriftField requiredField : childrenFields) {
      if (!isRequired(requiredField)) {
        continue;
      }
      if (!includedFieldsIds.contains(requiredField.getFieldId())) {
        fixedEvents.addAll(new DefaultProtocolEventsGenerator().createProtocolEventsForField(requiredField));
      }
    }

    acceptProtocol(DefaultProtocolEventsGenerator.READ_FIELD_STOP);
    acceptProtocol(eventIter.next()).readStructEnd();
  }

  private void checkField(byte type, Iterator<TProtocol> eventIter, ThriftField fieldDefinition) throws TException {
    switch (type) {
      case TType.STRUCT:
        checkStruct(eventIter, (ThriftType.StructType) fieldDefinition.getType());
        return;
      case TType.LIST:
        checkList(eventIter, fieldDefinition);
        return;
      case TType.MAP:
        checkMap(eventIter, fieldDefinition);
        return;
      case TType.SET:
        checkSet(eventIter, fieldDefinition);
        return;
    }
    checkPrimitiveField(type, eventIter);
  }

  /**
   * check each element of the Set, make sure all the element contain required fields
   *
   * @param eventIter
   * @param setFieldDefinition
   * @throws TException
   */
  private void checkSet(Iterator<TProtocol> eventIter, ThriftField setFieldDefinition) throws TException {
    TSet thriftSet = acceptProtocol(eventIter.next()).readSetBegin();
    ThriftField elementFieldDefinition = ((ThriftType.SetType) setFieldDefinition.getType()).getValues();
    int setSize = thriftSet.size;
    for (int i = 0; i < setSize; i++) {
      checkField(thriftSet.elemType, eventIter, elementFieldDefinition);
    }
    acceptProtocol(eventIter.next()).readSetEnd();
  }

  private void checkMap(Iterator<TProtocol> eventIter, ThriftField mapFieldForWriting) throws TException {
    TMap thriftMap = acceptProtocol(eventIter.next()).readMapBegin();
    ThriftField keyFieldForWriting = ((ThriftType.MapType) mapFieldForWriting.getType()).getKey();
    ThriftField valueFieldForWriting = ((ThriftType.MapType) mapFieldForWriting.getType()).getValue();

    int mapSize = thriftMap.size;
    for (int i = 0; i < mapSize; i++) {
      //readkey
      checkField(thriftMap.keyType, eventIter, keyFieldForWriting);
      //readValue
      checkField(thriftMap.valueType, eventIter, valueFieldForWriting);
    }
    acceptProtocol(eventIter.next()).readMapEnd();
  }

  private void checkList(Iterator<TProtocol> eventIter, ThriftField listFieldUsedForWriting) throws TException {
    ThriftField valueFieldForWriting = ((ThriftType.ListType) listFieldUsedForWriting.getType()).getValues();
    TList thriftList = acceptProtocol(eventIter.next()).readListBegin();
    int listSize = thriftList.size;
    for (int i = 0; i < listSize; i++) {
      checkField(thriftList.elemType, eventIter, valueFieldForWriting);
    }
    acceptProtocol(eventIter.next()).readListEnd();
  }

  /**
   * Once reached primitive field, just the copy the event.
   *
   * @param type
   * @param eventIter
   * @throws TException
   */
  private void checkPrimitiveField(byte type, Iterator<TProtocol> eventIter) throws TException {
    acceptProtocol(eventIter.next());
  }

  private boolean isStopField(TField field) {
    return field.type == TType.STOP;
  }

  private boolean isRequired(ThriftField requiredField) {
    return requiredField.getRequirement() == ThriftField.Requirement.REQUIRED;
  }

}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.parquet.format;

import static org.apache.parquet.format.FileMetaData._Fields.CREATED_BY;
import static org.apache.parquet.format.FileMetaData._Fields.KEY_VALUE_METADATA;
import static org.apache.parquet.format.FileMetaData._Fields.NUM_ROWS;
import static org.apache.parquet.format.FileMetaData._Fields.ROW_GROUPS;
import static org.apache.parquet.format.FileMetaData._Fields.SCHEMA;
import static org.apache.parquet.format.FileMetaData._Fields.VERSION;
import static org.apache.parquet.format.event.Consumers.fieldConsumer;
import static org.apache.parquet.format.event.Consumers.listElementsOf;
import static org.apache.parquet.format.event.Consumers.listOf;
import static org.apache.parquet.format.event.Consumers.struct;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.List;

import org.apache.thrift.TBase;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TCompactProtocol;
import org.apache.thrift.protocol.TProtocol;
import org.apache.thrift.transport.TIOStreamTransport;

import org.apache.parquet.format.event.Consumers.Consumer;
import org.apache.parquet.format.event.Consumers.DelegatingFieldConsumer;
import org.apache.parquet.format.event.EventBasedThriftReader;
import org.apache.parquet.format.event.TypedConsumer.I32Consumer;
import org.apache.parquet.format.event.TypedConsumer.I64Consumer;
import org.apache.parquet.format.event.TypedConsumer.StringConsumer;

/**
 * Utility to read/write metadata
 * We use the TCompactProtocol to serialize metadata
 */
public class Util {

  public static void writeColumnIndex(ColumnIndex columnIndex, OutputStream to) throws IOException {
    write(columnIndex, to);
  }

  public static ColumnIndex readColumnIndex(InputStream from) throws IOException {
    return read(from, new ColumnIndex());
  }

  public static void writeOffsetIndex(OffsetIndex offsetIndex, OutputStream to) throws IOException {
    write(offsetIndex, to);
  }

  public static OffsetIndex readOffsetIndex(InputStream from) throws IOException {
    return read(from, new OffsetIndex());
  }

  public static void writePageHeader(PageHeader pageHeader, OutputStream to) throws IOException {
    write(pageHeader, to);
  }

  public static PageHeader readPageHeader(InputStream from) throws IOException {
    return read(from, new PageHeader());
  }

  public static void writeFileMetaData(org.apache.parquet.format.FileMetaData fileMetadata, OutputStream to) throws IOException {
    write(fileMetadata, to);
  }

  public static FileMetaData readFileMetaData(InputStream from) throws IOException {
    return read(from, new FileMetaData());
  }
  /**
   * reads the meta data from the stream
   * @param from the stream to read the metadata from
   * @param skipRowGroups whether row groups should be skipped
   * @return the resulting metadata
   * @throws IOException if any I/O error occurs during the reading
   */
  public static FileMetaData readFileMetaData(InputStream from, boolean skipRowGroups) throws IOException {
    FileMetaData md = new FileMetaData();
    if (skipRowGroups) {
      readFileMetaData(from, new DefaultFileMetaDataConsumer(md), skipRowGroups);
    } else {
      read(from, md);
    }
    return md;
  }

  /**
   * To read metadata in a streaming fashion.
   *
   */
  public static abstract class FileMetaDataConsumer {
    abstract public void setVersion(int version);
    abstract public void setSchema(List<SchemaElement> schema);
    abstract public void setNumRows(long numRows);
    abstract public void addRowGroup(RowGroup rowGroup);
    abstract public void addKeyValueMetaData(KeyValue kv);
    abstract public void setCreatedBy(String createdBy);
  }

  /**
   * Simple default consumer that sets the fields
   *
   */
  public static final class DefaultFileMetaDataConsumer extends FileMetaDataConsumer {
    private final FileMetaData md;

    public DefaultFileMetaDataConsumer(FileMetaData md) {
      this.md = md;
    }

    @Override
    public void setVersion(int version) {
      md.setVersion(version);
    }

    @Override
    public void setSchema(List<SchemaElement> schema) {
      md.setSchema(schema);
    }

    @Override
    public void setNumRows(long numRows) {
      md.setNum_rows(numRows);
    }

    @Override
    public void setCreatedBy(String createdBy) {
      md.setCreated_by(createdBy);
    }

    @Override
    public void addRowGroup(RowGroup rowGroup) {
      md.addToRow_groups(rowGroup);
    }

    @Override
    public void addKeyValueMetaData(KeyValue kv) {
      md.addToKey_value_metadata(kv);
    }
  }

  public static void readFileMetaData(InputStream from, FileMetaDataConsumer consumer) throws IOException {
    readFileMetaData(from, consumer, false);
  }

  public static void readFileMetaData(InputStream from, final FileMetaDataConsumer consumer, boolean skipRowGroups) throws IOException {
    try {
      DelegatingFieldConsumer eventConsumer = fieldConsumer()
      .onField(VERSION, new I32Consumer() {
        @Override
        public void consume(int value) {
          consumer.setVersion(value);
        }
      }).onField(SCHEMA, listOf(SchemaElement.class, new Consumer<List<SchemaElement>>() {
        @Override
        public void consume(List<SchemaElement> schema) {
          consumer.setSchema(schema);
        }
      })).onField(NUM_ROWS, new I64Consumer() {
        @Override
        public void consume(long value) {
          consumer.setNumRows(value);
        }
      }).onField(KEY_VALUE_METADATA, listElementsOf(struct(KeyValue.class, new Consumer<KeyValue>() {
        @Override
        public void consume(KeyValue kv) {
          consumer.addKeyValueMetaData(kv);
        }
      }))).onField(CREATED_BY, new StringConsumer() {
        @Override
        public void consume(String value) {
          consumer.setCreatedBy(value);
        }
      });
      if (!skipRowGroups) {
        eventConsumer = eventConsumer.onField(ROW_GROUPS, listElementsOf(struct(RowGroup.class, new Consumer<RowGroup>() {
          @Override
          public void consume(RowGroup rowGroup) {
            consumer.addRowGroup(rowGroup);
          }
        })));
      }
      new EventBasedThriftReader(protocol(from)).readStruct(eventConsumer);

    } catch (TException e) {
      throw new IOException("can not read FileMetaData: " + e.getMessage(), e);
    }
  }

  private static TProtocol protocol(OutputStream to) {
    return protocol(new TIOStreamTransport(to));
  }

  private static TProtocol protocol(InputStream from) {
    return protocol(new TIOStreamTransport(from));
  }

  private static InterningProtocol protocol(TIOStreamTransport t) {
    return new InterningProtocol(new TCompactProtocol(t));
  }

  private static <T extends TBase<?,?>> T read(InputStream from, T tbase) throws IOException {
    try {
      tbase.read(protocol(from));
      return tbase;
    } catch (TException e) {
      throw new IOException("can not read " + tbase.getClass() + ": " + e.getMessage(), e);
    }
  }

  private static void write(TBase<?, ?> tbase, OutputStream to) throws IOException {
    try {
      tbase.write(protocol(to));
    } catch (TException e) {
      throw new IOException("can not write " + tbase, e);
    }
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.hadoop.thrift;

import java.io.Closeable;
import java.io.IOException;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.thrift.TBase;
import org.apache.thrift.protocol.TProtocolFactory;

import org.apache.parquet.thrift.FieldIgnoredHandler;

/**
 * To create a Parquet file from the Thrift binary of records
 */
public class ThriftToParquetFileWriter implements Closeable {

  private final RecordWriter<Void, BytesWritable> recordWriter;
  private final TaskAttemptContext taskAttemptContext;

  /**
   * defaults to buffered = true
   * @param fileToCreate the file to create. If null will create the default file name from the taskAttemptContext
   * @param taskAttemptContext The current taskAttemptContext
   * @param protocolFactory to create protocols to read the incoming bytes
   * @param thriftClass to produce the schema
   * @throws IOException if there was a problem writing
   * @throws InterruptedException from the underlying Hadoop API
   */
  public ThriftToParquetFileWriter(
      Path fileToCreate,
      TaskAttemptContext taskAttemptContext,
      TProtocolFactory protocolFactory,
      Class<? extends TBase<?,?>> thriftClass)
          throws IOException, InterruptedException {
    this(fileToCreate, taskAttemptContext, protocolFactory, thriftClass, true, null);
  }

  /**
   * defaults to buffered = true
   * @param fileToCreate the file to create. If null will create the default file name from the taskAttemptContext
   * @param taskAttemptContext The current taskAttemptContext
   * @param protocolFactory to create protocols to read the incoming bytes
   * @param thriftClass to produce the schema
   * @param errorHandler to define what to do when failing to read a record
   * @throws IOException if there was a problem writing
   * @throws InterruptedException from the underlying Hadoop API
   */
  public ThriftToParquetFileWriter(
      Path fileToCreate,
      TaskAttemptContext taskAttemptContext,
      TProtocolFactory protocolFactory,
      Class<? extends TBase<?,?>> thriftClass,
      FieldIgnoredHandler errorHandler) throws IOException, InterruptedException {
    this(fileToCreate, taskAttemptContext, protocolFactory, thriftClass, true, errorHandler);
  }

  /**
   * @param fileToCreate the file to create. If null will create the default file name from the taskAttemptContext
   * @param taskAttemptContext The current taskAttemptContext
   * @param protocolFactory to create protocols to read the incoming bytes
   * @param thriftClass to produce the schema
   * @param buffered buffer each record individually
   * @param errorHandler an error handler
   * @throws IOException if there was a problem writing
   * @throws InterruptedException from the underlying Hadoop API
   */
  public ThriftToParquetFileWriter(
      Path fileToCreate,
      TaskAttemptContext taskAttemptContext,
      TProtocolFactory protocolFactory,
      Class<? extends TBase<?,?>> thriftClass,
      boolean buffered,
      FieldIgnoredHandler errorHandler) throws IOException, InterruptedException {
    this.taskAttemptContext = taskAttemptContext;
    this.recordWriter = new ParquetThriftBytesOutputFormat(protocolFactory, thriftClass, buffered, errorHandler).getRecordWriter(taskAttemptContext, fileToCreate);
  }

  /**
   * write one record to the columnar store
   * @param bytes a bytes writable
   * @throws IOException if there is an error while writing
   * @throws InterruptedException if writing is interrupted
   */
  public void write(BytesWritable bytes) throws IOException, InterruptedException {
    recordWriter.write(null, bytes);
  }

  /**
   * close the file
   *
   * @see java.io.Closeable#close()
   */
  @Override
  public void close() throws IOException {
    try {
      recordWriter.close(taskAttemptContext);
    } catch (InterruptedException e) {
      Thread.interrupted();
      throw new IOException("The thread was interrupted", e);
    }
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.hadoop.example;

import org.apache.hadoop.mapreduce.Job;

import org.apache.parquet.example.data.Group;
import org.apache.parquet.hadoop.ParquetOutputFormat;
import org.apache.parquet.hadoop.util.ContextUtil;
import org.apache.parquet.schema.MessageType;

/**
 * An example output format
 *
 * must be provided the schema up front
 * @see ExampleOutputFormat#setSchema(Job, MessageType)
 * @see GroupWriteSupport#PARQUET_EXAMPLE_SCHEMA
 */
public class ExampleOutputFormat extends ParquetOutputFormat<Group> {

  /**
   * set the schema being written to the job conf
   * @param job a job
   * @param schema the schema of the data
   */
  public static void setSchema(Job job, MessageType schema) {
    GroupWriteSupport.setSchema(schema, ContextUtil.getConfiguration(job));
  }

  /**
   * retrieve the schema from the conf
   * @param job a job
   * @return the schema
   */
  public static MessageType getSchema(Job job) {
    return GroupWriteSupport.getSchema(ContextUtil.getConfiguration(job));
  }

  public ExampleOutputFormat() {
    super(new GroupWriteSupport());
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.filter2.predicate;

import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

import org.apache.parquet.filter2.predicate.Operators.Column;
import org.apache.parquet.hadoop.metadata.ColumnPath;
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;

/**
 * Contains all valid mappings from class -> parquet type (and vice versa) for use in
 * {@link FilterPredicate}s
 *
 * This is a bit ugly, but it allows us to provide good error messages at runtime
 * when there are type mismatches.
 *
 * TODO: this has some overlap with {@link PrimitiveTypeName#javaType}
 * TODO: (https://issues.apache.org/jira/browse/PARQUET-30)
 */
public class ValidTypeMap {
  private ValidTypeMap() { }

  // classToParquetType and parquetTypeToClass are used as a bi-directional map
  private static final Map<Class<?>, Set<PrimitiveTypeName>> classToParquetType = new HashMap<Class<?>, Set<PrimitiveTypeName>>();
  private static final Map<PrimitiveTypeName, Set<Class<?>>> parquetTypeToClass = new HashMap<PrimitiveTypeName, Set<Class<?>>>();

  // set up the mapping in both directions
  private static void add(Class<?> c, PrimitiveTypeName p) {
    Set<PrimitiveTypeName> descriptors = classToParquetType.get(c);
    if (descriptors == null) {
      descriptors = new HashSet<PrimitiveTypeName>();
      classToParquetType.put(c, descriptors);
    }
    descriptors.add(p);

    Set<Class<?>> classes = parquetTypeToClass.get(p);
    if (classes == null) {
      classes = new HashSet<Class<?>>();
      parquetTypeToClass.put(p, classes);
    }
    classes.add(c);
  }

  static {
    for (PrimitiveTypeName t: PrimitiveTypeName.values()) {
      Class<?> c = t.javaType;

      if (c.isPrimitive()) {
        c = PrimitiveToBoxedClass.get(c);
      }

      add(c, t);
    }
  }

  /**
   * Asserts that foundColumn was declared as a type that is compatible with the type for this column found
   * in the schema of the parquet file.
   *
   * @throws java.lang.IllegalArgumentException if the types do not align
   *
   * @param foundColumn the column as declared by the user
   * @param primitiveType the primitive type according to the schema
   * @param <T> the java Type of values in the column, must be Comparable
   */
  public static <T extends Comparable<T>> void assertTypeValid(Column<T> foundColumn, PrimitiveTypeName primitiveType) {
    Class<T> foundColumnType = foundColumn.getColumnType();
    ColumnPath columnPath = foundColumn.getColumnPath();

    Set<PrimitiveTypeName> validTypeDescriptors = classToParquetType.get(foundColumnType);

    if (validTypeDescriptors == null) {
      StringBuilder message = new StringBuilder();
      message
          .append("Column ")
          .append(columnPath.toDotString())
          .append(" was declared as type: ")
          .append(foundColumnType.getName())
          .append(" which is not supported in FilterPredicates.");

      Set<Class<?>> supportedTypes = parquetTypeToClass.get(primitiveType);
      if (supportedTypes != null) {
        message
          .append(" Supported types for this column are: ")
          .append(supportedTypes);
      } else {
        message.append(" There are no supported types for columns of " + primitiveType);
      }
      throw new IllegalArgumentException(message.toString());
    }

    if (!validTypeDescriptors.contains(primitiveType)) {
      StringBuilder message = new StringBuilder();
      message
          .append("FilterPredicate column: ")
          .append(columnPath.toDotString())
          .append("'s declared type (")
          .append(foundColumnType.getName())
          .append(") does not match the schema found in file metadata. Column ")
          .append(columnPath.toDotString())
          .append(" is of type: ")
          .append(primitiveType)
          .append("\nValid types for this column are: ")
          .append(parquetTypeToClass.get(primitiveType));
      throw new IllegalArgumentException(message.toString());
    }
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.parquet;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;

import org.apache.parquet.glob.GlobExpander;
import org.apache.parquet.glob.WildcardPath;

public final class Strings {
  private Strings() { }

  /**
   * Join an Iterable of Strings into a single string with a delimiter.
   * For example, join(Arrays.asList("foo","","bar","x"), "|") would return
   * "foo||bar|x"
   *
   * @param s an iterable of strings
   * @param on the delimiter
   * @return a single joined string
   */
  public static String join(Iterable<String> s, String on) {
    return join(s.iterator(), on);
  }

  /**
   * Join an Iterator of Strings into a single string with a delimiter.
   * For example, join(Arrays.asList("foo","","bar","x"), "|") would return
   * "foo||bar|x"
   *
   * @param iter an iterator of strings
   * @param on the delimiter
   * @return a single joined string
   */
  public static String join(Iterator<String> iter, String on) {
    StringBuilder sb = new StringBuilder();
    while (iter.hasNext()) {
      sb.append(iter.next());
      if (iter.hasNext()) {
        sb.append(on);
      }
    }
    return sb.toString();
  }

  /**
   * Join an Array of Strings into a single string with a delimiter.
   * For example, join(new String[] {"foo","","bar","x"}, "|") would return
   * "foo||bar|x"
   *
   * @param s an iterable of strings
   * @param on the delimiter
   * @return a single joined string
   */
  public static String join(String[] s, String on) {
    return join(Arrays.asList(s), on);
  }

  /**
   * Returns true if s.isEmpty() or s == null
   *
   * @param s a string that may be null or empty
   * @return true if the string s is null or is empty
   */
  public static boolean isNullOrEmpty(String s) {
    return s == null || s.isEmpty();
  }

  /**
   * Expands a string with braces ("{}") into all of its possible permutations.
   * We call anything inside of {} braces a "one-of" group.
   *
   * The only special characters in this glob syntax are '}', '{' and ','
   *
   * The top-level pattern must not contain any commas, but a "one-of" group separates
   * its elements with commas, and a one-of group may contain sub one-of groups.
   *
   * For example:
   * start{a,b,c}end -> startaend, startbend, startcend
   * start{a,{b,c},d} -> startaend, startbend, startcend, startdend
   * {a,b,c} -> a, b, c
   * start{a, b{x,y}} -> starta, startbx, startby
   *
   * @param globPattern a string in the format described above
   * @return a list of all the strings that would satisfy globPattern, including duplicates
   */
  public static List<String> expandGlob(String globPattern) {
    return GlobExpander.expand(globPattern);
  }

  /**
   * Expands a string according to {@link #expandGlob(String)}, and then constructs a {@link WildcardPath}
   * for each expanded result which can be used to match strings as described in {@link WildcardPath}.
   *
   * @param globPattern a String to be passed to {@link #expandGlob(String)}
   * @param delim the delimeter used by {@link WildcardPath}
   * @return a list of wildcard paths, one for each expanded result
   */
  public static List<WildcardPath> expandGlobToWildCardPaths(String globPattern, char delim) {
    List<WildcardPath> ret = new ArrayList<WildcardPath>();
    for (String expandedGlob : Strings.expandGlob(globPattern)) {
      ret.add(new WildcardPath(globPattern, expandedGlob, delim));
    }
    return ret;
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet;

/**
 * Utilities for working with ints
 */
@Deprecated
public final class Ints {
  private Ints() { }

  /**
   * Cast value to a an int, or throw an exception
   * if there is an overflow.
   *
   * @param value a long to be casted to an int
   * @return an int that is == to value
   * @throws IllegalArgumentException if value can't be casted to an int
   * @deprecated replaced by {@link java.lang.Math#toIntExact(long)}
   */
  public static int checkedCast(long value) {
    int valueI = (int) value;
    if (valueI != value) {
      throw new IllegalArgumentException(String.format("Overflow casting %d to an int", value));
    }
    return valueI;
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.hive;

import java.lang.reflect.Method;

import org.apache.parquet.hive.internal.Hive010Binding;
import org.apache.parquet.hive.internal.Hive012Binding;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Factory for creating HiveBinding objects based on the version of Hive
 * available in the classpath. This class does not provide static methods
 * to enable mocking.
 */
public class HiveBindingFactory {
  private static final Logger LOG = LoggerFactory.getLogger(HiveBindingFactory.class);
  private static final String HIVE_VERSION_CLASS_NAME = "org.apache.hive.common.util.HiveVersionInfo";
  private static final String HIVE_VERSION_METHOD_NAME = "getVersion";
  private static final String HIVE_UTILITIES_CLASS_NAME = "org.apache.hadoop.hive.ql.exec.Utilities";
  private static final String HIVE_012_INDICATOR_UTILITIES_GETMAPWORK = "getMapWork";
  private static final String HIVE_VERSION_NULL = "<null>";
  private static final Class<? extends HiveBinding> LATEST_BINDING = Hive012Binding.class;
  static final String HIVE_VERSION_UNKNOWN = "Unknown";
  static final String HIVE_VERSION_010 = "0.10";
  static final String HIVE_VERSION_011 = "0.11";
  static final String HIVE_VERSION_012 = "0.12";
  static final String HIVE_VERSION_013 = "0.13";

  /**
   * @return HiveBinding based on the Hive version in the classpath
   */
  public HiveBinding create() {
    Class<? extends HiveBinding> bindingClazz = create(HiveBindingFactory.class
        .getClassLoader());
    try {
      return bindingClazz.newInstance();
    } catch (Exception e) {
      throw new HiveBindingInstantiationError("Unexpected error creating instance"
          + " of " + bindingClazz.getCanonicalName(), e);
    }
  }

  /**
   * Internal method visible for testing purposes
   */
  @SuppressWarnings("rawtypes")
  Class<? extends HiveBinding> create(ClassLoader classLoader) {
    // HiveVersionInfo was added in 0.11, if the class does
    // not exist then return the hive binding for 0.10
    Class hiveVersionInfo;
    try {
      hiveVersionInfo = Class.forName(HIVE_VERSION_CLASS_NAME, true, classLoader);
    } catch (ClassNotFoundException e) {
      LOG.debug("Class " + HIVE_VERSION_CLASS_NAME + ", not found, returning {}",
          Hive010Binding.class.getSimpleName());
      return Hive010Binding.class;
    }
    return createInternal(hiveVersionInfo);
  }

  /**
   * Internal method visible for testing purposes
   */
  @SuppressWarnings({"unchecked", "rawtypes"})
  Class<? extends HiveBinding> createInternal(Class hiveVersionInfo) {
    String hiveVersion;
    try {
      Method getVersionMethod = hiveVersionInfo.
          getMethod(HIVE_VERSION_METHOD_NAME, (Class[])null);
      String rawVersion = (String)getVersionMethod.invoke(null, (Object[])null);
      LOG.debug("Raw Version from {} is '{}'", hiveVersionInfo.getSimpleName(), rawVersion);
      hiveVersion = trimVersion(rawVersion);
    } catch (Exception e) {
      throw new UnexpectedHiveVersionProviderError("Unexpected error whilst " +
          "determining Hive version", e);
    }
    if(hiveVersion.equalsIgnoreCase(HIVE_VERSION_UNKNOWN)) {
      LOG.debug("Unknown hive version, attempting to guess");
      return createBindingForUnknownVersion();
    }
    if(hiveVersion.startsWith(HIVE_VERSION_010)) {
      LOG.debug("Hive version {}, returning {}", hiveVersion, Hive010Binding.class.getSimpleName());
      return Hive010Binding.class;
    } else if(hiveVersion.startsWith(HIVE_VERSION_011)) {
      LOG.debug("Hive version " + hiveVersion + ", returning " +
          Hive010Binding.class.getSimpleName() + " as it's expected the 0.10 " +
          "binding will work with 0.11");
      return Hive010Binding.class;
    } else if(hiveVersion.startsWith(HIVE_VERSION_013)) {
      throw new HiveBindingInstantiationError("Hive 0.13 contains native Parquet support " +
          "and the parquet-hive jars from the parquet project should not be included " +
          "in Hive's classpath.");
    }
    LOG.debug("Hive version {}, returning {}", hiveVersion,
        Hive012Binding.class.getSimpleName());
    // as of 11/26/2013 it looks like the 0.12 binding will work for 0.13
    return Hive012Binding.class;
  }

  private Class<? extends HiveBinding> createBindingForUnknownVersion() {
    try {
      Class<?> utilitiesClass = Class.forName(HIVE_UTILITIES_CLASS_NAME);
      for(Method method : utilitiesClass.getDeclaredMethods()) {
        if(HIVE_012_INDICATOR_UTILITIES_GETMAPWORK.equals(method.getName())) {
          LOG.debug("Found " + HIVE_UTILITIES_CLASS_NAME + "." +
              HIVE_012_INDICATOR_UTILITIES_GETMAPWORK + " returning 0.12 binding");
          return Hive012Binding.class;
        }
      }
      // if the getMapWork method does not exist then it must be 0.10 or 0.11
      return Hive010Binding.class;
    } catch (ClassNotFoundException e) {
      LOG.debug("Could not find " + HIVE_UTILITIES_CLASS_NAME + ", returning" +
          " the latest binding since this class existed in 0.10, 0.11, and 0.12");
      return LATEST_BINDING;
    }
  }

  private static String trimVersion(String s) {
    if(s == null) {
      return HIVE_VERSION_NULL;
    }
    return s.trim();
  }
  static class HiveBindingInstantiationError extends Error {
    private static final long serialVersionUID = -9348060142128L;
    public HiveBindingInstantiationError(String msg) {
      super(msg);
    }
    public HiveBindingInstantiationError(String msg, Exception e) {
      super(msg, e);
    }
  }
  static class UnexpectedHiveVersionProviderError extends Error {
    private static final long serialVersionUID = -7344858060142118L;
    public UnexpectedHiveVersionProviderError(String msg, Exception e) {
      super(msg, e);
    }
  }
}