/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.bytes;

import org.apache.parquet.IOExceptionUtils;
import org.apache.parquet.ParquetRuntimeException;

import java.io.IOException;
import java.io.OutputStream;

/**
 * Based on DataOutputStream but in little endian and without the String/char methods
 */
public class LittleEndianDataOutputStream extends OutputStream {

  private final OutputStream out;

  /**
   * Creates a new data output stream to write data to the specified
   * underlying output stream. The counter written is
   * set to zero.
   *
   * @param   out   the underlying output stream, to be saved for later
   *                use.
   * @see     java.io.FilterOutputStream#out
   */
  public LittleEndianDataOutputStream(OutputStream out) {
    this.out = out;
  }

  /**
   * Writes the specified byte (the low eight bits of the argument
   * b) to the underlying output stream. If no exception
   * is thrown, the counter written is incremented by
   * 1.
   *

   * Implements the write method of OutputStream.
   *
   * @param      b   the byte to be written.
   * @exception  IOException  if an I/O error occurs.
   * @see        java.io.FilterOutputStream#out
   */
  public void write(int b) throws IOException {
    out.write(b);
  }

  /**
   * Writes len bytes from the specified byte array
   * starting at offset off to the underlying output stream.
   * If no exception is thrown, the counter written is
   * incremented by len.
   *
   * @param      b     the data.
   * @param      off   the start offset in the data.
   * @param      len   the number of bytes to write.
   * @exception  IOException  if an I/O error occurs.
   * @see        java.io.FilterOutputStream#out
   */
  public void write(byte b[], int off, int len) throws IOException {
    out.write(b, off, len);
  }

  /**
   * Flushes this data output stream. This forces any buffered output
   * bytes to be written out to the stream.
   *

   * The flush method of DataOutputStream
   * calls the flush method of its underlying output stream.
   *
   * @exception  IOException  if an I/O error occurs.
   * @see        java.io.FilterOutputStream#out
   * @see        java.io.OutputStream#flush()
   */
  public void flush() throws IOException {
    out.flush();
  }

  /**
   * Writes a boolean to the underlying output stream as
   * a 1-byte value. The value true is written out as the
   * value (byte)1; the value false is
   * written out as the value (byte)0. If no exception is
   * thrown, the counter written is incremented by
   * 1.
   *
   * @param      v   a boolean value to be written.
   * @exception  IOException  if an I/O error occurs.
   * @see        java.io.FilterOutputStream#out
   */
  public final void writeBoolean(boolean v) throws IOException {
    out.write(v ? 1 : 0);
  }

  /**
   * Writes out a byte to the underlying output stream as
   * a 1-byte value. If no exception is thrown, the counter
   * written is incremented by 1.
   *
   * @param      v   a byte value to be written.
   * @exception  IOException  if an I/O error occurs.
   * @see        java.io.FilterOutputStream#out
   */
  public final void writeByte(int v) throws IOException {
    out.write(v);
  }

  /**
   * Writes a short to the underlying output stream as two
   * bytes, low byte first. If no exception is thrown, the counter
   * written is incremented by 2.
   *
   * @param      v   a short to be written.
   * @exception  IOException  if an I/O error occurs.
   * @see        java.io.FilterOutputStream#out
   */
  public final void writeShort(int v) throws IOException {
    out.write((v >>> 0) & 0xFF);
    out.write((v >>> 8) & 0xFF);
  }

  /**
   * Writes an int to the underlying output stream as four
   * bytes, low byte first. If no exception is thrown, the counter
   * written is incremented by 4.
   *
   * @param      v   an int to be written.
   * @exception  IOException  if an I/O error occurs.
   * @see        java.io.FilterOutputStream#out
   */
  public final void writeInt(int v) throws IOException {
    // TODO: see note in LittleEndianDataInputStream: maybe faster
    // to use Integer.reverseBytes() and then writeInt, or a ByteBuffer
    // approach
    out.write((v >>>  0) & 0xFF);
    out.write((v >>>  8) & 0xFF);
    out.write((v >>> 16) & 0xFF);
    out.write((v >>> 24) & 0xFF);
  }

  private byte writeBuffer[] = new byte[8];

  /**
   * Writes a long to the underlying output stream as eight
   * bytes, low byte first. In no exception is thrown, the counter
   * written is incremented by 8.
   *
   * @param      v   a long to be written.
   * @exception  IOException  if an I/O error occurs.
   * @see        java.io.FilterOutputStream#out
   */
  public final void writeLong(long v) throws IOException {
    writeBuffer[7] = (byte)(v >>> 56);
    writeBuffer[6] = (byte)(v >>> 48);
    writeBuffer[5] = (byte)(v >>> 40);
    writeBuffer[4] = (byte)(v >>> 32);
    writeBuffer[3] = (byte)(v >>> 24);
    writeBuffer[2] = (byte)(v >>> 16);
    writeBuffer[1] = (byte)(v >>>  8);
    writeBuffer[0] = (byte)(v >>>  0);
    out.write(writeBuffer, 0, 8);
  }

  /**
   * Converts the float argument to an int using the
   * floatToIntBits method in class Float,
   * and then writes that int value to the underlying
   * output stream as a 4-byte quantity, low byte first. If no
   * exception is thrown, the counter written is
   * incremented by 4.
   *
   * @param      v   a float value to be written.
   * @exception  IOException  if an I/O error occurs.
   * @see        java.io.FilterOutputStream#out
   * @see        java.lang.Float#floatToIntBits(float)
   */
  public final void writeFloat(float v) throws IOException {
    writeInt(Float.floatToIntBits(v));
  }

  /**
   * Converts the double argument to a long using the
   * doubleToLongBits method in class Double,
   * and then writes that long value to the underlying
   * output stream as an 8-byte quantity, low byte first. If no
   * exception is thrown, the counter written is
   * incremented by 8.
   *
   * @param      v   a double value to be written.
   * @exception  IOException  if an I/O error occurs.
   * @see        java.io.FilterOutputStream#out
   * @see        java.lang.Double#doubleToLongBits(double)
   */
  public final void writeDouble(double v) throws IOException {
    writeLong(Double.doubleToLongBits(v));
  }

  public void close() {
    IOExceptionUtils.closeQuietly(out);
  }

}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.io;

import org.apache.parquet.column.ColumnReader;
import org.apache.parquet.column.impl.ColumnReadStoreImpl;
import org.apache.parquet.filter.RecordFilter;
import org.apache.parquet.filter.UnboundRecordFilter;
import org.apache.parquet.io.api.RecordMaterializer;

/**
 * Extends the
 */
class FilteredRecordReader<T> extends RecordReaderImplementation<T> {

  private final RecordFilter recordFilter;
  private final long recordCount;
  private long recordsRead = 0;

  /**
   * @param root          the root of the schema
   * @param validating
   * @param columnStore
   * @param unboundFilter Filter records, pass in NULL_FILTER to leave unfiltered.
   */
  public FilteredRecordReader(MessageColumnIO root, RecordMaterializer<T> recordMaterializer, boolean validating,
                              ColumnReadStoreImpl columnStore, UnboundRecordFilter unboundFilter, long recordCount) {
    super(root, recordMaterializer, validating, columnStore);
    this.recordCount = recordCount;
    if ( unboundFilter != null ) {
      recordFilter = unboundFilter.bind(getColumnReaders());
    } else {
      recordFilter = null;
    }
  }

  /**
   * Override read() method to provide skip.
   */
  @Override
  public T read() {
    skipToMatch();
    if (recordsRead == recordCount) {
      return null;
    }
    ++ recordsRead;
    return super.read();
  }

  // FilteredRecordReader skips forwards itself, it never asks the layer above to do the skipping for it.
  // This is different from how filtering is handled in the filter2 API
  @Override
  public boolean shouldSkipCurrentRecord() {
    return false;
  }

  /**
   * Skips forwards until the filter finds the first match. Returns false
   * if none found.
   */
  private void skipToMatch() {
    while (recordsRead < recordCount && !recordFilter.isMatch()) {
      State currentState = getState(0);
      do {
        ColumnReader columnReader = currentState.column;

        // currentLevel = depth + 1 at this point
        // set the current value
        if (columnReader.getCurrentDefinitionLevel() >= currentState.maxDefinitionLevel) {
          columnReader.skip();
        }
        columnReader.consume();

        // Based on repetition level work out next state to go to
        int nextR = currentState.maxRepetitionLevel == 0 ? 0 : columnReader.getCurrentRepetitionLevel();
        currentState = currentState.getNextState(nextR);
      } while (currentState != null);
      ++ recordsRead;
    }
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.hadoop.util;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.nio.charset.StandardCharsets;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;

import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.conf.Configuration;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Serialization utils copied from:
 * https://github.com/kevinweil/elephant-bird/blob/master/core/src/main/java/com/twitter/elephantbird/util/HadoopUtils.java
 *
 * TODO: Refactor elephant-bird so that we can depend on utils like this without extra baggage.
 */
public final class SerializationUtil {
  private static final Logger LOG = LoggerFactory.getLogger(SerializationUtil.class);

  private SerializationUtil() { }

  /**
   * Writes an object to a configuration.
   *
   * @param key for the configuration
   * @param obj the object to write
   * @param conf to read from
   * @throws IOException if there is an error while writing
   */
  public static void writeObjectToConfAsBase64(String key, Object obj, Configuration conf) throws IOException {
    try(ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
      try(GZIPOutputStream gos = new GZIPOutputStream(baos);
            ObjectOutputStream oos = new ObjectOutputStream(gos)) {
        oos.writeObject(obj);
      }
      conf.set(key, new String(Base64.encodeBase64(baos.toByteArray()), StandardCharsets.UTF_8));
    }
  }

  /**
   * Reads an object (that was written using
   * {@link #writeObjectToConfAsBase64}) from a configuration
   *
   * @param key for the configuration
   * @param conf to read from
   * @param <T> the Java type of the deserialized object
   * @return the read object, or null if key is not present in conf
   * @throws IOException if there is an error while reading
   */
  @SuppressWarnings("unchecked")
  public static <T> T readObjectFromConfAsBase64(String key, Configuration conf) throws IOException {
    String b64 = conf.get(key);
    if (b64 == null) {
      return null;
    }

    byte[] bytes = Base64.decodeBase64(b64.getBytes(StandardCharsets.UTF_8));

    try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
           GZIPInputStream gis = new GZIPInputStream(bais);
           ObjectInputStream ois  = new ObjectInputStream(gis)) {
      return (T) ois.readObject();
    } catch (ClassNotFoundException e) {
      throw new IOException("Could not read object from config with key " + key, e);
    } catch (ClassCastException e) {
      throw new IOException("Couldn't cast object read from config with key " + key, e);
    }
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.internal.column.columnindex;

import java.util.Formatter;

import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.ints.IntList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.longs.LongList;

/**
 * Builder implementation to create {@link OffsetIndex} objects during writing a parquet file.
 */
public class OffsetIndexBuilder {

  private static class OffsetIndexImpl implements OffsetIndex {
    private long[] offsets;
    private int[] compressedPageSizes;
    private long[] firstRowIndexes;

    @Override
    public String toString() {
      try (Formatter formatter = new Formatter()) {
        formatter.format("%-10s  %20s  %16s  %20s\n", "", "offset", "compressed size", "first row index");
        for (int i = 0, n = offsets.length; i < n; ++i) {
          formatter.format("page-%-5d  %20d  %16d  %20d\n", i, offsets[i], compressedPageSizes[i], firstRowIndexes[i]);
        }
        return formatter.toString();
      }
    }

    @Override
    public int getPageCount() {
      return offsets.length;
    }

    @Override
    public long getOffset(int pageIndex) {
      return offsets[pageIndex];
    }

    @Override
    public int getCompressedPageSize(int pageIndex) {
      return compressedPageSizes[pageIndex];
    }

    @Override
    public long getFirstRowIndex(int pageIndex) {
      return firstRowIndexes[pageIndex];
    }
  }

  private static final OffsetIndexBuilder NO_OP_BUILDER = new OffsetIndexBuilder() {
    @Override
    public void add(int compressedPageSize, long rowCount) {
    }

    @Override
    public void add(long offset, int compressedPageSize, long rowCount) {
    }
  };

  private final LongList offsets = new LongArrayList();
  private final IntList compressedPageSizes = new IntArrayList();
  private final LongList firstRowIndexes = new LongArrayList();
  private long previousOffset;
  private int previousPageSize;
  private long previousRowIndex;
  private long previousRowCount;

  /**
   * @return a no-op builder that does not collect values and therefore returns {@code null} at {@link #build(long)}
   */
  public static OffsetIndexBuilder getNoOpBuilder() {
    return NO_OP_BUILDER;
  }

  /**
   * @return an {@link OffsetIndexBuilder} instance to build an {@link OffsetIndex} object
   */
  public static OffsetIndexBuilder getBuilder() {
    return new OffsetIndexBuilder();
  }

  private OffsetIndexBuilder() {
  }

  /**
   * Adds the specified parameters to this builder. Used by the writers to building up {@link OffsetIndex} objects to be
   * written to the Parquet file.
   *
   * @param compressedPageSize
   *          the size of the page (including header)
   * @param rowCount
   *          the number of rows in the page
   */
  public void add(int compressedPageSize, long rowCount) {
    add(previousOffset + previousPageSize, compressedPageSize, previousRowIndex + previousRowCount);
    previousRowCount = rowCount;
  }

  /**
   * Adds the specified parameters to this builder. Used by the metadata converter to building up {@link OffsetIndex}
   * objects read from the Parquet file.
   *
   * @param offset
   *          the offset of the page in the file
   * @param compressedPageSize
   *          the size of the page (including header)
   * @param firstRowIndex
   *          the index of the first row in the page (within the row group)
   */
  public void add(long offset, int compressedPageSize, long firstRowIndex) {
    previousOffset = offset;
    offsets.add(offset);
    previousPageSize = compressedPageSize;
    compressedPageSizes.add(compressedPageSize);
    previousRowIndex = firstRowIndex;
    firstRowIndexes.add(firstRowIndex);
  }

  /**
   * Builds the offset index. Used by the metadata converter to building up {@link OffsetIndex}
   * objects read from the Parquet file.
   *
   * @return the newly created offset index or {@code null} if the {@link OffsetIndex} object would be empty
   */
  public OffsetIndex build() {
    return build(0);
  }

  /**
   * Builds the offset index. Used by the writers to building up {@link OffsetIndex} objects to be
   * written to the Parquet file.
   *
   * @param firstPageOffset
   *          the actual offset in the file to be used to translate all the collected offsets
   * @return the newly created offset index or {@code null} if the {@link OffsetIndex} object would be empty
   */
  public OffsetIndex build(long firstPageOffset) {
    if (compressedPageSizes.isEmpty()) {
      return null;
    }
    long[] offsets = this.offsets.toLongArray();
    if (firstPageOffset != 0) {
      for (int i = 0, n = offsets.length; i < n; ++i) {
        offsets[i] += firstPageOffset;
      }
    }
    OffsetIndexImpl offsetIndex = new OffsetIndexImpl();
    offsetIndex.offsets = offsets;
    offsetIndex.compressedPageSizes = compressedPageSizes.toIntArray();
    offsetIndex.firstRowIndexes = firstRowIndexes.toLongArray();

    return offsetIndex;
  }

}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.hadoop;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.LinkedHashMap;
import java.util.Map;

/**
 * A basic implementation of an LRU cache.  Besides evicting the least recently
 * used entries (either based on insertion or access order), this class also
 * checks for "stale" entries as entries are inserted or retrieved (note
 * "staleness" is defined by the entries themselves (see
 * {@link org.apache.parquet.hadoop.LruCache.Value}).
 *
 * @param <K> The key type. Acts as the key in a {@link java.util.LinkedHashMap}
 * @param <V> The value type.  Must extend {@link org.apache.parquet.hadoop.LruCache.Value}
 *           so that the "staleness" of the value can be easily determined.
 */
final class LruCache<K, V extends LruCache.Value<K, V>> {
  private static final Logger LOG = LoggerFactory.getLogger(LruCache.class);

  private static final float DEFAULT_LOAD_FACTOR = 0.75f;

  private final LinkedHashMap<K, V> cacheMap;

  /**
   * Constructs an access-order based LRU cache with {@code maxSize} entries.
   * @param maxSize The maximum number of entries to store in the cache.
   */
  public LruCache(final int maxSize) {
    this(maxSize, DEFAULT_LOAD_FACTOR, true);
  }

  /**
   * Constructs an LRU cache.
   *
   * @param maxSize The maximum number of entries to store in the cache.
   * @param loadFactor Used to determine the initial capacity.
   * @param accessOrder the ordering mode - {@code true} for access-order,
   * {@code false} for insertion-order
   */
  public LruCache(final int maxSize, final float loadFactor, final boolean accessOrder) {
    int initialCapacity = Math.round(maxSize / loadFactor);
    cacheMap =
            new LinkedHashMap<K, V>(initialCapacity, loadFactor, accessOrder) {
              @Override
              public boolean removeEldestEntry(final Map.Entry<K, V> eldest) {
                boolean result = size() > maxSize;
                if (result) {
                  if (LOG.isDebugEnabled()) {
                    LOG.debug("Removing eldest entry in cache: "
                            + eldest.getKey());
                  }
                }
                return result;
              }
            };
  }

  /**
   * Removes the mapping for the specified key from this cache if present.
   * @param key key whose mapping is to be removed from the cache
   * @return the previous value associated with key, or null if there was no
   * mapping for key.
   */
  public V remove(final K key) {
    V oldValue = cacheMap.remove(key);
    if (oldValue != null) {
      LOG.debug("Removed cache entry for '{}'", key);
    }
    return oldValue;
  }

  /**
   * Associates the specified value with the specified key in this cache. The
   * value is only inserted if it is not null and it is considered current. If
   * the cache previously contained a mapping for the key, the old value is
   * replaced only if the new value is "newer" than the old one.
   * @param key key with which the specified value is to be associated
   * @param newValue value to be associated with the specified key
   */
  public void put(final K key, final V newValue) {
    if (newValue == null || !newValue.isCurrent(key)) {
      if (LOG.isWarnEnabled()) {
        LOG.warn("Ignoring new cache entry for '{}' because it is {}", key,
                (newValue == null ? "null" : "not current"));
      }
      return;
    }

    V oldValue = cacheMap.get(key);
    if (oldValue != null && oldValue.isNewerThan(newValue)) {
      if (LOG.isWarnEnabled()) {
        LOG.warn("Ignoring new cache entry for '{}' because "
                + "existing cache entry is newer", key);
      }
      return;
    }

    // no existing value or new value is newer than old value
    oldValue = cacheMap.put(key, newValue);
    if (LOG.isDebugEnabled()) {
      if (oldValue == null) {
        LOG.debug("Added new cache entry for '{}'", key);
      } else {
        LOG.debug("Overwrote existing cache entry for '{}'", key);
      }
    }
  }

  /**
   * Removes all of the mappings from this cache. The cache will be empty
   * after this call returns.
   */
  public void clear() {
    cacheMap.clear();
  }

  /**
   * Returns the value to which the specified key is mapped, or null if 1) the
   * value is not current or 2) this cache contains no mapping for the key.
   * @param key the key whose associated value is to be returned
   * @return the value to which the specified key is mapped, or null if 1) the
   * value is not current or 2) this cache contains no mapping for the key
   */
  public V getCurrentValue(final K key) {
    V value = cacheMap.get(key);
    LOG.debug("Value for '{}' {} in cache", key, (value == null ? "not " : ""));
    if (value != null && !value.isCurrent(key)) {
      // value is not current; remove it and return null
      remove(key);
      return null;
    }

    return value;
  }

  /**
   * Returns the number of key-value mappings in this cache.
   * @return the number of key-value mappings in this cache.
   */
  public int size() {
    return cacheMap.size();
  }

  /**
   * {@link org.apache.parquet.hadoop.LruCache} expects all values to follow this
   * interface so the cache can determine 1) whether values are current (e.g.
   * the referenced data has not been modified/updated in such a way that the
   * value is no longer useful) and 2) whether a value is strictly "newer"
   * than another value.
   *
   * @param <K> The key type.
   * @param <V> Provides a bound for the {@link #isNewerThan(V)} method
   */
  interface Value<K, V> {
    /**
     * Is the value still current (e.g. has the referenced data been
     * modified/updated in such a way that the value is no longer useful)
     * @param key the key associated with this value
     * @return {@code true} the value is still current, {@code false} the value
     * is no longer useful
     */
    boolean isCurrent(K key);

    /**
     * Compares this value with the specified value to check for relative age.
     * @param otherValue the value to be compared.
     * @return {@code true} the value is strictly newer than the other value,
     * {@code false} the value is older or just
     * as new as the other value.
     */
    boolean isNewerThan(V otherValue);
  }

}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.avro;

import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.avro.Conversion;
import org.apache.avro.LogicalType;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericFixed;
import org.apache.avro.generic.IndexedRecord;
import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
import org.apache.parquet.hadoop.api.WriteSupport;
import org.apache.parquet.io.api.Binary;
import org.apache.parquet.io.api.RecordConsumer;
import org.apache.parquet.schema.GroupType;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.Type;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.parquet.Preconditions;

/**
 * Avro implementation of {@link WriteSupport} for generic, specific, and
 * reflect models. Use {@link AvroParquetWriter} or
 * {@link AvroParquetOutputFormat} rather than using this class directly.
 */
public class AvroWriteSupport<T> extends WriteSupport<T> {

  public static final String AVRO_DATA_SUPPLIER = "parquet.avro.write.data.supplier";

  public static void setAvroDataSupplier(
      Configuration configuration, Class<? extends AvroDataSupplier> suppClass) {
    configuration.set(AVRO_DATA_SUPPLIER, suppClass.getName());
  }

  static final String AVRO_SCHEMA = "parquet.avro.schema";
  private static final Schema MAP_KEY_SCHEMA = Schema.create(Schema.Type.STRING);

  public static final String WRITE_OLD_LIST_STRUCTURE =
      "parquet.avro.write-old-list-structure";
  static final boolean WRITE_OLD_LIST_STRUCTURE_DEFAULT = true;

  private static final String MAP_REPEATED_NAME = "key_value";
  private static final String MAP_KEY_NAME = "key";
  private static final String MAP_VALUE_NAME = "value";
  private static final String LIST_REPEATED_NAME = "list";
  private static final String OLD_LIST_REPEATED_NAME = "array";
  static final String LIST_ELEMENT_NAME = "element";

  private RecordConsumer recordConsumer;
  private MessageType rootSchema;
  private Schema rootAvroSchema;
  private LogicalType rootLogicalType;
  private Conversion<?> rootConversion;
  private GenericData model;
  private ListWriter listWriter;

  public AvroWriteSupport() {
  }

  /**
   * @param schema the write parquet schema
   * @param avroSchema the write avro schema
   * @deprecated will be removed in 2.0.0
   */
  @Deprecated
  public AvroWriteSupport(MessageType schema, Schema avroSchema) {
    this.rootSchema = schema;
    this.rootAvroSchema = avroSchema;
    this.rootLogicalType = rootAvroSchema.getLogicalType();
    this.model = null;
  }

  public AvroWriteSupport(MessageType schema, Schema avroSchema,
                          GenericData model) {
    this.rootSchema = schema;
    this.rootAvroSchema = avroSchema;
    this.rootLogicalType = rootAvroSchema.getLogicalType();
    this.model = model;
  }

  @Override
  public String getName() {
    return "avro";
  }

  /**
   * @param configuration a configuration
   * @param schema the write schema
   * @see org.apache.parquet.avro.AvroParquetOutputFormat#setSchema(org.apache.hadoop.mapreduce.Job, org.apache.avro.Schema)
   */
  public static void setSchema(Configuration configuration, Schema schema) {
    configuration.set(AVRO_SCHEMA, schema.toString());
  }

  @Override
  public WriteContext init(Configuration configuration) {
    if (rootAvroSchema == null) {
      this.rootAvroSchema = new Schema.Parser().parse(configuration.get(AVRO_SCHEMA));
      this.rootSchema = new AvroSchemaConverter().convert(rootAvroSchema);
    }

    if (model == null) {
      this.model = getDataModel(configuration);
    }

    boolean writeOldListStructure = configuration.getBoolean(
        WRITE_OLD_LIST_STRUCTURE, WRITE_OLD_LIST_STRUCTURE_DEFAULT);
    if (writeOldListStructure) {
      this.listWriter = new TwoLevelListWriter();
    } else {
      this.listWriter = new ThreeLevelListWriter();
    }

    Map<String, String> extraMetaData = new HashMap<String, String>();
    extraMetaData.put(AvroReadSupport.AVRO_SCHEMA_METADATA_KEY, rootAvroSchema.toString());
    return new WriteContext(rootSchema, extraMetaData);
  }

  @Override
  public void prepareForWrite(RecordConsumer recordConsumer) {
    this.recordConsumer = recordConsumer;
  }

  // overloaded version for backward compatibility
  @SuppressWarnings("unchecked")
  public void write(IndexedRecord record) {
    write((T) record);
  }

  @Override
  public void write(T record) {
    if (rootLogicalType != null) {
      Conversion<?> conversion = model.getConversionByClass(
          record.getClass(), rootLogicalType);

      recordConsumer.startMessage();
      writeRecordFields(rootSchema, rootAvroSchema,
          convert(rootAvroSchema, rootLogicalType, conversion, record));
      recordConsumer.endMessage();

    } else {
      recordConsumer.startMessage();
      writeRecordFields(rootSchema, rootAvroSchema, record);
      recordConsumer.endMessage();
    }
  }

  private void writeRecord(GroupType schema, Schema avroSchema,
                           Object record) {
    recordConsumer.startGroup();
    writeRecordFields(schema, avroSchema, record);
    recordConsumer.endGroup();
  }

  private void writeRecordFields(GroupType schema, Schema avroSchema,
                                 Object record) {
    List<Type> fields = schema.getFields();
    List<Schema.Field> avroFields = avroSchema.getFields();
    int index = 0; // parquet ignores Avro nulls, so index may differ
    for (int avroIndex = 0; avroIndex < avroFields.size(); avroIndex++) {
      Schema.Field avroField = avroFields.get(avroIndex);
      if (avroField.schema().getType().equals(Schema.Type.NULL)) {
        continue;
      }
      Type fieldType = fields.get(index);
      Object value = model.getField(record, avroField.name(), avroIndex);
      if (value != null) {
        recordConsumer.startField(fieldType.getName(), index);
        writeValue(fieldType, avroField.schema(), value);
        recordConsumer.endField(fieldType.getName(), index);
      } else if (fieldType.isRepetition(Type.Repetition.REQUIRED)) {
        throw new RuntimeException("Null-value for required field: " + avroField.name());
      }
      index++;
    }
  }

  private <V> void writeMap(GroupType schema, Schema avroSchema,
                            Map<CharSequence, V> map) {
    GroupType innerGroup = schema.getType(0).asGroupType();
    Type keyType = innerGroup.getType(0);
    Type valueType = innerGroup.getType(1);

    recordConsumer.startGroup(); // group wrapper (original type MAP)
    if (map.size() > 0) {
      recordConsumer.startField(MAP_REPEATED_NAME, 0);

      for (Map.Entry<CharSequence, V> entry : map.entrySet()) {
        recordConsumer.startGroup(); // repeated group key_value, middle layer
        recordConsumer.startField(MAP_KEY_NAME, 0);
        writeValue(keyType, MAP_KEY_SCHEMA, entry.getKey());
        recordConsumer.endField(MAP_KEY_NAME, 0);
        V value = entry.getValue();
        if (value != null) {
          recordConsumer.startField(MAP_VALUE_NAME, 1);
          writeValue(valueType, avroSchema.getValueType(), value);
          recordConsumer.endField(MAP_VALUE_NAME, 1);
        } else if (!valueType.isRepetition(Type.Repetition.OPTIONAL)) {
          throw new RuntimeException("Null map value for " + avroSchema.getName());
        }
        recordConsumer.endGroup();
      }

      recordConsumer.endField(MAP_REPEATED_NAME, 0);
    }
    recordConsumer.endGroup();
  }

  private void writeUnion(GroupType parquetSchema, Schema avroSchema,
                          Object value) {
    recordConsumer.startGroup();

    // ResolveUnion will tell us which of the union member types to
    // deserialise.
    int avroIndex = model.resolveUnion(avroSchema, value);

    // For parquet's schema we skip nulls
    GroupType parquetGroup = parquetSchema.asGroupType();
    int parquetIndex = avroIndex;
    for (int i = 0; i < avroIndex; i++) {
      if (avroSchema.getTypes().get(i).getType().equals(Schema.Type.NULL)) {
        parquetIndex--;
      }
    }

    // TODO: what if the value is null?

    // Sparsely populated method of encoding unions, each member has its own
    // set of columns.
    String memberName = "member" + parquetIndex;
    recordConsumer.startField(memberName, parquetIndex);
    writeValue(parquetGroup.getType(parquetIndex),
               avroSchema.getTypes().get(avroIndex), value);
    recordConsumer.endField(memberName, parquetIndex);

    recordConsumer.endGroup();
  }

  /**
   * Calls an appropriate write method based on the value.
   * Value MUST not be null.
   *
   * @param type the Parquet type
   * @param avroSchema the Avro schema
   * @param value a non-null value to write
   */
  private void writeValue(Type type, Schema avroSchema, Object value) {
    Schema nonNullAvroSchema = AvroSchemaConverter.getNonNull(avroSchema);
    LogicalType logicalType = nonNullAvroSchema.getLogicalType();
    if (logicalType != null) {
      Conversion<?> conversion = model.getConversionByClass(
          value.getClass(), logicalType);
      writeValueWithoutConversion(type, nonNullAvroSchema,
          convert(nonNullAvroSchema, logicalType, conversion, value));
    } else {
      writeValueWithoutConversion(type, nonNullAvroSchema, value);
    }
  }

  private <D> Object convert(Schema schema, LogicalType logicalType,
                             Conversion<D> conversion, Object datum) {
    if (conversion == null) {
      return datum;
    }
    Class<D> fromClass = conversion.getConvertedType();
    switch (schema.getType()) {
      case RECORD:  return conversion.toRecord(fromClass.cast(datum), schema, logicalType);
      case ENUM:    return conversion.toEnumSymbol(fromClass.cast(datum), schema, logicalType);
      case ARRAY:   return conversion.toArray(fromClass.cast(datum), schema, logicalType);
      case MAP:     return conversion.toMap(fromClass.cast(datum), schema, logicalType);
      case FIXED:   return conversion.toFixed(fromClass.cast(datum), schema, logicalType);
      case STRING:  return conversion.toCharSequence(fromClass.cast(datum), schema, logicalType);
      case BYTES:   return conversion.toBytes(fromClass.cast(datum), schema, logicalType);
      case INT:     return conversion.toInt(fromClass.cast(datum), schema, logicalType);
      case LONG:    return conversion.toLong(fromClass.cast(datum), schema, logicalType);
      case FLOAT:   return conversion.toFloat(fromClass.cast(datum), schema, logicalType);
      case DOUBLE:  return conversion.toDouble(fromClass.cast(datum), schema, logicalType);
      case BOOLEAN: return conversion.toBoolean(fromClass.cast(datum), schema, logicalType);
    }
    return datum;
  }

  /**
   * Calls an appropriate write method based on the value.
   * Value must not be null and the schema must not be nullable.
   *
   * @param type a Parquet type
   * @param avroSchema a non-nullable Avro schema
   * @param value a non-null value to write
   */
  @SuppressWarnings("unchecked")
  private void writeValueWithoutConversion(Type type, Schema avroSchema, Object value) {
    switch (avroSchema.getType()) {
      case BOOLEAN:
        recordConsumer.addBoolean((Boolean) value);
        break;
      case INT:
        if (value instanceof Character) {
          recordConsumer.addInteger((Character) value);
        } else {
          recordConsumer.addInteger(((Number) value).intValue());
        }
        break;
      case LONG:
        recordConsumer.addLong(((Number) value).longValue());
        break;
      case FLOAT:
        recordConsumer.addFloat(((Number) value).floatValue());
        break;
      case DOUBLE:
        recordConsumer.addDouble(((Number) value).doubleValue());
        break;
      case FIXED:
        recordConsumer.addBinary(Binary.fromReusedByteArray(((GenericFixed) value).bytes()));
        break;
      case BYTES:
        if (value instanceof byte[]) {
          recordConsumer.addBinary(Binary.fromReusedByteArray((byte[]) value));
        } else {
          recordConsumer.addBinary(Binary.fromReusedByteBuffer((ByteBuffer) value));
        }
        break;
      case STRING:
        recordConsumer.addBinary(fromAvroString(value));
        break;
      case RECORD:
        writeRecord(type.asGroupType(), avroSchema, value);
        break;
      case ENUM:
        recordConsumer.addBinary(Binary.fromString(value.toString()));
        break;
      case ARRAY:
        listWriter.writeList(type.asGroupType(), avroSchema, value);
        break;
      case MAP:
        writeMap(type.asGroupType(), avroSchema, (Map<CharSequence, ?>) value);
        break;
      case UNION:
        writeUnion(type.asGroupType(), avroSchema, value);
        break;
    }
  }

  private Binary fromAvroString(Object value) {
    if (value instanceof Utf8) {
      Utf8 utf8 = (Utf8) value;
      return Binary.fromReusedByteArray(utf8.getBytes(), 0, utf8.getByteLength());
    }
    return Binary.fromCharSequence((CharSequence) value);
  }

  private static GenericData getDataModel(Configuration conf) {
    Class<? extends AvroDataSupplier> suppClass = conf.getClass(
        AVRO_DATA_SUPPLIER, SpecificDataSupplier.class, AvroDataSupplier.class);
    return ReflectionUtils.newInstance(suppClass, conf).get();
  }

  private abstract class ListWriter {

    protected abstract void writeCollection(
        GroupType type, Schema schema, Collection<?> collection);

    protected abstract void writeObjectArray(
        GroupType type, Schema schema, Object[] array);

    protected abstract void startArray();

    protected abstract void endArray();

    public void writeList(GroupType schema, Schema avroSchema, Object value) {
      recordConsumer.startGroup(); // group wrapper (original type LIST)
      if (value instanceof Collection) {
        writeCollection(schema, avroSchema, (Collection) value);
      } else {
        Class<?> arrayClass = value.getClass();
        Preconditions.checkArgument(arrayClass.isArray(),
            "Cannot write unless collection or array: " + arrayClass.getName());
        writeJavaArray(schema, avroSchema, arrayClass, value);
      }
      recordConsumer.endGroup();
    }

    public void writeJavaArray(GroupType schema, Schema avroSchema,
                               Class<?> arrayClass, Object value) {
      Class<?> elementClass = arrayClass.getComponentType();

      if (!elementClass.isPrimitive()) {
        writeObjectArray(schema, avroSchema, (Object[]) value);
        return;
      }

      switch (avroSchema.getElementType().getType()) {
        case BOOLEAN:
          Preconditions.checkArgument(elementClass == boolean.class,
              "Cannot write as boolean array: " + arrayClass.getName());
          writeBooleanArray((boolean[]) value);
          break;
        case INT:
          if (elementClass == byte.class) {
            writeByteArray((byte[]) value);
          } else if (elementClass == char.class) {
            writeCharArray((char[]) value);
          } else if (elementClass == short.class) {
            writeShortArray((short[]) value);
          } else if (elementClass == int.class) {
            writeIntArray((int[]) value);
          } else {
            throw new IllegalArgumentException(
                "Cannot write as an int array: " + arrayClass.getName());
          }
          break;
        case LONG:
          Preconditions.checkArgument(elementClass == long.class,
              "Cannot write as long array: " + arrayClass.getName());
          writeLongArray((long[]) value);
          break;
        case FLOAT:
          Preconditions.checkArgument(elementClass == float.class,
              "Cannot write as float array: " + arrayClass.getName());
          writeFloatArray((float[]) value);
          break;
        case DOUBLE:
          Preconditions.checkArgument(elementClass == double.class,
              "Cannot write as double array: " + arrayClass.getName());
          writeDoubleArray((double[]) value);
          break;
        default:
          throw new IllegalArgumentException("Cannot write " +
              avroSchema.getElementType() + " array: " + arrayClass.getName());
      }
    }

    protected void writeBooleanArray(boolean[] array) {
      if (array.length > 0) {
        startArray();
        for (boolean element : array) {
          recordConsumer.addBoolean(element);
        }
        endArray();
      }
    }

    protected void writeByteArray(byte[] array) {
      if (array.length > 0) {
        startArray();
        for (byte element : array) {
          recordConsumer.addInteger(element);
        }
        endArray();
      }
    }

    protected void writeShortArray(short[] array) {
      if (array.length > 0) {
        startArray();
        for (short element : array) {
          recordConsumer.addInteger(element);
        }
        endArray();
      }
    }

    protected void writeCharArray(char[] array) {
      if (array.length > 0) {
        startArray();
        for (char element : array) {
          recordConsumer.addInteger(element);
        }
        endArray();
      }
    }

    protected void writeIntArray(int[] array) {
      if (array.length > 0) {
        startArray();
        for (int element : array) {
          recordConsumer.addInteger(element);
        }
        endArray();
      }
    }

    protected void writeLongArray(long[] array) {
      if (array.length > 0) {
        startArray();
        for (long element : array) {
          recordConsumer.addLong(element);
        }
        endArray();
      }
    }

    protected void writeFloatArray(float[] array) {
      if (array.length > 0) {
        startArray();
        for (float element : array) {
          recordConsumer.addFloat(element);
        }
        endArray();
      }
    }

    protected void writeDoubleArray(double[] array) {
      if (array.length > 0) {
        startArray();
        for (double element : array) {
          recordConsumer.addDouble(element);
        }
        endArray();
      }
    }
  }

  /**
   * For backward-compatibility. This preserves how lists were written in 1.x.
   */
  private class TwoLevelListWriter extends ListWriter {
    @Override
    public void writeCollection(GroupType schema, Schema avroSchema,
                                Collection<?> array) {
      if (array.size() > 0) {
        recordConsumer.startField(OLD_LIST_REPEATED_NAME, 0);
        try {
          for (Object elt : array) {
            writeValue(schema.getType(0), avroSchema.getElementType(), elt);
          }
        } catch (NullPointerException e) {
          // find the null element and throw a better error message
          int i = 0;
          for (Object elt : array) {
            if (elt == null) {
              throw new NullPointerException(
                  "Array contains a null element at " + i + "\n" +
                  "Set parquet.avro.write-old-list-structure=false to turn " +
                  "on support for arrays with null elements.");
            }
            i += 1;
          }
          // no element was null, throw the original exception
          throw e;
        }
        recordConsumer.endField(OLD_LIST_REPEATED_NAME, 0);
      }
    }

    @Override
    protected void writeObjectArray(GroupType type, Schema schema,
                                    Object[] array) {
      if (array.length > 0) {
        recordConsumer.startField(OLD_LIST_REPEATED_NAME, 0);
        try {
          for (Object element : array) {
            writeValue(type.getType(0), schema.getElementType(), element);
          }
        } catch (NullPointerException e) {
          // find the null element and throw a better error message
          for (int i = 0; i < array.length; i += 1) {
            if (array[i] == null) {
              throw new NullPointerException(
                  "Array contains a null element at " + i + "\n" +
                  "Set parquet.avro.write-old-list-structure=false to turn " +
                  "on support for arrays with null elements.");
            }
          }
          // no element was null, throw the original exception
          throw e;
        }
        recordConsumer.endField(OLD_LIST_REPEATED_NAME, 0);
      }
    }

    @Override
    protected void startArray() {
      recordConsumer.startField(OLD_LIST_REPEATED_NAME, 0);
    }

    @Override
    protected void endArray() {
      recordConsumer.endField(OLD_LIST_REPEATED_NAME, 0);
    }
  }

  private class ThreeLevelListWriter extends ListWriter {
    @Override
    protected void writeCollection(GroupType type, Schema schema, Collection<?> collection) {
      if (collection.size() > 0) {
        recordConsumer.startField(LIST_REPEATED_NAME, 0);
        GroupType repeatedType = type.getType(0).asGroupType();
        Type elementType = repeatedType.getType(0);
        for (Object element : collection) {
          recordConsumer.startGroup(); // repeated group array, middle layer
          if (element != null) {
            recordConsumer.startField(LIST_ELEMENT_NAME, 0);
            writeValue(elementType, schema.getElementType(), element);
            recordConsumer.endField(LIST_ELEMENT_NAME, 0);
          } else if (!elementType.isRepetition(Type.Repetition.OPTIONAL)) {
            throw new RuntimeException(
                "Null list element for " + schema.getName());
          }
          recordConsumer.endGroup();
        }
        recordConsumer.endField(LIST_REPEATED_NAME, 0);
      }
    }

    @Override
    protected void writeObjectArray(GroupType type, Schema schema,
                                    Object[] array) {
      if (array.length > 0) {
        recordConsumer.startField(LIST_REPEATED_NAME, 0);
        GroupType repeatedType = type.getType(0).asGroupType();
        Type elementType = repeatedType.getType(0);
        for (Object element : array) {
          recordConsumer.startGroup(); // repeated group array, middle layer
          if (element != null) {
            recordConsumer.startField(LIST_ELEMENT_NAME, 0);
            writeValue(elementType, schema.getElementType(), element);
            recordConsumer.endField(LIST_ELEMENT_NAME, 0);
          } else if (!elementType.isRepetition(Type.Repetition.OPTIONAL)) {
            throw new RuntimeException(
                "Null list element for " + schema.getName());
          }
          recordConsumer.endGroup();
        }
        recordConsumer.endField(LIST_REPEATED_NAME, 0);
      }
    }

    @Override
    protected void startArray() {
      recordConsumer.startField(LIST_REPEATED_NAME, 0);
      recordConsumer.startGroup(); // repeated group array, middle layer
      recordConsumer.startField(LIST_ELEMENT_NAME, 0);
    }

    @Override
    protected void endArray() {
      recordConsumer.endField(LIST_ELEMENT_NAME, 0);
      recordConsumer.endGroup();
      recordConsumer.endField(LIST_REPEATED_NAME, 0);
    }
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.thrift.projection.deprecated;

import org.apache.hadoop.fs.GlobPattern;

import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;

/**
 * Enhanced version of GlobPattern class that is defined in hadoop with extra capability of matching
 * full path separated by '/', and double star matching
 *
 * This is used for parsing values assigned to ThriftReadSupport.THRIFT_COLUMN_FILTER_KEY
 */
@Deprecated
public class PathGlobPattern {
  private static final char BACKSLASH = '\\';
  private static final char PATH_SEPARATOR = '/';
  private Pattern compiled;
  private boolean hasWildcard = false;

  /**
   * Construct the glob pattern object with a glob pattern string
   *
   * @param globPattern the glob pattern string
   */
  public PathGlobPattern(String globPattern) {
    set(globPattern);
  }

  /**
   * Compile glob pattern string
   *
   * @param globPattern the glob pattern
   * @return the pattern object
   */
  public static Pattern compile(String globPattern) {
    return new GlobPattern(globPattern).compiled();
  }

  private static void error(String message, String pattern, int pos) {
    throw new PatternSyntaxException(message, pattern, pos);
  }

  /**
   * @return the compiled pattern
   */
  public Pattern compiled() {
    return compiled;
  }

  /**
   * Match input against the compiled glob pattern
   *
   * @param s input chars
   * @return true for successful matches
   */
  public boolean matches(CharSequence s) {
    return compiled.matcher(s).matches();
  }

  /**
   * Set and compile a glob pattern
   *
   * @param glob the glob pattern string
   */
  public void set(String glob) {
    StringBuilder regex = new StringBuilder();
    int setOpen = 0;
    int curlyOpen = 0;
    int len = glob.length();
    hasWildcard = false;

    for (int i = 0; i < len; i++) {
      char c = glob.charAt(i);

      switch (c) {
        case BACKSLASH:
          if (++i >= len) {
            error("Missing escaped character", glob, i);
          }
          regex.append(c).append(glob.charAt(i));
          continue;
        case '.':
        case '$':
        case '(':
        case ')':
        case '|':
        case '+':
          // escape regex special chars that are not glob special chars
          regex.append(BACKSLASH);
          break;
        case '*':
          if (i + 1 < len && glob.charAt(i + 1) == '*') {
            regex.append('.');
            i++;
            break;
          }
          regex.append("[^" + PATH_SEPARATOR + "]");
          hasWildcard = true;
          break;
        case '?':
          regex.append('.');
          hasWildcard = true;
          continue;
        case '{': // start of a group
          regex.append("(?:"); // non-capturing
          curlyOpen++;
          hasWildcard = true;
          continue;
        case ',':
          regex.append(curlyOpen > 0 ? '|' : c);
          continue;
        case '}':
          if (curlyOpen > 0) {
            // end of a group
            curlyOpen--;
            regex.append(")");
            continue;
          }
          break;
        case '[':
          if (setOpen > 0) {
            error("Unclosed character class", glob, i);
          }
          setOpen++;
          hasWildcard = true;
          break;
        case '^': // ^ inside [...] can be unescaped
          if (setOpen == 0) {
            regex.append(BACKSLASH);
          }
          break;
        case '!': // [! needs to be translated to [^
          regex.append(setOpen > 0 && '[' == glob.charAt(i - 1) ? '^' : '!');
          continue;
        case ']':
          // Many set errors like [][] could not be easily detected here,
          // as []], []-] and [-] are all valid POSIX glob and java regex.
          // We'll just let the regex compiler do the real work.
          setOpen = 0;
          break;
        default:
      }
      regex.append(c);
    }

    if (setOpen > 0) {
      error("Unclosed character class", glob, len);
    }
    if (curlyOpen > 0) {
      error("Unclosed group", glob, len);
    }
    compiled = Pattern.compile(regex.toString());
  }

  @Override
  public String toString() {
    return compiled.toString();
  }

  /**
   * @return true if this is a wildcard pattern (with special chars)
   */
  public boolean hasWildcard() {
    return hasWildcard;
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.parquet.cli;

import com.beust.jcommander.internal.Lists;
import com.google.common.base.Preconditions;
import com.google.common.io.CharStreams;
import com.google.common.io.Resources;
import org.apache.avro.Schema;
import org.apache.avro.file.DataFileReader;
import org.apache.avro.file.SeekableInput;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.ChecksumFileSystem;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.parquet.avro.AvroParquetReader;
import org.apache.parquet.avro.AvroReadSupport;
import org.apache.parquet.cli.json.AvroJsonReader;
import org.apache.parquet.cli.util.Formats;
import org.apache.parquet.cli.util.GetClassLoader;
import org.apache.parquet.cli.util.Schemas;
import org.apache.parquet.cli.util.SeekableFSDataInputStream;
import org.apache.parquet.hadoop.ParquetReader;
import org.slf4j.Logger;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.security.AccessController;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;

public abstract class BaseCommand implements Command, Configurable {

  private static final String RESOURCE_URI_SCHEME = "resource";
  private static final String STDIN_AS_SOURCE = "stdin";

  protected final Logger console;

  private Configuration conf = null;
  private LocalFileSystem localFS = null;

  public BaseCommand(Logger console) {
    this.console = console;
  }

  /**
   * @return FileSystem to use when no file system scheme is present in a path
   * @throws IOException if there is an error loading the default fs
   */
  public FileSystem defaultFS() throws IOException {
    if (localFS == null) {
      this.localFS = FileSystem.getLocal(getConf());
    }
    return localFS;
  }

  /**
   * Output content to the console or a file.
   *
   * This will not produce checksum files.
   *
   * @param content String content to write
   * @param console A {@link Logger} for writing to the console
   * @param filename The destination {@link Path} as a String
   * @throws IOException if there is an error while writing
   */
  public void output(String content, Logger console, String filename)
      throws IOException {
    if (filename == null || "-".equals(filename)) {
      console.info(content);
    } else {
      FSDataOutputStream outgoing = create(filename);
      try {
        outgoing.write(content.getBytes(StandardCharsets.UTF_8));
      } finally {
        outgoing.close();
      }
    }
  }

  /**
   * Creates a file and returns an open {@link FSDataOutputStream}.
   *
   * If the file does not have a file system scheme, this uses the default FS.
   *
   * This will not produce checksum files and will overwrite a file that
   * already exists.
   *
   * @param filename The filename to create
   * @return An open FSDataOutputStream
   * @throws IOException if there is an error creating the file
   */
  public FSDataOutputStream create(String filename) throws IOException {
    return create(filename, true);
  }

  /**
   * Creates a file and returns an open {@link FSDataOutputStream}.
   *
   * If the file does not have a file system scheme, this uses the default FS.
   *
   * This will produce checksum files and will overwrite a file that already
   * exists.
   *
   * @param filename The filename to create
   * @return An open FSDataOutputStream
   * @throws IOException if there is an error creating the file
   */
  public FSDataOutputStream createWithChecksum(String filename)
      throws IOException {
    return create(filename, false);
  }

  private FSDataOutputStream create(String filename, boolean noChecksum)
      throws IOException {
    Path filePath = qualifiedPath(filename);
    // even though it was qualified using the default FS, it may not be in it
    FileSystem fs = filePath.getFileSystem(getConf());
    if (noChecksum && fs instanceof ChecksumFileSystem) {
      fs = ((ChecksumFileSystem) fs).getRawFileSystem();
    }
    return fs.create(filePath, true /* overwrite */);
  }

  /**
   * Returns a qualified {@link Path} for the {@code filename}.
   *
   * If the file does not have a file system scheme, this uses the default FS.
   *
   * @param filename The filename to qualify
   * @return A qualified Path for the filename
   * @throws IOException if there is an error creating a qualified path
   */
  public Path qualifiedPath(String filename) throws IOException {
    Path cwd = defaultFS().makeQualified(new Path("."));
    return new Path(filename).makeQualified(defaultFS().getUri(), cwd);
  }

  /**
   * Returns a {@link URI} for the {@code filename} that is a qualified Path or
   * a resource URI.
   *
   * If the file does not have a file system scheme, this uses the default FS.
   *
   * @param filename The filename to qualify
   * @return A qualified URI for the filename
   * @throws IOException if there is an error creating a qualified URI
   */
  public URI qualifiedURI(String filename) throws IOException {
    URI fileURI = URI.create(filename);
    if (RESOURCE_URI_SCHEME.equals(fileURI.getScheme())) {
      return fileURI;
    } else {
      return qualifiedPath(filename).toUri();
    }
  }

  /**
   * Opens an existing file or resource.
   *
   * If the file does not have a file system scheme, this uses the default FS.
   *
   * @param filename The filename to open.
   * @return An open InputStream with the file contents
   * @throws IOException if there is an error opening the file
   * @throws IllegalArgumentException If the file does not exist
   */
  public InputStream open(String filename) throws IOException {
    if (STDIN_AS_SOURCE.equals(filename)) {
      return System.in;
    }

    URI uri = qualifiedURI(filename);
    if (RESOURCE_URI_SCHEME.equals(uri.getScheme())) {
      return Resources.getResource(uri.getRawSchemeSpecificPart()).openStream();
    } else {
      Path filePath = new Path(uri);
      // even though it was qualified using the default FS, it may not be in it
      FileSystem fs = filePath.getFileSystem(getConf());
      return fs.open(filePath);
    }
  }

  public SeekableInput openSeekable(String filename) throws IOException {
    Path path = qualifiedPath(filename);
    // even though it was qualified using the default FS, it may not be in it
    FileSystem fs = path.getFileSystem(getConf());
    return new SeekableFSDataInputStream(fs, path);
  }

  @Override
  public void setConf(Configuration conf) {
    this.conf = conf;
    HadoopFileSystemURLStreamHandler.setDefaultConf(conf);
  }

  @Override
  public Configuration getConf() {
    return conf;
  }

  /**
   * Returns a {@link ClassLoader} for a set of jars and directories.
   *
   * @param jars A list of jar paths
   * @param paths A list of directories containing .class files
   * @return a classloader for the jars and paths
   * @throws MalformedURLException if  a jar or path is invalid
   */
  protected static ClassLoader loaderFor(List<String> jars, List<String> paths)
      throws MalformedURLException {
    return AccessController.doPrivileged(new GetClassLoader(urls(jars, paths)));
  }

  /**
   * Returns a {@link ClassLoader} for a set of jars.
   *
   * @param jars A list of jar paths
   * @return a classloader for the jars
   * @throws MalformedURLException if a URL is invalid
   */
  protected static ClassLoader loaderForJars(List<String> jars)
      throws MalformedURLException {
    return AccessController.doPrivileged(new GetClassLoader(urls(jars, null)));
  }

  /**
   * Returns a {@link ClassLoader} for a set of directories.
   *
   * @param paths A list of directories containing .class files
   * @return a classloader for the paths
   * @throws MalformedURLException if a path is invalid
   */
  protected static ClassLoader loaderForPaths(List<String> paths)
      throws MalformedURLException {
    return AccessController.doPrivileged(new GetClassLoader(urls(null, paths)));
  }

  private static List<URL> urls(List<String> jars, List<String> dirs)
      throws MalformedURLException {
    // check the additional jars and lib directories in the local FS
    final List<URL> urls = Lists.newArrayList();
    if (dirs != null) {
      for (String lib : dirs) {
        // final URLs must end in '/' for URLClassLoader
        File path = lib.endsWith("/") ? new File(lib) : new File(lib + "/");
        Preconditions.checkArgument(path.exists(),
            "Lib directory does not exist: " + lib);
        Preconditions.checkArgument(path.isDirectory(),
            "Not a directory: " + lib);
        Preconditions.checkArgument(path.canRead() && path.canExecute(),
            "Insufficient permissions to access lib directory: " + lib);
        urls.add(path.toURI().toURL());
      }
    }
    if (jars != null) {
      for (String jar : jars) {
        File path = new File(jar);
        Preconditions.checkArgument(path.exists(),
            "Jar files does not exist: " + jar);
        Preconditions.checkArgument(path.isFile(),
            "Not a file: " + jar);
        Preconditions.checkArgument(path.canRead(),
            "Cannot read jar file: " + jar);
        urls.add(path.toURI().toURL());
      }
    }
    return urls;
  }

  protected <D> Iterable<D> openDataFile(final String source, Schema projection)
      throws IOException {
    Formats.Format format = Formats.detectFormat(open(source));
    switch (format) {
      case PARQUET:
        Configuration conf = new Configuration(getConf());
        // TODO: add these to the reader builder
        AvroReadSupport.setRequestedProjection(conf, projection);
        AvroReadSupport.setAvroReadSchema(conf, projection);
        final ParquetReader<D> parquet = AvroParquetReader.<D>builder(qualifiedPath(source))
            .disableCompatibility()
            .withDataModel(GenericData.get())
            .withConf(conf)
            .build();
        return new Iterable<D>() {
          @Override
          public Iterator<D> iterator() {
            return new Iterator<D>() {
              private boolean hasNext = false;
              private D next = advance();

              @Override
              public boolean hasNext() {
                return hasNext;
              }

              @Override
              public D next() {
                if (!hasNext) {
                  throw new NoSuchElementException();
                }
                D toReturn = next;
                this.next = advance();
                return toReturn;
              }

              private D advance() {
                try {
                  D next = parquet.read();
                  this.hasNext = (next != null);
                  return next;
                } catch (IOException e) {
                  throw new RuntimeException(
                      "Failed while reading Parquet file: " + source, e);
                }
              }

              @Override
              public void remove() {
                throw new UnsupportedOperationException("Remove is not supported");
              }
            };
          }
        };

      case AVRO:
        Iterable<D> avroReader = (Iterable<D>) DataFileReader.openReader(
            openSeekable(source), new GenericDatumReader<>(projection));
        return avroReader;

      default:
        if (source.endsWith("json")) {
          return new AvroJsonReader<>(open(source), projection);
        } else {
          Preconditions.checkArgument(projection == null,
              "Cannot select columns from text files");
          Iterable text = CharStreams.readLines(new InputStreamReader(open(source)));
          return text;
        }
    }
  }

  protected Schema getAvroSchema(String source) throws IOException {
    Formats.Format format;
    try (SeekableInput in = openSeekable(source)) {
      format = Formats.detectFormat((InputStream) in);
      in.seek(0);

      switch (format) {
        case PARQUET:
          return Schemas.fromParquet(getConf(), qualifiedURI(source));
        case AVRO:
          return Schemas.fromAvro(open(source));
        case TEXT:
          if (source.endsWith("avsc")) {
            return Schemas.fromAvsc(open(source));
          } else if (source.endsWith("json")) {
            return Schemas.fromJSON("json", open(source));
          }
        default:
      }

      throw new IllegalArgumentException(String.format(
          "Could not determine file format of %s.", source));
    }
  }

}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.column.values.bitpacking;

import static org.apache.parquet.bytes.BytesUtils.getWidthFromMaxInt;
import static org.apache.parquet.column.Encoding.BIT_PACKED;
import static org.apache.parquet.column.values.bitpacking.BitPacking.getBitPackingWriter;

import java.io.IOException;

import org.apache.parquet.bytes.ByteBufferAllocator;
import org.apache.parquet.bytes.BytesInput;
import org.apache.parquet.bytes.CapacityByteArrayOutputStream;
import org.apache.parquet.column.Encoding;
import org.apache.parquet.column.values.ValuesWriter;
import org.apache.parquet.column.values.bitpacking.BitPacking.BitPackingWriter;
import org.apache.parquet.io.ParquetEncodingException;

/**
 * a column writer that packs the ints in the number of bits required based on the maximum size.
 */
public class BitPackingValuesWriter extends ValuesWriter {

  private CapacityByteArrayOutputStream out;
  private BitPackingWriter bitPackingWriter;
  private int bitsPerValue;

  /**
   * @param bound the maximum value stored by this column
   * @param initialCapacity initial capacity for the writer
   * @param pageSize the page size
   * @param allocator a buffer allocator
   */
  public BitPackingValuesWriter(int bound, int initialCapacity, int pageSize, ByteBufferAllocator allocator) {
    this.bitsPerValue = getWidthFromMaxInt(bound);
    this.out = new CapacityByteArrayOutputStream(initialCapacity, pageSize, allocator);
    init();
  }

  private void init() {
    this.bitPackingWriter = getBitPackingWriter(bitsPerValue, out);
  }

  /**
   * {@inheritDoc}
   * @see org.apache.parquet.column.values.ValuesWriter#writeInteger(int)
   */
  @Override
  public void writeInteger(int v) {
    try {
      bitPackingWriter.write(v);
    } catch (IOException e) {
      throw new ParquetEncodingException(e);
    }
  }

  /**
   * {@inheritDoc}
   * @see org.apache.parquet.column.values.ValuesWriter#getBufferedSize()
   */
  @Override
  public long getBufferedSize() {
    return out.size();
  }

  /**
   * {@inheritDoc}
   * @see org.apache.parquet.column.values.ValuesWriter#getBytes()
   */
  @Override
  public BytesInput getBytes() {
    try {
      this.bitPackingWriter.finish();
      return BytesInput.from(out);
    } catch (IOException e) {
      throw new ParquetEncodingException(e);
    }
  }

  /**
   * {@inheritDoc}
   * @see org.apache.parquet.column.values.ValuesWriter#reset()
   */
  @Override
  public void reset() {
    out.reset();
    init();
  }

  @Override
  public void close() {
    out.close();
  }

  /**
   * {@inheritDoc}
   * @see org.apache.parquet.column.values.ValuesWriter#getAllocatedSize()
   */
  @Override
  public long getAllocatedSize() {
    return out.getCapacity();
  }

  @Override
  public String memUsageString(String prefix) {
    return out.memUsageString(prefix);
  }

  @Override
  public Encoding getEncoding() {
    return BIT_PACKED;
  }

}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.filter;

import org.apache.parquet.column.ColumnReader;
import java.util.Arrays;
import static org.apache.parquet.Preconditions.checkNotNull;

/**
 * Record filter which applies the supplied predicate to the specified column.
 */
public final class ColumnRecordFilter implements RecordFilter {

  private final ColumnReader filterOnColumn;
  private final ColumnPredicates.Predicate filterPredicate;

  /**
   * Factory method for record filter which applies the supplied predicate to the specified column.
   * Note that if searching for a repeated sub-attribute it will only ever match against the
   * first instance of it in the object.
   *
   * @param columnPath Dot separated path specifier, e.g. "engine.capacity"
   * @param predicate Should call getBinary etc. and check the value
   * @return a column filter
   */
  public static final UnboundRecordFilter column(final String columnPath,
                                                 final ColumnPredicates.Predicate predicate) {
    checkNotNull(columnPath, "columnPath");
    checkNotNull(predicate,  "predicate");
    return new UnboundRecordFilter() {
      final String[] filterPath = columnPath.split("\\.");
      @Override
      public RecordFilter bind(Iterable<ColumnReader> readers) {
        for (ColumnReader reader : readers) {
          if ( Arrays.equals( reader.getDescriptor().getPath(), filterPath)) {
            return new ColumnRecordFilter(reader, predicate);
          }
        }
        throw new IllegalArgumentException( "Column " + columnPath + " does not exist.");
      }
    };
  }

  /**
   * Private constructor. Use column() instead.
   */
  private ColumnRecordFilter(ColumnReader filterOnColumn, ColumnPredicates.Predicate filterPredicate) {
    this.filterOnColumn  = filterOnColumn;
    this.filterPredicate = filterPredicate;
  }

  /**
   * @return true if the current value for the column reader matches the predicate.
   */
  @Override
  public boolean isMatch() {
    return filterPredicate.apply(filterOnColumn);
  }

}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.hadoop;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Formatter;
import java.util.List;

import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
import org.apache.parquet.internal.column.columnindex.OffsetIndex;
import org.apache.parquet.internal.filter2.columnindex.RowRanges;

import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.ints.IntList;

/**
 * Internal utility class to help at column index based filtering.
 */
class ColumnIndexFilterUtils {
  static class OffsetRange {
    private final long offset;
    private long length;

    private OffsetRange(long offset, int length) {
      this.offset = offset;
      this.length = length;
    }

    long getOffset() {
      return offset;
    }

    long getLength() {
      return length;
    }

    private boolean extend(long offset, int length) {
      if (this.offset + this.length == offset) {
        this.length += length;
        return true;
      } else {
        return false;
      }
    }
  }

  private static class FilteredOffsetIndex implements OffsetIndex {
    private final OffsetIndex offsetIndex;
    private final int[] indexMap;

    private FilteredOffsetIndex(OffsetIndex offsetIndex, int[] indexMap) {
      this.offsetIndex = offsetIndex;
      this.indexMap = indexMap;
    }

    @Override
    public int getPageCount() {
      return indexMap.length;
    }

    @Override
    public long getOffset(int pageIndex) {
      return offsetIndex.getOffset(indexMap[pageIndex]);
    }

    @Override
    public int getCompressedPageSize(int pageIndex) {
      return offsetIndex.getCompressedPageSize(indexMap[pageIndex]);
    }

    @Override
    public long getFirstRowIndex(int pageIndex) {
      return offsetIndex.getFirstRowIndex(indexMap[pageIndex]);
    }

    @Override
    public long getLastRowIndex(int pageIndex, long totalRowCount) {
      int nextIndex = indexMap[pageIndex] + 1;
      return (nextIndex >= offsetIndex.getPageCount() ? totalRowCount : offsetIndex.getFirstRowIndex(nextIndex)) - 1;
    }

    @Override
    public String toString() {
      try (Formatter formatter = new Formatter()) {
        formatter.format("%-12s  %20s  %16s  %20s\n", "", "offset", "compressed size", "first row index");
        for (int i = 0, n = offsetIndex.getPageCount(); i < n; ++i) {
          int index = Arrays.binarySearch(indexMap, i);
          boolean isHidden = index < 0;
          formatter.format("%spage-%-5d  %20d  %16d  %20d\n",
              isHidden ? "- " : "  ",
              isHidden ? i : index,
              offsetIndex.getOffset(i),
              offsetIndex.getCompressedPageSize(i),
              offsetIndex.getFirstRowIndex(i));
        }
        return formatter.toString();
      }
    }
  }

  /*
   * Returns the filtered offset index containing only the pages which are overlapping with rowRanges.
   */
  static OffsetIndex filterOffsetIndex(OffsetIndex offsetIndex, RowRanges rowRanges, long totalRowCount) {
    IntList indexMap = new IntArrayList();
    for (int i = 0, n = offsetIndex.getPageCount(); i < n; ++i) {
      long from = offsetIndex.getFirstRowIndex(i);
      if (rowRanges.isOverlapping(from, offsetIndex.getLastRowIndex(i, totalRowCount))) {
        indexMap.add(i);
      }
    }
    return new FilteredOffsetIndex(offsetIndex, indexMap.toIntArray());
  }

  static List<OffsetRange> calculateOffsetRanges(OffsetIndex offsetIndex, ColumnChunkMetaData cm,
      long firstPageOffset) {
    List<OffsetRange> ranges = new ArrayList<>();
    int n = offsetIndex.getPageCount();
    if (n > 0) {
      OffsetRange currentRange = null;

      // Add a range for the dictionary page if required
      long rowGroupOffset = cm.getStartingPos();
      if (rowGroupOffset < firstPageOffset) {
        currentRange = new OffsetRange(rowGroupOffset, (int) (firstPageOffset - rowGroupOffset));
        ranges.add(currentRange);
      }

      for (int i = 0; i < n; ++i) {
        long offset = offsetIndex.getOffset(i);
        int length = offsetIndex.getCompressedPageSize(i);
        if (currentRange == null || !currentRange.extend(offset, length)) {
          currentRange = new OffsetRange(offset, length);
          ranges.add(currentRange);
        }
      }
    }
    return ranges;
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.thrift;

import com.twitter.elephantbird.thrift.TStructDescriptor;
import com.twitter.elephantbird.thrift.TStructDescriptor.Field;
import java.util.HashSet;
import java.util.Set;
import org.apache.thrift.TBase;
import org.apache.thrift.TEnum;
import org.apache.thrift.TUnion;

import org.apache.parquet.schema.Type;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.thrift.projection.FieldProjectionFilter;
import org.apache.parquet.thrift.struct.ThriftField;
import org.apache.parquet.thrift.struct.ThriftField.Requirement;
import org.apache.parquet.thrift.struct.ThriftType;
import org.apache.parquet.thrift.struct.ThriftType.*;
import org.apache.parquet.thrift.struct.ThriftType.StructType.StructOrUnionType;
import org.apache.parquet.thrift.struct.ThriftTypeID;
import org.apache.thrift.meta_data.FieldMetaData;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;

import static org.apache.parquet.schema.Type.Repetition.REPEATED;

/**
 * Given a thrift class, this class converts it to parquet schema,
 * a {@link FieldProjectionFilter} can be specified for projection pushdown.
 */
public class ThriftSchemaConverter {
  private final FieldProjectionFilter fieldProjectionFilter;

  public ThriftSchemaConverter() {
    this(FieldProjectionFilter.ALL_COLUMNS);
  }

  public ThriftSchemaConverter(FieldProjectionFilter fieldProjectionFilter) {
    this.fieldProjectionFilter = fieldProjectionFilter;
  }

  public MessageType convert(Class<? extends TBase<?, ?>> thriftClass) {
    return convert(toStructType(thriftClass));
  }

  /**
   * struct is assumed to contain valid structOrUnionType metadata when used with this method.
   * This method may throw if structOrUnionType is unknown.
   *
   * Use convertWithoutProjection below to convert a StructType to MessageType
   *
   * @param struct the thrift type descriptor
   * @return the struct as a Parquet message type
   */
  public MessageType convert(StructType struct) {
    MessageType messageType = ThriftSchemaConvertVisitor.convert(struct, fieldProjectionFilter, true);
    fieldProjectionFilter.assertNoUnmatchedPatterns();
    return messageType;
  }

  /**
   * struct is not required to have known structOrUnionType, which is useful
   * for converting a StructType from an (older) file schema to a MessageType
   *
   * @param struct the thrift type descriptor
   * @return the struct as a Parquet message type
   */
  public static MessageType convertWithoutProjection(StructType struct) {
    return ThriftSchemaConvertVisitor.convert(struct, FieldProjectionFilter.ALL_COLUMNS, false);
  }

  public static <T extends TBase<?,?>> StructOrUnionType structOrUnionType(Class<T> klass) {
    return TUnion.class.isAssignableFrom(klass) ? StructOrUnionType.UNION : StructOrUnionType.STRUCT;
  }

  public static ThriftType.StructType toStructType(Class<? extends TBase<?, ?>> thriftClass) {
    final TStructDescriptor struct = TStructDescriptor.getInstance(thriftClass);
    return toStructType(struct);
  }

  private static StructType toStructType(TStructDescriptor struct) {
    List<Field> fields = struct.getFields();
    List<ThriftField> children = new ArrayList<ThriftField>(fields.size());
    for (Field field : fields) {
      Requirement req =
          field.getFieldMetaData() == null ?
              Requirement.OPTIONAL :
              Requirement.fromType(field.getFieldMetaData().requirementType);
      children.add(toThriftField(field.getName(), field, req));
    }
    return new StructType(children, structOrUnionType(struct.getThriftClass()));
  }

  /**
   * Returns whether the given type is the element type of a list or is a
   * synthetic group with one field that is the element type. This is
   * determined by checking whether the type can be a synthetic group and by
   * checking whether a potential synthetic group matches the expected
   * ThriftField.
   * <p>
   * This method never guesses because the expected ThriftField is known.
   *
   * @param repeatedType a type that may be the element type
   * @param thriftElement the expected Schema for list elements
   * @return {@code true} if the repeatedType is the element schema
   */
  static boolean isListElementType(Type repeatedType,
                                   ThriftField thriftElement) {
    if (repeatedType.isPrimitive() ||
        (repeatedType.asGroupType().getFieldCount() != 1) ||
        (repeatedType.asGroupType().getType(0).isRepetition(REPEATED))) {
      // The repeated type must be the element type because it is an invalid
      // synthetic wrapper. Must be a group with one optional or required field
      return true;
    } else if (thriftElement != null && thriftElement.getType() instanceof StructType) {
      Set<String> fieldNames = new HashSet<String>();
      for (ThriftField field : ((StructType) thriftElement.getType()).getChildren()) {
        fieldNames.add(field.getName());
      }
      // If the repeated type is a subset of the structure of the ThriftField,
      // then it must be the element type.
      return fieldNames.contains(repeatedType.asGroupType().getFieldName(0));
    }
    return false;
  }

  private static ThriftField toThriftField(String name, Field field, ThriftField.Requirement requirement) {
    ThriftType type;
    switch (ThriftTypeID.fromByte(field.getType())) {
      case STOP:
      case VOID:
      default:
        throw new UnsupportedOperationException("can't convert type of " + field);
      case BOOL:
        type = new BoolType();
        break;
      case BYTE:
        type = new ByteType();
        break;
      case DOUBLE:
        type = new DoubleType();
        break;
      case I16:
        type = new I16Type();
        break;
      case I32:
        type = new I32Type();
        break;
      case I64:
        type = new I64Type();
        break;
      case STRING:
        StringType stringType = new StringType();
        FieldMetaData fieldMetaData = field.getFieldMetaData();
        // There is no real binary type (see THRIFT-1920) in Thrift,
        // binary data is represented by String type with an additional binary flag.
        if (fieldMetaData != null && fieldMetaData.valueMetaData.isBinary()) {
          stringType.setBinary(true);
        }
        type = stringType;
        break;
      case STRUCT:
        type = toStructType(field.gettStructDescriptor());
        break;
      case MAP:
        final Field mapKeyField = field.getMapKeyField();
        final Field mapValueField = field.getMapValueField();
        type = new ThriftType.MapType(
            toThriftField(mapKeyField.getName(), mapKeyField, requirement),
            toThriftField(mapValueField.getName(), mapValueField, requirement));
        break;
      case SET:
        final Field setElemField = field.getSetElemField();
        type = new ThriftType.SetType(toThriftField(setElemField.getName(), setElemField, requirement));
        break;
      case LIST:
        final Field listElemField = field.getListElemField();
        type = new ThriftType.ListType(toThriftField(listElemField.getName(), listElemField, requirement));
        break;
      case ENUM:
        Collection<TEnum> enumValues = field.getEnumValues();
        List<EnumValue> values = new ArrayList<ThriftType.EnumValue>();
        for (TEnum tEnum : enumValues) {
          values.add(new EnumValue(tEnum.getValue(), tEnum.toString()));
        }
        type = new EnumType(values);
        break;
    }
    return new ThriftField(name, field.getId(), requirement, type);
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.schema;

import org.apache.parquet.Preconditions;
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
import org.apache.parquet.schema.Type.Repetition;

import static org.apache.parquet.schema.LogicalTypeAnnotation.stringType;

/**
 * Utility functions to convert from Java-like map and list types
 * to equivalent Parquet types.
 */
public abstract class ConversionPatterns {

  private static final String ELEMENT_NAME = "element";

  /**
   * to preserve the difference between empty list and null when optional
   *
   * @param repetition   repetition for the list or map
   * @param alias        name of the field
   * @param logicalTypeAnnotation logical type for the list or map
   * @param nested       the nested repeated field
   * @return a group type
   */
  private static GroupType listWrapper(Repetition repetition, String alias, LogicalTypeAnnotation logicalTypeAnnotation, Type nested) {
    if (!nested.isRepetition(Repetition.REPEATED)) {
      throw new IllegalArgumentException("Nested type should be repeated: " + nested);
    }
    return new GroupType(repetition, alias, logicalTypeAnnotation, nested);
  }

  public static GroupType mapType(Repetition repetition, String alias, Type keyType, Type valueType) {
    return mapType(repetition, alias, "map", keyType, valueType);
  }

  public static GroupType stringKeyMapType(Repetition repetition, String alias, String mapAlias, Type valueType) {
    return mapType(repetition, alias, mapAlias, new PrimitiveType(Repetition.REQUIRED, PrimitiveTypeName.BINARY, "key", stringType()), valueType);
  }

  public static GroupType stringKeyMapType(Repetition repetition, String alias, Type valueType) {
    return stringKeyMapType(repetition, alias, "map", valueType);
  }

  public static GroupType mapType(Repetition repetition, String alias, String mapAlias, Type keyType, Type valueType) {
    //support projection only on key of a map
    if (valueType == null) {
      return listWrapper(
              repetition,
              alias,
              LogicalTypeAnnotation.mapType(),
              new GroupType(
                      Repetition.REPEATED,
                      mapAlias,
                      LogicalTypeAnnotation.MapKeyValueTypeAnnotation.getInstance(),
                      keyType)
      );
    } else {
      if (!valueType.getName().equals("value")) {
        throw new RuntimeException(valueType.getName() + " should be value");
      }
      return listWrapper(
              repetition,
              alias,
              LogicalTypeAnnotation.mapType(),
              new GroupType(
                      Repetition.REPEATED,
                      mapAlias,
                      LogicalTypeAnnotation.MapKeyValueTypeAnnotation.getInstance(),
                      keyType,
                      valueType)
      );
    }
  }

  /**
   * @param repetition repetition for the list
   * @param alias      name of the field
   * @param nestedType type of elements in the list
   * @return a group representing the list using a 2-level representation
   * @deprecated use listOfElements instead
   */
  @Deprecated
  public static GroupType listType(Repetition repetition, String alias, Type nestedType) {
    return listWrapper(
            repetition,
            alias,
            LogicalTypeAnnotation.listType(),
            nestedType
    );
  }

  /**
   * Creates a 3-level list structure annotated with LIST with elements of the
   * given elementType. The repeated level is inserted automatically and the
   * elementType's repetition should be the correct repetition of the elements,
   * required for non-null and optional for nullable.
   *
   * @param listRepetition the repetition of the entire list structure
   * @param name the name of the list structure type
   * @param elementType the type of elements contained by the list
   * @return a GroupType that represents the list
   */
  public static GroupType listOfElements(Repetition listRepetition, String name, Type elementType) {
    Preconditions.checkArgument(elementType.getName().equals(ELEMENT_NAME),
        "List element type must be named 'element'");
    return listWrapper(
        listRepetition,
        name,
        LogicalTypeAnnotation.listType(),
        new GroupType(Repetition.REPEATED, "list", elementType)
    );
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.pig;

import java.io.IOException;
import java.util.Properties;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.OutputFormat;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.pig.ResourceSchema;
import org.apache.pig.ResourceStatistics;
import org.apache.pig.StoreFunc;
import org.apache.pig.StoreMetadata;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.util.UDFContext;
import org.apache.pig.impl.util.Utils;
import org.apache.pig.parser.ParserException;

import org.apache.parquet.hadoop.ParquetOutputFormat;
import org.apache.parquet.io.ParquetEncodingException;

/**
 * A pig storer implementation for the Parquet file format.
 * see {@link ParquetOutputFormat} for available parameters.
 *
 * It uses a TupleWriteSupport to write Tuples into the ParquetOutputFormat
 * The Pig schema is automatically converted to the Parquet schema using {@link PigSchemaConverter}
 * and stored in the file
 */
public class ParquetStorer extends StoreFunc implements StoreMetadata {

  private static final String SCHEMA = "schema";

  private RecordWriter<Void, Tuple> recordWriter;

  private String signature;

  private Properties getProperties() {
    UDFContext udfc = UDFContext.getUDFContext();
    Properties p =
        udfc.getUDFProperties(this.getClass(), new String[]{ signature });
    return p;
  }

  private Schema getSchema() {
    try {
      final String schemaString = getProperties().getProperty(SCHEMA);
      if (schemaString == null) {
        throw new ParquetEncodingException("Can not store relation in Parquet as the schema is unknown");
      }
      return Utils.getSchemaFromString(schemaString);
    } catch (ParserException e) {
      throw new ParquetEncodingException("can not get schema from context", e);
    }
  }

  public ParquetStorer() {
  }

  /**
   * {@inheritDoc}
   */
  @Override
  public void setStoreFuncUDFContextSignature(String signature) {
    super.setStoreFuncUDFContextSignature(signature);
    this.signature = signature;
  }

  /**
   * {@inheritDoc}
   */
  @Override
  public void checkSchema(ResourceSchema s) throws IOException {
    getProperties().setProperty(SCHEMA, s.toString());
  }

  /**
   * {@inheritDoc}
   */
  @Override
  public OutputFormat<Void, Tuple> getOutputFormat() throws IOException {
    Schema pigSchema = getSchema();
    return new ParquetOutputFormat<Tuple>(new TupleWriteSupport(pigSchema));
  }

  /**
   * {@inheritDoc}
   */
  @SuppressWarnings({ "rawtypes", "unchecked" }) // that's how the base class is defined
  @Override
  public void prepareToWrite(RecordWriter recordWriter) throws IOException {
    this.recordWriter = recordWriter;
  }

  /**
   * {@inheritDoc}
   */
  @Override
  public void putNext(Tuple tuple) throws IOException {
    try {
      this.recordWriter.write(null, tuple);
    } catch (InterruptedException e) {
      Thread.interrupted();
      throw new ParquetEncodingException("Interrupted while writing", e);
    }
  }

  /**
   * {@inheritDoc}
   */
  @Override
  public void setStoreLocation(String location, Job job) throws IOException {
    FileOutputFormat.setOutputPath(job, new Path(location));
  }

  /**
   * {@inheritDoc}
   */
  @Override
  public void storeSchema(ResourceSchema schema, String location, Job job)
      throws IOException {
  }

  /**
   * {@inheritDoc}
   */
  @Override
  public void storeStatistics(ResourceStatistics resourceStatistics, String location, Job job)
      throws IOException {
  }

}