/*
 * Copyright 2014-2019 Real Logic Ltd.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package io.aeron;

import io.aeron.exceptions.AeronException;
import io.aeron.exceptions.ConfigurationException;
import io.aeron.exceptions.DriverTimeoutException;
import io.aeron.logbuffer.FragmentHandler;
import org.agrona.DirectBuffer;
import org.agrona.ErrorHandler;
import org.agrona.IoUtil;
import org.agrona.concurrent.*;
import org.agrona.concurrent.broadcast.BroadcastReceiver;
import org.agrona.concurrent.broadcast.CopyBroadcastReceiver;
import org.agrona.concurrent.ringbuffer.ManyToOneRingBuffer;
import org.agrona.concurrent.ringbuffer.RingBuffer;
import org.agrona.concurrent.status.CountersReader;

import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;

import static java.nio.channels.FileChannel.MapMode.READ_WRITE;
import static java.nio.file.StandardOpenOption.READ;
import static java.nio.file.StandardOpenOption.WRITE;
import static java.util.concurrent.atomic.AtomicIntegerFieldUpdater.newUpdater;
import static org.agrona.BitUtil.SIZE_OF_INT;
import static org.agrona.SystemUtil.getDurationInNanos;

/**
 * Aeron entry point for communicating to the Media Driver for creating {@link Publication}s and {@link Subscription}s.
 * Use an {@link Aeron.Context} to configure the Aeron object.
 *

 * A client application requires only one Aeron object per Media Driver.
 *

 * Note: If {@link Aeron.Context#errorHandler(ErrorHandler)} is not set and a {@link DriverTimeoutException}
 * occurs then the process will face the wrath of {@link System#exit(int)}.
 * See {@link Aeron.Configuration#DEFAULT_ERROR_HANDLER}.
 */
public class Aeron implements AutoCloseable
{
    /**
     * Used to represent a null value for when some value is not yet set.
     */
    public static final int NULL_VALUE = -1;

    /**
     * Using an integer because there is no support for boolean. 1 is closed, 0 is not closed.
     */
    private static final AtomicIntegerFieldUpdater<Aeron> IS_CLOSED_UPDATER = newUpdater(Aeron.class, "isClosed");

    @SuppressWarnings("unused") private volatile int isClosed;
    private final long clientId;
    private final ClientConductor conductor;
    private final RingBuffer commandBuffer;
    private final AgentInvoker conductorInvoker;
    private final AgentRunner conductorRunner;
    private final Context ctx;

    Aeron(final Context ctx)
    {
        ctx.conclude();

        this.ctx = ctx;
        clientId = ctx.clientId();
        commandBuffer = ctx.toDriverBuffer();
        conductor = new ClientConductor(ctx);

        if (ctx.useConductorAgentInvoker())
        {
            conductorInvoker = new AgentInvoker(ctx.errorHandler(), null, conductor);
            conductorRunner = null;
        }
        else
        {
            conductorInvoker = null;
            conductorRunner = new AgentRunner(ctx.idleStrategy(), ctx.errorHandler(), null, conductor);
        }
    }

    /**
     * Create an Aeron instance and connect to the media driver with a default {@link Context}.
     *

     * Threads required for interacting with the media driver are created and managed within the Aeron instance.
     *
     * @return the new {@link Aeron} instance connected to the Media Driver.
     */
    public static Aeron connect()
    {
        return connect(new Context());
    }

    /**
     * Create an Aeron instance and connect to the media driver.
     *

     * Threads required for interacting with the media driver are created and managed within the Aeron instance.
     *

     * If an exception occurs while trying to establish a connection then the {@link Context#close()} method
     * will be called on the passed context.
     *
     * @param ctx for configuration of the client.
     * @return the new {@link Aeron} instance connected to the Media Driver.
     */
    public static Aeron connect(final Context ctx)
    {
        try
        {
            final Aeron aeron = new Aeron(ctx);

            if (ctx.useConductorAgentInvoker())
            {
                aeron.conductorInvoker.start();
            }
            else
            {
                AgentRunner.startOnThread(aeron.conductorRunner, ctx.threadFactory());
            }

            return aeron;
        }
        catch (final Exception ex)
        {
            ctx.close();
            throw ex;
        }
    }

    /**
     * Print out the values from {@link #countersReader()} which can be useful for debugging.
     *
     *  @param out to where the counters get printed.
     */
    public void printCounters(final PrintStream out)
    {
        final CountersReader counters = countersReader();
        counters.forEach((value, id, label) -> out.format("%3d: %,20d - %s%n", id, value, label));
    }

    /**
     * Has the client been closed? If not then the CnC file may not be unmapped.
     *
     * @return true if the client has been explicitly closed otherwise false.
     */
    public boolean isClosed()
    {
        return 1 == isClosed;
    }

    /**
     * Get the {@link Aeron.Context} that is used by this client.
     *
     * @return the {@link Aeron.Context} that is use by this client.
     */
    public Context context()
    {
        return ctx;
    }

    /**
     * Get the client identity that has been allocated for communicating with the media driver.
     *
     * @return the client identity that has been allocated for communicating with the media driver.
     */
    public long clientId()
    {
        return clientId;
    }

    /**
     * Get the {@link AgentInvoker} for the client conductor.
     *
     * @return the {@link AgentInvoker} for the client conductor.
     */
    public AgentInvoker conductorAgentInvoker()
    {
        return conductorInvoker;
    }

    /**
     * Clean up and release all Aeron client resources and shutdown conductor thread if not using
     * {@link Context#useConductorAgentInvoker(boolean)}.
     *

     * This will close all currently open {@link Publication}s, {@link Subscription}s, and {@link Counter}s created
     * from this client.
     */
    public void close()
    {
        if (IS_CLOSED_UPDATER.compareAndSet(this, 0, 1))
        {
            if (null != conductorRunner)
            {
                conductorRunner.close();
            }
            else
            {
                conductorInvoker.close();
            }
        }
    }

    /**
     * Add a {@link Publication} for publishing messages to subscribers. The publication returned is threadsafe.
     *
     * @param channel  for sending the messages known to the media layer.
     * @param streamId within the channel scope.
     * @return a new {@link ConcurrentPublication}.
     */
    public ConcurrentPublication addPublication(final String channel, final int streamId)
    {
        return conductor.addPublication(channel, streamId);
    }

    /**
     * Add an {@link ExclusivePublication} for publishing messages to subscribers from a single thread.
     *
     * @param channel  for sending the messages known to the media layer.
     * @param streamId within the channel scope.
     * @return a new {@link ExclusivePublication}.
     */
    public ExclusivePublication addExclusivePublication(final String channel, final int streamId)
    {
        return conductor.addExclusivePublication(channel, streamId);
    }

    /**
     * Add a new {@link Subscription} for subscribing to messages from publishers.
     *

     * The method will set up the {@link Subscription} to use the
     * {@link Aeron.Context#availableImageHandler(AvailableImageHandler)} and
     * {@link Aeron.Context#unavailableImageHandler(UnavailableImageHandler)} from the {@link Aeron.Context}.
     *
     * @param channel  for receiving the messages known to the media layer.
     * @param streamId within the channel scope.
     * @return the {@link Subscription} for the channel and streamId pair.
     */
    public Subscription addSubscription(final String channel, final int streamId)
    {
        return conductor.addSubscription(channel, streamId);
    }

    /**
     * Add a new {@link Subscription} for subscribing to messages from publishers.
     *

     * This method will override the default handlers from the {@link Aeron.Context}, i.e.
     * {@link Aeron.Context#availableImageHandler(AvailableImageHandler)} and
     * {@link Aeron.Context#unavailableImageHandler(UnavailableImageHandler)}. Null values are valid and will
     * result in no action being taken.
     *
     * @param channel                 for receiving the messages known to the media layer.
     * @param streamId                within the channel scope.
     * @param availableImageHandler   called when {@link Image}s become available for consumption. Null is valid if no
     *                                action is to be taken.
     * @param unavailableImageHandler called when {@link Image}s go unavailable for consumption. Null is valid if no
     *                                action is to be taken.
     * @return the {@link Subscription} for the channel and streamId pair.
     */
    public Subscription addSubscription(
        final String channel,
        final int streamId,
        final AvailableImageHandler availableImageHandler,
        final UnavailableImageHandler unavailableImageHandler)
    {
        return conductor.addSubscription(channel, streamId, availableImageHandler, unavailableImageHandler);
    }

    /**
     * Generate the next correlation id that is unique for the connected Media Driver.
     *

     * This is useful generating correlation identifiers for pairing requests with responses in a clients own
     * application protocol.
     *

     * This method is thread safe and will work across processes that all use the same media driver.
     *
     * @return next correlation id that is unique for the Media Driver.
     */
    public long nextCorrelationId()
    {
        if (1 == isClosed)
        {
            throw new AeronException("client is closed");
        }

        return commandBuffer.nextCorrelationId();
    }

    /**
     * Get the {@link CountersReader} for the Aeron media driver counters.
     *
     * @return new {@link CountersReader} for the Aeron media driver in use.
     */
    public CountersReader countersReader()
    {
        if (1 == isClosed)
        {
            throw new AeronException("client is closed");
        }

        return conductor.countersReader();
    }

    /**
     * Allocate a counter on the media driver and return a {@link Counter} for it.
     *

     * The counter should be freed by calling {@link Counter#close()}.
     *
     * @param typeId      for the counter.
     * @param keyBuffer   containing the optional key for the counter.
     * @param keyOffset   within the keyBuffer at which the key begins.
     * @param keyLength   of the key in the keyBuffer.
     * @param labelBuffer containing the mandatory label for the counter. The label should not be length prefixed.
     * @param labelOffset within the labelBuffer at which the label begins.
     * @param labelLength of the label in the labelBuffer.
     * @return the newly allocated counter.
     * @see org.agrona.concurrent.status.CountersManager#allocate(int, DirectBuffer, int, int, DirectBuffer, int, int)
     */
    public Counter addCounter(
        final int typeId,
        final DirectBuffer keyBuffer,
        final int keyOffset,
        final int keyLength,
        final DirectBuffer labelBuffer,
        final int labelOffset,
        final int labelLength)
    {
        return conductor.addCounter(typeId, keyBuffer, keyOffset, keyLength, labelBuffer, labelOffset, labelLength);
    }

    /**
     * Allocate a counter on the media driver and return a {@link Counter} for it.
     *

     * The counter should be freed by calling {@link Counter#close()}.
     *
     * @param typeId for the counter.
     * @param label  for the counter. It should be US-ASCII.
     * @return the newly allocated counter.
     * @see org.agrona.concurrent.status.CountersManager#allocate(String, int)
     */
    public Counter addCounter(final int typeId, final String label)
    {
        return conductor.addCounter(typeId, label);
    }

    /**
     * Configuration options for the {@link Aeron} client.
     */
    public static class Configuration
    {
        /**
         * Duration in milliseconds for which the client conductor will sleep between duty cycles.
         */
        static final long IDLE_SLEEP_MS = 16;

        /**
         * Duration in nanoseconds for which the client conductor will sleep between duty cycles.
         */
        static final long IDLE_SLEEP_NS = TimeUnit.MILLISECONDS.toNanos(IDLE_SLEEP_MS);

        /**
         * Default interval between sending keepalive control messages to the driver.
         */
        static final long KEEPALIVE_INTERVAL_NS = TimeUnit.MILLISECONDS.toNanos(500);

        /**
         * Duration to wait while lingering a entity such as an {@link Image} before deleting underlying resources
         * such as memory mapped files.
         */
        public static final String RESOURCE_LINGER_DURATION_PROP_NAME = "aeron.client.resource.linger.duration";

        /**
         * Default duration a resource should linger before deletion.
         */
        public static final long RESOURCE_LINGER_DURATION_DEFAULT = TimeUnit.SECONDS.toNanos(3);

        /**
         * The Default handler for Aeron runtime exceptions.
         * When a {@link DriverTimeoutException} is encountered, this handler will
         * exit the program.
         *

         * The error handler can be overridden by supplying an {@link Context} with a custom handler.
         *
         * @see Context#errorHandler(ErrorHandler)
         */
        public static final ErrorHandler DEFAULT_ERROR_HANDLER =
            (throwable) ->
            {
                throwable.printStackTrace();
                if (throwable instanceof DriverTimeoutException)
                {
                    System.err.printf(
                        "%n***%n*** timeout for the Media Driver - is it currently running? exiting%n***%n");
                    System.exit(-1);
                }
            };

        /**
         * Duration to wait while lingering a entity such as an {@link Image} before deleting underlying resources
         * such as memory mapped files.
         *
         * @return duration in nanoseconds to wait before deleting a expired resource.
         * @see #RESOURCE_LINGER_DURATION_PROP_NAME
         */
        public static long resourceLingerDurationNs()
        {
            return getDurationInNanos(RESOURCE_LINGER_DURATION_PROP_NAME, RESOURCE_LINGER_DURATION_DEFAULT);
        }
    }

    /**
     * This class provides configuration for the {@link Aeron} class via the {@link Aeron#connect(Aeron.Context)}
     * method and its overloads. It gives applications some control over the interactions with the Aeron Media Driver.
     * It can also set up error handling as well as application callbacks for image information from the
     * Media Driver.
     *

     * A number of the properties are for testing and should not be set by end users.
     *

     * Note: Do not reuse instances of the context across different {@link Aeron} clients.
     *

     * The context will be owned by {@link ClientConductor} after a successful
     * {@link Aeron#connect(Context)} and closed via {@link Aeron#close()}.
     */
    public static class Context extends CommonContext
    {
        private long clientId;
        private boolean useConductorAgentInvoker = false;
        private AgentInvoker driverAgentInvoker;
        private Lock clientLock;
        private EpochClock epochClock;
        private NanoClock nanoClock;
        private IdleStrategy idleStrategy;
        private CopyBroadcastReceiver toClientBuffer;
        private RingBuffer toDriverBuffer;
        private DriverProxy driverProxy;
        private MappedByteBuffer cncByteBuffer;
        private AtomicBuffer cncMetaDataBuffer;
        private LogBuffersFactory logBuffersFactory;
        private ErrorHandler errorHandler;
        private AvailableImageHandler availableImageHandler;
        private UnavailableImageHandler unavailableImageHandler;
        private AvailableCounterHandler availableCounterHandler;
        private UnavailableCounterHandler unavailableCounterHandler;
        private long keepAliveIntervalNs = Configuration.KEEPALIVE_INTERVAL_NS;
        private long interServiceTimeoutNs = 0;
        private long resourceLingerDurationNs = Configuration.resourceLingerDurationNs();

        private ThreadFactory threadFactory = Thread::new;

        /**
         * Perform a shallow copy of the object.
         *
         * @return a shallow copy of the object.
         */
        public Context clone()
        {
            return (Context)super.clone();
        }

        /**
         * This is called automatically by {@link Aeron#connect(Aeron.Context)} and its overloads.
         * There is no need to call it from a client application. It is responsible for providing default
         * values for options that are not individually changed through field setters.
         *
         * @return this for a fluent API.
         */
        public Context conclude()
        {
            super.conclude();

            if (null == clientLock)
            {
                clientLock = new ReentrantLock();
            }

            if (null == epochClock)
            {
                epochClock = new SystemEpochClock();
            }

            if (null == nanoClock)
            {
                nanoClock = new SystemNanoClock();
            }

            if (null == idleStrategy)
            {
                idleStrategy = new SleepingMillisIdleStrategy(Configuration.IDLE_SLEEP_MS);
            }

            if (cncFile() != null)
            {
                connectToDriver();
            }

            interServiceTimeoutNs = CncFileDescriptor.clientLivenessTimeout(cncMetaDataBuffer);
            if (interServiceTimeoutNs <= keepAliveIntervalNs)
            {
                throw new ConfigurationException("interServiceTimeoutNs=" + interServiceTimeoutNs +
                    " <= keepAliveIntervalNs=" + keepAliveIntervalNs);
            }

            if (null == toDriverBuffer)
            {
                toDriverBuffer = new ManyToOneRingBuffer(
                    CncFileDescriptor.createToDriverBuffer(cncByteBuffer, cncMetaDataBuffer));
            }

            if (null == toClientBuffer)
            {
                toClientBuffer = new CopyBroadcastReceiver(new BroadcastReceiver(
                    CncFileDescriptor.createToClientsBuffer(cncByteBuffer, cncMetaDataBuffer)));
            }

            if (countersMetaDataBuffer() == null)
            {
                countersMetaDataBuffer(
                    CncFileDescriptor.createCountersMetaDataBuffer(cncByteBuffer, cncMetaDataBuffer));
            }

            if (countersValuesBuffer() == null)
            {
                countersValuesBuffer(CncFileDescriptor.createCountersValuesBuffer(cncByteBuffer, cncMetaDataBuffer));
            }

            if (null == logBuffersFactory)
            {
                logBuffersFactory = new MappedLogBuffersFactory();
            }

            if (null == errorHandler)
            {
                errorHandler = Configuration.DEFAULT_ERROR_HANDLER;
            }

            if (null == driverProxy)
            {
                clientId = toDriverBuffer.nextCorrelationId();
                driverProxy = new DriverProxy(toDriverBuffer, clientId);
            }

            return this;
        }

        /**
         * Get the client identity that has been allocated for communicating with the media driver.
         *
         * @return the client identity that has been allocated for communicating with the media driver.
         */
        public long clientId()
        {
            return clientId;
        }

        /**
         * Should an {@link AgentInvoker} be used for running the {@link ClientConductor} rather than run it on
         * a thread with a {@link AgentRunner}.
         *
         * @param useConductorAgentInvoker use {@link AgentInvoker} be used for running the {@link ClientConductor}?
         * @return this for a fluent API.
         */
        public Context useConductorAgentInvoker(final boolean useConductorAgentInvoker)
        {
            this.useConductorAgentInvoker = useConductorAgentInvoker;
            return this;
        }

        /**
         * Should an {@link AgentInvoker} be used for running the {@link ClientConductor} rather than run it on
         * a thread with a {@link AgentRunner}.
         *
         * @return true if the {@link ClientConductor} will be run with an {@link AgentInvoker} otherwise false.
         */
        public boolean useConductorAgentInvoker()
        {
            return useConductorAgentInvoker;
        }

        /**
         * Set the {@link AgentInvoker} for the Media Driver to be used while awaiting a synchronous response.
         *

         * Useful for when running on a low thread count scenario.
         *
         * @param driverAgentInvoker to be invoked while awaiting a response in the client.
         * @return this for a fluent API.
         */
        public Context driverAgentInvoker(final AgentInvoker driverAgentInvoker)
        {
            this.driverAgentInvoker = driverAgentInvoker;
            return this;
        }

        /**
         * Get the {@link AgentInvoker} that is used to run the Media Driver while awaiting a synchronous response.
         *
         * @return the {@link AgentInvoker} that is used for running the Media Driver.
         */
        public AgentInvoker driverAgentInvoker()
        {
            return driverAgentInvoker;
        }

        /**
         * The {@link Lock} that is used to provide mutual exclusion in the Aeron client.
         *

         * If the {@link #useConductorAgentInvoker()} is set and only one thread accesses the client
         * then the lock can be set to {@link NoOpLock} to elide the lock overhead.
         *
         * @param lock that is used to provide mutual exclusion in the Aeron client.
         * @return this for a fluent API.
         */
        public Context clientLock(final Lock lock)
        {
            clientLock = lock;
            return this;
        }

        /**
         * Get the {@link Lock} that is used to provide mutual exclusion in the Aeron client.
         *
         * @return the {@link Lock} that is used to provide mutual exclusion in the Aeron client.
         */
        public Lock clientLock()
        {
            return clientLock;
        }

        /**
         * Set the {@link EpochClock} to be used for tracking wall clock time when interacting with the driver.
         *
         * @param clock {@link EpochClock} to be used for tracking wall clock time when interacting with the driver.
         * @return this for a fluent API.
         */
        public Context epochClock(final EpochClock clock)
        {
            this.epochClock = clock;
            return this;
        }

        /**
         * Get the {@link EpochClock} used by the client for the epoch time in milliseconds.
         *
         * @return the {@link EpochClock} used by the client for the epoch time in milliseconds.
         */
        public EpochClock epochClock()
        {
            return epochClock;
        }

        /**
         * Set the {@link NanoClock} to be used for tracking high resolution time.
         *
         * @param clock {@link NanoClock} to be used for tracking high resolution time.
         * @return this for a fluent API.
         */
        public Context nanoClock(final NanoClock clock)
        {
            this.nanoClock = clock;
            return this;
        }

        /**
         * Get the {@link NanoClock} to be used for tracking high resolution time.
         *
         * @return the {@link NanoClock} to be used for tracking high resolution time.
         */
        public NanoClock nanoClock()
        {
            return nanoClock;
        }

        /**
         * Provides an IdleStrategy for the thread responsible for communicating with the Aeron Media Driver.
         *
         * @param idleStrategy Thread idle strategy for communication with the Media Driver.
         * @return this for a fluent API.
         */
        public Context idleStrategy(final IdleStrategy idleStrategy)
        {
            this.idleStrategy = idleStrategy;
            return this;
        }

        /**
         * Get the {@link IdleStrategy} employed by the client conductor thread.
         *
         * @return the {@link IdleStrategy} employed by the client conductor thread.
         */
        public IdleStrategy idleStrategy()
        {
            return idleStrategy;
        }

        /**
         * This method is used for testing and debugging.
         *
         * @param toClientBuffer Injected CopyBroadcastReceiver
         * @return this for a fluent API.
         */
        Context toClientBuffer(final CopyBroadcastReceiver toClientBuffer)
        {
            this.toClientBuffer = toClientBuffer;
            return this;
        }

        /**
         * The buffer used for communicating from the media driver to the Aeron client.
         *
         * @return the buffer used for communicating from the media driver to the Aeron client.
         */
        public CopyBroadcastReceiver toClientBuffer()
        {
            return toClientBuffer;
        }

        /**
         * This method is used for testing and debugging.
         *
         * @param toDriverBuffer Injected RingBuffer.
         * @return this for a fluent API.
         */
        Context toDriverBuffer(final RingBuffer toDriverBuffer)
        {
            this.toDriverBuffer = toDriverBuffer;
            return this;
        }

        /**
         * Get the {@link RingBuffer} used for sending commands to the media driver.
         *
         * @return the {@link RingBuffer} used for sending commands to the media driver.
         */
        public RingBuffer toDriverBuffer()
        {
            return toDriverBuffer;
        }

        /**
         * Set the proxy for communicating with the media driver.
         *
         * @param driverProxy for communicating with the media driver.
         * @return this for a fluent API.
         */
        Context driverProxy(final DriverProxy driverProxy)
        {
            this.driverProxy = driverProxy;
            return this;
        }

        /**
         * Get the proxy for communicating with the media driver.
         *
         * @return the proxy for communicating with the media driver.
         */
        public DriverProxy driverProxy()
        {
            return driverProxy;
        }

        /**
         * This method is used for testing and debugging.
         *
         * @param logBuffersFactory Injected LogBuffersFactory
         * @return this for a fluent API.
         */
        Context logBuffersFactory(final LogBuffersFactory logBuffersFactory)
        {
            this.logBuffersFactory = logBuffersFactory;
            return this;
        }

        /**
         * Get the factory for making log buffers.
         *
         * @return the factory for making log buffers.
         */
        public LogBuffersFactory logBuffersFactory()
        {
            return logBuffersFactory;
        }

        /**
         * Handle Aeron exceptions in a callback method. The default behavior is defined by
         * {@link Configuration#DEFAULT_ERROR_HANDLER}. This is the error handler which will be used if an error occurs
         * during the callback for poll operations such as {@link Subscription#poll(FragmentHandler, int)}.
         *

         * The error handler can be reset after {@link Aeron#connect()} and the latest version will always be used
         * so that the boot strapping process can be performed such as replacing the default one with a
         * {@link CountedErrorHandler}.
         *
         * @param errorHandler Method to handle objects of type Throwable.
         * @return this for a fluent API.
         * @see io.aeron.exceptions.DriverTimeoutException
         * @see io.aeron.exceptions.RegistrationException
         */
        public Context errorHandler(final ErrorHandler errorHandler)
        {
            this.errorHandler = errorHandler;
            return this;
        }

        /**
         * Get the error handler that will be called for errors reported back from the media driver.
         *
         * @return the error handler that will be called for errors reported back from the media driver.
         */
        public ErrorHandler errorHandler()
        {
            return errorHandler;
        }

        /**
         * Setup a default callback for when an {@link Image} is available.
         *
         * @param handler Callback method for handling available image notifications.
         * @return this for a fluent API.
         */
        public Context availableImageHandler(final AvailableImageHandler handler)
        {
            this.availableImageHandler = handler;
            return this;
        }

        /**
         * Get the default callback handler for notifying when {@link Image}s become available.
         *
         * @return the callback handler for notifying when {@link Image}s become available.
         */
        public AvailableImageHandler availableImageHandler()
        {
            return availableImageHandler;
        }

        /**
         * Setup a default callback for when an {@link Image} is unavailable.
         *
         * @param handler Callback method for handling unavailable image notifications.
         * @return this for a fluent API.
         */
        public Context unavailableImageHandler(final UnavailableImageHandler handler)
        {
            this.unavailableImageHandler = handler;
            return this;
        }

        /**
         * Get the callback handler for when an {@link Image} is unavailable.
         *
         * @return the callback handler for when an {@link Image} is unavailable.
         */
        public UnavailableImageHandler unavailableImageHandler()
        {
            return unavailableImageHandler;
        }

        /**
         * Setup a callback for when a counter is available.
         *
         * @param handler to be called for handling available counter notifications.
         * @return this for a fluent API.
         */
        public Context availableCounterHandler(final AvailableCounterHandler handler)
        {
            this.availableCounterHandler = handler;
            return this;
        }

        /**
         * Get the callback handler for when a counter is available.
         *
         * @return the callback handler for when a counter is available.
         */
        public AvailableCounterHandler availableCounterHandler()
        {
            return availableCounterHandler;
        }

        /**
         * Setup a callback for when a counter is unavailable.
         *
         * @param handler to be called for handling unavailable counter notifications.
         * @return this for a fluent API.
         */
        public Context unavailableCounterHandler(final UnavailableCounterHandler handler)
        {
            this.unavailableCounterHandler = handler;
            return this;
        }

        /**
         * Get the callback handler for when a counter is unavailable.
         *
         * @return the callback handler for when a counter is unavailable.
         */
        public UnavailableCounterHandler unavailableCounterHandler()
        {
            return unavailableCounterHandler;
        }

        /**
         * Set the interval in nanoseconds for which the client will perform keep-alive operations.
         *
         * @param value the interval in nanoseconds for which the client will perform keep-alive operations.
         * @return this for a fluent API.
         */
        public Context keepAliveIntervalNs(final long value)
        {
            keepAliveIntervalNs = value;
            return this;
        }

        /**
         * Get the interval in nanoseconds for which the client will perform keep-alive operations.
         *
         * @return the interval in nanoseconds for which the client will perform keep-alive operations.
         */
        public long keepAliveIntervalNs()
        {
            return keepAliveIntervalNs;
        }

        /**
         * Set the amount of time, in milliseconds, that this client will wait until it determines the
         * Media Driver is unavailable. When this happens a
         * {@link io.aeron.exceptions.DriverTimeoutException} will be generated for the error handler.
         *
         * @param value Number of milliseconds.
         * @return this for a fluent API.
         * @see #errorHandler(ErrorHandler)
         */
        public Context driverTimeoutMs(final long value)
        {
            super.driverTimeoutMs(value);
            return this;
        }

        /**
         * Set the timeout between service calls the to {@link ClientConductor} duty cycles in nanoseconds.
         *
         * @param interServiceTimeout the timeout (ns) between service calls the to {@link ClientConductor} duty cycle.
         * @return this for a fluent API.
         */
        Context interServiceTimeoutNs(final long interServiceTimeout)
        {
            this.interServiceTimeoutNs = interServiceTimeout;
            return this;
        }

        /**
         * Return the timeout between service calls to the duty cycle for the client.
         *

         * When exceeded, {@link #errorHandler()} will be called and the active {@link Publication}s and {@link Image}s
         * closed.
         *

         * This value is controlled by the driver and included in the CnC file. It can be configured by adjusting
         * the {@code aeron.client.liveness.timeout} property on the media driver.
         *
         * @return the timeout in nanoseconds between service calls as an allowed maximum.
         */
        public long interServiceTimeoutNs()
        {
            return interServiceTimeoutNs;
        }

        /**
         * Duration to wait while lingering a entity such as an {@link Image} before deleting underlying resources
         * such as memory mapped files.
         *
         * @param resourceLingerDurationNs to wait before deleting a expired resource.
         * @return this for a fluent API.
         * @see Configuration#RESOURCE_LINGER_DURATION_PROP_NAME
         */
        public Context resourceLingerDurationNs(final long resourceLingerDurationNs)
        {
            this.resourceLingerDurationNs = resourceLingerDurationNs;
            return this;
        }

        /**
         * Duration to wait while lingering a entity such as an {@link Image} before deleting underlying resources
         * such as memory mapped files.
         *
         * @return duration in nanoseconds to wait before deleting a expired resource.
         * @see Configuration#RESOURCE_LINGER_DURATION_PROP_NAME
         */
        public long resourceLingerDurationNs()
        {
            return resourceLingerDurationNs;
        }

        /**
         * {@inheritDoc}
         */
        public Context aeronDirectoryName(final String dirName)
        {
            super.aeronDirectoryName(dirName);
            return this;
        }

        /**
         * Specify the thread factory to use when starting the conductor thread.
         *
         * @param threadFactory thread factory to construct the thread.
         * @return this for a fluent API.
         */
        public Context threadFactory(final ThreadFactory threadFactory)
        {
            this.threadFactory = threadFactory;
            return this;
        }

        /**
         * The thread factory to be use to construct the conductor thread
         *
         * @return the specified thread factory or {@link Thread#Thread(Runnable)} if none is provided
         */
        public ThreadFactory threadFactory()
        {
            return threadFactory;
        }

        /**
         * Clean up all resources that the client uses to communicate with the Media Driver.
         */
        public void close()
        {
            final MappedByteBuffer cncByteBuffer = this.cncByteBuffer;
            this.cncByteBuffer = null;
            IoUtil.unmap(cncByteBuffer);
            super.close();
        }

        private void connectToDriver()
        {
            final long startTimeMs = epochClock.time();
            final long deadlineMs = startTimeMs + driverTimeoutMs();
            final File cncFile = cncFile();

            while (null == toDriverBuffer)
            {
                while (!cncFile.exists() || cncFile.length() <= 0)
                {
                    if (epochClock.time() > deadlineMs)
                    {
                        throw new DriverTimeoutException("CnC file not created: " + cncFile.getAbsolutePath());
                    }

                    sleep(Configuration.IDLE_SLEEP_MS);
                }

                cncByteBuffer = waitForFileMapping(cncFile, deadlineMs, epochClock);
                cncMetaDataBuffer = CncFileDescriptor.createMetaDataBuffer(cncByteBuffer);

                int cncVersion;
                while (0 == (cncVersion = cncMetaDataBuffer.getIntVolatile(CncFileDescriptor.cncVersionOffset(0))))
                {
                    if (epochClock.time() > deadlineMs)
                    {
                        throw new DriverTimeoutException("CnC file is created but not initialised");
                    }

                    sleep(1);
                }

                if (CncFileDescriptor.CNC_VERSION != cncVersion)
                {
                    throw new AeronException("CnC file version not supported: version=" + cncVersion);
                }

                final ManyToOneRingBuffer ringBuffer = new ManyToOneRingBuffer(
                    CncFileDescriptor.createToDriverBuffer(cncByteBuffer, cncMetaDataBuffer));

                while (0 == ringBuffer.consumerHeartbeatTime())
                {
                    if (epochClock.time() > deadlineMs)
                    {
                        throw new DriverTimeoutException("no driver heartbeat detected");
                    }

                    sleep(1);
                }

                final long timeMs = epochClock.time();
                if (ringBuffer.consumerHeartbeatTime() < (timeMs - driverTimeoutMs()))
                {
                    if (timeMs > deadlineMs)
                    {
                        throw new DriverTimeoutException("no driver heartbeat detected");
                    }

                    IoUtil.unmap(cncByteBuffer);
                    cncByteBuffer = null;
                    cncMetaDataBuffer = null;

                    sleep(100);
                    continue;
                }

                toDriverBuffer = ringBuffer;
            }
        }
    }

    private static MappedByteBuffer waitForFileMapping(
        final File cncFile, final long deadlineMs, final EpochClock epochClock)
    {
        try (FileChannel fileChannel = FileChannel.open(cncFile.toPath(), READ, WRITE))
        {
            while (fileChannel.size() < CncFileDescriptor.CNC_VERSION_FIELD_OFFSET + SIZE_OF_INT)
            {
                if (epochClock.time() > deadlineMs)
                {
                    throw new AeronException("CnC file is created but not populated");
                }

                sleep(Configuration.IDLE_SLEEP_MS);
            }

            return fileChannel.map(READ_WRITE, 0, fileChannel.size());
        }
        catch (final IOException ex)
        {
            throw new AeronException("cannot open CnC file", ex);
        }
    }

    static void sleep(final long durationMs)
    {
        try
        {
            Thread.sleep(durationMs);
        }
        catch (final InterruptedException ignore)
        {
            Thread.currentThread().interrupt();
        }
    }
}


/*
 * Copyright 2014-2019 Real Logic Ltd.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package io.aeron.archive.client;

import io.aeron.FragmentAssembler;
import io.aeron.Subscription;
import io.aeron.archive.codecs.ControlResponseDecoder;
import io.aeron.archive.codecs.MessageHeaderDecoder;
import io.aeron.archive.codecs.MessageHeaderEncoder;
import io.aeron.archive.codecs.RecordingDescriptorDecoder;
import io.aeron.logbuffer.FragmentHandler;
import io.aeron.logbuffer.Header;
import org.agrona.DirectBuffer;

/**
 * Encapsulate the polling, decoding, and dispatching of archive control protocol response messages.
 */
public class ControlResponseAdapter implements FragmentHandler
{
    private final MessageHeaderDecoder messageHeaderDecoder = new MessageHeaderDecoder();
    private final ControlResponseDecoder controlResponseDecoder = new ControlResponseDecoder();
    private final RecordingDescriptorDecoder recordingDescriptorDecoder = new RecordingDescriptorDecoder();

    private final int fragmentLimit;
    private final ControlResponseListener listener;
    private final Subscription subscription;
    private final FragmentAssembler fragmentAssembler = new FragmentAssembler(this);

    /**
     * Create an adapter for a given subscription to an archive for control response messages.
     *
     * @param listener      to which responses are dispatched.
     * @param subscription  to poll for new events.
     * @param fragmentLimit to apply for each polling operation.
     */
    public ControlResponseAdapter(
        final ControlResponseListener listener,
        final Subscription subscription,
        final int fragmentLimit)
    {
        this.fragmentLimit = fragmentLimit;
        this.listener = listener;
        this.subscription = subscription;
    }

    /**
     * Poll for recording events and dispatch them to the {@link ControlResponseListener} for this instance.
     *
     * @return the number of fragments read during the operation. Zero if no events are available.
     */
    public int poll()
    {
        return subscription.poll(fragmentAssembler, fragmentLimit);
    }

    /**
     * Dispatch a descriptor message to a consumer by reading the fields in the correct order.
     *
     * @param decoder  which wraps the encoded message ready for reading.
     * @param consumer to which the decoded fields should be passed.
     */
    public static void dispatchDescriptor(
        final RecordingDescriptorDecoder decoder, final RecordingDescriptorConsumer consumer)
    {
        consumer.onRecordingDescriptor(
            decoder.controlSessionId(),
            decoder.correlationId(),
            decoder.recordingId(),
            decoder.startTimestamp(),
            decoder.stopTimestamp(),
            decoder.startPosition(),
            decoder.stopPosition(),
            decoder.initialTermId(),
            decoder.segmentFileLength(),
            decoder.termBufferLength(),
            decoder.mtuLength(),
            decoder.sessionId(),
            decoder.streamId(),
            decoder.strippedChannel(),
            decoder.originalChannel(),
            decoder.sourceIdentity());
    }

    public void onFragment(
        final DirectBuffer buffer,
        final int offset,
        @SuppressWarnings("unused") final int length,
        @SuppressWarnings("unused") final Header header)
    {
        messageHeaderDecoder.wrap(buffer, offset);

        final int schemaId = messageHeaderDecoder.schemaId();
        if (schemaId != MessageHeaderDecoder.SCHEMA_ID)
        {
            throw new ArchiveException("expected schemaId=" + MessageHeaderDecoder.SCHEMA_ID + ", actual=" + schemaId);
        }

        final int templateId = messageHeaderDecoder.templateId();
        switch (templateId)
        {
            case ControlResponseDecoder.TEMPLATE_ID:
                handleControlResponse(listener, buffer, offset);
                break;

            case RecordingDescriptorDecoder.TEMPLATE_ID:
                handleRecordingDescriptor(listener, buffer, offset);
                break;
        }
    }

    private void handleControlResponse(
        final ControlResponseListener listener,
        final DirectBuffer buffer,
        final int offset)
    {
        controlResponseDecoder.wrap(
            buffer,
            offset + MessageHeaderEncoder.ENCODED_LENGTH,
            messageHeaderDecoder.blockLength(),
            messageHeaderDecoder.version());

        listener.onResponse(
            controlResponseDecoder.controlSessionId(),
            controlResponseDecoder.correlationId(),
            controlResponseDecoder.relevantId(),
            controlResponseDecoder.code(),
            controlResponseDecoder.errorMessage());
    }

    private void handleRecordingDescriptor(
        final ControlResponseListener listener,
        final DirectBuffer buffer,
        final int offset)
    {
        recordingDescriptorDecoder.wrap(
            buffer,
            offset + MessageHeaderEncoder.ENCODED_LENGTH,
            messageHeaderDecoder.blockLength(),
            messageHeaderDecoder.version());

        dispatchDescriptor(recordingDescriptorDecoder, listener);
    }
}


/*
 * Copyright 2014-2019 Real Logic Ltd.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package io.aeron.logbuffer;

import org.agrona.concurrent.UnsafeBuffer;

import static io.aeron.protocol.DataHeaderFlyweight.HEADER_LENGTH;

/**
 * Rebuild a term buffer from received frames which can be out-of-order. The resulting data structure will only
 * monotonically increase in state.
 */
public class TermRebuilder
{
    /**
     * Insert a packet of frames into the log at the appropriate termOffset as indicated by the term termOffset header.
     * <p>
     * If the packet has already been inserted then this is a noop.
     *
     * @param termBuffer into which the packet should be inserted.
     * @param termOffset in the term at which the packet should be inserted.
     * @param packet     containing a sequence of frames.
     * @param length     of the packet of frames in bytes.
     */
    public static void insert(
        final UnsafeBuffer termBuffer, final int termOffset, final UnsafeBuffer packet, final int length)
    {
        if (0 == termBuffer.getInt(termOffset))
        {
            termBuffer.putBytes(termOffset + HEADER_LENGTH, packet, HEADER_LENGTH, length - HEADER_LENGTH);

            termBuffer.putLong(termOffset + 24, packet.getLong(24));
            termBuffer.putLong(termOffset + 16, packet.getLong(16));
            termBuffer.putLong(termOffset + 8, packet.getLong(8));

            termBuffer.putLongOrdered(termOffset, packet.getLong(0));
        }
    }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.bytes;

import static java.lang.Math.max;
import static java.lang.Math.pow;
import static java.lang.String.format;
import static org.apache.parquet.Preconditions.checkArgument;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;

import org.apache.parquet.OutputStreamCloseException;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Similar to a {@link ByteArrayOutputStream}, but uses a different strategy for growing that does not involve copying.
 * Where ByteArrayOutputStream is backed by a single array that "grows" by copying into a new larger array, this output
 * stream grows by allocating a new array (slab) and adding it to a list of previous arrays.
 *
 * Each new slab is allocated to be the same size as all the previous slabs combined, so these allocations become
 * exponentially less frequent, just like ByteArrayOutputStream, with one difference. This output stream accepts a
 * max capacity hint, which is a hint describing the max amount of data that will be written to this stream. As the
 * total size of this stream nears this max, this stream starts to grow linearly instead of exponentially.
 * So new slabs are allocated to be 1/5th of the max capacity hint,
 * instead of equal to the total previous size of all slabs. This is useful because it prevents allocating roughly
 * twice the needed space when a new slab is added just before the stream is done being used.
 *
 * When reusing this stream it will adjust the initial slab size based on the previous data size, aiming for fewer
 * allocations, with the assumption that a similar amount of data will be written to this stream on re-use.
 * See ({@link CapacityByteArrayOutputStream#reset()}).
 */
public class CapacityByteArrayOutputStream extends OutputStream {
  private static final Logger LOG = LoggerFactory.getLogger(CapacityByteArrayOutputStream.class);
  private static final ByteBuffer EMPTY_SLAB = ByteBuffer.wrap(new byte[0]);

  private int initialSlabSize;
  private final int maxCapacityHint;
  private final List<ByteBuffer> slabs = new ArrayList<ByteBuffer>();

  private ByteBuffer currentSlab;
  private int currentSlabIndex;
  private int bytesAllocated = 0;
  private int bytesUsed = 0;
  private ByteBufferAllocator allocator;

  /**
   * Return an initial slab size such that a CapacityByteArrayOutputStream constructed with it
   * will end up allocating targetNumSlabs in order to reach targetCapacity. This aims to be
   * a balance between the overhead of creating new slabs and wasting memory by eagerly making
   * initial slabs too big.
   *
   * Note that targetCapacity here need not match maxCapacityHint in the constructor of
   * CapacityByteArrayOutputStream, though often that would make sense.
   *
   * @param minSlabSize no matter what we shouldn't make slabs any smaller than this
   * @param targetCapacity after we've allocated targetNumSlabs how much capacity should we have?
   * @param targetNumSlabs how many slabs should it take to reach targetCapacity?
   * @return an initial slab size
   */
  public static int initialSlabSizeHeuristic(int minSlabSize, int targetCapacity, int targetNumSlabs) {
    // initialSlabSize = (targetCapacity / (2^targetNumSlabs)) means we double targetNumSlabs times
    // before reaching the targetCapacity
    // eg for page size of 1MB we start at 1024 bytes.
    // we also don't want to start too small, so we also apply a minimum.
    return max(minSlabSize, ((int) (targetCapacity / pow(2, targetNumSlabs))));
  }

  public static CapacityByteArrayOutputStream withTargetNumSlabs(
      int minSlabSize, int maxCapacityHint, int targetNumSlabs) {
    return withTargetNumSlabs(minSlabSize, maxCapacityHint, targetNumSlabs, new HeapByteBufferAllocator());
  }

  /**
   * Construct a CapacityByteArrayOutputStream configured such that its initial slab size is
   * determined by {@link #initialSlabSizeHeuristic}, with targetCapacity == maxCapacityHint
   *
   * @param minSlabSize a minimum slab size
   * @param maxCapacityHint a hint for the maximum required capacity
   * @param targetNumSlabs the target number of slabs
   * @param allocator an allocator to use when creating byte buffers for slabs
   * @return a capacity baos
   */
  public static CapacityByteArrayOutputStream withTargetNumSlabs(
      int minSlabSize, int maxCapacityHint, int targetNumSlabs, ByteBufferAllocator allocator) {

    return new CapacityByteArrayOutputStream(
        initialSlabSizeHeuristic(minSlabSize, maxCapacityHint, targetNumSlabs),
        maxCapacityHint, allocator);
  }

  /**
   * Defaults maxCapacityHint to 1MB
   * @param initialSlabSize an initial slab size
   * @deprecated use {@link CapacityByteArrayOutputStream#CapacityByteArrayOutputStream(int, int, ByteBufferAllocator)}
   */
  @Deprecated
  public CapacityByteArrayOutputStream(int initialSlabSize) {
    this(initialSlabSize, 1024 * 1024, new HeapByteBufferAllocator());
  }

  /**
   * Defaults maxCapacityHint to 1MB
   * @param initialSlabSize an initial slab size
   * @param allocator an allocator to use when creating byte buffers for slabs
   * @deprecated use {@link CapacityByteArrayOutputStream#CapacityByteArrayOutputStream(int, int, ByteBufferAllocator)}
   */
  @Deprecated
  public CapacityByteArrayOutputStream(int initialSlabSize, ByteBufferAllocator allocator) {
    this(initialSlabSize, 1024 * 1024, allocator);
  }

  /**
   * @param initialSlabSize the size to make the first slab
   * @param maxCapacityHint a hint (not guarantee) of the max amount of data written to this stream
   * @deprecated use {@link CapacityByteArrayOutputStream#CapacityByteArrayOutputStream(int, int, ByteBufferAllocator)}
   */
  @Deprecated
  public CapacityByteArrayOutputStream(int initialSlabSize, int maxCapacityHint) {
    this(initialSlabSize, maxCapacityHint, new HeapByteBufferAllocator());
  }

  /**
   * @param initialSlabSize the size to make the first slab
   * @param maxCapacityHint a hint (not guarantee) of the max amount of data written to this stream
   * @param allocator an allocator to use when creating byte buffers for slabs
   */
  public CapacityByteArrayOutputStream(int initialSlabSize, int maxCapacityHint, ByteBufferAllocator allocator) {
    checkArgument(initialSlabSize > 0, "initialSlabSize must be > 0");
    checkArgument(maxCapacityHint > 0, "maxCapacityHint must be > 0");
    checkArgument(maxCapacityHint >= initialSlabSize, String.format("maxCapacityHint can't be less than initialSlabSize %d %d", initialSlabSize, maxCapacityHint));
    this.initialSlabSize = initialSlabSize;
    this.maxCapacityHint = maxCapacityHint;
    this.allocator = allocator;
    reset();
  }

  /**
   * the new slab is guaranteed to be at least minimumSize
   * @param minimumSize the size of the data we want to copy in the new slab
   */
  private void addSlab(int minimumSize) {
    int nextSlabSize;

    if (bytesUsed == 0) {
      nextSlabSize = initialSlabSize;
    } else if (bytesUsed > maxCapacityHint / 5) {
      // to avoid an overhead of up to twice the needed size, we get linear when approaching target page size
      nextSlabSize = maxCapacityHint / 5;
    } else {
      // double the size every time
      nextSlabSize = bytesUsed;
    }

    if (nextSlabSize < minimumSize) {
      LOG.debug("slab size {} too small for value of size {}. Bumping up slab size", nextSlabSize, minimumSize);
      nextSlabSize = minimumSize;
    }

    LOG.debug("used {} slabs, adding new slab of size {}", slabs.size(), nextSlabSize);

    this.currentSlab = allocator.allocate(nextSlabSize);
    this.slabs.add(currentSlab);
    this.bytesAllocated += nextSlabSize;
    this.currentSlabIndex = 0;
  }

  @Override
  public void write(int b) {
    if (!currentSlab.hasRemaining()) {
      addSlab(1);
    }
    currentSlab.put(currentSlabIndex, (byte) b);
    currentSlabIndex += 1;
    currentSlab.position(currentSlabIndex);
    bytesUsed += 1;
  }

  @Override
  public void write(byte b[], int off, int len) {
    if ((off < 0) || (off > b.length) || (len < 0) ||
        ((off + len) - b.length > 0)) {
      throw new IndexOutOfBoundsException(
          String.format("Given byte array of size %d, with requested length(%d) and offset(%d)", b.length, len, off));
    }
    if (len >= currentSlab.remaining()) {
      final int length1 = currentSlab.remaining();
      currentSlab.put(b, off, length1);
      bytesUsed += length1;
      currentSlabIndex += length1;
      final int length2 = len - length1;
      addSlab(length2);
      currentSlab.put(b, off + length1, length2);
      currentSlabIndex = length2;
      bytesUsed += length2;
    } else {
      currentSlab.put(b, off, len);
      currentSlabIndex += len;
      bytesUsed += len;
    }
  }

  private void writeToOutput(OutputStream out, ByteBuffer buf, int len) throws IOException {
    if (buf.hasArray()) {
      out.write(buf.array(), buf.arrayOffset(), len);
    } else {
      // The OutputStream interface only takes a byte[], unfortunately this means that a ByteBuffer
      // not backed by a byte array must be copied to fulfil this interface
      byte[] copy = new byte[len];
      buf.flip();
      buf.get(copy);
      out.write(copy);
    }
  }

  /**
   * Writes the complete contents of this buffer to the specified output stream argument. the output
   * stream's write method out.write(slab, 0, slab.length)) will be called once per slab.
   *
   * @param      out   the output stream to which to write the data.
   * @exception  IOException  if an I/O error occurs.
   */
  public void writeTo(OutputStream out) throws IOException {
    for (int i = 0; i < slabs.size() - 1; i++) {
      writeToOutput(out, slabs.get(i), slabs.get(i).position());
    }
    writeToOutput(out, currentSlab, currentSlabIndex);
  }

  /**
   * @return The total size in bytes of data written to this stream.
   */
  public long size() {
    return bytesUsed;
  }

  /**
   *
   * @return The total size in bytes currently allocated for this stream.
   */
  public int getCapacity() {
    return bytesAllocated;
  }

  /**
   * When re-using an instance with reset, it will adjust slab size based on previous data size.
   * The intent is to reuse the same instance for the same type of data (for example, the same column).
   * The assumption is that the size in the buffer will be consistent.
   */
  public void reset() {
    // readjust slab size.
    // 7 = 2^3 - 1 so that doubling the initial size 3 times will get to the same size
    this.initialSlabSize = max(bytesUsed / 7, initialSlabSize);
    LOG.debug("initial slab of size {}", initialSlabSize);
    for (ByteBuffer slab : slabs) {
      allocator.release(slab);
    }
    this.slabs.clear();
    this.bytesAllocated = 0;
    this.bytesUsed = 0;
    this.currentSlab = EMPTY_SLAB;
    this.currentSlabIndex = 0;
  }

  /**
   * @return the index of the last value written to this stream, which
   * can be passed to {@link #setByte(long, byte)} in order to change it
   */
  public long getCurrentIndex() {
    checkArgument(bytesUsed > 0, "This is an empty stream");
    return bytesUsed - 1;
  }

  /**
   * Replace the byte stored at position index in this stream with value
   *
   * @param index which byte to replace
   * @param value the value to replace it with
   */
  public void setByte(long index, byte value) {
    checkArgument(index < bytesUsed, "Index: " + index + " is >= the current size of: " + bytesUsed);

    long seen = 0;
    for (int i = 0; i < slabs.size(); i++) {
      ByteBuffer slab = slabs.get(i);
      if (index < seen + slab.limit()) {
        // ok found index
        slab.put((int)(index-seen), value);
        break;
      }
      seen += slab.limit();
    }
  }

  /**
   * @param prefix  a prefix to be used for every new line in the string
   * @return a text representation of the memory usage of this structure
   */
  public String memUsageString(String prefix) {
    return format("%s %s %d slabs, %,d bytes", prefix, getClass().getSimpleName(), slabs.size(), getCapacity());
  }

  /**
   * @return the total number of allocated slabs
   */
  int getSlabCount() {
    return slabs.size();
  }

  @Override
  public void close() {
    for (ByteBuffer slab : slabs) {
      allocator.release(slab);
    }
    try {
      super.close();
    }catch(IOException e){
      throw new OutputStreamCloseException(e);
    }
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.internal.column.columnindex;

import static java.util.Objects.requireNonNull;

import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Formatter;
import java.util.List;
import java.util.PrimitiveIterator;
import java.util.function.IntPredicate;

import org.apache.parquet.column.statistics.Statistics;
import org.apache.parquet.filter2.predicate.Operators.And;
import org.apache.parquet.filter2.predicate.Operators.Eq;
import org.apache.parquet.filter2.predicate.Operators.Gt;
import org.apache.parquet.filter2.predicate.Operators.GtEq;
import org.apache.parquet.filter2.predicate.Operators.LogicalNotUserDefined;
import org.apache.parquet.filter2.predicate.Operators.Lt;
import org.apache.parquet.filter2.predicate.Operators.LtEq;
import org.apache.parquet.filter2.predicate.Operators.Not;
import org.apache.parquet.filter2.predicate.Operators.NotEq;
import org.apache.parquet.filter2.predicate.Operators.Or;
import org.apache.parquet.filter2.predicate.Operators.UserDefined;
import org.apache.parquet.filter2.predicate.UserDefinedPredicate;
import org.apache.parquet.io.api.Binary;
import org.apache.parquet.schema.PrimitiveComparator;
import org.apache.parquet.schema.PrimitiveStringifier;
import org.apache.parquet.schema.PrimitiveType;

import it.unimi.dsi.fastutil.booleans.BooleanArrayList;
import it.unimi.dsi.fastutil.booleans.BooleanList;
import it.unimi.dsi.fastutil.booleans.BooleanLists;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.ints.IntList;
import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
import it.unimi.dsi.fastutil.ints.IntSet;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.longs.LongList;
import it.unimi.dsi.fastutil.longs.LongLists;

/**
 * Builder implementation to create {@link ColumnIndex} objects.
 */
public abstract class ColumnIndexBuilder {

  static abstract class ColumnIndexBase<C> implements ColumnIndex {
    /*
     * A class containing the value to be compared to the min/max values. This way we only need to do the deboxing once
     * per predicate execution instead for every comparison.
     */
    abstract class ValueComparator {
      abstract int compareValueToMin(int arrayIndex);

      abstract int compareValueToMax(int arrayIndex);

      int arrayLength() {
        return pageIndexes.length;
      }

      int translate(int arrayIndex) {
        return pageIndexes[arrayIndex];
      }
    }

    private static final ByteBuffer EMPTY_BYTE_BUFFER = ByteBuffer.allocate(0);
    private static final int MAX_VALUE_LENGTH_FOR_TOSTRING = 40;
    private static final String TOSTRING_TRUNCATION_MARKER = "(...)";
    private static final int TOSTRING_TRUNCATION_START_POS = (MAX_VALUE_LENGTH_FOR_TOSTRING
        - TOSTRING_TRUNCATION_MARKER.length()) / 2;
    private static final int TOSTRING_TRUNCATION_END_POS = MAX_VALUE_LENGTH_FOR_TOSTRING
        - TOSTRING_TRUNCATION_MARKER.length() - TOSTRING_TRUNCATION_START_POS;
    private static final String TOSTRING_MISSING_VALUE_MARKER = "<none>";

    final PrimitiveStringifier stringifier;
    final PrimitiveComparator<C> comparator;
    private boolean[] nullPages;
    private BoundaryOrder boundaryOrder;
    // Storing the page index for each array index (min/max values are not stored for null-pages)
    private int[] pageIndexes;
    // might be null
    private long[] nullCounts;

    static String truncate(String str) {
      if (str.length() <= MAX_VALUE_LENGTH_FOR_TOSTRING) {
        return str;
      }
      return str.substring(0, TOSTRING_TRUNCATION_START_POS) + TOSTRING_TRUNCATION_MARKER
          + str.substring(str.length() - TOSTRING_TRUNCATION_END_POS);
    }

    ColumnIndexBase(PrimitiveType type) {
      comparator = type.comparator();
      stringifier = type.stringifier();
    }

    @Override
    public BoundaryOrder getBoundaryOrder() {
      return boundaryOrder;
    }

    @Override
    public List<Long> getNullCounts() {
      if (nullCounts == null) {
        return null;
      }
      return LongLists.unmodifiable(LongArrayList.wrap(nullCounts));
    }

    @Override
    public List<Boolean> getNullPages() {
      return BooleanLists.unmodifiable(BooleanArrayList.wrap(nullPages));
    }

    @Override
    public List<ByteBuffer> getMinValues() {
      List<ByteBuffer> list = new ArrayList<>(getPageCount());
      int arrayIndex = 0;
      for (int i = 0, n = getPageCount(); i < n; ++i) {
        if (isNullPage(i)) {
          list.add(EMPTY_BYTE_BUFFER);
        } else {
          list.add(getMinValueAsBytes(arrayIndex++));
        }
      }
      return list;
    }

    @Override
    public List<ByteBuffer> getMaxValues() {
      List<ByteBuffer> list = new ArrayList<>(getPageCount());
      int arrayIndex = 0;
      for (int i = 0, n = getPageCount(); i < n; ++i) {
        if (isNullPage(i)) {
          list.add(EMPTY_BYTE_BUFFER);
        } else {
          list.add(getMaxValueAsBytes(arrayIndex++));
        }
      }
      return list;
    }

    @Override
    public String toString() {
      try (Formatter formatter = new Formatter()) {
        formatter.format("Boudary order: %s\n", boundaryOrder);
        String minMaxPart = "  %-" + MAX_VALUE_LENGTH_FOR_TOSTRING + "s  %-" + MAX_VALUE_LENGTH_FOR_TOSTRING + "s\n";
        formatter.format("%-10s  %20s" + minMaxPart, "", "null count", "min", "max");
        String format = "page-%-5d  %20s" + minMaxPart;
        int arrayIndex = 0;
        for (int i = 0, n = nullPages.length; i < n; ++i) {
          String nullCount = nullCounts == null ? TOSTRING_MISSING_VALUE_MARKER : Long.toString(nullCounts[i]);
          String min, max;
          if (nullPages[i]) {
            min = max = TOSTRING_MISSING_VALUE_MARKER;
          } else {
            min = truncate(getMinValueAsString(arrayIndex));
            max = truncate(getMaxValueAsString(arrayIndex++));
          }
          formatter.format(format, i, nullCount, min, max);
        }
        return formatter.toString();
      }
    }

    int getPageCount() {
      return nullPages.length;
    }

    boolean isNullPage(int pageIndex) {
      return nullPages[pageIndex];
    }

    /*
     * Returns the min value for arrayIndex as a ByteBuffer. (Min values are not stored for null-pages so arrayIndex
     * might not equal to pageIndex.)
     */
    abstract ByteBuffer getMinValueAsBytes(int arrayIndex);

    /*
     * Returns the max value for arrayIndex as a ByteBuffer. (Max values are not stored for null-pages so arrayIndex
     * might not equal to pageIndex.)
     */
    abstract ByteBuffer getMaxValueAsBytes(int arrayIndex);

    /*
     * Returns the min value for arrayIndex as a String. (Min values are not stored for null-pages so arrayIndex might
     * not equal to pageIndex.)
     */
    abstract String getMinValueAsString(int arrayIndex);

    /*
     * Returns the max value for arrayIndex as a String. (Max values are not stored for null-pages so arrayIndex might
     * not equal to pageIndex.)
     */
    abstract String getMaxValueAsString(int arrayIndex);

    /* Creates a Statistics object for filtering. Used for user defined predicates. */
    abstract <T extends Comparable<T>> org.apache.parquet.filter2.predicate.Statistics<T> createStats(int arrayIndex);

    /* Creates a ValueComparator object containing the specified value to be compared for min/max values */
    abstract ValueComparator createValueComparator(Object value);

    @Override
    public PrimitiveIterator.OfInt visit(And and) {
      throw new UnsupportedOperationException("AND shall not be used on column index directly");
    }

    @Override
    public PrimitiveIterator.OfInt visit(Not not) {
      throw new UnsupportedOperationException("NOT shall not be used on column index directly");
    }

    @Override
    public PrimitiveIterator.OfInt visit(Or or) {
      throw new UnsupportedOperationException("OR shall not be used on column index directly");
    }

    @Override
    public <T extends Comparable<T>> PrimitiveIterator.OfInt visit(Eq<T> eq) {
      T value = eq.getValue();
      if (value == null) {
        if (nullCounts == null) {
          // Searching for nulls so if we don't have null related statistics we have to return all pages
          return IndexIterator.all(getPageCount());
        } else {
          return IndexIterator.filter(getPageCount(), pageIndex -> nullCounts[pageIndex] > 0);
        }
      }
      return getBoundaryOrder().eq(createValueComparator(value));
    }

    @Override
    public <T extends Comparable<T>> PrimitiveIterator.OfInt visit(Gt<T> gt) {
      return getBoundaryOrder().gt(createValueComparator(gt.getValue()));
    }

    @Override
    public <T extends Comparable<T>> PrimitiveIterator.OfInt visit(GtEq<T> gtEq) {
      return getBoundaryOrder().gtEq(createValueComparator(gtEq.getValue()));
    }

    @Override
    public <T extends Comparable<T>> PrimitiveIterator.OfInt visit(Lt<T> lt) {
      return getBoundaryOrder().lt(createValueComparator(lt.getValue()));
    }

    @Override
    public <T extends Comparable<T>> PrimitiveIterator.OfInt visit(LtEq<T> ltEq) {
      return getBoundaryOrder().ltEq(createValueComparator(ltEq.getValue()));
    }

    @Override
    public <T extends Comparable<T>> PrimitiveIterator.OfInt visit(NotEq<T> notEq) {
      T value = notEq.getValue();
      if (value == null) {
        return IndexIterator.filter(getPageCount(), pageIndex -> !nullPages[pageIndex]);
      }

      if (nullCounts == null) {
        // Nulls match so if we don't have null related statistics we have to return all pages
        return IndexIterator.all(getPageCount());
      }

      // Merging value filtering with pages containing nulls
      IntSet matchingIndexes = new IntOpenHashSet();
      getBoundaryOrder().notEq(createValueComparator(value))
          .forEachRemaining((int index) -> matchingIndexes.add(index));
      return IndexIterator.filter(getPageCount(),
          pageIndex -> nullCounts[pageIndex] > 0 || matchingIndexes.contains(pageIndex));
    }

    @Override
    public <T extends Comparable<T>, U extends UserDefinedPredicate<T>> PrimitiveIterator.OfInt visit(
        UserDefined<T, U> udp) {
      final UserDefinedPredicate<T> predicate = udp.getUserDefinedPredicate();
      final boolean acceptNulls = predicate.keep(null);

      if (acceptNulls && nullCounts == null) {
        // Nulls match so if we don't have null related statistics we have to return all pages
        return IndexIterator.all(getPageCount());
      }

      return IndexIterator.filter(getPageCount(), new IntPredicate() {
        private int arrayIndex = -1;

        @Override
        public boolean test(int pageIndex) {
          if (isNullPage(pageIndex)) {
            return acceptNulls;
          } else {
            ++arrayIndex;
            if (acceptNulls && nullCounts[pageIndex] > 0) {
              return true;
            }
            org.apache.parquet.filter2.predicate.Statistics<T> stats = createStats(arrayIndex);
            return !predicate.canDrop(stats);
          }
        }
      });
    }

    @Override
    public <T extends Comparable<T>, U extends UserDefinedPredicate<T>> PrimitiveIterator.OfInt visit(
        LogicalNotUserDefined<T, U> udp) {
      final UserDefinedPredicate<T> inversePredicate = udp.getUserDefined().getUserDefinedPredicate();
      final boolean acceptNulls = !inversePredicate.keep(null);

      if (acceptNulls && nullCounts == null) {
        // Nulls match so if we don't have null related statistics we have to return all pages
        return IndexIterator.all(getPageCount());
      }

      return IndexIterator.filter(getPageCount(), new IntPredicate() {
        private int arrayIndex = -1;

        @Override
        public boolean test(int pageIndex) {
          if (isNullPage(pageIndex)) {
            return acceptNulls;
          } else {
            ++arrayIndex;
            if (acceptNulls && nullCounts[pageIndex] > 0) {
              return true;
            }
            org.apache.parquet.filter2.predicate.Statistics<T> stats = createStats(arrayIndex);
            return !inversePredicate.inverseCanDrop(stats);
          }
        }
      });
    }
  }

  private static final ColumnIndexBuilder NO_OP_BUILDER = new ColumnIndexBuilder() {
    @Override
    public ColumnIndex build() {
      return null;
    }

    @Override
    public void add(Statistics<?> stats) {
    }

    @Override
    void addMinMax(Object min, Object max) {
    }

    @Override
    ColumnIndexBase<?> createColumnIndex(PrimitiveType type) {
      return null;
    }

    @Override
    void clearMinMax() {
    }

    @Override
    void addMinMaxFromBytes(ByteBuffer min, ByteBuffer max) {
    }

    @Override
    int compareMinValues(PrimitiveComparator<Binary> comparator, int index1, int index2) {
      return 0;
    }

    @Override
    int compareMaxValues(PrimitiveComparator<Binary> comparator, int index1, int index2) {
      return 0;
    }

    @Override
    int sizeOf(Object value) {
      return 0;
    }
  };

  private PrimitiveType type;
  private final BooleanList nullPages = new BooleanArrayList();
  private final LongList nullCounts = new LongArrayList();
  private long minMaxSize;
  private final IntList pageIndexes = new IntArrayList();
  private int nextPageIndex;

  /**
   * @return a no-op builder that does not collect statistics objects and therefore returns {@code null} at
   *         {@link #build()}.
   */
  public static ColumnIndexBuilder getNoOpBuilder() {
    return NO_OP_BUILDER;
  }

  /**
   * @param type
   *          the type this builder is to be created for
   * @param truncateLength
   *          the length to be used for truncating binary values if possible
   * @return a {@link ColumnIndexBuilder} instance to be used for creating {@link ColumnIndex} objects
   */
  public static ColumnIndexBuilder getBuilder(PrimitiveType type, int truncateLength) {
    ColumnIndexBuilder builder = createNewBuilder(type, truncateLength);
    builder.type = type;
    return builder;
  }

  private static ColumnIndexBuilder createNewBuilder(PrimitiveType type, int truncateLength) {
    switch (type.getPrimitiveTypeName()) {
      case BINARY:
      case FIXED_LEN_BYTE_ARRAY:
      case INT96:
        return new BinaryColumnIndexBuilder(type, truncateLength);
      case BOOLEAN:
        return new BooleanColumnIndexBuilder();
      case DOUBLE:
        return new DoubleColumnIndexBuilder();
      case FLOAT:
        return new FloatColumnIndexBuilder();
      case INT32:
        return new IntColumnIndexBuilder();
      case INT64:
        return new LongColumnIndexBuilder();
      default:
        throw new IllegalArgumentException("Unsupported type for column index: " + type);
    }
  }

  /**
   * @param type
   *          the primitive type
   * @param boundaryOrder
   *          the boundary order of the min/max values
   * @param nullPages
   *          the null pages (one boolean value for each page that signifies whether the page consists of nulls
   *          entirely)
   * @param nullCounts
   *          the number of null values for each page
   * @param minValues
   *          the min values for each page
   * @param maxValues
   *          the max values for each page
   * @return the newly created {@link ColumnIndex} object based on the specified arguments
   */
  public static ColumnIndex build(
      PrimitiveType type,
      BoundaryOrder boundaryOrder,
      List<Boolean> nullPages,
      List<Long> nullCounts,
      List<ByteBuffer> minValues,
      List<ByteBuffer> maxValues) {

    ColumnIndexBuilder builder = createNewBuilder(type, Integer.MAX_VALUE);

    builder.fill(nullPages, nullCounts, minValues, maxValues);
    ColumnIndexBase<?> columnIndex = builder.build(type);
    columnIndex.boundaryOrder = requireNonNull(boundaryOrder);
    return columnIndex;
  }

  ColumnIndexBuilder() {
    // Shall be able to be created inside this package only
  }

  /**
   * Adds the data from the specified statistics to this builder
   *
   * @param stats
   *          the statistics to be added
   */
  public void add(Statistics<?> stats) {
    if (stats.hasNonNullValue()) {
      nullPages.add(false);
      Object min = stats.genericGetMin();
      Object max = stats.genericGetMax();
      addMinMax(min, max);
      pageIndexes.add(nextPageIndex);
      minMaxSize += sizeOf(min);
      minMaxSize += sizeOf(max);
    } else {
      nullPages.add(true);
    }
    nullCounts.add(stats.getNumNulls());
    ++nextPageIndex;
  }

  abstract void addMinMaxFromBytes(ByteBuffer min, ByteBuffer max);

  abstract void addMinMax(Object min, Object max);

  private void fill(List<Boolean> nullPages, List<Long> nullCounts, List<ByteBuffer> minValues,
      List<ByteBuffer> maxValues) {
    clear();
    int pageCount = nullPages.size();
    if ((nullCounts != null && nullCounts.size() != pageCount) || minValues.size() != pageCount
        || maxValues.size() != pageCount) {
      throw new IllegalArgumentException(
          String.format("Not all sizes are equal (nullPages:%d, nullCounts:%s, minValues:%d, maxValues:%d",
              nullPages.size(), nullCounts == null ? "null" : nullCounts.size(), minValues.size(), maxValues.size()));
    }
    this.nullPages.addAll(nullPages);
    // Nullcounts is optional in the format
    if (nullCounts != null) {
      this.nullCounts.addAll(nullCounts);
    }

    for (int i = 0; i < pageCount; ++i) {
      if (!nullPages.get(i)) {
        ByteBuffer min = minValues.get(i);
        ByteBuffer max = maxValues.get(i);
        addMinMaxFromBytes(min, max);
        pageIndexes.add(i);
        minMaxSize += min.remaining();
        minMaxSize += max.remaining();
      }
    }
  }

  /**
   * @return the newly created column index or {@code null} if the {@link ColumnIndex} would be empty
   */
  public ColumnIndex build() {
    ColumnIndexBase<?> columnIndex = build(type);
    if (columnIndex == null) {
      return null;
    }
    columnIndex.boundaryOrder = calculateBoundaryOrder(type.comparator());
    return columnIndex;
  }

  private ColumnIndexBase<?> build(PrimitiveType type) {
    if (nullPages.isEmpty()) {
      return null;
    }
    ColumnIndexBase<?> columnIndex = createColumnIndex(type);
    if (columnIndex == null) {
      // Might happen if the specialized builder discovers invalid min/max values
      return null;
    }
    columnIndex.nullPages = nullPages.toBooleanArray();
    // Null counts is optional so keep it null if the builder has no values
    if (!nullCounts.isEmpty()) {
      columnIndex.nullCounts = nullCounts.toLongArray();
    }
    columnIndex.pageIndexes = pageIndexes.toIntArray();

    return columnIndex;
  }

  private BoundaryOrder calculateBoundaryOrder(PrimitiveComparator<Binary> comparator) {
    if (isAscending(comparator)) {
      return BoundaryOrder.ASCENDING;
    } else if (isDescending(comparator)) {
      return BoundaryOrder.DESCENDING;
    } else {
      return BoundaryOrder.UNORDERED;
    }
  }

  // min[i] <= min[i+1] && max[i] <= max[i+1]
  private boolean isAscending(PrimitiveComparator<Binary> comparator) {
    for (int i = 1, n = pageIndexes.size(); i < n; ++i) {
      if (compareMinValues(comparator, i - 1, i) > 0 || compareMaxValues(comparator, i - 1, i) > 0) {
        return false;
      }
    }
    return true;
  }

  // min[i] >= min[i+1] && max[i] >= max[i+1]
  private boolean isDescending(PrimitiveComparator<Binary> comparator) {
    for (int i = 1, n = pageIndexes.size(); i < n; ++i) {
      if (compareMinValues(comparator, i - 1, i) < 0 || compareMaxValues(comparator, i - 1, i) < 0) {
        return false;
      }
    }
    return true;
  }

  abstract int compareMinValues(PrimitiveComparator<Binary> comparator, int index1, int index2);

  abstract int compareMaxValues(PrimitiveComparator<Binary> comparator, int index1, int index2);

  private void clear() {
    nullPages.clear();
    nullCounts.clear();
    clearMinMax();
    minMaxSize = 0;
    nextPageIndex = 0;
    pageIndexes.clear();
  }

  abstract void clearMinMax();

  abstract ColumnIndexBase<?> createColumnIndex(PrimitiveType type);

  abstract int sizeOf(Object value);

  /**
   * @return the number of pages added so far to this builder
   */
  public int getPageCount() {
    return nullPages.size();
  }

  /**
   * @return the sum of size in bytes of the min/max values added so far to this builder
   */
  public long getMinMaxSize() {
    return minMaxSize;
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.column.values;

import java.io.IOException;
import java.nio.ByteBuffer;

import org.apache.parquet.bytes.ByteBufferInputStream;
import org.apache.parquet.io.ParquetDecodingException;
import org.apache.parquet.io.api.Binary;

/**
 * Base class to implement an encoding for a given column type.
 *
 * A ValuesReader is provided with a page (byte-buffer) and is responsible
 * for deserializing the primitive values stored in that page.
 *
 * Given that pages are homogeneous (store only a single type), typical subclasses
 * will only override one of the read*() methods.
 */
public abstract class ValuesReader {

  // To be used to maintain the deprecated behavior of getNextOffset(); -1 means undefined
  private int actualOffset = -1;
  private int nextOffset;

  /**
   * Called to initialize the column reader from a part of a page.
   *
   * The underlying implementation knows how much data to read, so a length
   * is not provided.
   *
   * Each page may contain several sections:
   *

   *
 repetition levels column
   *
 definition levels column
   *
 data column
   *

   *
   * This function is called with 'offset' pointing to the beginning of one of these sections,
   * and should return the offset to the section following it.
   *
   * @param valueCount count of values in this page
   * @param page the array to read from containing the page data (repetition levels, definition levels, data)
   * @param offset where to start reading from in the page
   *
   * @throws IOException
   * @deprecated Will be removed in 2.0.0
   */
  @Deprecated
  public void initFromPage(int valueCount, ByteBuffer page, int offset) throws IOException {
    if (offset < 0) {
      throw new IllegalArgumentException("Illegal offset: " + offset);
    }
    actualOffset = offset;
    ByteBuffer pageWithOffset = page.duplicate();
    pageWithOffset.position(offset);
    initFromPage(valueCount, ByteBufferInputStream.wrap(pageWithOffset));
    actualOffset = -1;
  }

  /**
   * Same functionality as method of the same name that takes a ByteBuffer instead of a byte[].
   *
   * This method is only provided for backward compatibility and will be removed in a future release.
   * Please update any code using it as soon as possible.
   * @see #initFromPage(int, ByteBuffer, int)
   */
  @Deprecated
  public void initFromPage(int valueCount, byte[] page, int offset) throws IOException {
    this.initFromPage(valueCount, ByteBuffer.wrap(page), offset);
  }

  /**
   * Called to initialize the column reader from a part of a page.
   *
   * Implementations must consume all bytes from the input stream, leaving the
   * stream ready to read the next section of data. The underlying
   * implementation knows how much data to read, so a length is not provided.
   *
   * Each page may contain several sections:
   *

   *
 repetition levels column
   *
 definition levels column
   *
 data column
   *

   *
   * @param valueCount count of values in this page
   * @param in an input stream containing the page data at the correct offset
   *
   * @throws IOException if there is an exception while reading from the input stream
   */
  public void initFromPage(int valueCount, ByteBufferInputStream in) throws IOException {
    if (actualOffset != -1) {
      throw new UnsupportedOperationException(
          "Either initFromPage(int, ByteBuffer, int) or initFromPage(int, ByteBufferInputStream) must be implemented in "
              + getClass().getName());
    }
    initFromPage(valueCount, in.slice(valueCount), 0);
  }

  /**
   * Called to return offset of the next section
   * @return offset of the next section
   * @deprecated Will be removed in 2.0.0
   */
  @Deprecated
  public int getNextOffset() {
    if (nextOffset == -1) {
      throw new ParquetDecodingException("Unsupported: cannot get offset of the next section.");
    } else {
      return nextOffset;
    }
  }

  // To be used to maintain the deprecated behavior of getNextOffset();
  // bytesRead is the number of bytes read in the last initFromPage call
  protected void updateNextOffset(int bytesRead) {
    nextOffset = actualOffset == -1 ? -1 : actualOffset + bytesRead;
  }

  /**
   * usable when the encoding is dictionary based
   * @return the id of the next value from the page
   */
  public int readValueDictionaryId() {
    throw new UnsupportedOperationException();
  }

  /**
   * @return the next boolean from the page
   */
  public boolean readBoolean() {
    throw new UnsupportedOperationException();
  }

  /**
   * @return the next Binary from the page
   */
  public Binary readBytes() {
    throw new UnsupportedOperationException();
  }

  /**
   * @return the next float from the page
   */
  public float readFloat() {
    throw new UnsupportedOperationException();
  }

  /**
   * @return the next double from the page
   */
  public double readDouble() {
    throw new UnsupportedOperationException();
  }

  /**
   * @return the next integer from the page
   */
  public int readInteger() {
    throw new UnsupportedOperationException();
  }

  /**
   * @return the next long from the page
   */
  public long readLong() {
    throw new UnsupportedOperationException();
  }

  /**
   * Skips the next value in the page
   */
  abstract public void skip();

  /**
   * Skips the next n values in the page
   *
   * @param n
   *          the number of values to be skipped
   */
  public void skip(int n) {
    for (int i = 0; i < n; ++i) {
      skip();
    }
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.avro;

import org.apache.avro.Schema;
import org.apache.avro.generic.IndexedRecord;
import org.apache.hadoop.mapreduce.Job;
import org.apache.parquet.hadoop.ParquetInputFormat;
import org.apache.parquet.hadoop.util.ContextUtil;

/**
 * A Hadoop {@link org.apache.hadoop.mapreduce.InputFormat} for Parquet files.
 *
 * @param <T> the Java type of objects produced by this InputFormat
 */
public class AvroParquetInputFormat<T> extends ParquetInputFormat<T> {
  public AvroParquetInputFormat() {
    super(AvroReadSupport.class);
  }

  /**
   * Set the subset of columns to read (projection pushdown). Specified as an Avro
   * schema, the requested projection is converted into a Parquet schema for Parquet
   * column projection.
   *

   * This is useful if the full schema is large and you only want to read a few
   * columns, since it saves time by not reading unused columns.
   *

   * If a requested projection is set, then the Avro schema used for reading
   * must be compatible with the projection. For instance, if a column is not included
   * in the projection then it must either not be included or be optional in the read
   * schema. Use {@link #setAvroReadSchema(org.apache.hadoop.mapreduce.Job,
   * org.apache.avro.Schema)} to set a read schema, if needed.
   * @param job a job
   * @param requestedProjection the requested projection schema
   * @see #setAvroReadSchema(org.apache.hadoop.mapreduce.Job, org.apache.avro.Schema)
   * @see org.apache.parquet.avro.AvroParquetOutputFormat#setSchema(org.apache.hadoop.mapreduce.Job, org.apache.avro.Schema)
   */
  public static void setRequestedProjection(Job job, Schema requestedProjection) {
    AvroReadSupport.setRequestedProjection(ContextUtil.getConfiguration(job),
        requestedProjection);
  }

  /**
   * Override the Avro schema to use for reading. If not set, the Avro schema used for
   * writing is used.
   *

   * Differences between the read and write schemas are resolved using
   * Avro's schema resolution rules.
   * @param job a job
   * @param avroReadSchema the requested schema
   * @see #setRequestedProjection(org.apache.hadoop.mapreduce.Job, org.apache.avro.Schema)
   * @see org.apache.parquet.avro.AvroParquetOutputFormat#setSchema(org.apache.hadoop.mapreduce.Job, org.apache.avro.Schema)
   */
  public static void setAvroReadSchema(Job job, Schema avroReadSchema) {
    AvroReadSupport.setAvroReadSchema(ContextUtil.getConfiguration(job), avroReadSchema);
  }

  /**
   * Uses an instance of the specified {@link AvroDataSupplier} class to control how the
   * {@link org.apache.avro.specific.SpecificData} instance that is used to find
   * Avro specific records is created.
   * @param job a job
   * @param supplierClass an avro data supplier class
   */
  public static void setAvroDataSupplier(Job job,
      Class<? extends AvroDataSupplier> supplierClass) {
    AvroReadSupport.setAvroDataSupplier(ContextUtil.getConfiguration(job), supplierClass);
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.filter2.recordlevel;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;

import org.apache.parquet.hadoop.metadata.ColumnPath;
import org.apache.parquet.filter2.recordlevel.IncrementallyUpdatedFilterPredicate.ValueInspector;
import org.apache.parquet.io.PrimitiveColumnIO;
import org.apache.parquet.io.api.Converter;
import org.apache.parquet.io.api.GroupConverter;

import static org.apache.parquet.Preconditions.checkArgument;
import static org.apache.parquet.Preconditions.checkNotNull;

/**
 * See {@link FilteringRecordMaterializer}
 */
public class FilteringGroupConverter extends GroupConverter {
  // the real converter
  private final GroupConverter delegate;

  // the path, from the root of the schema, to this converter
  // used ultimately by the primitive converter proxy to figure
  // out which column it represents.
  private final List<Integer> indexFieldPath;

  // for a given column, which nodes in the filter expression need
  // to be notified of this column's value
  private final Map<ColumnPath, List<ValueInspector>> valueInspectorsByColumn;

  // used to go from our indexFieldPath to the PrimitiveColumnIO for that column
  private final Map<List<Integer>, PrimitiveColumnIO> columnIOsByIndexFieldPath;

  public FilteringGroupConverter(
      GroupConverter delegate,
      List<Integer> indexFieldPath,
      Map<ColumnPath, List<ValueInspector>> valueInspectorsByColumn, Map<List<Integer>,
      PrimitiveColumnIO> columnIOsByIndexFieldPath) {

    this.delegate = checkNotNull(delegate, "delegate");
    this.indexFieldPath = checkNotNull(indexFieldPath, "indexFieldPath");
    this.columnIOsByIndexFieldPath = checkNotNull(columnIOsByIndexFieldPath, "columnIOsByIndexFieldPath");
    this.valueInspectorsByColumn = checkNotNull(valueInspectorsByColumn, "valueInspectorsByColumn");
  }

  // When a converter is asked for, we get the real one from the delegate, then wrap it
  // in a filtering pass-through proxy.
  // TODO: making the assumption that getConverter(i) is only called once, is that valid?
  @Override
  public Converter getConverter(int fieldIndex) {

    // get the real converter from the delegate
    Converter delegateConverter = checkNotNull(delegate.getConverter(fieldIndex), "delegate converter");

    // determine the indexFieldPath for the converter proxy we're about to make, which is
    // this converter's path + the requested fieldIndex
    List<Integer> newIndexFieldPath = new ArrayList<Integer>(indexFieldPath.size() + 1);
    newIndexFieldPath.addAll(indexFieldPath);
    newIndexFieldPath.add(fieldIndex);

    if (delegateConverter.isPrimitive()) {
      PrimitiveColumnIO columnIO = getColumnIO(newIndexFieldPath);
      ColumnPath columnPath = ColumnPath.get(columnIO.getColumnDescriptor().getPath());
      ValueInspector[] valueInspectors = getValueInspectors(columnPath);
      return new FilteringPrimitiveConverter(delegateConverter.asPrimitiveConverter(), valueInspectors);
    } else {
      return new FilteringGroupConverter(delegateConverter.asGroupConverter(), newIndexFieldPath, valueInspectorsByColumn, columnIOsByIndexFieldPath);
    }

  }

  private PrimitiveColumnIO getColumnIO(List<Integer> indexFieldPath) {
    PrimitiveColumnIO found = columnIOsByIndexFieldPath.get(indexFieldPath);
    checkArgument(found != null, "Did not find PrimitiveColumnIO for index field path" + indexFieldPath);
    return found;
  }

  private ValueInspector[] getValueInspectors(ColumnPath columnPath) {
    List<ValueInspector> inspectorsList = valueInspectorsByColumn.get(columnPath);
    if (inspectorsList == null) {
      return new ValueInspector[] {};
    } else {
      return inspectorsList.toArray(new ValueInspector[inspectorsList.size()]);
    }
  }

  @Override
  public void start() {
    delegate.start();
  }

  @Override
  public void end() {
    delegate.end();
  }
}