package it.unimi.dsi.fastutil;

/*
 * Copyright (C) 2002-2017 Sebastiano Vigna
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/** Common code for all hash-based classes. */

public class HashCommon {

	protected HashCommon() {}

	/** 232 · φ, φ = (√5 − 1)/2. */
	private static final int INT_PHI = 0x9E3779B9;
	/** The reciprocal of {@link #INT_PHI} modulo 232. */
	private static final int INV_INT_PHI = 0x144cbc89;
	/** 264 · φ, φ = (√5 − 1)/2. */
	private static final long LONG_PHI = 0x9E3779B97F4A7C15L;
	/** The reciprocal of {@link #LONG_PHI} modulo 264. */
	private static final long INV_LONG_PHI = 0xf1de83e19937733dL;

	/** Avalanches the bits of an integer by applying the finalisation step of MurmurHash3.
	 *
	 *
This method implements the finalisation step of Austin Appleby's MurmurHash3.
	 * Its purpose is to avalanche the bits of the argument to within 0.25% bias.
	 *
	 * @param x an integer.
	 * @return a hash value with good avalanching properties.
	 */
	public static int murmurHash3(int x) {
		x ^= x >>> 16;
		x *= 0x85ebca6b;
		x ^= x >>> 13;
		x *= 0xc2b2ae35;
		x ^= x >>> 16;
		return x;
	}

	/** Avalanches the bits of a long integer by applying the finalisation step of MurmurHash3.
	 *
	 *
This method implements the finalisation step of Austin Appleby's MurmurHash3.
	 * Its purpose is to avalanche the bits of the argument to within 0.25% bias.
	 *
	 * @param x a long integer.
	 * @return a hash value with good avalanching properties.
	 */
	public static long murmurHash3(long x) {
		x ^= x >>> 33;
		x *= 0xff51afd7ed558ccdL;
		x ^= x >>> 33;
		x *= 0xc4ceb9fe1a85ec53L;
		x ^= x >>> 33;
		return x;
	}

	/** Quickly mixes the bits of an integer.
	 *
	 *
This method mixes the bits of the argument by multiplying by the golden ratio and
	 * xorshifting the result. It is borrowed from Koloboke, and
	 * it has slightly worse behaviour than {@link #murmurHash3(int)} (in open-addressing hash tables the average number of probes
	 * is slightly larger), but it's much faster.
	 *
	 * @param x an integer.
	 * @return a hash value obtained by mixing the bits of {@code x}.
	 * @see #invMix(int)
	 */
	public static int mix(final int x) {
		final int h = x * INT_PHI;
		return h ^ (h >>> 16);
	}

	/** The inverse of {@link #mix(int)}. This method is mainly useful to create unit tests.
	 *
	 * @param x an integer.
	 * @return a value that passed through {@link #mix(int)} would give {@code x}.
	 */
	public static int invMix(final int x) {
		return (x ^ x >>> 16) * INV_INT_PHI;
	}

	/** Quickly mixes the bits of a long integer.
	 *
	 *
This method mixes the bits of the argument by multiplying by the golden ratio and
	 * xorshifting twice the result. It is borrowed from Koloboke, and
	 * it has slightly worse behaviour than {@link #murmurHash3(long)} (in open-addressing hash tables the average number of probes
	 * is slightly larger), but it's much faster.
	 *
	 * @param x a long integer.
	 * @return a hash value obtained by mixing the bits of {@code x}.
	 */
	public static long mix(final long x) {
		long h = x * LONG_PHI;
		h ^= h >>> 32;
		return h ^ (h >>> 16);
	}

	/** The inverse of {@link #mix(long)}. This method is mainly useful to create unit tests.
	 *
	 * @param x a long integer.
	 * @return a value that passed through {@link #mix(long)} would give {@code x}.
	 */
	public static long invMix(long x) {
		x ^= x >>> 32;
		x ^= x >>> 16;
		return (x ^ x >>> 32) * INV_LONG_PHI;
	}

	/** Returns the hash code that would be returned by {@link Float#hashCode()}.
	 *
	 * @param f a float.
	 * @return the same code as {@link Float#hashCode() new Float(f).hashCode()}.
	 */

	public static int float2int(final float f) {
		return Float.floatToRawIntBits(f);
	}

	/** Returns the hash code that would be returned by {@link Double#hashCode()}.
	 *
	 * @param d a double.
	 * @return the same code as {@link Double#hashCode() new Double(f).hashCode()}.
	 */

	public static int double2int(final double d) {
		final long l = Double.doubleToRawLongBits(d);
		return (int)(l ^ (l >>> 32));
	}

	/** Returns the hash code that would be returned by {@link Long#hashCode()}.
	 *
	 * @param l a long.
	 * @return the same code as {@link Long#hashCode() new Long(f).hashCode()}.
	 */
	public static int long2int(final long l) {
		return (int)(l ^ (l >>> 32));
	}

	/** Returns the least power of two greater than or equal to the specified value.
	 *
	 *
Note that this function will return 1 when the argument is 0.
	 *
	 * @param x an integer smaller than or equal to 230.
	 * @return the least power of two greater than or equal to the specified value.
	 */
	public static int nextPowerOfTwo(int x) {
		if (x == 0) return 1;
		x--;
		x |= x >> 1;
		x |= x >> 2;
		x |= x >> 4;
		x |= x >> 8;
		return (x | x >> 16) + 1;
	}

	/** Returns the least power of two greater than or equal to the specified value.
	 *
	 *
Note that this function will return 1 when the argument is 0.
	 *
	 * @param x a long integer smaller than or equal to 262.
	 * @return the least power of two greater than or equal to the specified value.
	 */
	public static long nextPowerOfTwo(long x) {
		if (x == 0) return 1;
		x--;
		x |= x >> 1;
		x |= x >> 2;
		x |= x >> 4;
		x |= x >> 8;
		x |= x >> 16;
		return (x | x >> 32) + 1;
	}

	/** Returns the maximum number of entries that can be filled before rehashing.
	 *
	 * @param n the size of the backing array.
	 * @param f the load factor.
	 * @return the maximum number of entries before rehashing.
	 */
	public static int maxFill(final int n, final float f) {
		/* We must guarantee that there is always at least
		 * one free entry (even with pathological load factors). */
		return Math.min((int)Math.ceil(n * f), n - 1);
	}

	/** Returns the maximum number of entries that can be filled before rehashing.
	 *
	 * @param n the size of the backing array.
	 * @param f the load factor.
	 * @return the maximum number of entries before rehashing.
	 */
	public static long maxFill(final long n, final float f) {
		/* We must guarantee that there is always at least
		 * one free entry (even with pathological load factors). */
		return Math.min((long)Math.ceil(n * f), n - 1);
	}

	/** Returns the least power of two smaller than or equal to 230 and larger than or equal to {@code Math.ceil(expected / f)}.
	 *
	 * @param expected the expected number of elements in a hash table.
	 * @param f the load factor.
	 * @return the minimum possible size for a backing array.
	 * @throws IllegalArgumentException if the necessary size is larger than 230.
	 */
	public static int arraySize(final int expected, final float f) {
		final long s = Math.max(2, nextPowerOfTwo((long)Math.ceil(expected / f)));
		if (s > (1 << 30)) throw new IllegalArgumentException("Too large (" + expected + " expected elements with load factor " + f + ")");
		return (int)s;
	}

	/** Returns the least power of two larger than or equal to {@code Math.ceil(expected / f)}.
	 *
	 * @param expected the expected number of elements in a hash table.
	 * @param f the load factor.
	 * @return the minimum possible size for a backing big array.
	 */
	public static long bigArraySize(final long expected, final float f) {
		return nextPowerOfTwo((long)Math.ceil(expected / f));
	}
}


package it.unimi.dsi.fastutil.io;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.channels.WritableByteChannel;

/*
 * Copyright (C) 2005-2017 Sebastiano Vigna
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import it.unimi.dsi.fastutil.bytes.ByteArrays;

/** A {@linkplain RepositionableStream repositionable} {@link MeasurableInputStream} based on
 * cached data received by a {@link WritableByteChannel} whose first bytes can be inspected directly.
 *
 *
An instance of this class acts as a buffer holding the bytes written through its
 * {@link WritableByteChannel} interface (which can be easily turned into an {@link OutputStream} using
 * {@link Channels#newOutputStream(WritableByteChannel)}). The data can be discarded at any time using
 * {@link #clear()}. The first {@link #inspectable} bytes of {@link #buffer} contains the first
 * bytes written. When {@link #buffer} is full, the bytes are written to an overflow
 * file.
 *
 *
At any time, the stream of bytes written since creation (or since the last {@link #clear()})
 * are available as a fully implemented {@link MeasurableInputStream} which also implements
 * {@link RepositionableStream} and {@linkplain #mark(int) supports marking}.
 * Note that you must arbitrate carefully write and read accesses,
 * as it is always possible to call {@link #write(ByteBuffer)}
 * and thus modify the {@linkplain #length() length} of the {@link MeasurableInputStream}.
 *
 *
The method {@link #close()} makes the {@link MeasurableInputStream} and {@link WritableByteChannel} state-changing methods temporarily throw an {@link IOException}, but
 * does not otherwise modify the state of the stream. You can {@linkplain #reopen() reopen} the stream
 * later, or {@linkplain #clear() clear} it.
 * The method {@link #dispose()} can be used to release
 * the resources associated with the stream.
 *
 *
Buffering

 *
 *
This class provides no form of buffering except for the memory buffer described above, both
 * when reading and when writing. Users should consider wrapping instances of this class with a
 * {@link FastBufferedInputStream}, as reads after the buffer has been exhausted will be performed
 * directly on a {@link RandomAccessFile}. */
public class InspectableFileCachedInputStream extends MeasurableInputStream implements RepositionableStream, WritableByteChannel {
	public static final boolean DEBUG = false;

	/** The default buffer size (64KiB). */
	public static final int DEFAULT_BUFFER_SIZE = 64 * 1024;

	/** The inspection buffer. The first {@link #inspectable} bytes contain the first part of the input stream.
	 * The buffer is available for inspection, but users should not modify its content. */
	public final byte[] buffer;

	/** The number of valid bytes currently in {@link #buffer}. */
	public int inspectable;

	/** The overflow file used by this stream: it is created at construction time, and deleted on {@link #close()}. */
	private final File overflowFile;

	/** The random access file used to access the overflow file. */
	private final RandomAccessFile randomAccessFile;

	/** {@link #randomAccessFile randomAccessFile#getChannel()}, cached. */
	private final FileChannel fileChannel;

	/** The position on this stream (i.e., the index of the next byte to be returned). */
	private long position;

	/** The {@linkplain #mark(int) mark}, if set, or -1. */
	private long mark;

	/** The write position of the {@link #randomAccessFile overflow file}. When {@link #inspectable} is equal
	 * to {@link #buffer buffer.length}, the length of the stream is {@link #inspectable} + {@link #writePosition}.  */
	private long writePosition;

	/** Creates a new instance with specified buffer size and overlow-file directory.
	 *
	 * @param bufferSize the buffer size, in bytes.
	 * @param overflowFile the directory where the overflow file should be created, or {@code null} for the default temporary directory.
	 */
	public InspectableFileCachedInputStream(final int bufferSize, final File overflowFile) throws IOException {
		if (bufferSize <= 0) throw new IllegalArgumentException("Illegal buffer size " + bufferSize);
		if (overflowFile != null) this.overflowFile = overflowFile;
		else (this.overflowFile = File.createTempFile(getClass().getSimpleName(), "overflow")).deleteOnExit();
		buffer = new byte[bufferSize];
		randomAccessFile = new RandomAccessFile(this.overflowFile, "rw");
		fileChannel = randomAccessFile.getChannel();
		mark = -1;
	}

	/** Creates a new instance with specified buffer size and default overflow-file directory.
	 *
	 * @param bufferSize the buffer size, in bytes.
	 */
	public InspectableFileCachedInputStream(final int bufferSize) throws IOException {
		this(bufferSize, null);
	}

	/** Creates a new instance with default buffer size and overflow-file directory. */
	public InspectableFileCachedInputStream() throws IOException {
		this(DEFAULT_BUFFER_SIZE);
	}

	private void ensureOpen() throws IOException {
		if (position == -1) throw new IOException("This " + getClass().getSimpleName() + " is closed");
	}

	/** Clears the content of this {@link InspectableFileCachedInputStream}, zeroing the length of the represented
	 * stream. */
	public void clear() throws IOException {
		if (! fileChannel.isOpen()) throw new IOException("This " + getClass().getSimpleName() + " is closed");
		writePosition = position = inspectable = 0;
		mark = -1;
	}

	/** Appends the content of a specified buffer to the end of the currently represented stream.
	 *
	 * @param byteBuffer a byte buffer.
	 * @return the number of bytes appended (i.e., {@link ByteBuffer#remaining() byteBuffer.remaining()}).
	 */
	@Override
	public int write(final ByteBuffer byteBuffer) throws IOException {
		ensureOpen();
		final int remaining = byteBuffer.remaining();

		if (inspectable < buffer.length) {
			// Still some space in the inspectable buffer.
			final int toBuffer = Math.min(buffer.length - inspectable, remaining);
			byteBuffer.get(buffer, inspectable, toBuffer);
			inspectable += toBuffer;
		}

		if (byteBuffer.hasRemaining()) {
			fileChannel.position(writePosition);
			writePosition += fileChannel.write(byteBuffer);
		}

		return remaining;
	}

	/** Truncates the overflow file to a given size if possible.
	 *
	 * @param size the new size; the final size is the maximum between the current write position (i.e., the length
	 * of the represented stream minus the length of the inspection buffer) and this value.
	 */
	public void truncate(final long size) throws FileNotFoundException, IOException {
		fileChannel.truncate(Math.max(size, writePosition));
	}

	/** Makes the stream unreadable until the next {@link #clear()}.
	 *
	 * @see #reopen()
	 */
	@Override
	public void close() {
		position = -1;
	}

	/** Makes the stream readable again after a {@link #close()}.
	 *
	 * @see #close()
	 */
	public void reopen() throws IOException {
		if (! fileChannel.isOpen()) throw new IOException("This " + getClass().getSimpleName() + " is closed");
		position = 0;
	}

	/** Disposes this stream, deleting the overflow file. After that, the stream is unusable. */
	public void dispose() throws IOException {
		position = -1;
		randomAccessFile.close();
		overflowFile.delete();
	}

	@SuppressWarnings("deprecation")
	@Override
	protected void finalize() throws Throwable {
		try {
			dispose();
		}
		finally {
			super.finalize();
		}
	}

	@Override
	public int available() throws IOException {
		ensureOpen();
		return (int)Math.min(Integer.MAX_VALUE, length() - position);
	}

	@Override
	public int read(final byte[] b, int offset, int length) throws IOException {
		ensureOpen();
		if (length == 0) return 0;
		if (position == length()) return -1; // Nothing to read.
		ByteArrays.ensureOffsetLength(b, offset, length);
		int read = 0;

		if (position < inspectable) {
			/* The first min(inspectable - readPosition, length) bytes should be taken from the buffer. */
			final int toCopy = Math.min(inspectable - (int)position, length);
			System.arraycopy(buffer, (int)position, b, offset, toCopy);
			length -= toCopy;
			offset += toCopy;
			position += toCopy;
			read = toCopy;
		}

		if (length > 0) { // We want to read more.
			if (position == length()) return read != 0 ? read : -1; // There's nothing more to read.
			fileChannel.position(position - inspectable);
			final int toRead = (int)Math.min(length() - position, length);
			// This is *intentionally* not a readFully(). Let the language to its stuff.
			final int t = randomAccessFile.read(b, offset, toRead);
			position += t;
			read += t;
		}

		return read;
	}

	@Override
	public int read(final byte[] b) throws IOException {
		return read(b, 0, b.length);
	}

	@Override
	public long skip(final long n) throws IOException {
		ensureOpen();
		final long toSkip = Math.min(n, length() - position);
		position += toSkip;
		return toSkip;
	}

	@Override
	public int read() throws IOException {
		ensureOpen();
		if (position == length()) return -1; // Nothing to read
		if (position < inspectable) return buffer[(int)position++] & 0xFF;
		fileChannel.position(position - inspectable);
		position++;
		return randomAccessFile.read();
	}

	@Override
	public long length() throws IOException {
		ensureOpen();
		return inspectable + writePosition;
	}

	@Override
	public long position() throws IOException {
		ensureOpen();
		return position;
	}

	/** Positions the input stream.
	 *
	 * @param position the new position (will be minimized with {@link #length()}).
	 */
	@Override
	public void position(final long position) throws IOException {
		this.position = Math.min(position, length());
	}

	@Override
	public boolean isOpen() {
		return position != -1;
	}

	@Override
	public void mark(final int readlimit) {
		mark = position;
	}

	@Override
	public void reset() throws IOException {
		ensureOpen();
		if (mark == -1) throw new IOException("Mark has not been set");
		position(mark);
	}

	@Override
	public boolean markSupported() {
		return true;
	}
}


package it.unimi.dsi.fastutil.io;

/*
 * Copyright (C) 2005-2017 Sebastiano Vigna
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/** Simple, fast and repositionable byte-array input stream.
 *
 *
Warning: this class implements the correct semantics
 * of {@link #read(byte[], int, int)} as described in {@link java.io.InputStream}.
 * The implementation given in {@link java.io.ByteArrayInputStream} is broken,
 * but it will never be fixed because it's too late.
 *
 * @author Sebastiano Vigna
 */

public class FastByteArrayInputStream extends MeasurableInputStream implements RepositionableStream {

	/** The array backing the input stream. */
	public byte[] array;

	/** The first valid entry. */
	public int offset;

	/** The number of valid bytes in {@link #array} starting from {@link #offset}. */
	public int length;

	/** The current position as a distance from {@link #offset}. */
	private int position;

	/** The current mark as a position, or -1 if no mark exists. */
	private int mark;

	/** Creates a new array input stream using a given array fragment.
	 *
	 * @param array the backing array.
	 * @param offset the first valid entry of the array.
	 * @param length the number of valid bytes.
	 */
	public FastByteArrayInputStream(final byte[] array, final int offset, final int length) {
		this.array = array;
		this.offset = offset;
		this.length = length;
	}

	/** Creates a new array input stream using a given array.
	 *
	 * @param array the backing array.
	 */
	public FastByteArrayInputStream(final byte[] array) {
		this(array, 0, array.length);
	}

	@Override
	public boolean markSupported() {
		return true;
	}

	@Override
	public void reset() {
		position = mark;
	}

	/** Closing a fast byte array input stream has no effect. */
	@Override
	public void close() {}

	@Override
	public void mark(final int dummy) {
		mark = position;
	}

	@Override
	public int available() {
		return length - position;
	}

	@Override
	public long skip(long n) {
		if (n <= length - position) {
			position += (int)n;
			return n;
		}
		n = length - position;
		position = length;
		return n;
	}

	@Override
	public int read() {
		if (length == position) return -1;
		return array[offset + position++] & 0xFF;
	}

	/** Reads bytes from this byte-array input stream as
	 * specified in {@link java.io.InputStream#read(byte[], int, int)}.
	 * Note that the implementation given in {@link java.io.ByteArrayInputStream#read(byte[], int, int)}
	 * will return -1 on a zero-length read at EOF, contrarily to the specification. We won't.
	 */

	@Override
	public int read(final byte b[], final int offset, final int length) {
		if (this.length == this.position) return length == 0 ? 0 : -1;
		final int n = Math.min(length, this.length - this.position);
		System.arraycopy(array, this.offset + this.position, b, offset, n);
		this.position += n;
		return n;
	}

	@Override
	public long position() {
		return position;
	}

	@Override
	public void position(final long newPosition) {
		position = (int)Math.min(newPosition, length);
	}

	@Override
	public long length() {
		return length;
	}
}


package it.unimi.dsi.fastutil.io;

/*
 * Copyright (C) 2005-2017 Sebastiano Vigna
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.IOException;
import java.io.OutputStream;
import java.nio.channels.FileChannel;

/** Lightweight, unsynchronized output stream buffering class with
 *  {@linkplain MeasurableStream measurability} and
 *  {@linkplain RepositionableStream repositionability}.
 *
 *
This class provides buffering for output streams, but it does so with
 * purposes and an internal logic that are radically different from the ones
 * adopted in {@link java.io.BufferedOutputStream}. The main features follow.
 *
 *

 *

All methods are unsychronized.
 *
 *

As an additional feature, this class implements the {@link
 * RepositionableStream} and {@link MeasurableStream} interfaces.
 * An instance of this class will try to cast
 * the underlying byte stream to a {@link RepositionableStream} and to fetch by
 * reflection the {@link java.nio.channels.FileChannel} underlying the given
 * output stream, in this order. If either reference can be successfully
 * fetched, you can use {@link #position(long)} to reposition the stream.
 * Much in the same way, an instance of this class will try to cast the
 * the underlying byte stream to a {@link MeasurableStream}, and if this
 * operation is successful, or if a {@link java.nio.channels.FileChannel} can
 * be detected, then {@link #position()} and {@link #length()} will work as expected.
 *

 * @since 4.4
 */

public class FastBufferedOutputStream extends MeasurableOutputStream implements RepositionableStream {
	private static final boolean ASSERTS = false;

	/** The default size of the internal buffer in bytes (8Ki). */
	public static final int DEFAULT_BUFFER_SIZE = 8 * 1024;

	/** The internal buffer. */
	protected byte buffer[];

	/** The current position in the buffer. */
	protected int pos;

	/** The number of buffer bytes available starting from {@link #pos}
	 * (it must be always equal to {@code buffer.length - pos}). */
	protected int avail;

	/** The underlying output stream. */
	protected OutputStream os;

	/** The cached file channel underlying {@link #os}, if any. */
	private FileChannel fileChannel;

	/** {@link #os} cast to a positionable stream, if possible. */
	private RepositionableStream repositionableStream;

	/** {@link #os} cast to a measurable stream, if possible. */
	private MeasurableStream measurableStream;

	private static int ensureBufferSize(final int bufferSize) {
		if (bufferSize <= 0) throw new IllegalArgumentException("Illegal buffer size: " + bufferSize);
		return bufferSize;
	}

	/** Creates a new fast buffered output stream by wrapping a given output stream with a given buffer.
	 *
	 * @param os an output stream to wrap.
	 * @param buffer a buffer of positive length.
	 */
	public FastBufferedOutputStream(final OutputStream os, final byte[] buffer) {
		this.os = os;
		ensureBufferSize(buffer.length);
		this.buffer = buffer;
		avail = buffer.length;

		if (os instanceof RepositionableStream) repositionableStream = (RepositionableStream)os;
		if (os instanceof MeasurableStream) measurableStream = (MeasurableStream)os;

		if (repositionableStream == null) {

			try {
				fileChannel = (FileChannel)(os.getClass().getMethod("getChannel", new Class<?>[] {})).invoke(os);
			}
			catch(IllegalAccessException e) {}
			catch(IllegalArgumentException e) {}
			catch(NoSuchMethodException e) {}
			catch(java.lang.reflect.InvocationTargetException e) {}
			catch(ClassCastException e) {}
		}

	}

	/** Creates a new fast buffered output stream by wrapping a given output stream with a given buffer size.
	 *
	 * @param os an output stream to wrap.
	 * @param bufferSize the size in bytes of the internal buffer.
	 */
	public FastBufferedOutputStream(final OutputStream os, final int bufferSize) {
		this(os, new byte[ensureBufferSize(bufferSize)]);
	}

	/** Creates a new fast buffered ouptut stream by wrapping a given output stream with a buffer of {@link #DEFAULT_BUFFER_SIZE} bytes.
	 *
	 * @param os an output stream to wrap.
	 */
	public FastBufferedOutputStream(final OutputStream os) {
		this(os, DEFAULT_BUFFER_SIZE);
	}

	private void dumpBuffer(final boolean ifFull) throws IOException {
		if (! ifFull || avail == 0) {
			os.write(buffer, 0, pos);
			pos = 0;
			avail = buffer.length;
		}
	}

	@Override
	public void write(final int b) throws IOException {
		if (ASSERTS) assert avail > 0;
		avail--;
		buffer[pos++] = (byte)b;
		dumpBuffer(true);
	}

	@Override
	public void write(final byte b[], final int offset, final int length) throws IOException {
		if (length >= buffer.length) {
			dumpBuffer(false);
			os.write(b, offset, length);
			return;
		}

		if (length <= avail) {
			// Copy in buffer
			System.arraycopy(b, offset, buffer, pos, length);
			pos += length;
			avail -= length;
			dumpBuffer(true);
			return;
		}

		dumpBuffer(false);
		System.arraycopy(b, offset, buffer, 0, length);
		pos = length;
		avail -= length;
	}

	@Override
	public void flush() throws IOException {
		dumpBuffer(false);
		os.flush();
	}

	@Override
	public void close() throws IOException {
		if (os == null) return;
		flush();
		if (os != System.out) os.close();
		os = null;
		buffer = null;
	}

	@Override
	public long position() throws IOException {
		if (repositionableStream != null) return repositionableStream.position() + pos;
		else if (measurableStream != null) return measurableStream.position() + pos;
		else if (fileChannel != null) return fileChannel.position() + pos;
		else throw new UnsupportedOperationException("position() can only be called if the underlying byte stream implements the MeasurableStream or RepositionableStream interface or if the getChannel() method of the underlying byte stream exists and returns a FileChannel");
	}

	/** Repositions the stream.
	 *
	 *
Note that this method performs a {@link #flush()} before changing the underlying stream position.
	 */

	@Override
	public void position(final long newPosition) throws IOException {
		flush();
		if (repositionableStream != null) repositionableStream.position(newPosition);
		else if (fileChannel != null) fileChannel.position(newPosition);
		else throw new UnsupportedOperationException("position() can only be called if the underlying byte stream implements the RepositionableStream interface or if the getChannel() method of the underlying byte stream exists and returns a FileChannel");
	}

	/** Returns the length of the underlying output stream, if it is {@linkplain MeasurableStream measurable}.
	 *
	 *
Note that this method performs a {@link #flush()} before detecting the length.
	 *
	 * @return the length of the underlying output stream.
	 * @throws UnsupportedOperationException if the underlying output stream is not {@linkplain MeasurableStream measurable} and
	 * cannot provide a {@link FileChannel}.
	 */

	@Override
	public long length() throws IOException {
		flush();
		if (measurableStream != null) return measurableStream.length();
		if (fileChannel != null) return fileChannel.size();
		throw new UnsupportedOperationException();
	}
}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.api.consumer.metrics;

import static com.netflix.hollow.core.HollowConstants.VERSION_NONE;

import com.netflix.hollow.api.consumer.HollowConsumer;
import com.netflix.hollow.api.consumer.HollowConsumer.AbstractRefreshListener;
import com.netflix.hollow.api.consumer.HollowConsumer.Blob.BlobType;
import com.netflix.hollow.api.custom.HollowAPI;
import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
import java.util.List;
import java.util.OptionalLong;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 * A class for computing Hollow Consumer refresh metrics, requires plugging in metrics reporting implementation.
 *

 * This class computes Hollow Consumer refresh metrics by listening to refresh events. At the end of every refresh, whether
 * the refresh succeeded or failed, the refresh metrics in {@code ConsumerRefreshMetrics} and refresh details in {@code UpdatePlanDetails}
 * are reporte using the {@code RefreshMetricsReporting} interface. This interface makes it mandatory for concrete subclasses
 * to implement custom metrics reporting behavior.
 */
public abstract class AbstractRefreshMetricsListener extends AbstractRefreshListener implements RefreshMetricsReporting {

    private static final Logger log = Logger.getLogger(AbstractRefreshMetricsListener.class.getName());

    private OptionalLong lastRefreshTimeNanoOptional;
    private long refreshStartTimeNano;
    private long consecutiveFailures;
    private BlobType overallRefreshType;    // Indicates whether the overall refresh (that could comprise of multiple transitions)
                                            // is classified as snapshot, delta, or reverse delta. Note that if a snapshot
                                            // transition is present then the overall refresh type is snapshot.
    private ConsumerRefreshMetrics.UpdatePlanDetails updatePlanDetails;  // Some details about the transitions comprising a refresh
    // visible for testing
    ConsumerRefreshMetrics.Builder refreshMetricsBuilder;

    public AbstractRefreshMetricsListener() {
        lastRefreshTimeNanoOptional = OptionalLong.empty();
        consecutiveFailures = 0l;
    }

    @Override
    public final void refreshStarted(long currentVersion, long requestedVersion) {
        updatePlanDetails = new ConsumerRefreshMetrics.UpdatePlanDetails();
        refreshStartTimeNano = System.nanoTime();
        refreshMetricsBuilder = new ConsumerRefreshMetrics.Builder();
        refreshMetricsBuilder.setIsInitialLoad(currentVersion == VERSION_NONE);
        refreshMetricsBuilder.setUpdatePlanDetails(updatePlanDetails);
    }

    /**
     * This method acquires details of individual transitions that comprise a larger refresh.
     *

     * Details of transitions in a refresh such as count and type can be useful to understand consumer performance and
     * to troubleshoot issues relating to refresh failure.
     *

     * @param beforeVersion The version when refresh started
     * @param desiredVersion The version that the consumer refresh tries update to, even though it might not be attainable eg. HollowConstants.VERSION_LATEST
     * @param isSnapshotPlan Indicates whether the refresh involves a snapshot transition
     * @param transitionSequence List of transitions comprising the refresh
     */
    @Override
    public final void transitionsPlanned(long beforeVersion, long desiredVersion, boolean isSnapshotPlan, List<HollowConsumer.Blob.BlobType> transitionSequence) {
        updatePlanDetails.beforeVersion = beforeVersion;
        updatePlanDetails.desiredVersion = desiredVersion;
        updatePlanDetails.transitionSequence = transitionSequence;
        if (isSnapshotPlan) {
            overallRefreshType = BlobType.SNAPSHOT;
        } else {
            overallRefreshType = desiredVersion > beforeVersion ? BlobType.DELTA : BlobType.REVERSE_DELTA;
        }
        refreshMetricsBuilder.setOverallRefreshType(overallRefreshType);
    }

    @Override
    public final void blobLoaded(HollowConsumer.Blob transition) {
        updatePlanDetails.numSuccessfulTransitions ++;
    }

    /**
     * Metrics reporting implementation is provided by the extending subclass. If exceptions are not gracefully handled
     * in the extending subclass then an exception there can fail the consumer refresh, even though metrics reporting
     * might not be mission critical. This method protects against that scenario by catching all exceptions, logging
     * that there was an exception, and continuing with the consumer refresh.
     * @param refreshMetrics Consumer refresh metrics being reported
     */
    private final void noFailRefreshEndMetricsReporting(ConsumerRefreshMetrics refreshMetrics) {
        try {
            refreshEndMetricsReporting(refreshMetrics);
        } catch (Exception e) {
            // Metric reporting is not considered critical to consumer refresh. Log exceptions and continue.
            log.log(Level.SEVERE, "Encountered an exception in reporting consumer refresh metrics, ignoring exception and continuing with consumer refresh", e);
        }
    }

    @Override
    public final void refreshSuccessful(long beforeVersion, long afterVersion, long requestedVersion) {
        long refreshEndTimeNano = System.nanoTime();

        long durationMillis = TimeUnit.NANOSECONDS.toMillis(refreshEndTimeNano - refreshStartTimeNano);
        consecutiveFailures = 0l;
        lastRefreshTimeNanoOptional = OptionalLong.of(refreshEndTimeNano);

        refreshMetricsBuilder.setDurationMillis(durationMillis)
                .setIsRefreshSuccess(true)
                .setConsecutiveFailures(consecutiveFailures)
                .setRefreshSuccessAgeMillisOptional(0l)
                .setRefreshEndTimeNano(refreshEndTimeNano);

        noFailRefreshEndMetricsReporting(refreshMetricsBuilder.build());
    }

    @Override
    public final void refreshFailed(long beforeVersion, long afterVersion, long requestedVersion, Throwable failureCause) {
        long  refreshEndTimeNano = System.nanoTime();
        long durationMillis = TimeUnit.NANOSECONDS.toMillis(refreshEndTimeNano - refreshStartTimeNano);
        consecutiveFailures ++;

        refreshMetricsBuilder.setDurationMillis(durationMillis)
                .setIsRefreshSuccess(false)
                .setConsecutiveFailures(consecutiveFailures)
                .setRefreshEndTimeNano(refreshEndTimeNano);
        if (lastRefreshTimeNanoOptional.isPresent()) {
            refreshMetricsBuilder.setRefreshSuccessAgeMillisOptional(TimeUnit.NANOSECONDS.toMillis(refreshEndTimeNano - lastRefreshTimeNanoOptional.getAsLong()));
        }

        noFailRefreshEndMetricsReporting(refreshMetricsBuilder.build());
    }

    @Override
    public final void snapshotUpdateOccurred(HollowAPI refreshAPI, HollowReadStateEngine stateEngine, long version) {
        // no-op
    }

    @Override
    public final void deltaUpdateOccurred(HollowAPI refreshAPI, HollowReadStateEngine stateEngine, long version) {
        // no-op
    }

    @Override
    public final void snapshotApplied(HollowAPI api, HollowReadStateEngine stateEngine, long version) throws Exception {
        // no-op
    }

    @Override
    public final void deltaApplied(HollowAPI api, HollowReadStateEngine stateEngine, long version) throws Exception {
        // no-op
    }

}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.util;

import static com.netflix.hollow.core.util.Threads.daemonThread;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.RunnableFuture;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

/**
 *
 * A convenience wrapper around ThreadPoolExecutor. Provides sane defaults to
 * constructor arguments and allows for awaitUninterruptibly().
 *
 */
public class SimultaneousExecutor extends ThreadPoolExecutor {

    private static final String DEFAULT_THREAD_NAME = "simultaneous-executor";

    private final List<Future<?>> futures = new ArrayList<Future<?>>();

    /**
     * Creates an executor with a thread per processor.
     *

     * Equivalent to constructing a {@code SimultaneousExecutor} with {@code 1.0d}
     * threads per CPU.
     */
    public SimultaneousExecutor(Class<?> context, String description) {
        this(1.0d, context, description);
    }

    /**
     * Creates an executor with a thread per processor.
     *

     * Equivalent to calling {@code SimultaneousExecutor(1.0d)}
     *
     * @deprecated use {@link #SimultaneousExecutor(Class, String)}
     */
    @Deprecated
    public SimultaneousExecutor() {
        this(1.0d, SimultaneousExecutor.class, DEFAULT_THREAD_NAME);
    }

    /**
     * Creates an executor with number of threads calculated from the
     * specified factor.
     *
     * @param threadsPerCpu calculated as {@code processors * threadsPerCpu} then used as {@code corePoolSize} and {@code maximumPoolSize}
     * @param context used to name created threads
     */
    public SimultaneousExecutor(double threadsPerCpu, Class<?> context) {
        this(threadsPerCpu, context, DEFAULT_THREAD_NAME);
    }

    /**
     * Creates an executor with number of threads calculated from the
     * specified factor.
     *
     * @param threadsPerCpu calculated as {@code processors * threadsPerCpu} then used as {@code corePoolSize} and {@code maximumPoolSize}
     *
     * @deprecated use {@link #SimultaneousExecutor(double, Class)}
     */
    @Deprecated
    public SimultaneousExecutor(double threadsPerCpu) {
        this(threadsPerCpu, SimultaneousExecutor.class, DEFAULT_THREAD_NAME);
    }

    /**
     * Creates an executor with number of threads calculated from the
     * specified factor and threads named according to {@code context} and {@code description}.
     *
     * @param threadsPerCpu calculated as {@code processors * threadsPerCpu} then used as {@code corePoolSize} and {@code maximumPoolSize}
     * @param context combined with {@code description} to name created threads
     * @param description brief description used to name created threads; combined with {@code context}
     */
    public SimultaneousExecutor(double threadsPerCpu, Class<?> context, String description) {
        this((int) ((double) Runtime.getRuntime().availableProcessors() * threadsPerCpu), context, description);
    }

    /**
     * Creates an executor with number of threads calculated from the
     * specified factor and threads named according to {@code description}.
     *
     * @param threadsPerCpu calculated as {@code processors * threadsPerCpu} then used as {@code corePoolSize} and {@code maximumPoolSize}
     * @param description brief description used to name created threads
     *
     * @deprecated use {@link #SimultaneousExecutor(double, Class, String)}
     */
    @Deprecated
    public SimultaneousExecutor(double threadsPerCpu, String description) {
        this((int) ((double) Runtime.getRuntime().availableProcessors() * threadsPerCpu), SimultaneousExecutor.class, description);
    }

    /**
     * Creates an executor with the specified number of threads and threads named
     * according to {@code context}.
     *
     * @param numThreads used as {@code corePoolSize} and {@code maximumPoolSize}
     * @param context used to name created threads
     */
    public SimultaneousExecutor(int numThreads, Class<?> context) {
        this(numThreads, context, DEFAULT_THREAD_NAME);
    }

    /**
     * Creates an executor with the specified number of threads.
     *
     * @param numThreads used as {@code corePoolSize} and {@code maximumPoolSize}
     *
     * @deprecated use {@link #SimultaneousExecutor(int, Class)}
     */
    @Deprecated
    public SimultaneousExecutor(int numThreads) {
        this(numThreads, SimultaneousExecutor.class, DEFAULT_THREAD_NAME);
    }

    /**
     * Creates an executor with the specified number of threads and threads named
     * according to {@code context} and {@code description}.
     *
     * @param numThreads used as {@code corePoolSize} and {@code maximumPoolSize}
     * @param context combined with {@code description} to name created threads
     * @param description brief description used to name created threads; combined with {@code context}
     */
    public SimultaneousExecutor(int numThreads, Class<?> context, String description) {
        super(numThreads, numThreads, 100, TimeUnit.SECONDS, new LinkedBlockingQueue<>(),
                r -> daemonThread(r, context, description));
    }

    /**
     * Creates an executor with the specified number of threads and threads named
     * according to {@code description}.
     *
     * @param numThreads used as {@code corePoolSize} and {@code maximumPoolSize}
     * @param description brief description used to name created threads
     *
     * @deprecated use {@link #SimultaneousExecutor(int, Class, String)}
     */
    @Deprecated
    public SimultaneousExecutor(int numThreads, final String description) {
        this(numThreads, SimultaneousExecutor.class, description);
    }

    @Override
    public void execute(Runnable command) {
        if(command instanceof RunnableFuture) {
            super.execute(command);
        } else {
            super.execute(newTaskFor(command, Boolean.TRUE));
        }
    }

    /**
     * Awaits completion of all submitted tasks.
     *
     * After this call completes, the thread pool will be shut down.
     */
    public void awaitUninterruptibly() {
        shutdown();
        while (!isTerminated()) {
            try {
                awaitTermination(1, TimeUnit.DAYS);
            } catch (final InterruptedException e) { }
        }
    }

    @Override
    protected final <T> RunnableFuture<T> newTaskFor(final Runnable runnable, final T value) {
        final RunnableFuture<T> task = super.newTaskFor(runnable, value);
        futures.add(task);
        return task;
    }

    @Override
    protected final <T> RunnableFuture<T> newTaskFor(final Callable<T> callable) {
        final RunnableFuture<T> task = super.newTaskFor(callable);
        futures.add(task);
        return task;
    }

    /**
     * Await successful completion of all submitted tasks. Throw exception of the first failed task
     * if 1 or more tasks failed.
     *
     * After this call completes, the thread pool will be shut down.
     *
     * @throws ExecutionException if a computation threw an
     * exception
     * @throws InterruptedException if the current thread was interrupted
     * while waiting
     */
    public void awaitSuccessfulCompletion() throws InterruptedException, ExecutionException {
        awaitUninterruptibly();
        for (final Future<?> f : futures) {
            f.get();
        }
    }

    /**
     * Await successful completion of all previously submitted tasks.  Throw exception of the first failed task
     * if 1 or more tasks failed.
     *
     * After this call completes, the thread pool will not be shut down and can be reused.
     *
     * @throws ExecutionException if a computation threw an
     * exception
     * @throws InterruptedException if the current thread was interrupted
     * while waiting
     */
    public void awaitSuccessfulCompletionOfCurrentTasks() throws InterruptedException, ExecutionException {
        for(Future<?> f : futures) {
            f.get();
        }

        futures.clear();
    }

}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.api.consumer.index;

import com.netflix.hollow.api.consumer.HollowConsumer;
import com.netflix.hollow.api.objects.HollowRecord;
import java.util.Objects;

/**
 * A type safe hash index for indexing non-primary-key data.
 *

 * This type of index can map multiple keys to a single matching record,
 * and/or multiple records to a single key.
 *

 * If the index is {@link HollowConsumer#addRefreshListener(HollowConsumer.RefreshListener) registered} with its
 * associated {@link HollowConsumer} then the index will track updates and changes will be reflected in matched results
 * (performed after such updates).  When a registered index is no longer needed it should be
 * {@link HollowConsumer#removeRefreshListener(HollowConsumer.RefreshListener) deregistered} to avoid unnecessary
 * index recalculation and to ensure the index is reclaimed by the garbage collector.
 *
 * @param <T> the root, select, and result type
 * @param <Q> the query type
 */
public final class HashIndex<T extends HollowRecord, Q> extends HashIndexSelect<T, T, Q> {

    HashIndex(
            HollowConsumer consumer,
            Class<T> rootType,
            Class<Q> matchedFieldsType) {
        super(consumer,
                rootType,
                rootType, "",
                matchedFieldsType);
    }

    HashIndex(
            HollowConsumer consumer,
            Class<T> rootType,
            String fieldPath, Class<Q> matchedFieldType) {
        super(consumer,
                rootType,
                rootType, "",
                fieldPath, matchedFieldType);
    }

    /**
     * Starts the building of a {@link HashIndex}.
     *
     * @param consumer the consumer containing instances of the given root type
     * @param rootType the root type to match and select from
     * @param <T> the root type
     * @return a builder
     */
    public static <T extends HollowRecord> Builder<T> from(HollowConsumer consumer, Class<T> rootType) {
        Objects.requireNonNull(consumer);
        Objects.requireNonNull(rootType);
        return new Builder<>(consumer, rootType);
    }

    /**
     * The builder of a {@link HashIndex} or a {@link HashIndexSelect}.
     *
     * @param <T> the root type
     */
    public static final class Builder<T extends HollowRecord> {
        final HollowConsumer consumer;
        final Class<T> rootType;

        Builder(HollowConsumer consumer, Class<T> rootType) {
            this.consumer = consumer;
            this.rootType = rootType;
        }

        /**
         * Creates a {@link HashIndex} for matching with field paths and types declared by
         * {@link FieldPath} annotated fields or methods on the given query type.
         *
         * @param queryType the query type
         * @param <Q> the query type
         * @return a {@code HashIndex}
         * @throws IllegalArgumentException if the query type declares one or more invalid field paths
         * or invalid types given resolution of corresponding field paths
         */
        public <Q> HashIndex<T, Q> usingBean(Class<Q> queryType) {
            Objects.requireNonNull(queryType);
            return new HashIndex<>(consumer, rootType, queryType);
        }

        /**
         * Creates a {@link HashIndex} for matching with a single query field path and type.
         *
         * @param queryFieldPath the query field path
         * @param queryFieldType the query type
         * @param <Q> the query type
         * @return a {@code HashIndex}
         * @throws IllegalArgumentException if the query field path is empty or invalid
         * @throws IllegalArgumentException if the query field type is invalid given resolution of the
         * query field path
         */
        public <Q> HashIndex<T, Q> usingPath(String queryFieldPath, Class<Q> queryFieldType) {
            Objects.requireNonNull(queryFieldPath);
            if (queryFieldPath.isEmpty()) {
                throw new IllegalArgumentException("queryFieldPath argument is an empty String");
            }
            Objects.requireNonNull(queryFieldType);
            return new HashIndex<>(consumer, rootType, queryFieldPath, queryFieldType);
        }

        /**
         * Transitions to build a hash index with result selection.
         *
         * @param selectFieldPath the select field path
         * @param selectFieldType the select, and result, field type associated with the
         * resolved select field path
         * @param <S> the select type
         * @return a builder of a {@link HashIndexSelect}
         */
        public <S extends HollowRecord> BuilderWithSelect<T, S> selectField(
                String selectFieldPath, Class<S> selectFieldType) {
            Objects.requireNonNull(selectFieldPath);
            Objects.requireNonNull(selectFieldType);
            return new BuilderWithSelect<>(consumer, rootType, selectFieldPath, selectFieldType);
        }
    }
}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.api.consumer.index;

import static java.util.stream.Collectors.joining;

import com.netflix.hollow.api.consumer.HollowConsumer;
import com.netflix.hollow.api.custom.HollowAPI;
import com.netflix.hollow.api.objects.HollowRecord;
import com.netflix.hollow.core.index.FieldPaths;
import com.netflix.hollow.core.index.HollowHashIndex;
import com.netflix.hollow.core.index.HollowHashIndexResult;
import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.function.Function;
import java.util.stream.Stream;

/**
 * A type safe hash index, with result selection, for indexing non-primary-key data.
 *

 * This type of index can map multiple keys to a single matching record,
 * and/or multiple records to a single key.
 *

 * If the index is {@link HollowConsumer#addRefreshListener(HollowConsumer.RefreshListener) registered} with its
 * associated {@link HollowConsumer} then the index will track updates and changes will be reflected in matched results
 * (performed after such updates).  When a registered index is no longer needed it should be
 * {@link HollowConsumer#removeRefreshListener(HollowConsumer.RefreshListener) deregistered} to avoid unnecessary
 * index recalculation and to ensure the index is reclaimed by the garbage collector.
 *
 * @param <T> the root type
 * @param <S> the select and result type
 * @param <Q> the query type
 */
public class HashIndexSelect<T extends HollowRecord, S extends HollowRecord, Q>
        implements HollowConsumer.RefreshListener, HollowConsumer.RefreshRegistrationListener, Function<Q, Stream<S>> {
    final HollowConsumer consumer;
    HollowAPI api;
    final SelectFieldPathResultExtractor<S> selectField;
    final List<MatchFieldPathArgumentExtractor<Q>> matchFields;
    final String rootTypeName;
    final String selectFieldPath;
    final String[] matchFieldPaths;
    HollowHashIndex hhi;

    HashIndexSelect(
            HollowConsumer consumer,
            Class<T> rootType,
            SelectFieldPathResultExtractor<S> selectField,
            List<MatchFieldPathArgumentExtractor<Q>> matchFields) {
        this.consumer = consumer;
        this.api = consumer.getAPI();
        this.selectField = selectField;
        this.matchFields = matchFields;

        // Validate select field path
        // @@@ Add method to FieldPath
        this.selectFieldPath = selectField.fieldPath.getSegments().stream().map(FieldPaths.FieldSegment::getName)
                .collect(joining("."));

        // Validate match field paths
        this.matchFieldPaths = matchFields.stream()
                // @@@ Add method to FieldPath
                .map(mf -> mf.fieldPath.getSegments().stream().map(FieldPaths.FieldSegment::getName)
                        .collect(joining(".")))
                .toArray(String[]::new);
        this.rootTypeName = rootType.getSimpleName();

        this.hhi = new HollowHashIndex(consumer.getStateEngine(), rootTypeName, selectFieldPath, matchFieldPaths);
    }

    HashIndexSelect(
            HollowConsumer consumer,
            Class<T> rootType,
            Class<S> selectType, String selectField,
            Class<Q> matchFieldsType) {
        this(consumer,
                rootType,
                SelectFieldPathResultExtractor
                        .from(consumer.getAPI().getClass(), consumer.getStateEngine(), rootType, selectField,
                                selectType),
                MatchFieldPathArgumentExtractor
                        .fromHolderClass(consumer.getStateEngine(), rootType, matchFieldsType,
                                FieldPaths::createFieldPathForHashIndex));
    }

    HashIndexSelect(
            HollowConsumer consumer,
            Class<T> rootType,
            Class<S> selectType, String selectField,
            String fieldPath, Class<Q> matchFieldType) {
        this(consumer,
                rootType,
                SelectFieldPathResultExtractor
                        .from(consumer.getAPI().getClass(), consumer.getStateEngine(), rootType, selectField,
                                selectType),
                Collections.singletonList(
                        MatchFieldPathArgumentExtractor
                                .fromPathAndType(consumer.getStateEngine(), rootType, fieldPath, matchFieldType,
                                        FieldPaths::createFieldPathForHashIndex)));
    }

    /**
     * Finds matches for a given query.
     *
     * @param query the query
     * @return a stream of matching records (may be empty if there are no matches)
     */
    @Override
    public Stream<S> apply(Q query) {
        return findMatches(query);
    }

    /**
     * Finds matches for a given query.
     *
     * @param query the query
     * @return a stream of matching records (may be empty if there are no matches)
     */
    public Stream<S> findMatches(Q query) {
        Object[] queryArray = matchFields.stream().map(mf -> mf.extract(query)).toArray();

        HollowHashIndexResult matches = hhi.findMatches(queryArray);
        if (matches == null) {
            return Stream.empty();
        }

        return matches.stream().mapToObj(i -> selectField.extract(api, i));
    }

    // HollowConsumer.RefreshListener

    @Override public void refreshStarted(long currentVersion, long requestedVersion) {
    }

    @Override public void snapshotUpdateOccurred(HollowAPI api, HollowReadStateEngine stateEngine, long version) {
        HollowHashIndex hhi = this.hhi;
        hhi.detachFromDeltaUpdates();
        hhi = new HollowHashIndex(consumer.getStateEngine(), rootTypeName, selectFieldPath, matchFieldPaths);
        hhi.listenForDeltaUpdates();
        this.hhi = hhi;
        this.api = api;
    }

    @Override public void deltaUpdateOccurred(HollowAPI api, HollowReadStateEngine stateEngine, long version) {
        this.api = api;
    }

    @Override public void blobLoaded(HollowConsumer.Blob transition) {
    }

    @Override public void refreshSuccessful(long beforeVersion, long afterVersion, long requestedVersion) {
    }

    @Override public void refreshFailed(
            long beforeVersion, long afterVersion, long requestedVersion, Throwable failureCause) {
    }

    // HollowConsumer.RefreshRegistrationListener

    @Override public void onBeforeAddition(HollowConsumer c) {
        if (c != consumer) {
            throw new IllegalStateException("The index's consumer and the listener's consumer are not the same");
        }
        hhi.listenForDeltaUpdates();
    }

    @Override public void onAfterRemoval(HollowConsumer c) {
        hhi.detachFromDeltaUpdates();
    }

    /**
     * The builder of a {@link HashIndexSelect}.
     *
     * @param <T> the root type
     * @param <S> the select, and result, type
     */
    public static class BuilderWithSelect<T extends HollowRecord, S extends HollowRecord> {
        final HollowConsumer consumer;
        final Class<T> rootType;
        final String selectFieldPath;
        final Class<S> selectFieldType;

        BuilderWithSelect(
                HollowConsumer consumer, Class<T> rootType,
                String selectFieldPath, Class<S> selectFieldType) {
            this.consumer = consumer;
            this.rootType = rootType;
            this.selectFieldPath = selectFieldPath;
            this.selectFieldType = selectFieldType;
        }

        /**
         * Creates a {@link HashIndexSelect} for matching with field paths and types declared by
         * {@link FieldPath} annotated fields or methods on the given query type.
         *
         * @param queryType the query type
         * @param <Q> the query type
         * @return a {@code HashIndexSelect}
         * @throws IllegalArgumentException if the query type declares one or more invalid field paths
         * or invalid types given resolution of corresponding field paths
         * @throws IllegalArgumentException if the select field path is invalid, or the select field type
         * is invalid given resolution of the select field path.
         */
        public <Q> HashIndexSelect<T, S, Q> usingBean(Class<Q> queryType) {
            Objects.requireNonNull(queryType);
            return new HashIndexSelect<>(consumer, rootType, selectFieldType, selectFieldPath,
                    queryType);
        }

        /**
         * Creates a {@link HashIndexSelect} for matching with a single query field path and type.
         *
         * @param queryFieldPath the query field path
         * @param queryFieldType the query type
         * @param <Q> the query type
         * @return a {@code HashIndexSelect}
         * @throws IllegalArgumentException if the query field path is empty or invalid
         * @throws IllegalArgumentException if the query field type is invalid given resolution of the
         * query field path
         * @throws IllegalArgumentException if the select field path is invalid, or the select field type
         * is invalid given resolution of the select field path.
         */
        public <Q> HashIndexSelect<T, S, Q> usingPath(String queryFieldPath, Class<Q> queryFieldType) {
            Objects.requireNonNull(queryFieldPath);
            if (queryFieldPath.isEmpty()) {
                throw new IllegalArgumentException("selectFieldPath argument is an empty String");
            }
            Objects.requireNonNull(queryFieldType);
            return new HashIndexSelect<>(consumer, rootType, selectFieldType, selectFieldPath,
                    queryFieldPath, queryFieldType);
        }
    }
}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.tools.compact;

import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
import com.netflix.hollow.core.read.engine.HollowTypeReadState;
import com.netflix.hollow.core.read.engine.PopulatedOrdinalListener;
import com.netflix.hollow.core.schema.HollowMapSchema;
import com.netflix.hollow.core.schema.HollowSchema;
import com.netflix.hollow.core.schema.HollowSchemaSorter;
import com.netflix.hollow.core.schema.HollowSetSchema;
import com.netflix.hollow.core.util.IntMap;
import com.netflix.hollow.core.write.HollowTypeWriteState;
import com.netflix.hollow.core.write.HollowWriteRecord;
import com.netflix.hollow.core.write.HollowWriteStateEngine;
import com.netflix.hollow.core.write.copy.HollowRecordCopier;
import com.netflix.hollow.tools.patch.delta.PartialOrdinalRemapper;
import com.netflix.hollow.tools.traverse.TransitiveSetTraverser;
import java.util.BitSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
 * During a long delta chain, it's possible that a large number of holes in the ordinal space will exist in some types.
 *

 * The HollowCompactor can reclaim space by moving records off of the high end of the ordinal space into these holes.
 *

 * This is accomplished by producing deltas which only include removals and additions of identical records
 * allocated to more optimal ordinals.
 *

 * This must sometimes be accomplished with a series of deltas, because the remapping of one state will cause some removals/additions
 * in referencing states (since they will point to new ordinals).  In a single delta transition, the HollowCompactor will
 * only attempt to compact a set of types which are not referencing each other (either directly or transitively).
 *
 */
public class HollowCompactor {

    private final HollowWriteStateEngine writeEngine;
    private final HollowReadStateEngine readEngine;

    private long minCandidateHoleCostInBytes;
    private int minCandidateHolePercentage;

    /**
     * Provide the state engines on which to operate, and the criteria to identify when a compaction is necessary
     *
     * @param writeEngine the HollowWriteStateEngine to compact
     * @param readEngine  a HollowReadStateEngine at the same data state as the writeEngine
     * @param config      The criteria to identify when a compaction is necessary.
     */
    public HollowCompactor(HollowWriteStateEngine writeEngine, HollowReadStateEngine readEngine, CompactionConfig config) {
        this(writeEngine, readEngine, config.getMinCandidateHoleCostInBytes(), config.getMinCandidateHolePercentage());
    }

    /**
     * Provide the state engines on which to operate, and the criteria to identify when a compaction is necessary
     *
     * @param writeEngine                 the HollowWriteStateEngine to compact
     * @param readEngine                  a HollowReadStateEngine at the same data state as the writeEngine
     * @param minCandidateHoleCostInBytes identify a type as a candidate for compaction only when the bytes used by ordinal holes exceeds this value
     * @param minCandidateHolePercentage  identify a type as a candidate for compaction only when the percentage of space used by ordinal holes exceeds this value
     */
    public HollowCompactor(HollowWriteStateEngine writeEngine, HollowReadStateEngine readEngine, long minCandidateHoleCostInBytes, int minCandidateHolePercentage) {
        this.writeEngine = writeEngine;
        this.readEngine = readEngine;
        this.minCandidateHoleCostInBytes = minCandidateHoleCostInBytes;
        this.minCandidateHolePercentage = minCandidateHolePercentage;
    }

    /**
     * Determine whether a compaction is necessary, based on the criteria specified in the constructor.
     * @return {@code true} if compaction is necessary, otherwise {@code false}
     */
    public boolean needsCompaction() {
        return !findCompactionTargets().isEmpty();
    }

    /**
     * Perform a compaction.  It is expected that:
     *
     *

     *
the {@link HollowWriteStateEngine} supplied in the constructor is unmodified since the
     *       last call to {@link HollowWriteStateEngine#prepareForNextCycle()}

     *
the {@link HollowReadStateEngine} supplied in the constructor reflects the same state as
     *       the HollowWriteStateEngine.

     *

     *
     */
    public void compact() {
        Set<String> compactionTargets = findCompactionTargets();

        Map<String, BitSet> relocatedOrdinals = new HashMap<String, BitSet>();
        PartialOrdinalRemapper remapper = new PartialOrdinalRemapper();

        for(String compactionTarget : compactionTargets) {
            HollowTypeReadState typeState = readEngine.getTypeState(compactionTarget);
            HollowTypeWriteState writeState = writeEngine.getTypeState(compactionTarget);
            BitSet populatedOrdinals = typeState.getListener(PopulatedOrdinalListener.class).getPopulatedOrdinals();
            BitSet typeRelocatedOrdinals = new BitSet(populatedOrdinals.length());
            int populatedCardinality = populatedOrdinals.cardinality();

            writeState.addAllObjectsFromPreviousCycle();

            int numRelocations = 0;
            int ordinalToRelocate = populatedOrdinals.nextSetBit(populatedCardinality);
            while(ordinalToRelocate != -1) {
                numRelocations++;
                ordinalToRelocate = populatedOrdinals.nextSetBit(ordinalToRelocate+1);
            }

            HollowRecordCopier copier = HollowRecordCopier.createCopier(typeState);
            IntMap remappedOrdinals = new IntMap(numRelocations);

            ordinalToRelocate = populatedOrdinals.length();
            int relocatePosition = -1;

            try {

                for(int i=0;i<numRelocations;i++) {
                    while(!populatedOrdinals.get(--ordinalToRelocate));
                    relocatePosition = populatedOrdinals.nextClearBit(relocatePosition + 1);
                    typeRelocatedOrdinals.set(ordinalToRelocate);
                    writeState.removeOrdinalFromThisCycle(ordinalToRelocate);
                    HollowWriteRecord rec = copier.copy(ordinalToRelocate);
                    writeState.mapOrdinal(rec, relocatePosition, false, true);
                    remappedOrdinals.put(ordinalToRelocate, relocatePosition);
                }

            } finally {
                writeState.recalculateFreeOrdinals();
            }

            remapper.addOrdinalRemapping(compactionTarget, remappedOrdinals);
            relocatedOrdinals.put(compactionTarget, typeRelocatedOrdinals);
        }

        /// find the referencing dependents
        TransitiveSetTraverser.addReferencingOutsideClosure(readEngine, relocatedOrdinals);

        /// copy all forward except remapped and transitive dependents of remapped
        for(HollowSchema schema : HollowSchemaSorter.dependencyOrderedSchemaList(writeEngine.getSchemas())) {
            if(!compactionTargets.contains(schema.getName())) {
                HollowTypeWriteState writeState = writeEngine.getTypeState(schema.getName());

                writeState.addAllObjectsFromPreviousCycle();

                BitSet typeRelocatedOrdinals = relocatedOrdinals.get(schema.getName());
                if(typeRelocatedOrdinals != null) {
                    HollowTypeReadState readState = readEngine.getTypeState(schema.getName());
                    IntMap remappedOrdinals = new IntMap(typeRelocatedOrdinals.cardinality());

                    boolean preserveHashPositions = shouldPreserveHashPositions(schema);
                    HollowRecordCopier copier = HollowRecordCopier.createCopier(readState, remapper, preserveHashPositions);

                    int remapOrdinal = typeRelocatedOrdinals.nextSetBit(0);
                    while(remapOrdinal != -1) {
                        HollowWriteRecord rec = copier.copy(remapOrdinal);
                        int newOrdinal = writeState.add(rec);
                        remappedOrdinals.put(remapOrdinal, newOrdinal);
                        writeState.removeOrdinalFromThisCycle(remapOrdinal);

                        remapOrdinal = typeRelocatedOrdinals.nextSetBit(remapOrdinal + 1);
                    }

                    remapper.addOrdinalRemapping(schema.getName(), remappedOrdinals);
                }
            }
        }
    }

    /**
     * Find candidate types for compaction.  No two types in the returned set will have a dependency relationship, either
     * directly or transitively.
     */
    private Set<String> findCompactionTargets() {
        List<HollowSchema> schemas = HollowSchemaSorter.dependencyOrderedSchemaList(readEngine.getSchemas());
        Set<String> typesToCompact = new HashSet<String>();

        for(HollowSchema schema : schemas) {
            if(isCompactionCandidate(schema.getName())) {
                if(!candidateIsDependentOnAnyTargetedType(schema.getName(), typesToCompact))
                    typesToCompact.add(schema.getName());
            }
        }

        return typesToCompact;
    }

    private boolean isCompactionCandidate(String typeName) {
        HollowTypeReadState typeState = readEngine.getTypeState(typeName);
        BitSet populatedOrdinals = typeState.getListener(PopulatedOrdinalListener.class).getPopulatedOrdinals();
        double numOrdinals = populatedOrdinals.length();
        double numHoles = populatedOrdinals.length() - populatedOrdinals.cardinality();
        double holePercentage = numHoles / numOrdinals * 100d;
        long approximateHoleCostInBytes = typeState.getApproximateHoleCostInBytes();
        boolean isCompactionCandidate = holePercentage > (double)minCandidateHolePercentage && approximateHoleCostInBytes > minCandidateHoleCostInBytes;
        return isCompactionCandidate;
    }

    private boolean candidateIsDependentOnAnyTargetedType(String type, Set<String> targetedTypes) {
        for(String targetedType : targetedTypes) {
            if(HollowSchemaSorter.typeIsTransitivelyDependent(readEngine, type, targetedType))
                return true;
        }

        return false;
    }

    private boolean shouldPreserveHashPositions(HollowSchema schema) {
        switch(schema.getSchemaType()) {
        case MAP:
            return readEngine.getTypesWithDefinedHashCodes().contains(((HollowMapSchema)schema).getKeyType());
        case SET:
            return readEngine.getTypesWithDefinedHashCodes().contains(((HollowSetSchema)schema).getElementType());
        default:
            return false;
        }
    }

    /**
     * A configuration that specifies when a type is a candidate for compaction.
     */
    public static class CompactionConfig {
        private final long minCandidateHoleCostInBytes;
        private final int minCandidateHolePercentage;

        /**
         * Create a new compaction.  Both of the criteria specified by the following parameters must be met in order for a type
         * to be considered a candidate for compaction.
         *
         * @param minCandidateHoleCostInBytes identify a type as a candidate for compaction only when the bytes used by ordinal holes exceeds this value
         * @param minCandidateHolePercentage identify a type as a candidate for compaction only when the percentage of space used by ordinal holes exceeds this value
         */
        public CompactionConfig(long minCandidateHoleCostInBytes, int minCandidateHolePercentage) {
            this.minCandidateHoleCostInBytes = minCandidateHoleCostInBytes;
            this.minCandidateHolePercentage = minCandidateHolePercentage;
        }

        public long getMinCandidateHoleCostInBytes() {
            return minCandidateHoleCostInBytes;
        }

        public int getMinCandidateHolePercentage() {
            return minCandidateHolePercentage;
        }
    }
}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.index;

import static com.netflix.hollow.core.HollowConstants.ORDINAL_NONE;

import com.netflix.hollow.core.index.key.HollowPrimaryKeyValueDeriver;
import com.netflix.hollow.core.index.key.PrimaryKey;
import com.netflix.hollow.core.memory.encoding.FixedLengthElementArray;
import com.netflix.hollow.core.memory.encoding.HashCodes;
import com.netflix.hollow.core.memory.pool.ArraySegmentRecycler;
import com.netflix.hollow.core.memory.pool.WastefulRecycler;
import com.netflix.hollow.core.read.HollowReadFieldUtils;
import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
import com.netflix.hollow.core.read.engine.HollowTypeStateListener;
import com.netflix.hollow.core.read.engine.PopulatedOrdinalListener;
import com.netflix.hollow.core.read.engine.object.HollowObjectTypeReadState;
import com.netflix.hollow.core.schema.HollowObjectSchema;
import com.netflix.hollow.core.schema.HollowObjectSchema.FieldType;
import com.netflix.hollow.core.schema.HollowSchema;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.BitSet;
import java.util.Collection;
import java.util.Collections;
import java.util.List;

/**
 * A HollowPrimaryKeyIndex is the go-to mechanism for indexing and querying data in a Hollow blob.
 *

 * A primary key index can be used to index and query a type by a {@link PrimaryKey}.  The provided {@link PrimaryKey} does
 * not have to be the same as declared as the default in the data model.
 */
public class HollowPrimaryKeyIndex implements HollowTypeStateListener {

    private final HollowObjectTypeReadState typeState;
    private final int[][] fieldPathIndexes;
    private final FieldType[] fieldTypes;
    private final PrimaryKey primaryKey;
    private final HollowPrimaryKeyValueDeriver keyDeriver;

    private final ArraySegmentRecycler memoryRecycler;

    private final BitSet specificOrdinalsToIndex;

    private volatile PrimaryKeyIndexHashTable hashTableVolatile;

    public HollowPrimaryKeyIndex(HollowReadStateEngine stateEngine, String type, String... fieldPaths) {
        this(stateEngine, WastefulRecycler.DEFAULT_INSTANCE, type, fieldPaths);
    }

    public HollowPrimaryKeyIndex(HollowReadStateEngine stateEngine, PrimaryKey primaryKey) {
        this(stateEngine, primaryKey, WastefulRecycler.DEFAULT_INSTANCE);
    }

    public HollowPrimaryKeyIndex(HollowReadStateEngine stateEngine, ArraySegmentRecycler memoryRecycler, String type, String... fieldPaths) {
        this(stateEngine, createPrimaryKey(stateEngine, type, fieldPaths), memoryRecycler);
    }

    public HollowPrimaryKeyIndex(HollowReadStateEngine stateEngine, PrimaryKey primaryKey, ArraySegmentRecycler memoryRecycler) {
        this(stateEngine, primaryKey, memoryRecycler, null);
    }

    /**
     * This initializer can be used to create a HollowPrimaryKeyIndex which will only index a subset of the records in the specified type.
     *
     * @param stateEngine the read state engine
     * @param primaryKey the primary key
     * @param memoryRecycler the memory recycler
     * @param specificOrdinalsToIndex the bit set
     */
    public HollowPrimaryKeyIndex(HollowReadStateEngine stateEngine, PrimaryKey primaryKey, ArraySegmentRecycler memoryRecycler, BitSet specificOrdinalsToIndex) {
        if (primaryKey==null) throw new IllegalArgumentException("primaryKey can't not be null");

        this.primaryKey = primaryKey;
        this.typeState = (HollowObjectTypeReadState) stateEngine.getTypeState(primaryKey.getType());
        this.fieldPathIndexes = new int[primaryKey.numFields()][];
        this.fieldTypes = new FieldType[primaryKey.numFields()];

        this.memoryRecycler = memoryRecycler;

        for(int i=0;i<primaryKey.numFields();i++) {
            fieldPathIndexes[i] = primaryKey.getFieldPathIndex(stateEngine, i);
            fieldTypes[i] = primaryKey.getFieldType(stateEngine, i);
        }

        this.keyDeriver = new HollowPrimaryKeyValueDeriver(typeState, fieldPathIndexes, fieldTypes);
        this.specificOrdinalsToIndex = specificOrdinalsToIndex;

        reindex();
    }

    private static PrimaryKey createPrimaryKey(HollowReadStateEngine stateEngine, String type, String... fieldPaths) {
        if (fieldPaths != null && fieldPaths.length != 0) {
            return new PrimaryKey(type, fieldPaths);
        }

        HollowSchema schema = stateEngine.getSchema(type);
        if (schema instanceof HollowObjectSchema) {
            return ((HollowObjectSchema) schema).getPrimaryKey();
        }
        return null;
    }

    /**
     * Once called, this HollowPrimaryKeyIndex will be kept up-to-date when deltas are applied to the indexed state engine.
     *

     * This method should be called before any subsequent deltas occur after the index is created.
     *

     * In order to prevent memory leaks, if this method is called and the index is no longer needed, call detachFromDeltaUpdates() before
     * discarding the index.
     */
    public void listenForDeltaUpdates() {
        if(specificOrdinalsToIndex != null)
            throw new IllegalStateException("Cannot listen for delta updates when indexing only specified ordinals!");

        typeState.addListener(this);
    }

    /**
     * Once called, this HollowPrimaryKeyIndex will no longer be kept up-to-date when deltas are applied to the indexed state engine.
     *

     * Call this method before discarding indexes which are currently listening for delta updates.
     */
    public void detachFromDeltaUpdates() {
        typeState.removeListener(this);
    }

    public HollowObjectTypeReadState getTypeState() {
        return typeState;
    }

    public PrimaryKey getPrimaryKey() {
        return primaryKey;
    }

    public List<FieldType> getFieldTypes() {
        return Arrays.asList(fieldTypes);
    }

    /**
     * Query an index with a single specified field.  The returned value with be the ordinal of the matching record.
     *

     * Use a generated API or the Generic Object API to use the returned ordinal.
     *
     * @param key the field key
     * @return the matching ordinal for the key, otherwise -1 if the key is not present
     */
    public int getMatchingOrdinal(Object key) {
        PrimaryKeyIndexHashTable hashTable = hashTableVolatile;
        if(fieldPathIndexes.length != 1 || hashTable.bitsPerElement == 0)
            return -1;

        int hashCode = keyHashCode(key, 0);

        int ordinal = -1;

        do {
            hashTable = this.hashTableVolatile;
            int bucket = hashCode & hashTable.hashMask;
            ordinal = readOrdinal(hashTable, bucket);
            while(ordinal != -1) {
                if(keyDeriver.keyMatches(key, ordinal, 0))
                    break;

                bucket++;
                bucket &= hashTable.hashMask;
                ordinal = readOrdinal(hashTable, bucket);
            }
        } while(hashTableVolatile != hashTable);

        return ordinal;
    }

    /**
     * Query an index with two specified fields.  The returned value with be the ordinal of the matching record.
     *

     * Use a generated API or the Generic Object API to use the returned ordinal.
     *
     * @param key1 the first field key
     * @param key2 the second field key
     * @return the matching ordinal for the two keys, otherwise -1 if the key is not present
     */
    public int getMatchingOrdinal(Object key1, Object key2) {
        PrimaryKeyIndexHashTable hashTable = hashTableVolatile;
        if(fieldPathIndexes.length != 2 || hashTable.bitsPerElement == 0)
            return -1;

        int hashCode = keyHashCode(key1, 0);
        hashCode ^= keyHashCode(key2, 1);

        int ordinal = -1;

        do {
            hashTable = this.hashTableVolatile;
            int bucket = hashCode & hashTable.hashMask;
            ordinal = readOrdinal(hashTable, bucket);
            while(ordinal != -1) {
                if(keyDeriver.keyMatches(key1, ordinal, 0) && keyDeriver.keyMatches(key2, ordinal, 1))
                    break;

                bucket++;
                bucket &= hashTable.hashMask;
                ordinal = readOrdinal(hashTable, bucket);
            }
        } while(hashTableVolatile != hashTable);

        return ordinal;
    }

    /**
     * Query an index with three specified fields.  The returned value with be the ordinal of the matching record.
     *

     * Use a generated API or the Generic Object API to use the returned ordinal.
     *
     * @param key1 the first field key
     * @param key2 the second field key
     * @param key3 the third field key
     * @return the matching ordinal for the three keys, otherwise -1 if the key is not present
     */
    public int getMatchingOrdinal(Object key1, Object key2, Object key3) {
        PrimaryKeyIndexHashTable hashTable = hashTableVolatile;
        if(fieldPathIndexes.length != 3 || hashTable.bitsPerElement == 0)
            return -1;

        int hashCode = keyHashCode(key1, 0);
        hashCode ^= keyHashCode(key2, 1);
        hashCode ^= keyHashCode(key3, 2);

        int ordinal = -1;

        do {
            hashTable = this.hashTableVolatile;
            int bucket = hashCode & hashTable.hashMask;
            ordinal = readOrdinal(hashTable, bucket);
            while(ordinal != -1) {
                if(keyDeriver.keyMatches(key1, ordinal, 0) && keyDeriver.keyMatches(key2, ordinal, 1) && keyDeriver.keyMatches(key3, ordinal, 2))
                    break;

                bucket++;
                bucket &= hashTable.hashMask;
                ordinal = readOrdinal(hashTable, bucket);
            }
        } while(hashTableVolatile != hashTable);

        return ordinal;
    }

    /**
     * Query an index with four or more specified fields.  The returned value with be the ordinal of the matching record.
     *

     * Use a generated API or the Generic Object API to use the returned ordinal.
     *
     * @param keys the field keys
     * @return the matching ordinal for the keys, otherwise -1 if the key is not present
     */
    public int getMatchingOrdinal(Object... keys) {
        PrimaryKeyIndexHashTable hashTable = hashTableVolatile;
        if(fieldPathIndexes.length != keys.length || hashTable.bitsPerElement == 0)
            return -1;

        int hashCode = 0;
        for(int i=0;i<keys.length;i++)
            hashCode ^= keyHashCode(keys[i], i);

        int ordinal = -1;

        do {
            hashTable = this.hashTableVolatile;
            int bucket = hashCode & hashTable.hashMask;
            ordinal = readOrdinal(hashTable, bucket);
            while(ordinal != -1) {
                if(keyDeriver.keyMatches(ordinal, keys))
                    break;

                bucket++;
                bucket &= hashTable.hashMask;
                ordinal = readOrdinal(hashTable, bucket);
            }
        } while(hashTableVolatile != hashTable);

        return ordinal;
    }

    private int readOrdinal(PrimaryKeyIndexHashTable hashTable, int bucket) {
        return (int)hashTable.hashTable.getElementValue((long)hashTable.bitsPerElement * (long)bucket, hashTable.bitsPerElement) - 1;
    }

    private int keyHashCode(Object key, int fieldIdx) {
        switch(fieldTypes[fieldIdx]) {
            case BOOLEAN:
                return HashCodes.hashInt(HollowReadFieldUtils.booleanHashCode((Boolean)key));
            case DOUBLE:
                return HashCodes.hashInt(HollowReadFieldUtils.doubleHashCode(((Double)key).doubleValue()));
            case FLOAT:
                return HashCodes.hashInt(HollowReadFieldUtils.floatHashCode(((Float)key).floatValue()));
            case INT:
                return HashCodes.hashInt(HollowReadFieldUtils.intHashCode(((Integer)key).intValue()));
            case LONG:
                return HashCodes.hashInt(HollowReadFieldUtils.longHashCode(((Long)key).longValue()));
            case REFERENCE:
                return HashCodes.hashInt(((Integer)key).intValue());
            case BYTES:
                return HashCodes.hashCode((byte[])key);
            case STRING:
                return HashCodes.hashCode((String)key);
        }

        throw new IllegalArgumentException("I don't know how to hash a " + fieldTypes[fieldIdx]);
    }

    private void setHashTable(PrimaryKeyIndexHashTable hashTable) {
        this.hashTableVolatile = hashTable;
    }

    /**
     * @return whether or not this index contains duplicate records (two or more records mapping to a single primary key).
     */
    public boolean containsDuplicates() {
        return !getDuplicateKeys().isEmpty();
    }

    /**
     * @return any keys which are mapped to two or more records.
     */
    public synchronized Collection<Object[]> getDuplicateKeys() {
        PrimaryKeyIndexHashTable hashTable = hashTableVolatile;
        if(hashTable.bitsPerElement == 0)
            return Collections.emptyList();

        List<Object[]> duplicateKeys = new ArrayList<Object[]>();

        for(int i=0;i<hashTable.hashTableSize;i++) {
            int ordinal = (int)hashTable.hashTable.getElementValue((long)i * (long)hashTable.bitsPerElement, hashTable.bitsPerElement) - 1;

            if(ordinal != -1) {
                int compareBucket = (i+1) & hashTable.hashMask;
                int compareOrdinal = (int)hashTable.hashTable.getElementValue((long)compareBucket * (long)hashTable.bitsPerElement, hashTable.bitsPerElement) - 1;
                while(compareOrdinal != -1) {
                    if(recordsHaveEqualKeys(ordinal, compareOrdinal))
                        duplicateKeys.add(keyDeriver.getRecordKey(ordinal));

                    compareBucket = (compareBucket + 1) & hashTable.hashMask;
                    compareOrdinal = (int)hashTable.hashTable.getElementValue((long)compareBucket * (long)hashTable.bitsPerElement, hashTable.bitsPerElement) - 1;
                }
            }
        }

        return duplicateKeys;
    }

    @Override
    public void beginUpdate() { }

    @Override
    public void addedOrdinal(int ordinal) { }

    @Override
    public void removedOrdinal(int ordinal) { }

    @Override
    public synchronized void endUpdate() {
        BitSet ordinals = typeState.getListener(PopulatedOrdinalListener.class).getPopulatedOrdinals();

        int hashTableSize = HashCodes.hashTableSize(ordinals.cardinality());
        int bitsPerElement = (32 - Integer.numberOfLeadingZeros(typeState.maxOrdinal() + 1));

        PrimaryKeyIndexHashTable hashTable = hashTableVolatile;
        if(hashTableSize == hashTable.hashTableSize
                && bitsPerElement == hashTable.bitsPerElement
                && shouldPerformDeltaUpdate()) {
            deltaUpdate(hashTableSize, bitsPerElement);
        } else {
            reindex();
        }
    }

    public void destroy() {
        PrimaryKeyIndexHashTable hashTable = hashTableVolatile;
        if(hashTable != null)
            hashTable.hashTable.destroy(memoryRecycler);
    }

    private synchronized void reindex() {
        PrimaryKeyIndexHashTable hashTable = hashTableVolatile;
        // Could be null on first reindex
        if(hashTable != null) {
            hashTable.hashTable.destroy(memoryRecycler);
        }

        BitSet ordinals = specificOrdinalsToIndex;

        if(ordinals == null) {
            PopulatedOrdinalListener listener = typeState.getListener(PopulatedOrdinalListener.class);
            ordinals = listener.getPopulatedOrdinals();
        }

        int hashTableSize = HashCodes.hashTableSize(ordinals.cardinality());
        int bitsPerElement = (32 - Integer.numberOfLeadingZeros(typeState.maxOrdinal() + 1));

        FixedLengthElementArray hashedArray = new FixedLengthElementArray(memoryRecycler, (long)hashTableSize * (long)bitsPerElement);

        int hashMask = hashTableSize - 1;

        int ordinal = ordinals.nextSetBit(0);
        while(ordinal != ORDINAL_NONE) {
            int hashCode = recordHash(ordinal);
            int bucket = hashCode & hashMask;

            while(hashedArray.getElementValue((long)bucket * (long)bitsPerElement, bitsPerElement) != 0)
                bucket = (bucket + 1) & hashMask;

            hashedArray.setElementValue((long)bucket * (long)bitsPerElement, bitsPerElement, ordinal + 1);

            ordinal = ordinals.nextSetBit(ordinal + 1);
        }

        setHashTable(new PrimaryKeyIndexHashTable(hashedArray, hashTableSize, hashMask, bitsPerElement));

        memoryRecycler.swap();
    }

    private void deltaUpdate(int hashTableSize, int bitsPerElement) {
        // For a delta update hashTableVolatile cannot be null
        PrimaryKeyIndexHashTable hashTable = hashTableVolatile;
        hashTable.hashTable.destroy(memoryRecycler);

        PopulatedOrdinalListener listener = typeState.getListener(PopulatedOrdinalListener.class);
        BitSet prevOrdinals = listener.getPreviousOrdinals();
        BitSet ordinals = listener.getPopulatedOrdinals();

        long totalBitsInHashTable = (long)hashTableSize * (long)bitsPerElement;
        FixedLengthElementArray hashedArray = new FixedLengthElementArray(memoryRecycler, totalBitsInHashTable);
        hashedArray.copyBits(hashTable.hashTable, 0, 0, totalBitsInHashTable);

        int hashMask = hashTableSize - 1;

        int prevOrdinal = prevOrdinals.nextSetBit(0);
        while(prevOrdinal != ORDINAL_NONE) {
            if(!ordinals.get(prevOrdinal)) {
                /// remove this ordinal
                int hashCode = recordHash(prevOrdinal);
                int bucket = hashCode & hashMask;

                while(hashedArray.getElementValue((long)bucket * (long)bitsPerElement, bitsPerElement) != prevOrdinal + 1)
                    bucket = (bucket + 1) & hashMask;

                hashedArray.clearElementValue((long)bucket * (long)bitsPerElement, bitsPerElement);
                int emptyBucket = bucket;
                bucket = (bucket + 1) & hashMask;
                int moveOrdinal = (int)hashedArray.getElementValue((long)bucket * (long)bitsPerElement, bitsPerElement) - 1;

                while(moveOrdinal != ORDINAL_NONE) {
                    int naturalHash = recordHash(moveOrdinal);
                    int naturalBucket = naturalHash & hashMask;

                    if(!bucketInRange(emptyBucket, bucket, naturalBucket)) {
                        hashedArray.setElementValue((long)emptyBucket * (long)bitsPerElement, bitsPerElement, moveOrdinal + 1);
                        hashedArray.clearElementValue((long)bucket * (long)bitsPerElement, bitsPerElement);
                        emptyBucket = bucket;
                    }

                    bucket = (bucket + 1) & hashMask;
                    moveOrdinal = (int)hashedArray.getElementValue((long)bucket * (long)bitsPerElement, bitsPerElement) - 1;
                }

            }

            prevOrdinal = prevOrdinals.nextSetBit(prevOrdinal + 1);
        }

        int ordinal = ordinals.nextSetBit(0);
        while(ordinal != ORDINAL_NONE) {
            if(!prevOrdinals.get(ordinal)) {
                int hashCode = recordHash(ordinal);
                int bucket = hashCode & hashMask;

                while(hashedArray.getElementValue((long)bucket * (long)bitsPerElement, bitsPerElement) != 0) {
                    bucket = (bucket + 1) & hashMask;
                }

                hashedArray.setElementValue((long)bucket * (long)bitsPerElement, bitsPerElement, ordinal + 1);
            }

            ordinal = ordinals.nextSetBit(ordinal + 1);
        }

        setHashTable(new PrimaryKeyIndexHashTable(hashedArray, hashTableSize, hashMask, bitsPerElement));

        memoryRecycler.swap();
    }

    private boolean bucketInRange(int fromBucket, int toBucket, int testBucket) {
        if(toBucket > fromBucket) {
            return testBucket > fromBucket && testBucket <= toBucket;
        } else {
            return testBucket > fromBucket || testBucket <= toBucket;
        }
    }

    private int recordHash(int ordinal) {
        int hashCode = 0;
        for(int i=0;i<fieldPathIndexes.length;i++) {
            hashCode ^= fieldHash(ordinal, i);
            // hashCode ^= HashCodes.hashInt(hashCode);
        }
        return hashCode;
    }

    private int fieldHash(int ordinal, int fieldIdx) {
        HollowObjectTypeReadState typeState = this.typeState;
        HollowObjectSchema schema = typeState.getSchema();

        int lastFieldPath = fieldPathIndexes[fieldIdx].length - 1;
        for(int i=0;i<lastFieldPath;i++) {
            int fieldPosition = fieldPathIndexes[fieldIdx][i];
            ordinal = typeState.readOrdinal(ordinal, fieldPosition);
            typeState = (HollowObjectTypeReadState) schema.getReferencedTypeState(fieldPosition);
            schema = typeState.getSchema();
        }

        int hashCode = HollowReadFieldUtils.fieldHashCode(typeState, ordinal, fieldPathIndexes[fieldIdx][lastFieldPath]);

        switch(fieldTypes[fieldIdx]) {
            case STRING:
            case BYTES:
                return hashCode;
            default:
                return HashCodes.hashInt(hashCode);
        }
    }

    public Object[] getRecordKey(int ordinal) {
        return keyDeriver.getRecordKey(ordinal);
    }

    private boolean recordsHaveEqualKeys(int ordinal1, int ordinal2) {
        for(int i=0;i<fieldPathIndexes.length;i++) {
            if(!fieldsAreEqual(ordinal1, ordinal2, i))
                return false;
        }
        return true;
    }

    private boolean fieldsAreEqual(int ordinal1, int ordinal2, int fieldIdx) {
        HollowObjectTypeReadState typeState = this.typeState;
        HollowObjectSchema schema = typeState.getSchema();

        int lastFieldPath = fieldPathIndexes[fieldIdx].length - 1;
        for(int i=0;i<lastFieldPath;i++) {
            int fieldPosition = fieldPathIndexes[fieldIdx][i];
            ordinal1 = typeState.readOrdinal(ordinal1, fieldPosition);
            ordinal2 = typeState.readOrdinal(ordinal2, fieldPosition);
            typeState = (HollowObjectTypeReadState) schema.getReferencedTypeState(fieldPosition);
            schema = typeState.getSchema();
        }

        if(fieldTypes[fieldIdx] == FieldType.REFERENCE)
            return typeState.readOrdinal(ordinal1, fieldPathIndexes[fieldIdx][lastFieldPath]) == typeState.readOrdinal(ordinal2, fieldPathIndexes[fieldIdx][lastFieldPath]);

        return HollowReadFieldUtils.fieldsAreEqual(typeState, ordinal1, fieldPathIndexes[fieldIdx][lastFieldPath], typeState, ordinal2, fieldPathIndexes[fieldIdx][lastFieldPath]);
    }

    private boolean shouldPerformDeltaUpdate() {
        BitSet previousOrdinals = typeState.getListener(PopulatedOrdinalListener.class).getPreviousOrdinals();
        BitSet ordinals = typeState.getListener(PopulatedOrdinalListener.class).getPopulatedOrdinals();

        int prevCardinality = 0;
        int removedRecords = 0;

        int prevOrdinal = previousOrdinals.nextSetBit(0);
        while(prevOrdinal != ORDINAL_NONE) {
            prevCardinality++;
            if(!ordinals.get(prevOrdinal))
                removedRecords++;

            prevOrdinal = previousOrdinals.nextSetBit(prevOrdinal + 1);
        }

        if(removedRecords > prevCardinality * 0.1d)
            return false;
        return true;
    }

    private static class PrimaryKeyIndexHashTable {
        private final FixedLengthElementArray hashTable;
        private final int hashTableSize;
        private final int hashMask;
        private final int bitsPerElement;

        public PrimaryKeyIndexHashTable(FixedLengthElementArray hashTable, int hashTableSize, int hashMask, int bitsPerElement) {
            this.hashTable = hashTable;
            this.hashTableSize = hashTableSize;
            this.hashMask = hashMask;
            this.bitsPerElement = bitsPerElement;
        }
    }
}