/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.index;

import com.netflix.hollow.core.memory.encoding.FixedLengthElementArray;
import com.netflix.hollow.core.memory.encoding.HashCodes;
import com.netflix.hollow.core.read.HollowReadFieldUtils;
import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
import com.netflix.hollow.core.read.engine.HollowTypeReadState;
import com.netflix.hollow.core.read.engine.HollowTypeStateListener;
import com.netflix.hollow.core.read.engine.object.HollowObjectTypeReadState;

/**
 * A HollowHashIndex is used for indexing non-primary-key data.  This type of index can map multiple keys to a single matching record, and/or
 * multiple records to a single key.
 *

 * The field definitions in a hash key may be hierarchical (traverse multiple record types) via dot-notation.  For example,
 * the field definition actors.element.actorId may be used to traverse a child LIST or SET type record referenced by the field
 * actors, each elements contained therein, and finally each actors actorId field.
 */
public class HollowHashIndex implements HollowTypeStateListener {

    private volatile HollowHashIndexState hashStateVolatile;

    private final HollowReadStateEngine stateEngine;
    private final HollowObjectTypeReadState typeState;
    private final String type;
    private final String selectField;
    private final String[] matchFields;

    /**
     * Define a {@link HollowHashIndex}.
     *
     * @param stateEngine The state engine to index
     * @param type The query starts with the specified type
     * @param selectField The query will select records at this field (specify "" to select the specified type).
     * The selectField may span collection elements and/or map keys or values, which can result in multiple matches per record of the specified start type.
     * @param matchFields The query will match on the specified match fields.  The match fields may span collection elements and/or map keys or values.
     */
    public HollowHashIndex(HollowReadStateEngine stateEngine, String type, String selectField, String... matchFields) {
        this.stateEngine = stateEngine;
        this.type = type;
        this.typeState = (HollowObjectTypeReadState) stateEngine.getTypeState(type);
        this.selectField = selectField;
        this.matchFields = matchFields;

        reindexHashIndex();
    }

    /**
     * Recreate the hash index entirely
     */
    private void reindexHashIndex() {
        HollowHashIndexBuilder builder = new HollowHashIndexBuilder(stateEngine, type, selectField, matchFields);

        builder.buildIndex();

        this.hashStateVolatile = new HollowHashIndexState(builder);
    }

    /**
     * Query the index.
     *
     * @param query the query
     * @return the hash index result to gather the matched ordinals. A {@code null} value indicated no matches were
     * found.
     */
    public HollowHashIndexResult findMatches(Object... query) {
        int hashCode = 0;

        for(int i=0;i<query.length;i++) {
            if(query[i] == null)
                throw new IllegalArgumentException("querying by null unsupported; i=" + i);
            hashCode ^= HashCodes.hashInt(keyHashCode(query[i], i));
        }

        HollowHashIndexResult result;
        HollowHashIndexState hashState;
        do {
            result = null;
            hashState = hashStateVolatile;
            long bucket = hashCode & hashState.getMatchHashMask();
            long hashBucketBit = bucket * hashState.getBitsPerMatchHashEntry();
            boolean bucketIsEmpty = hashState.getMatchHashTable().getElementValue(hashBucketBit, hashState.getBitsPerTraverserField()[0]) == 0;

            while (!bucketIsEmpty) {
                if (matchIsEqual(hashState.getMatchHashTable(), hashBucketBit, query)) {
                    int selectSize = (int) hashState.getMatchHashTable().getElementValue(hashBucketBit + hashState.getBitsPerMatchHashKey(), hashState.getBitsPerSelectTableSize());
                    long selectBucketPointer = hashState.getMatchHashTable().getElementValue(hashBucketBit + hashState.getBitsPerMatchHashKey() + hashState.getBitsPerSelectTableSize(), hashState.getBitsPerSelectTablePointer());

                    result = new HollowHashIndexResult(hashState, selectBucketPointer, selectSize);
                    break;
                }

                bucket = (bucket + 1) & hashState.getMatchHashMask();
                hashBucketBit = bucket * hashState.getBitsPerMatchHashEntry();
                bucketIsEmpty = hashState.getMatchHashTable().getElementValue(hashBucketBit, hashState.getBitsPerTraverserField()[0]) == 0;
            }
        } while (hashState != hashStateVolatile);

        return result;
    }

    private int keyHashCode(Object key, int fieldIdx) {
        HollowHashIndexState hashState = hashStateVolatile;
        switch(hashState.getMatchFields()[fieldIdx].getFieldType()) {
        case BOOLEAN:
            return HollowReadFieldUtils.booleanHashCode((Boolean)key);
        case DOUBLE:
            return HollowReadFieldUtils.doubleHashCode((Double) key);
        case FLOAT:
            return HollowReadFieldUtils.floatHashCode((Float) key);
        case INT:
            return HollowReadFieldUtils.intHashCode((Integer) key);
        case LONG:
            return HollowReadFieldUtils.longHashCode((Long) key);
        case REFERENCE:
            return (Integer) key;
        case BYTES:
            return HashCodes.hashCode((byte[])key);
        case STRING:
            return HashCodes.hashCode((String)key);
        }

        throw new IllegalArgumentException("I don't know how to hash a " + hashState.getMatchFields()[fieldIdx].getFieldType());
    }

    private boolean matchIsEqual(FixedLengthElementArray matchHashTable, long hashBucketBit, Object[] query) {
        HollowHashIndexState hashState = hashStateVolatile;
        for(int i = 0; i< hashState.getMatchFields().length; i++) {
            HollowHashIndexField field = hashState.getMatchFields()[i];
            int hashOrdinal = (int)matchHashTable.getElementValue(hashBucketBit + hashState.getOffsetPerTraverserField()[field.getBaseIteratorFieldIdx()], hashState.getBitsPerTraverserField()[field.getBaseIteratorFieldIdx()]) - 1;

            HollowTypeReadState readState = field.getBaseDataAccess();
            int[] fieldPath = field.getSchemaFieldPositionPath();

            if(fieldPath.length == 0) {
                if (!query[i].equals(hashOrdinal))
                    return false;
            } else {
                for(int j=0;j<fieldPath.length - 1;j++) {
                    HollowObjectTypeReadState objectAccess = (HollowObjectTypeReadState)readState;
                    readState = objectAccess.getSchema().getReferencedTypeState(fieldPath[j]);
                    hashOrdinal = objectAccess.readOrdinal(hashOrdinal, fieldPath[j]);
                }

                HollowObjectTypeReadState objectAccess = (HollowObjectTypeReadState)readState;
                int fieldIdx = fieldPath[fieldPath.length-1];
                if(hashOrdinal == -1 || !HollowReadFieldUtils.fieldValueEquals(objectAccess, hashOrdinal, fieldIdx, query[i])) {
                    return false;
                }
            }
        }

        return true;
    }

    /**
     * Once called, this HollowHashIndex will be kept up-to-date when deltas are applied to the indexed state engine.
     *

     * This method should be called before any subsequent deltas occur after the index is created.
     *

     * In order to prevent memory leaks, if this method is called and the index is no longer needed, call detachFromDeltaUpdates() before
     * discarding the index.
     */
    public void listenForDeltaUpdates() {
        typeState.addListener(this);
    }

    /**
     * Once called, this HollowHashIndex will no longer be kept up-to-date when deltas are applied to the indexed state engine.
     *

     * Call this method before discarding indexes which are currently listening for delta updates.
     */
    public void detachFromDeltaUpdates() {
        typeState.removeListener(this);
    }

    @Override
    public void beginUpdate() { }

    @Override
    public void addedOrdinal(int ordinal) { }

    @Override
    public void removedOrdinal(int ordinal) { }

    @Override
    public void endUpdate() {
       reindexHashIndex();
    }

    public HollowReadStateEngine getStateEngine() {
        return stateEngine;
    }

    public String getType() {
        return type;
    }

    public String getSelectField() {
        return selectField;
    }

    public String[] getMatchFields() {
        return matchFields;
    }

    protected static class HollowHashIndexState {

        final FixedLengthElementArray selectHashArray;
        final int bitsPerSelectHashEntry;
        private final FixedLengthElementArray matchHashTable;
        private final HollowHashIndexField[] matchFields;
        private final int matchHashMask;
        private final int bitsPerMatchHashKey;
        private final int bitsPerMatchHashEntry;
        private final int[] bitsPerTraverserField;
        private final int[] offsetPerTraverserField;
        private final int bitsPerSelectTableSize;
        private final int bitsPerSelectTablePointer;

        public HollowHashIndexState(HollowHashIndexBuilder builder) {
            matchHashTable = builder.getFinalMatchHashTable();
            selectHashArray = builder.getFinalSelectHashArray();
            matchFields = builder.getMatchFields();
            matchHashMask = (int) builder.getFinalMatchHashMask();
            bitsPerMatchHashKey = builder.getBitsPerMatchHashKey();
            bitsPerMatchHashEntry = builder.getFinalBitsPerMatchHashEntry();
            bitsPerTraverserField = builder.getBitsPerTraverserField();
            offsetPerTraverserField = builder.getOffsetPerTraverserField();
            bitsPerSelectTableSize = builder.getFinalBitsPerSelectTableSize();
            bitsPerSelectTablePointer = builder.getFinalBitsPerSelectTablePointer();
            bitsPerSelectHashEntry = builder.getBitsPerSelectHashEntry();
        }

        public FixedLengthElementArray getSelectHashArray() {
            return selectHashArray;
        }

        public int getBitsPerSelectHashEntry() {
            return bitsPerSelectHashEntry;
        }

        public FixedLengthElementArray getMatchHashTable() {
            return matchHashTable;
        }

        public HollowHashIndexField[] getMatchFields() {
            return matchFields;
        }

        public int getMatchHashMask() {
            return matchHashMask;
        }

        public int getBitsPerMatchHashKey() {
            return bitsPerMatchHashKey;
        }

        public int getBitsPerMatchHashEntry() {
            return bitsPerMatchHashEntry;
        }

        public int[] getBitsPerTraverserField() {
            return bitsPerTraverserField;
        }

        public int[] getOffsetPerTraverserField() {
            return offsetPerTraverserField;
        }

        public int getBitsPerSelectTableSize() {
            return bitsPerSelectTableSize;
        }

        public int getBitsPerSelectTablePointer() {
            return bitsPerSelectTablePointer;
        }
    }
}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.api.client;

import static com.netflix.hollow.core.util.Threads.daemonThread;

import com.netflix.hollow.api.consumer.HollowConsumer;
import com.netflix.hollow.core.HollowConstants;

import java.util.Random;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 * Implementations of this class are responsible for two things:
 *
 * 1) Tracking the latest announced data state version.
 * 2) Keeping the client up to date by calling triggerAsyncRefresh() on self when the latest version changes.
 *
 * A default implementation {@link HollowAnnouncementWatcher.DefaultWatcher} is available.  If this implementation
 * is used, calling {@link HollowClient#triggerRefresh()} will always attempt to get to the latest state, unless an
 * explicit state was specified via {@link HollowClient#triggerRefreshTo(long)}.
 *
 * @deprecated Implement the {@link HollowConsumer.AnnouncementWatcher} for use with the {@link HollowConsumer} instead.
 *
 */
@Deprecated
public abstract class HollowAnnouncementWatcher {
    private static final Logger log = Logger.getLogger(HollowAnnouncementWatcher.class.getName());
    private final ExecutorService refreshExecutor;

    /**
     * Construct a HollowAnnouncementWatcher with a default ExecutorService.
     */
    public HollowAnnouncementWatcher() {
        refreshExecutor = Executors.newFixedThreadPool(1,
                r -> daemonThread(r, getClass(), "watch"));
    }

    /**
     * Construct a HollowAnnouncementWatcher with the specified ExecutorService.
     *
     * @param refreshExecutor the ExecutorService to use for asynchronous state refresh.
     */
    public HollowAnnouncementWatcher(ExecutorService refreshExecutor) {
        this.refreshExecutor = refreshExecutor;
    }

    /**
     * @return the latest announced version.
     */
    public abstract long getLatestVersion();

    /**
     * If some push announcement mechanism is to be provided by this HollowAnnouncementWatcher, subscribe here.
     * Alternatively, if some polling announcement mechanism is to be provided, setup the polling cycle here.
     *
     * When announcements are received, or polling reveals a new version, a call should be placed to triggerRefresh().
     */
    public abstract void subscribeToEvents();

    /**
     * Override this method ONLY if it is legal to explicitly update to a specific version.
     *
     * @param latestVersion the latest version
     */
    public void setLatestVersion(long latestVersion) {
        throw new UnsupportedOperationException("Cannot explicitly set latest version on a " + this.getClass());
    }

    /**
     * Will force a double snapshot refresh on the next update.
     */
    protected void forceDoubleSnapshotNextUpdate() {
        client.forceDoubleSnapshotNextUpdate();
    }

    /**
     * Triggers a refresh in a new thread immediately.
     */
    public void triggerAsyncRefresh() {
        triggerAsyncRefreshWithDelay(0);
    }

    /**
     * Triggers async refresh after some random number of milliseconds have passed,
     * between now and the specified maximum number of milliseconds.
     *
     * Any subsequent calls for async refresh will not begin until after the specified delay
     * has completed.
     *
     * @param maxDelayMillis the maximum delay in milliseconds
     */
    public void triggerAsyncRefreshWithRandomDelay(int maxDelayMillis) {
        Random rand = new Random();
        int delayMillis = maxDelayMillis > 0 ? rand.nextInt(maxDelayMillis) : 0;
        triggerAsyncRefreshWithDelay(delayMillis);
    }

    /**
     * Triggers async refresh after the specified number of milliseconds has passed.
     *
     * Any subsequent calls for async refresh will not begin until after the specified delay
     * has completed.
     *
     * @param delayMillis the delay in milliseconds
     */
    public void triggerAsyncRefreshWithDelay(int delayMillis) {
        final HollowClient client = this.client;
        final long targetBeginTime = System.currentTimeMillis() + delayMillis;

        refreshExecutor.execute(new Runnable() {
            public void run() {
                try {
                    long delay = targetBeginTime - System.currentTimeMillis();
                    if(delay > 0)
                        Thread.sleep(delay);
                    client.triggerRefresh();
                } catch(Throwable th) {
                    log.log(Level.SEVERE, "Async refresh failed", th);
                }
            }
        });
    }

    private HollowClient client;

    protected HollowClient getClientToNotify() { return client; }

    void setClientToNotify(HollowClient client) {
        this.client = client;
        subscribeToEvents();
    }

    public static class DefaultWatcher extends HollowAnnouncementWatcher {

        private long latestVersion = HollowConstants.VERSION_LATEST;

        public DefaultWatcher() {
            super();
        }

        public DefaultWatcher(ExecutorService refreshExecutor) {
            super(refreshExecutor);
        }

        @Override
        public long getLatestVersion() {
            /// by default, just try to fetch the maximum available version
            return latestVersion;
        }

        @Override
        public void subscribeToEvents() {
            // by default, update events not available.
        }

        @Override
        public void setLatestVersion(long latestVersion) {
            this.latestVersion = latestVersion;
        }
    };

}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.tools.diff;

import com.netflix.hollow.core.index.key.PrimaryKey;
import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
import com.netflix.hollow.core.schema.HollowObjectSchema;
import com.netflix.hollow.core.schema.HollowSchema;
import com.netflix.hollow.core.util.SimultaneousExecutor;
import com.netflix.hollow.tools.diff.exact.DiffEqualityMapping;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.logging.Logger;

/**
 * Calculate a detailed accounting for the differences between two data states.
 *

 * The differences between two states are broken down by specified types.  Records for each evaluated type are matched based on primary keys.
 * The matched records are traversed in tandem to determine which individual fields/branches in their hierarchies differ.
 * The difference between matched records is measured per field in the hierarchy as an integer value indicating the number
 * of unmatched values in either the from or the to state.
 *

 * Unmatched records are accounted for separately -- for the purposes of the diff, it is sufficient to mark these records as unmatched.
 *
 */
public class HollowDiff {

    private final Logger log = Logger.getLogger(HollowDiff.class.getName());
    private final HollowReadStateEngine fromStateEngine;
    private final HollowReadStateEngine toStateEngine;

    private final DiffEqualityMapping equalityMapping;

    private final Map<String, HollowTypeDiff> typeDiffs = new LinkedHashMap<>();

    /**
     * Instantiate a HollowDiff.  By default, all OBJECT types with a defined PrimaryKey will be
     * configured to be diffed.
     *

     * To calculate the diff, call calculateDiff().
     *
     * @param from the "from" state
     * @param to the "to" state
     */
    public HollowDiff(HollowReadStateEngine from, HollowReadStateEngine to) {
        this(from, to, true);
    }

    /**
     * Instantiate a HollowDiff.
     *

     * To calculate the diff, call calculateDiff().
     *
     * @param from the "from" state
     * @param to the "to" state
     * @param isAutoDiscoverTypeDiff If true, all OBJECT types with a defined PrimaryKey will be configured to be diffed.
     */
    public HollowDiff(HollowReadStateEngine from, HollowReadStateEngine to, boolean isAutoDiscoverTypeDiff) {
        this.fromStateEngine = from;
        this.toStateEngine = to;
        this.equalityMapping = new DiffEqualityMapping(from, to);

        if (isAutoDiscoverTypeDiff) { // Auto Discover TypeDiff from both from and to StateEngine
            List<HollowSchema> schemas = new ArrayList<>();
            schemas.addAll(fromStateEngine.getSchemas());
            schemas.addAll(toStateEngine.getSchemas());
            for (HollowSchema schema : schemas) {
                if (schema instanceof HollowObjectSchema) {
                    PrimaryKey pKey = ((HollowObjectSchema) schema).getPrimaryKey();
                    if (pKey == null) continue;

                    addTypeDiff(schema.getName(), pKey.getFieldPaths());
                }
            }
        }
    }

    /**
     * Add a type to be included in the diff report
     *
     * @param type the type name
     * @param primaryKeyPaths the path(s) to the field(s) which comprise the type's primary key
     * @return the diff type
     */
    public HollowTypeDiff addTypeDiff(String type, String... primaryKeyPaths) {
        HollowTypeDiff typeDiff = new HollowTypeDiff(this, type, primaryKeyPaths);
        if(typeDiff.hasAnyData())
            typeDiffs.put(type, typeDiff);
        return typeDiff;
    }

    public List<HollowTypeDiff> getTypeDiffs() {
        return new ArrayList<>(typeDiffs.values());
    }

    /**
     * Retrieve a diff report for a specific type in order to inspect the calculated differences
     * @param type the type name
     * @return the diff type
     */
    public HollowTypeDiff getTypeDiff(String type) {
        return typeDiffs.get(type);
    }

    public HollowReadStateEngine getFromStateEngine() {
        return fromStateEngine;
    }

    public HollowReadStateEngine getToStateEngine() {
        return toStateEngine;
    }

    /**
     * Run the diff
     */
    public void calculateDiffs() {
        long startTime = System.currentTimeMillis();

        prepareForDiffCalculation();

        long endTime = System.currentTimeMillis();

        log.info("PREPARED IN " + (endTime - startTime) + "ms");

        for(HollowTypeDiff typeDiff : typeDiffs.values()) {
            typeDiff.calculateDiffs();
        }
    }

    public DiffEqualityMapping getEqualityMapping() {
        return equalityMapping;
    }

    private void prepareForDiffCalculation() {
        SimultaneousExecutor executor = new SimultaneousExecutor(1 + typeDiffs.size(), getClass(), "prepare");

        executor.execute(() -> {
            for(HollowTypeDiff typeDiff : typeDiffs.values()) {
                equalityMapping.getEqualOrdinalMap(typeDiff.getTypeName());
            }
        });

        for(final HollowTypeDiff typeDiff : typeDiffs.values()) {
            executor.execute(typeDiff::calculateMatches);
        }

        try {
            executor.awaitSuccessfulCompletion();
        } catch (InterruptedException | ExecutionException e) {
            throw new RuntimeException(e);
        }

        equalityMapping.markPrepared();
    }

}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.tools.history;

import com.netflix.hollow.core.index.key.PrimaryKey;
import com.netflix.hollow.core.read.dataaccess.HollowDataAccess;
import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
import com.netflix.hollow.core.read.engine.PopulatedOrdinalListener;
import com.netflix.hollow.core.read.engine.object.HollowObjectTypeReadState;
import com.netflix.hollow.core.schema.HollowObjectSchema;
import com.netflix.hollow.core.schema.HollowSchema;
import com.netflix.hollow.core.util.RemovedOrdinalIterator;
import com.netflix.hollow.core.util.SimultaneousExecutor;
import com.netflix.hollow.tools.diff.exact.DiffEqualOrdinalMap;
import com.netflix.hollow.tools.diff.exact.DiffEqualOrdinalMap.OrdinalIdentityTranslator;
import com.netflix.hollow.tools.diff.exact.DiffEqualityMapping;
import com.netflix.hollow.tools.history.keyindex.HollowHistoricalStateKeyOrdinalMapping;
import com.netflix.hollow.tools.history.keyindex.HollowHistoricalStateTypeKeyOrdinalMapping;
import com.netflix.hollow.tools.history.keyindex.HollowHistoryKeyIndex;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;

/**
 * Retains, in memory, the changes in a dataset over many states.  Indexes data for efficient retrieval from any
 * point in time.
 *

 * This historical data is maintained by retaining and indexing all of the changes for the delta chain in memory.
 * Because only changes over time are retained, rather than complete states, a great length of history can often
 * be held in memory.  Additionally, because the data is held indexed in memory, it can be accessed very quickly,
 * so that changes to specific records over time can be precisely investigated quickly.
 *

 * Each retained state is accessible via a {@link HollowHistoricalState}, from which a {@link HollowDataAccess} can
 * be obtained and used interchangeably with a (current) {@link HollowReadStateEngine} for many operations.
 *
 */
public class HollowHistory {

    private final HollowHistoryKeyIndex keyIndex;
    private final HollowHistoricalStateCreator creator;
    private final int maxHistoricalStatesToKeep;

    private HollowReadStateEngine latestHollowReadStateEngine;
    private Map<String, String> latestHeaderEntries;
    private final List<HollowHistoricalState> historicalStates;

    private final Map<Long, HollowHistoricalState> historicalStateLookupMap;

    private long latestVersion;

    private boolean ignoreListOrderingOnDoubleSnapshot = false;

    /**
     * @param initialHollowStateEngine The HollowReadStateEngine at an initial (earliest) state.
     * @param initialVersion The initial version of the HollowReadStateEngine
     * @param maxHistoricalStatesToKeep The number of historical states to keep in memory
     */
    public HollowHistory(HollowReadStateEngine initialHollowStateEngine, long initialVersion, int maxHistoricalStatesToKeep) {
        this(initialHollowStateEngine, initialVersion, maxHistoricalStatesToKeep, true);
    }

    /**
     * @param initialHollowStateEngine The HollowReadStateEngine at an initial (earliest) state.
     * @param initialVersion The initial version of the HollowReadStateEngine
     * @param maxHistoricalStatesToKeep The number of historical states to keep in memory
     * @param isAutoDiscoverTypeIndex true if scheme types are auto-discovered from the initiate state engine
     */
    public HollowHistory(HollowReadStateEngine initialHollowStateEngine, long initialVersion, int maxHistoricalStatesToKeep, boolean isAutoDiscoverTypeIndex) {
        this.keyIndex = new HollowHistoryKeyIndex(this);
        this.creator = new HollowHistoricalStateCreator(this);
        this.latestHollowReadStateEngine = initialHollowStateEngine;
        this.latestHeaderEntries = latestHollowReadStateEngine.getHeaderTags();
        this.historicalStates = new ArrayList<HollowHistoricalState>();
        this.historicalStateLookupMap = new HashMap<Long, HollowHistoricalState>();
        this.maxHistoricalStatesToKeep = maxHistoricalStatesToKeep;
        this.latestVersion = initialVersion;

        if (isAutoDiscoverTypeIndex) {
            for (HollowSchema schema : initialHollowStateEngine.getSchemas()) {
                if (schema instanceof HollowObjectSchema) {
                    PrimaryKey pKey = ((HollowObjectSchema) schema).getPrimaryKey();
                    if (pKey == null) continue;

                    keyIndex.addTypeIndex(pKey);
                    keyIndex.indexTypeField(pKey);
                }
            }
        }
    }

    /**
     * Call this method to indicate that list ordering changes should be identified as modified records when
     * a double snapshot occurs.  By default, these will not be identified as updates.
     */
    public void ignoreListOrderingOnDoubleSnapshot() {
        this.ignoreListOrderingOnDoubleSnapshot = true;
    }

    /**
     * @return The {@link HollowHistoryKeyIndex}, responsible for identifying keyOrdinals.
     */
    public HollowHistoryKeyIndex getKeyIndex() {
        return keyIndex;
    }

    /**
     * @return The {@link HollowReadStateEngine} backing the latest state.
     */
    public HollowReadStateEngine getLatestState() {
        return latestHollowReadStateEngine;
    }

    /**
     * @return An array of each historical state.
     */
    public HollowHistoricalState[] getHistoricalStates() {
        return historicalStates.toArray(new HollowHistoricalState[historicalStates.size()]);
    }

    /**
     * @return the number of historical states
     */
    public int getNumberOfHistoricalStates() {
        return historicalStates.size();
    }

    /**
     * @param version A version in the past
     * @return The {@link HollowHistoricalState} for the specified version, if it exists.
     */
    public HollowHistoricalState getHistoricalState(long version) {
        if(latestVersion == version)
            return historicalStates.get(0);
        return historicalStateLookupMap.get(version);
    }

    /**
     * Call this method after each time a delta occurs in the backing {@link HollowReadStateEngine}.  This
     * is how the HollowHistory knows how to create a new {@link HollowHistoricalState}.
     *
     * @param newVersion The version of the new state
     */
    public void deltaOccurred(long newVersion) {
        keyIndex.update(latestHollowReadStateEngine, true);

        HollowHistoricalStateDataAccess historicalDataAccess = creator.createBasedOnNewDelta(latestVersion, latestHollowReadStateEngine);
        historicalDataAccess.setNextState(latestHollowReadStateEngine);

        HollowHistoricalStateKeyOrdinalMapping keyOrdinalMapping = createKeyOrdinalMappingFromDelta();
        HollowHistoricalState historicalState = new HollowHistoricalState(newVersion, keyOrdinalMapping, historicalDataAccess, latestHeaderEntries);

        addHistoricalState(historicalState);
        this.latestVersion = newVersion;
        this.latestHeaderEntries = latestHollowReadStateEngine.getHeaderTags();
    }

    /**
     * Call this method after each time a double snapshot occurs.
     *

     * This method will replace the previous backing {@link HollowReadStateEngine} with the newly
     * supplied one, stitch together all of the existing history with the new state currently in the
     * new {@link HollowReadStateEngine}, and create a new {@link HollowHistoricalState} to represent
     * the transition.
     *
     * @param newHollowStateEngine the new state engine
     * @param newVersion the new version
     */
    public void doubleSnapshotOccurred(HollowReadStateEngine newHollowStateEngine, long newVersion) {
        if(!keyIndex.isInitialized())
            keyIndex.update(latestHollowReadStateEngine, false);

        keyIndex.update(newHollowStateEngine, false);

        HollowHistoricalStateDataAccess historicalDataAccess;

        DiffEqualityMapping mapping = new DiffEqualityMapping(latestHollowReadStateEngine, newHollowStateEngine, true, !ignoreListOrderingOnDoubleSnapshot);
        DiffEqualityMappingOrdinalRemapper remapper = new DiffEqualityMappingOrdinalRemapper(mapping);

        historicalDataAccess = creator.createHistoricalStateFromDoubleSnapshot(latestVersion, latestHollowReadStateEngine, newHollowStateEngine, remapper);

        HollowHistoricalStateDataAccess nextRemappedDataAccess = historicalDataAccess;
        HollowHistoricalState nextRemappedState = null;

        HollowHistoricalStateDataAccess[] remappedDataAccesses = new HollowHistoricalStateDataAccess[historicalStates.size()];
        HollowHistoricalStateKeyOrdinalMapping[] remappedKeyOrdinalMappings = new HollowHistoricalStateKeyOrdinalMapping[historicalStates.size()];

        remapHistoricalStateOrdinals(remapper, remappedDataAccesses, remappedKeyOrdinalMappings);

        for(int i=0;i<historicalStates.size();i++) {
            HollowHistoricalState historicalStateToRemap = historicalStates.get(i);
            HollowHistoricalStateDataAccess remappedDataAccess = remappedDataAccesses[i];
            HollowHistoricalStateKeyOrdinalMapping remappedKeyOrdinalMapping = remappedKeyOrdinalMappings[i];

            remappedDataAccess.setNextState(nextRemappedDataAccess);
            nextRemappedDataAccess = remappedDataAccess;
            HollowHistoricalState remappedState = new HollowHistoricalState(historicalStateToRemap.getVersion(), remappedKeyOrdinalMapping, remappedDataAccess, historicalStateToRemap.getHeaderEntries());
            remappedState.setNextState(nextRemappedState);
            nextRemappedState = remappedState;
            historicalStates.set(i, remappedState);
            historicalStateLookupMap.put(remappedState.getVersion(), remappedState);
        }

        historicalDataAccess.setNextState(newHollowStateEngine);

        HollowHistoricalStateKeyOrdinalMapping keyOrdinalMapping = createKeyOrdinalMappingFromDoubleSnapshot(newHollowStateEngine, remapper);
        HollowHistoricalState historicalState = new HollowHistoricalState(newVersion, keyOrdinalMapping, historicalDataAccess, latestHeaderEntries);

        addHistoricalState(historicalState);
        this.latestVersion = newVersion;
        this.latestHollowReadStateEngine = newHollowStateEngine;
        this.latestHeaderEntries = latestHollowReadStateEngine.getHeaderTags();
    }

    private void remapHistoricalStateOrdinals(final DiffEqualityMappingOrdinalRemapper remapper, final HollowHistoricalStateDataAccess[] remappedDataAccesses, final HollowHistoricalStateKeyOrdinalMapping[] remappedKeyOrdinalMappings) {
        SimultaneousExecutor executor = new SimultaneousExecutor(getClass(), "remap");
        final int numThreads = executor.getCorePoolSize();

        for(int i=0;i<executor.getCorePoolSize();i++) {
            final int threadNumber = i;
            executor.execute(() -> {
                for(int t=threadNumber;t<historicalStates.size();t+=numThreads) {
                    HollowHistoricalState historicalStateToRemap = historicalStates.get(t);
                    remappedDataAccesses[t] = creator.copyButRemapOrdinals(historicalStateToRemap.getDataAccess(), remapper);
                    remappedKeyOrdinalMappings[t] = historicalStateToRemap.getKeyOrdinalMapping().remap(remapper);
                }
            });
        }

        try {
            executor.awaitSuccessfulCompletion();
        } catch (InterruptedException | ExecutionException e) {
            throw new RuntimeException(e);
        }
    }

    private HollowHistoricalStateKeyOrdinalMapping createKeyOrdinalMappingFromDelta() {
        HollowHistoricalStateKeyOrdinalMapping keyOrdinalMapping = new HollowHistoricalStateKeyOrdinalMapping(keyIndex);

        for(String keyType : keyIndex.getTypeKeyIndexes().keySet()) {
            HollowHistoricalStateTypeKeyOrdinalMapping typeMapping = keyOrdinalMapping.getTypeMapping(keyType);
            HollowObjectTypeReadState typeState = (HollowObjectTypeReadState) latestHollowReadStateEngine.getTypeState(keyType);
            if (typeState==null) {
                // The type is present in the history's primary key index but is not present
                // in the latest read state; ensure the mapping is initialized to the default state
                typeMapping.prepare(0, 0);
                typeMapping.finish();
                continue;
            }

            PopulatedOrdinalListener listener = typeState.getListener(PopulatedOrdinalListener.class);

            RemovedOrdinalIterator removalIterator = new RemovedOrdinalIterator(listener);
            RemovedOrdinalIterator additionsIterator = new RemovedOrdinalIterator(listener.getPopulatedOrdinals(), listener.getPreviousOrdinals());

            typeMapping.prepare(additionsIterator.countTotal(), removalIterator.countTotal());

            int removedOrdinal = removalIterator.next();
            while(removedOrdinal != -1) {
                typeMapping.removed(typeState, removedOrdinal);
                removedOrdinal = removalIterator.next();
            }

            int addedOrdinal = additionsIterator.next();
            while(addedOrdinal != -1) {
                typeMapping.added(typeState, addedOrdinal);
                addedOrdinal = additionsIterator.next();
            }

            typeMapping.finish();
        }

        return keyOrdinalMapping;
    }

    private HollowHistoricalStateKeyOrdinalMapping createKeyOrdinalMappingFromDoubleSnapshot(HollowReadStateEngine newStateEngine, DiffEqualityMappingOrdinalRemapper ordinalRemapper) {
        HollowHistoricalStateKeyOrdinalMapping keyOrdinalMapping = new HollowHistoricalStateKeyOrdinalMapping(keyIndex);
        DiffEqualityMapping mapping = ordinalRemapper.getDiffEqualityMapping();

        for(String keyType : keyIndex.getTypeKeyIndexes().keySet()) {
            HollowHistoricalStateTypeKeyOrdinalMapping typeMapping = keyOrdinalMapping.getTypeMapping(keyType);
            HollowObjectTypeReadState fromTypeState = (HollowObjectTypeReadState) latestHollowReadStateEngine.getTypeState(keyType);
            HollowObjectTypeReadState toTypeState = (HollowObjectTypeReadState) newStateEngine.getTypeState(keyType);
            DiffEqualOrdinalMap equalOrdinalMap = mapping.getEqualOrdinalMap(keyType);

            BitSet fromOrdinals = fromTypeState == null ? new BitSet() : fromTypeState.getListener(PopulatedOrdinalListener.class).getPopulatedOrdinals();
            BitSet toOrdinals = toTypeState == null ? new BitSet() : toTypeState.getListener(PopulatedOrdinalListener.class).getPopulatedOrdinals();

            int removedOrdinalsCount = countUnmatchedOrdinals(fromOrdinals, equalOrdinalMap.getFromOrdinalIdentityTranslator());
            int addedOrdinalsCount = countUnmatchedOrdinals(toOrdinals, equalOrdinalMap.getToOrdinalIdentityTranslator());

            typeMapping.prepare(addedOrdinalsCount, removedOrdinalsCount);

            int fromOrdinal = fromOrdinals.nextSetBit(0);
            while(fromOrdinal != -1) {
                if(equalOrdinalMap.getIdentityFromOrdinal(fromOrdinal) == -1)
                    typeMapping.removed(fromTypeState, fromOrdinal, ordinalRemapper.getMappedOrdinal(keyType, fromOrdinal));

                fromOrdinal = fromOrdinals.nextSetBit(fromOrdinal + 1);
            }

            int toOrdinal = toOrdinals.nextSetBit(0);
            while(toOrdinal != -1) {
                if(equalOrdinalMap.getIdentityToOrdinal(toOrdinal) == -1)
                    typeMapping.added(toTypeState, toOrdinal);

                toOrdinal = toOrdinals.nextSetBit(toOrdinal + 1);
            }

            typeMapping.finish();
        }

        return keyOrdinalMapping;
    }

    private int countUnmatchedOrdinals(BitSet ordinals, OrdinalIdentityTranslator translator) {
        int count = 0;
        int ordinal = ordinals.nextSetBit(0);
        while(ordinal != -1) {
            if(translator.getIdentityOrdinal(ordinal) == -1)
                count++;
            ordinal = ordinals.nextSetBit(ordinal + 1);
        }
        return count;
    }

    private void addHistoricalState(HollowHistoricalState historicalState) {
        if(historicalStates.size() > 0) {
            historicalStates.get(0).getDataAccess().setNextState(historicalState.getDataAccess());
            historicalStates.get(0).setNextState(historicalState);
        }

        historicalStates.add(0, historicalState);
        historicalStateLookupMap.put(historicalState.getVersion(), historicalState);

        if(historicalStates.size() > maxHistoricalStatesToKeep) {
            removeHistoricalStates(1);
        }
    }

    /**
     * Removes the last {@code n} historical states.
     *
     * @param n the number of historical states to remove
     * @throws IllegalArgumentException if the {@code n} is less than {@code 0} or
     * greater than the {@link #getNumberOfHistoricalStates() number} of historical
     * states.
     */
    public void removeHistoricalStates(int n) {
        if (n < 0) {
            throw new IllegalArgumentException(String.format(
                    "Number of states to remove is negative: %d", n));
        }
        if (n > historicalStates.size()) {
            throw new IllegalArgumentException(String.format(
                    "Number of states to remove, %d, is greater than the number of states. %d",
                    n, historicalStates.size()));
        }

        while (n-- > 0) {
            HollowHistoricalState removedState = historicalStates.remove(historicalStates.size() - 1);
            historicalStateLookupMap.remove(removedState.getVersion());
        }
    }
}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.memory.encoding;

import com.netflix.hollow.core.memory.ByteData;
import com.netflix.hollow.core.memory.ByteDataBuffer;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;

/**
 * Variable-byte integer encoding and decoding logic
 *
 * @author dkoszewnik
 */
public class VarInt {

    /**
     * Write a 'null' variable length integer into the supplied {@link ByteDataBuffer}
     *
     * @param buf the buffer to write to
     */
    public static void writeVNull(ByteDataBuffer buf) {
        buf.write((byte)0x80);
        return;
    }

    /**
     * Encode the specified long as a variable length integer into the supplied {@link ByteDataBuffer}
     *
     * @param buf the buffer to write to
     * @param value the long value
     */
    public static void writeVLong(ByteDataBuffer buf, long value) {
        if(value < 0)                                buf.write((byte)0x81);
        if(value > 0xFFFFFFFFFFFFFFL || value < 0)   buf.write((byte)(0x80 | ((value >>> 56) & 0x7FL)));
        if(value > 0x1FFFFFFFFFFFFL || value < 0)    buf.write((byte)(0x80 | ((value >>> 49) & 0x7FL)));
        if(value > 0x3FFFFFFFFFFL || value < 0)      buf.write((byte)(0x80 | ((value >>> 42) & 0x7FL)));
        if(value > 0x7FFFFFFFFL || value < 0)        buf.write((byte)(0x80 | ((value >>> 35) & 0x7FL)));
        if(value > 0xFFFFFFFL || value < 0)          buf.write((byte)(0x80 | ((value >>> 28) & 0x7FL)));
        if(value > 0x1FFFFFL || value < 0)           buf.write((byte)(0x80 | ((value >>> 21) & 0x7FL)));
        if(value > 0x3FFFL || value < 0)             buf.write((byte)(0x80 | ((value >>> 14) & 0x7FL)));
        if(value > 0x7FL || value < 0)               buf.write((byte)(0x80 | ((value >>>  7) & 0x7FL)));

        buf.write((byte)(value & 0x7FL));
    }

    /**
     * Encode the specified long as a variable length integer into the supplied OuputStream
     *
     * @param out the output stream to write to
     * @param value the long value
     * @throws IOException if the value cannot be written to the output stream
     */
    public static void writeVLong(OutputStream out, long value) throws IOException {
        if(value < 0)                                out.write((byte)0x81);
        if(value > 0xFFFFFFFFFFFFFFL || value < 0)   out.write((byte)(0x80 | ((value >>> 56) & 0x7FL)));
        if(value > 0x1FFFFFFFFFFFFL || value < 0)    out.write((byte)(0x80 | ((value >>> 49) & 0x7FL)));
        if(value > 0x3FFFFFFFFFFL || value < 0)      out.write((byte)(0x80 | ((value >>> 42) & 0x7FL)));
        if(value > 0x7FFFFFFFFL || value < 0)        out.write((byte)(0x80 | ((value >>> 35) & 0x7FL)));
        if(value > 0xFFFFFFFL || value < 0)          out.write((byte)(0x80 | ((value >>> 28) & 0x7FL)));
        if(value > 0x1FFFFFL || value < 0)           out.write((byte)(0x80 | ((value >>> 21) & 0x7FL)));
        if(value > 0x3FFFL || value < 0)             out.write((byte)(0x80 | ((value >>> 14) & 0x7FL)));
        if(value > 0x7FL || value < 0)               out.write((byte)(0x80 | ((value >>>  7) & 0x7FL)));

        out.write((byte)(value & 0x7FL));
    }

    /**
     * Encode the specified int as a variable length integer into the supplied {@link ByteDataBuffer}
     *
     * @param buf the buffer to write to
     * @param value the int value
     */
    public static void writeVInt(ByteDataBuffer buf, int value) {
        if(value > 0x0FFFFFFF || value < 0) buf.write((byte)(0x80 | ((value >>> 28))));
        if(value > 0x1FFFFF || value < 0)   buf.write((byte)(0x80 | ((value >>> 21) & 0x7F)));
        if(value > 0x3FFF || value < 0)     buf.write((byte)(0x80 | ((value >>> 14) & 0x7F)));
        if(value > 0x7F || value < 0)       buf.write((byte)(0x80 | ((value >>>  7) & 0x7F)));

        buf.write((byte)(value & 0x7F));
    }

    /**
     * Encode the specified int as a variable length integer into the supplied OutputStream
     *
     * @param out the output stream to write to
     * @param value the int value
     * @throws IOException if the value cannot be written to the output stream
     */
    public static void writeVInt(OutputStream out, int value) throws IOException {
        if(value > 0x0FFFFFFF || value < 0) out.write((byte)(0x80 | ((value >>> 28))));
        if(value > 0x1FFFFF || value < 0)   out.write((byte)(0x80 | ((value >>> 21) & 0x7F)));
        if(value > 0x3FFF || value < 0)     out.write((byte)(0x80 | ((value >>> 14) & 0x7F)));
        if(value > 0x7F || value < 0)       out.write((byte)(0x80 | ((value >>>  7) & 0x7F)));

        out.write((byte)(value & 0x7F));
    }

    /**
     * Write the value as a VarInt into the array, starting at the specified position.
     *
     * @param data the byte array to write to
     * @param pos the position in the byte array
     * @param value the int value
     * @return the next position after the VarInt has been written.
     */
    public static int writeVInt(byte data[], int pos, int value) {
        if(value > 0x0FFFFFFF || value < 0) data[pos++] = ((byte)(0x80 | ((value >>> 28))));
        if(value > 0x1FFFFF || value < 0)   data[pos++] = ((byte)(0x80 | ((value >>> 21) & 0x7F)));
        if(value > 0x3FFF || value < 0)     data[pos++] = ((byte)(0x80 | ((value >>> 14) & 0x7F)));
        if(value > 0x7F || value < 0)       data[pos++] = ((byte)(0x80 | ((value >>>  7) & 0x7F)));

        data[pos++] = (byte)(value & 0x7F);

        return pos;
    }

    /**
     * Determine whether or not the value at the specified position in the supplied {@link ByteData} is
     * a 'null' variable length integer.
     *
     * @param arr the byte data to read from
     * @param position the position in the byte data to read from
     * @return true if the value is null
     */
    public static boolean readVNull(ByteData arr, long position) {
        return arr.get(position) == (byte)0x80;
    }

    /**
     * Read a variable length integer from the supplied {@link ByteData} starting at the specified position.
     * @param arr the byte data to read from
     * @param position the position in the byte data to read from
     * @return the int value
     */
    public static int readVInt(ByteData arr, long position) {
        byte b = arr.get(position++);

        if(b == (byte) 0x80)
            throw new RuntimeException("Attempting to read null value as int");

        int value = b & 0x7F;
        while ((b & 0x80) != 0) {
          b = arr.get(position++);
          value <<= 7;
          value |= (b & 0x7F);
        }

        return value;
    }

    /**
     * Read a variable length integer from the supplied InputStream
     * @param in the input stream to read from
     * @return the int value
     * @throws IOException if the value cannot be read from the input stream
     */
    public static int readVInt(InputStream in) throws IOException {
        byte b = (byte)in.read();

        if(b == (byte) 0x80)
            throw new RuntimeException("Attempting to read null value as int");

        int value = b & 0x7F;
        while ((b & 0x80) != 0) {
          b = (byte)in.read();
          value <<= 7;
          value |= (b & 0x7F);
        }

        return value;
    }

    /**
     * Read a variable length long from the supplied {@link ByteData} starting at the specified position.
     * @param arr the byte data to read from
     * @param position the position in the byte data to read from
     * @return the long value
     */
    public static long readVLong(ByteData arr, long position) {
        byte b = arr.get(position++);

        if(b == (byte) 0x80)
            throw new RuntimeException("Attempting to read null value as long");

        long value = b & 0x7F;
        while ((b & 0x80) != 0) {
          b = arr.get(position++);
          value <<= 7;
          value |= (b & 0x7F);
        }

        return value;
    }

    /**
     * Determine the size (in bytes) of the variable length long in the supplied {@link ByteData}, starting at the specified position.
     * @param arr the byte data to read from
     * @param position the position in the byte data to read from
     * @return the long value
     */
    public static int nextVLongSize(ByteData arr, long position) {
        byte b = arr.get(position++);

        if(b == (byte) 0x80)
            return 1;

        int length = 1;

        while((b & 0x80) != 0) {
            b = arr.get(position++);
            length++;
        }

        return length;
    }

    /**
     * Read a variable length long from the supplied InputStream.
     * @param in the input stream to read from
     * @return the long value
     * @throws IOException if the value cannot be read from the input stream
     */
    public static long readVLong(InputStream in) throws IOException {
        byte b = (byte)in.read();

        if(b == (byte) 0x80)
            throw new RuntimeException("Attempting to read null value as long");

        long value = b & 0x7F;
        while ((b & 0x80) != 0) {
          b = (byte)in.read();
          value <<= 7;
          value |= (b & 0x7F);
        }

        return value;
    }

    /**
     * Determine the size (in bytes) of the specified value when encoded as a variable length integer.
     * @param value the int value
     * @return the size (int bytes) of the value when encoded
     */
    public static int sizeOfVInt(int value) {
        if(value < 0)
            return 5;
        if(value < 0x80)
            return 1;
        if(value < 0x4000)
            return 2;
        if(value < 0x200000)
            return 3;
        if(value < 0x10000000)
            return 4;
        return 5;
    }

    /**
     * Determine the size (in bytes) of the specified value when encoded as a variable length integer.
     * @param value the long value
     * @return the size (int bytes) of the value when encoded
     */
    public static int sizeOfVLong(long value) {
        if(value < 0L)
            return 10;
        if(value < 0x80L)
            return 1;
        if(value < 0x4000L)
            return 2;
        if(value < 0x200000L)
            return 3;
        if(value < 0x10000000L)
            return 4;
        if(value < 0x800000000L)
            return 5;
        if(value < 0x40000000000L)
            return 6;
        if(value < 0x2000000000000L)
            return 7;
        if(value < 0x100000000000000L)
            return 8;
        return 9;
    }

    /**
     * Count the number of variable length integers encoded in the supplied {@link ByteData} in the specified range.
     * @param byteData the byte data
     * @param fieldPosition the field position
     * @param length the length
     * @return number of variable length integers encoded over a range in the byte data
     */
    public static int countVarIntsInRange(ByteData byteData, long fieldPosition, int length) {
        int numInts = 0;

        boolean insideInt = false;

        for(int i=0;i<length;i++) {
            byte b = byteData.get(fieldPosition + i);

            if((b & 0x80) == 0) {
                numInts++;
                insideInt = false;
            } else if(!insideInt && b == (byte)0x80) {
                numInts++;
            } else {
                insideInt = true;
            }
        }

        return numInts;
    }

}


/*******************************************************************************
 * Copyright 2011, 2012 Chris Banes.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *******************************************************************************/
package com.handmark.pulltorefresh.samples;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import android.app.ExpandableListActivity;
import android.os.AsyncTask;
import android.os.Bundle;
import android.widget.ExpandableListView;
import android.widget.SimpleExpandableListAdapter;

import com.handmark.pulltorefresh.library.PullToRefreshBase;
import com.handmark.pulltorefresh.library.PullToRefreshBase.OnRefreshListener;
import com.handmark.pulltorefresh.library.PullToRefreshExpandableListView;

public final class PullToRefreshExpandableListActivity extends ExpandableListActivity {
	private static final String KEY = "key";

	private List<Map<String, String>> groupData = new ArrayList<Map<String, String>>();
	private List<List<Map<String, String>>> childData = new ArrayList<List<Map<String, String>>>();

	private PullToRefreshExpandableListView mPullRefreshListView;
	private SimpleExpandableListAdapter mAdapter;

	/** Called when the activity is first created. */
	@Override
	public void onCreate(Bundle savedInstanceState) {
		super.onCreate(savedInstanceState);
		setContentView(R.layout.activity_ptr_expandable_list);

		mPullRefreshListView = (PullToRefreshExpandableListView) findViewById(R.id.pull_refresh_expandable_list);

		// Set a listener to be invoked when the list should be refreshed.
		mPullRefreshListView.setOnRefreshListener(new OnRefreshListener<ExpandableListView>() {
			@Override
			public void onRefresh(PullToRefreshBase<ExpandableListView> refreshView) {
				// Do work to refresh the list here.
				new GetDataTask().execute();
			}
		});

		for (String group : mGroupStrings) {
			Map<String, String> groupMap1 = new HashMap<String, String>();
			groupData.add(groupMap1);
			groupMap1.put(KEY, group);

			List<Map<String, String>> childList = new ArrayList<Map<String, String>>();
			for (String string : mChildStrings) {
				Map<String, String> childMap = new HashMap<String, String>();
				childList.add(childMap);
				childMap.put(KEY, string);
			}
			childData.add(childList);
		}

		mAdapter = new SimpleExpandableListAdapter(this, groupData, android.R.layout.simple_expandable_list_item_1,
				new String[] { KEY }, new int[] { android.R.id.text1 }, childData,
				android.R.layout.simple_expandable_list_item_2, new String[] { KEY }, new int[] { android.R.id.text1 });
		setListAdapter(mAdapter);
	}

	private class GetDataTask extends AsyncTask<Void, Void, String[]> {

		@Override
		protected String[] doInBackground(Void... params) {
			// Simulates a background job.
			try {
				Thread.sleep(2000);
			} catch (InterruptedException e) {
			}
			return mChildStrings;
		}

		@Override
		protected void onPostExecute(String[] result) {
			Map<String, String> newMap = new HashMap<String, String>();
			newMap.put(KEY, "Added after refresh...");
			groupData.add(newMap);

			List<Map<String, String>> childList = new ArrayList<Map<String, String>>();
			for (String string : mChildStrings) {
				Map<String, String> childMap = new HashMap<String, String>();
				childMap.put(KEY, string);
				childList.add(childMap);
			}
			childData.add(childList);

			mAdapter.notifyDataSetChanged();

			// Call onRefreshComplete when the list has been refreshed.
			mPullRefreshListView.onRefreshComplete();

			super.onPostExecute(result);
		}
	}

	private String[] mChildStrings = { "Child One", "Child Two", "Child Three", "Child Four", "Child Five", "Child Six" };

	private String[] mGroupStrings = { "Group One", "Group Two", "Group Three" };
}