/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.tools.history;

import com.netflix.hollow.core.read.dataaccess.HollowDataAccess;
import com.netflix.hollow.core.read.engine.HollowBlobReader;
import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
import com.netflix.hollow.core.read.engine.HollowTypeReadState;
import com.netflix.hollow.core.read.engine.PopulatedOrdinalListener;
import com.netflix.hollow.core.read.engine.list.HollowListDeltaHistoricalStateCreator;
import com.netflix.hollow.core.read.engine.list.HollowListTypeReadState;
import com.netflix.hollow.core.read.engine.map.HollowMapDeltaHistoricalStateCreator;
import com.netflix.hollow.core.read.engine.map.HollowMapTypeReadState;
import com.netflix.hollow.core.read.engine.object.HollowObjectDeltaHistoricalStateCreator;
import com.netflix.hollow.core.read.engine.object.HollowObjectTypeReadState;
import com.netflix.hollow.core.read.engine.set.HollowSetDeltaHistoricalStateCreator;
import com.netflix.hollow.core.read.engine.set.HollowSetTypeReadState;
import com.netflix.hollow.core.schema.HollowSchema;
import com.netflix.hollow.core.schema.HollowSchemaSorter;
import com.netflix.hollow.core.util.HollowWriteStateCreator;
import com.netflix.hollow.core.util.IntMap;
import com.netflix.hollow.core.util.IntMap.IntMapEntryIterator;
import com.netflix.hollow.core.util.SimultaneousExecutor;
import com.netflix.hollow.core.write.HollowBlobWriter;
import com.netflix.hollow.core.write.HollowTypeWriteState;
import com.netflix.hollow.core.write.HollowWriteRecord;
import com.netflix.hollow.core.write.HollowWriteStateEngine;
import com.netflix.hollow.core.write.copy.HollowRecordCopier;
import com.netflix.hollow.tools.combine.IdentityOrdinalRemapper;
import com.netflix.hollow.tools.combine.OrdinalRemapper;
import com.netflix.hollow.tools.diff.exact.DiffEqualOrdinalMap;
import com.netflix.hollow.tools.diff.exact.DiffEqualityMapping;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.Closeable;
import java.io.IOException;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;

/**
 * Used to create a historical {@link HollowDataAccess}, even in the absence of a {@link HollowHistory}.
 *
 */
public class HollowHistoricalStateCreator {

    private final HollowHistory totalHistory;

    public HollowHistoricalStateCreator() {
        this(null);
    }

    public HollowHistoricalStateCreator(HollowHistory totalHistory) {
        this.totalHistory = totalHistory;
    }

    /**
     * Create a {@link HollowDataAccess} for the prior state of the supplied {@link HollowReadStateEngine} after a delta
     * has been applied.
     *
     * @param version The state's version
     * @param stateEngine The current {@link HollowReadStateEngine} to which a delta has been applied.
     * @return a data access for history
     */
    public HollowHistoricalStateDataAccess createBasedOnNewDelta(long version, HollowReadStateEngine stateEngine) {
        IntMapOrdinalRemapper typeRemovedOrdinalMapping = new IntMapOrdinalRemapper();

        List<HollowTypeReadState> historicalTypeStates = new ArrayList<HollowTypeReadState>(stateEngine.getTypeStates().size());

        for(HollowTypeReadState typeState : stateEngine.getTypeStates()) {
            createDeltaHistoricalTypeState(typeRemovedOrdinalMapping, historicalTypeStates, typeState);
        }

        HollowHistoricalStateDataAccess dataAccess = new HollowHistoricalStateDataAccess(totalHistory, version, stateEngine, historicalTypeStates, typeRemovedOrdinalMapping, Collections.<String, HollowHistoricalSchemaChange>emptyMap());
        dataAccess.setNextState(stateEngine);

        return dataAccess;
    }

    private void createDeltaHistoricalTypeState(IntMapOrdinalRemapper typeRemovedOrdinalMapping, List<HollowTypeReadState> historicalTypeStates, HollowTypeReadState typeState) {
        if(typeState instanceof HollowObjectTypeReadState) {
            HollowObjectDeltaHistoricalStateCreator deltaHistoryCreator = new HollowObjectDeltaHistoricalStateCreator((HollowObjectTypeReadState)typeState);
            deltaHistoryCreator.populateHistory();
            typeRemovedOrdinalMapping.addOrdinalRemapping(typeState.getSchema().getName(), deltaHistoryCreator.getOrdinalMapping());
            historicalTypeStates.add(deltaHistoryCreator.createHistoricalTypeReadState());
        } else if(typeState instanceof HollowListTypeReadState) {
            HollowListDeltaHistoricalStateCreator deltaHistoryCreator = new HollowListDeltaHistoricalStateCreator((HollowListTypeReadState)typeState);
            deltaHistoryCreator.populateHistory();
            typeRemovedOrdinalMapping.addOrdinalRemapping(typeState.getSchema().getName(), deltaHistoryCreator.getOrdinalMapping());
            historicalTypeStates.add(deltaHistoryCreator.createHistoricalTypeReadState());
        } else if(typeState instanceof HollowSetTypeReadState) {
            HollowSetDeltaHistoricalStateCreator deltaHistoryCreator = new HollowSetDeltaHistoricalStateCreator((HollowSetTypeReadState)typeState);
            deltaHistoryCreator.populateHistory();
            typeRemovedOrdinalMapping.addOrdinalRemapping(typeState.getSchema().getName(), deltaHistoryCreator.getOrdinalMapping());
            historicalTypeStates.add(deltaHistoryCreator.createHistoricalTypeReadState());
        } else if(typeState instanceof HollowMapTypeReadState) {
            HollowMapDeltaHistoricalStateCreator deltaHistoryCreator = new HollowMapDeltaHistoricalStateCreator((HollowMapTypeReadState)typeState);
            deltaHistoryCreator.populateHistory();
            typeRemovedOrdinalMapping.addOrdinalRemapping(typeState.getSchema().getName(), deltaHistoryCreator.getOrdinalMapping());
            historicalTypeStates.add(deltaHistoryCreator.createHistoricalTypeReadState());
        }
    }

    /**
     * Create a {@link HollowDataAccess} for a {@link HollowHistory}.  Remap ordinal spaces for all prior historical
     * versions in the {@link HollowHistory} for consistency.
     *
     * @param version the version
     * @param previous the prior read state
     * @return the data access for a history
     */
    public HollowHistoricalStateDataAccess createConsistentOrdinalHistoricalStateFromDoubleSnapshot(long version, HollowReadStateEngine previous) {
        return new HollowHistoricalStateDataAccess(totalHistory, version, previous, IdentityOrdinalRemapper.INSTANCE, Collections.<String, HollowHistoricalSchemaChange>emptyMap());
    }

    /**
     * Create a {@link HollowDataAccess} for a historical state after a double snapshot occurs, without a {@link HollowHistory}.
     *
     * @param version the version
     * @param previous the previous state
     * @param current the current state
     * @param ordinalRemapper the ordinal remapper
     * @return the data access for a history
     */
    public HollowHistoricalStateDataAccess createHistoricalStateFromDoubleSnapshot(long version, HollowReadStateEngine previous, HollowReadStateEngine current, DiffEqualityMappingOrdinalRemapper ordinalRemapper) {
        HollowWriteStateEngine writeEngine = HollowWriteStateCreator.createWithSchemas(schemasWithoutKeys(previous.getSchemas()));
        IntMapOrdinalRemapper typeRemovedOrdinalLookupMaps = new IntMapOrdinalRemapper();

        for(HollowSchema previousSchema : HollowSchemaSorter.dependencyOrderedSchemaList(previous)) {
            HollowTypeReadState previousTypeState = previous.getTypeState(previousSchema.getName());
            String typeName = previousTypeState.getSchema().getName();
            IntMap ordinalLookupMap;

            if(current.getTypeState(typeName) == null) {
                ordinalLookupMap = copyAllRecords(previousTypeState, ordinalRemapper, writeEngine);
            } else {
                HollowTypeReadState currentTypeState = current.getTypeState(typeName);
                BitSet currentlyPopulatedOrdinals = currentTypeState.getListener(PopulatedOrdinalListener.class).getPopulatedOrdinals();
                ordinalLookupMap = copyUnmatchedRecords(previousTypeState, ordinalRemapper, currentlyPopulatedOrdinals, writeEngine);
            }
            typeRemovedOrdinalLookupMaps.addOrdinalRemapping(typeName, ordinalLookupMap);
        }

        Map<String, HollowHistoricalSchemaChange> schemaChanges = calculateSchemaChanges(previous, current, ordinalRemapper.getDiffEqualityMapping());

        return new HollowHistoricalStateDataAccess(totalHistory, version, roundTripStateEngine(writeEngine), typeRemovedOrdinalLookupMaps, schemaChanges);
    }

    private Map<String, HollowHistoricalSchemaChange> calculateSchemaChanges(HollowReadStateEngine previous, HollowReadStateEngine current, DiffEqualityMapping equalityMapping) {
        Map<String, HollowHistoricalSchemaChange> schemaChanges = new HashMap<String, HollowHistoricalSchemaChange>();
        for(HollowTypeReadState previousTypeState : previous.getTypeStates()) {
            String typeName = previousTypeState.getSchema().getName();
            HollowTypeReadState currentTypeState = current.getTypeState(typeName);
            if(currentTypeState == null) {
                schemaChanges.put(typeName, new HollowHistoricalSchemaChange(previousTypeState.getSchema(), null));
            } else if (equalityMapping.requiresMissingFieldTraversal(typeName)) {
                schemaChanges.put(typeName, new HollowHistoricalSchemaChange(previousTypeState.getSchema(), currentTypeState.getSchema()));
            }
        }

        for(HollowTypeReadState currentTypeState : current.getTypeStates()) {
            String typeName = currentTypeState.getSchema().getName();
            HollowTypeReadState previousTypeState = previous.getTypeState(typeName);
            if(previousTypeState == null) {
                schemaChanges.put(typeName, new HollowHistoricalSchemaChange(null, currentTypeState.getSchema()));
            }
        }

        return schemaChanges;
    }

    private IntMap copyUnmatchedRecords(HollowTypeReadState previousTypeState, DiffEqualityMappingOrdinalRemapper ordinalRemapper, BitSet currentlyPopulatedOrdinals, HollowWriteStateEngine writeEngine) {
        String typeName = previousTypeState.getSchema().getName();
        PopulatedOrdinalListener previousListener = previousTypeState.getListener(PopulatedOrdinalListener.class);
        HollowRecordCopier recordCopier = HollowRecordCopier.createCopier(previousTypeState, ordinalRemapper, false);  ///NOTE: This will invalidate custom hash codes
        DiffEqualOrdinalMap equalityMap = ordinalRemapper.getDiffEqualityMapping().getEqualOrdinalMap(typeName);

        boolean shouldCopyAllRecords = ordinalRemapper.getDiffEqualityMapping().requiresMissingFieldTraversal(typeName);

        BitSet previouslyPopulatedOrdinals = previousListener.getPopulatedOrdinals();

        int ordinalSpaceLength = Math.max(currentlyPopulatedOrdinals.length(), previouslyPopulatedOrdinals.length());
        int unmatchedOrdinalCount = ordinalSpaceLength - countMatchedRecords(previouslyPopulatedOrdinals, equalityMap);
        int unmatchedRecordCount = countUnmatchedRecords(previouslyPopulatedOrdinals, equalityMap);
        int nextFreeOrdinal = ordinalSpaceLength;

        ordinalRemapper.hintUnmatchedOrdinalCount(typeName, unmatchedOrdinalCount * 2);
        IntMap ordinalLookupMap = new IntMap(shouldCopyAllRecords ? previouslyPopulatedOrdinals.cardinality() : unmatchedRecordCount);

        BitSet mappedFromOrdinals = new BitSet(ordinalSpaceLength);
        BitSet mappedToOrdinals = new BitSet(ordinalSpaceLength);

        int fromOrdinal = previouslyPopulatedOrdinals.nextSetBit(0);
        while(fromOrdinal != -1) {
            int matchedToOrdinal = equalityMap.getIdentityFromOrdinal(fromOrdinal);
            if(matchedToOrdinal != -1) {
                mappedFromOrdinals.set(fromOrdinal);
                mappedToOrdinals.set(matchedToOrdinal);

                if(shouldCopyAllRecords) {
                    HollowWriteRecord rec = recordCopier.copy(fromOrdinal);
                    int removedMappedOrdinal = writeEngine.add(typeName, rec);

                    ordinalLookupMap.put(matchedToOrdinal, removedMappedOrdinal);
                }
            }

            fromOrdinal = previouslyPopulatedOrdinals.nextSetBit(fromOrdinal + 1);
        }

        fromOrdinal = mappedFromOrdinals.nextClearBit(0);
        int toOrdinal = mappedToOrdinals.nextClearBit(0);
        while(fromOrdinal < ordinalSpaceLength) {
            ordinalRemapper.remapOrdinal(typeName, fromOrdinal, nextFreeOrdinal);
            ordinalRemapper.remapOrdinal(typeName, nextFreeOrdinal, toOrdinal);

            if(previouslyPopulatedOrdinals.get(fromOrdinal)) {
                HollowWriteRecord rec = recordCopier.copy(fromOrdinal);
                int removedMappedOrdinal = writeEngine.add(typeName, rec);

                ordinalLookupMap.put(nextFreeOrdinal, removedMappedOrdinal);
            }

            fromOrdinal = mappedFromOrdinals.nextClearBit(fromOrdinal + 1);
            toOrdinal = mappedToOrdinals.nextClearBit(toOrdinal + 1);
            nextFreeOrdinal++;
        }

        return ordinalLookupMap;
    }

    private int countMatchedRecords(BitSet populatedOrdinals, DiffEqualOrdinalMap equalityMap) {
        int matchedRecordCount = 0;
        int ordinal = populatedOrdinals.nextSetBit(0);
        while(ordinal != -1) {
            if(equalityMap.getIdentityFromOrdinal(ordinal) != -1)
                matchedRecordCount++;
            ordinal = populatedOrdinals.nextSetBit(ordinal + 1);
        }
        return matchedRecordCount;
    }

    private int countUnmatchedRecords(BitSet populatedOrdinals, DiffEqualOrdinalMap equalityMap) {
        int unmatchedRecordCount = 0;
        int ordinal = populatedOrdinals.nextSetBit(0);
        while(ordinal != -1) {
            if(equalityMap.getIdentityFromOrdinal(ordinal) == -1)
                unmatchedRecordCount++;
            ordinal = populatedOrdinals.nextSetBit(ordinal + 1);
        }
        return unmatchedRecordCount;
    }

    private IntMap copyAllRecords(HollowTypeReadState typeState, DiffEqualityMappingOrdinalRemapper ordinalRemapper, HollowWriteStateEngine writeEngine) {
        String typeName = typeState.getSchema().getName();
        HollowRecordCopier recordCopier = HollowRecordCopier.createCopier(typeState, ordinalRemapper, false);  ///NOTE: This will invalidate custom hash codes
        PopulatedOrdinalListener listener = typeState.getListener(PopulatedOrdinalListener.class);
        IntMap ordinalLookupMap = new IntMap(listener.getPopulatedOrdinals().cardinality());
        int ordinal = listener.getPopulatedOrdinals().nextSetBit(0);
        while(ordinal != -1) {
            HollowWriteRecord rec = recordCopier.copy(ordinal);
            int mappedOrdinal = writeEngine.add(typeName, rec);
            ordinalLookupMap.put(ordinal, mappedOrdinal);
            ordinal = listener.getPopulatedOrdinals().nextSetBit(ordinal + 1);
        }
        return ordinalLookupMap;
    }

    public HollowHistoricalStateDataAccess copyButRemapOrdinals(HollowHistoricalStateDataAccess previous, OrdinalRemapper ordinalRemapper) {
        HollowWriteStateEngine writeEngine = HollowWriteStateCreator.createWithSchemas(schemasWithoutKeys(previous.getSchemas()));

        IntMapOrdinalRemapper typeRemovedOrdinalRemapping = new IntMapOrdinalRemapper();

        for(String typeName : previous.getAllTypes()) {
            HollowHistoricalTypeDataAccess typeDataAccess = (HollowHistoricalTypeDataAccess) previous.getTypeDataAccess(typeName);
            copyRemappedRecords(typeDataAccess.getRemovedRecords(), ordinalRemapper, writeEngine);

            IntMap ordinalLookupMap = remapPreviousOrdinalMapping(typeDataAccess.getOrdinalRemap(), typeName, ordinalRemapper);
            typeRemovedOrdinalRemapping.addOrdinalRemapping(typeName, ordinalLookupMap);
        }

        return new HollowHistoricalStateDataAccess(totalHistory, previous.getVersion(), roundTripStateEngine(writeEngine), typeRemovedOrdinalRemapping, previous.getSchemaChanges());
    }

    private void copyRemappedRecords(HollowTypeReadState readTypeState, OrdinalRemapper ordinalRemapper, HollowWriteStateEngine writeEngine) {
        String typeName = readTypeState.getSchema().getName();
        HollowTypeWriteState typeState = writeEngine.getTypeState(typeName);
        HollowRecordCopier copier = HollowRecordCopier.createCopier(readTypeState, ordinalRemapper, false);  ///NOTE: This will invalidate custom hash codes

        for(int i=0;i<=readTypeState.maxOrdinal();i++) {
            HollowWriteRecord rec = copier.copy(i);
            typeState.add(rec);
        }
    }

    private IntMap remapPreviousOrdinalMapping(IntMap previousOrdinalMapping, String typeName, OrdinalRemapper ordinalRemapper) {
        IntMapEntryIterator ordinalMappingIter = previousOrdinalMapping.iterator();
        IntMap ordinalLookupMap = new IntMap(previousOrdinalMapping.size());

        while(ordinalMappingIter.next())
            ordinalLookupMap.put(ordinalRemapper.getMappedOrdinal(typeName, ordinalMappingIter.getKey()), ordinalMappingIter.getValue());
        return ordinalLookupMap;
    }

    private static HollowReadStateEngine roundTripStateEngine(HollowWriteStateEngine writeEngine) {
        HollowBlobWriter writer = new HollowBlobWriter(writeEngine);
        HollowReadStateEngine removedRecordCopies = new HollowReadStateEngine();
        HollowBlobReader reader = new HollowBlobReader(removedRecordCopies);

        // Use a pipe to write and read concurrently to avoid writing
        // to temporary files or allocating memory
        // @@@ for small states it's more efficient to sequentially write to
        // and read from a byte array but it is tricky to estimate the size
        SimultaneousExecutor executor = new SimultaneousExecutor(1, HollowHistoricalStateCreator.class, "round-trip");
        Exception pipeException = null;
        // Ensure read-side is closed after completion of read
        try (PipedInputStream in = new PipedInputStream(1 << 15)) {
            BufferedOutputStream out = new BufferedOutputStream(new PipedOutputStream(in));
            executor.execute(() -> {
                // Ensure write-side is closed after completion of write
                try (Closeable ac = out) {
                    writer.writeSnapshot(out);
                } catch (IOException e) {
                    throw new RuntimeException(e);
                }
            });

            reader.readSnapshot(new BufferedInputStream(in));
        } catch (Exception e) {
            pipeException = e;
        }

        // Ensure no underlying writer exception is lost due to broken pipe
        try {
            executor.awaitSuccessfulCompletion();
        } catch (InterruptedException | ExecutionException e) {
            if (pipeException == null) {
                throw new RuntimeException(e);
            }

            pipeException.addSuppressed(e);
        }
        if (pipeException != null)
            throw new RuntimeException(pipeException);

        return removedRecordCopies;
    }

    private List<HollowSchema> schemasWithoutKeys(List<HollowSchema> schemas) {
        List<HollowSchema> baldSchemas = new ArrayList<HollowSchema>();
        for(HollowSchema prevSchema : schemas)
            baldSchemas.add(HollowSchema.withoutKeys(prevSchema));
        return baldSchemas;
    }

}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.api.producer;

import static com.netflix.hollow.api.producer.HollowIncrementalCyclePopulator.AddIfAbsent;
import static com.netflix.hollow.api.producer.HollowIncrementalCyclePopulator.DELETE_RECORD;

import com.netflix.hollow.api.consumer.HollowConsumer;
import com.netflix.hollow.api.consumer.HollowConsumer.BlobRetriever;
import com.netflix.hollow.api.consumer.fs.HollowFilesystemAnnouncementWatcher;
import com.netflix.hollow.core.util.SimultaneousExecutor;
import com.netflix.hollow.core.write.objectmapper.RecordPrimaryKey;
import com.netflix.hollow.core.write.objectmapper.flatrecords.FlatRecord;

import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

/**
 * Warning: This is a BETA API and is subject to breaking changes.
 * @deprecated see {@link HollowProducer.Incremental}
 * @see HollowProducer.Incremental
 */
@Deprecated
public class HollowIncrementalProducer {

    private static final long FAILED_VERSION = Long.MIN_VALUE;

    private final HollowProducer producer;
    private final ConcurrentHashMap<RecordPrimaryKey, Object> mutations;
    private final HollowProducer.Populator populator;
    private final ListenerSupport listeners;
    private final Map<String, Object> cycleMetadata;
    private final Class<?>[] dataModel;
    private final HollowConsumer.AnnouncementWatcher announcementWatcher;
    private final HollowConsumer.BlobRetriever blobRetriever;
    private final double threadsPerCpu;
    private long lastSucessfulCycle;

    public HollowIncrementalProducer(HollowProducer producer) {
        this(producer, 1.0d, null, null, new ArrayList<IncrementalCycleListener>());
    }

    //For backwards compatible. TODO: @Deprecated ??
    public HollowIncrementalProducer(HollowProducer producer, double threadsPerCpu) {
        this(producer, threadsPerCpu, null, null, new ArrayList<IncrementalCycleListener>());
    }

    protected HollowIncrementalProducer(HollowProducer producer, double threadsPerCpu, HollowConsumer.AnnouncementWatcher announcementWatcher, HollowConsumer.BlobRetriever blobRetriever, List<IncrementalCycleListener> listeners, Class<?>... classes) {
        this.producer = producer;
        this.mutations = new ConcurrentHashMap<RecordPrimaryKey, Object>();
        this.populator = new HollowIncrementalCyclePopulator(mutations, threadsPerCpu);
        this.dataModel = classes;
        this.announcementWatcher = announcementWatcher;
        this.blobRetriever = blobRetriever;
        this.listeners = new ListenerSupport();
        this.cycleMetadata = new HashMap<String, Object>();
        this.threadsPerCpu = threadsPerCpu;

        for (IncrementalCycleListener listener : listeners)
            this.listeners.add(listener);
    }

    /**
     * Initializes the data model and restores from existing state.
     */
    public void restoreFromLastState() {
        producer.initializeDataModel(dataModel);
        long latestAnnouncedVersion = announcementWatcher.getLatestVersion();
        if (latestAnnouncedVersion == HollowFilesystemAnnouncementWatcher.NO_ANNOUNCEMENT_AVAILABLE || latestAnnouncedVersion < 0) {
            return;
        }

        restore(latestAnnouncedVersion, blobRetriever);
    }

    public void restore(long versionDesired, BlobRetriever blobRetriever) {
        producer.hardRestore(versionDesired, blobRetriever);
    }

    public void addOrModify(Object obj) {
        RecordPrimaryKey pk = extractRecordPrimaryKey(obj);
        mutations.put(pk, obj);
    }

    public void addIfAbsent(Object obj) {
        RecordPrimaryKey pk = extractRecordPrimaryKey(obj);
        mutations.putIfAbsent(pk, new AddIfAbsent(obj));
    }

    public void addOrModify(Collection<Object> objList) {
        for(Object obj : objList) {
            addOrModify(obj);
        }
    }

    public void addOrModify(FlatRecord flatRecord) {
        RecordPrimaryKey pk = flatRecord.getRecordPrimaryKey();
        mutations.put(pk, flatRecord);
    }

    public void addIfAbsent(FlatRecord flatRecord) {
        RecordPrimaryKey pk = flatRecord.getRecordPrimaryKey();
        mutations.putIfAbsent(pk, new AddIfAbsent(flatRecord));
    }

    public void addOrModifyInParallel(Collection<Object> objList) {
        executeInParallel(objList, "add-or-modify", this::addOrModify);
    }

    public void delete(Object obj) {
        RecordPrimaryKey pk = extractRecordPrimaryKey(obj);
        delete(pk);
    }

    public void delete(Collection<Object> objList) {
        for(Object obj : objList) {
            delete(obj);
        }
    }

    public void deleteInParallel(Collection<Object> objList) {
        executeInParallel(objList, "delete", this::delete);
    }

    public void discard(Object obj) {
        RecordPrimaryKey pk = extractRecordPrimaryKey(obj);
        discard(pk);
    }

    public void discard(Collection<Object> objList) {
        for(Object obj : objList) {
            discard(obj);
        }
    }

    public void discardInParallel(Collection<Object> objList) {
        executeInParallel(objList, "discard", this::discard);
    }

    public void delete(RecordPrimaryKey key) {
        mutations.put(key, DELETE_RECORD);
    }

    public void discard(RecordPrimaryKey key) {
        mutations.remove(key);
    }

    public void clearChanges() {
        this.mutations.clear();
    }

    public boolean hasChanges() {
        return this.mutations.size() > 0;
    }

    public void addCycleMetadata(String key, Object value) {
        this.cycleMetadata.put(key, value);
    }

    public void addAllCycleMetadata(Map<String, Object> metadata) {
        this.cycleMetadata.putAll(metadata);
    }

    public void removeFromCycleMetadata(String key) {
        this.cycleMetadata.remove(key);
    }

    public void clearCycleMetadata() {
        this.cycleMetadata.clear();
    }

    public boolean hasMetadata() {
        return !this.cycleMetadata.isEmpty();
    }

    public void addListener(IncrementalCycleListener listener) {
        this.listeners.add(listener);
    }

    public void removeListener(IncrementalCycleListener listener) {
        this.listeners.remove(listener);
    }

    /**
     * Runs a Hollow Cycle, if successful, cleans the mutations map.
     *
     * @return the version of the cycle if successful, otherwise the {@link #FAILED_VERSION}
     * @since 2.9.9
     */
    public long runCycle() {
        long recordsRemoved = countRecordsToRemove();
        long recordsAddedOrModified = this.mutations.values().size() - recordsRemoved;
        try {
            long version = producer.runCycle(populator);
            if(version == lastSucessfulCycle) {
                return version;
            }
            listeners.fireIncrementalCycleComplete(version, recordsAddedOrModified, recordsRemoved, new HashMap<String, Object>(cycleMetadata));
            //Only clean changes when the version is new.
            clearChanges();
            lastSucessfulCycle = version;
            return version;
        } catch (Exception e) {
            listeners.fireIncrementalCycleFail(e, recordsAddedOrModified, recordsRemoved, new HashMap<String, Object>(cycleMetadata));
            return FAILED_VERSION;
        } finally {
            clearCycleMetadata();
        }
    }

    private long countRecordsToRemove() {
        long recordsToRemove = 0L;
        Collection<Object> records = mutations.values();
        for (Object record : records) {
            if (record == HollowIncrementalCyclePopulator.DELETE_RECORD) recordsToRemove++;
        }
        return recordsToRemove;
    }

    private RecordPrimaryKey extractRecordPrimaryKey(Object obj) {
        return producer.getObjectMapper().extractPrimaryKey(obj);
    }

    public static HollowIncrementalProducer.Builder withProducer(HollowProducer hollowProducer) {
        Builder builder = new Builder();
        return builder.withProducer(hollowProducer);
    }

    public static class Builder<B extends HollowIncrementalProducer.Builder<B>> {
        protected HollowProducer producer;
        protected double threadsPerCpu = 1.0d;
        protected HollowConsumer.AnnouncementWatcher announcementWatcher;
        protected HollowConsumer.BlobRetriever blobRetriever;
        protected Class<?>[] dataModel;
        protected List<IncrementalCycleListener> listeners = new ArrayList<IncrementalCycleListener>();

        public B withProducer(HollowProducer producer) {
            this.producer = producer;
            return (B) this;
        }

        public B withThreadsPerCpu(double threadsPerCpu) {
            this.threadsPerCpu = threadsPerCpu;
            return (B) this;
        }

        public B withAnnouncementWatcher(HollowConsumer.AnnouncementWatcher announcementWatcher) {
            this.announcementWatcher = announcementWatcher;
            return (B) this;
        }

        public B withBlobRetriever(HollowConsumer.BlobRetriever blobRetriever) {
            this.blobRetriever = blobRetriever;
            return (B) this;
        }

        public B withDataModel(Class<?>... classes) {
            this.dataModel = classes;
            return (B) this;
        }

        public B withListener(IncrementalCycleListener listener) {
            this.listeners.add(listener);
            return (B) this;
        }

        public B withListeners(IncrementalCycleListener... listeners) {
            for (IncrementalCycleListener listener : listeners)
                this.listeners.add(listener);
            return (B) this;
        }

        protected void checkArguments() {
            if (producer == null)
                throw new IllegalArgumentException("HollowProducer must be specified.");
        }

        public HollowIncrementalProducer build() {
            checkArguments();
            return new HollowIncrementalProducer(producer, threadsPerCpu, announcementWatcher, blobRetriever, listeners, dataModel);
        }
    }

    /**
     * Parallel execution. Modifies the mutation ConcurrentHashMap in parallel based on a Callback.
     * <p>
     * Note: This could be replaced with Java 8 parallelStream and lambadas instead of Callback interface
     * </p>
     * @param objList
     * @param callback
     */
    private void executeInParallel(Collection<Object> objList, String description, final Callback callback) {
        SimultaneousExecutor executor = new SimultaneousExecutor(threadsPerCpu, getClass(), description);
        for(final Object obj : objList) {
            executor.execute(() -> callback.call(obj));
        }

        try {
            executor.awaitSuccessfulCompletion();
        } catch(Exception e) {
            throw new RuntimeException(e);
        }
    }

    private interface Callback {
        void call(Object obj);
    }
}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.tools.combine;

import com.netflix.hollow.core.index.HollowPrimaryKeyIndex;
import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
import com.netflix.hollow.core.read.engine.HollowTypeReadState;
import com.netflix.hollow.tools.traverse.TransitiveSetTraverser;
import java.util.BitSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

/**
 * Specifies a set of exclusions for a {@link HollowCombiner}'s operation over one or more inputs.
 *

 * Exclusions are specified based on record primary keys.
 *

 * This is likely the most useful implementation of a {@link HollowCombinerCopyDirector}.
 *
 *
 * @author dkoszewnik
 *
 */
public class HollowCombinerExcludePrimaryKeysCopyDirector implements HollowCombinerCopyDirector {

    private final HollowCombinerCopyDirector baseDirector;
    private final Map<HollowTypeReadState, BitSet> excludedOrdinals;

    public HollowCombinerExcludePrimaryKeysCopyDirector() {
        this(HollowCombinerCopyDirector.DEFAULT_DIRECTOR);
    }

    /**
     * @param baseDirector if primary keys are not matched, delegate to the provided director for the answer to {@link #shouldCopy(HollowTypeReadState, int) }
     */
    public HollowCombinerExcludePrimaryKeysCopyDirector(HollowCombinerCopyDirector baseDirector) {
        this.excludedOrdinals = new HashMap<HollowTypeReadState, BitSet>();
        this.baseDirector = baseDirector;
    }

    /**
     * Exclude the record which matches the specified key.
     *
     * @param idx the index in which to query for the key
     * @param key the key
     */
    public void excludeKey(HollowPrimaryKeyIndex idx, Object... key) {
        int excludeOrdinal = idx.getMatchingOrdinal(key);

        if(excludeOrdinal >= 0) {
            BitSet excludedOrdinals = this.excludedOrdinals.get(idx.getTypeState());

            if(excludedOrdinals == null) {
                excludedOrdinals = new BitSet(idx.getTypeState().maxOrdinal()+1);
                this.excludedOrdinals.put(idx.getTypeState(), excludedOrdinals);
            }

            excludedOrdinals.set(excludeOrdinal);
        }
    }

    /**
     * Exclude any objects which are referenced by excluded objects.
     */
    public void excludeReferencedObjects() {
        Set<HollowReadStateEngine> stateEngines = new HashSet<HollowReadStateEngine>();
        for(Map.Entry<HollowTypeReadState, BitSet> entry : excludedOrdinals.entrySet())
            stateEngines.add(entry.getKey().getStateEngine());

        for(HollowReadStateEngine stateEngine : stateEngines) {
            Map<String, BitSet> typeBitSetsForStateEngine = new HashMap<String, BitSet>();

            for(Map.Entry<HollowTypeReadState, BitSet> entry : excludedOrdinals.entrySet()) {
                if(entry.getKey().getStateEngine() == stateEngine) {
                    String type = entry.getKey().getSchema().getName();
                    typeBitSetsForStateEngine.put(type, BitSet.valueOf(entry.getValue().toLongArray()));
                }
            }

            TransitiveSetTraverser.addTransitiveMatches(stateEngine, typeBitSetsForStateEngine);

            for(Map.Entry<String, BitSet> entry : typeBitSetsForStateEngine.entrySet())
                excludedOrdinals.put(stateEngine.getTypeState(entry.getKey()), entry.getValue());
        }
    }

    @Override
    public boolean shouldCopy(HollowTypeReadState typeState, int ordinal) {
        BitSet bitSet = excludedOrdinals.get(typeState);
        if(bitSet != null && bitSet.get(ordinal))
            return false;
        return baseDirector.shouldCopy(typeState, ordinal);
    }

}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.write;

import com.netflix.hollow.core.HollowBlobHeader;
import com.netflix.hollow.core.memory.encoding.VarInt;
import com.netflix.hollow.core.schema.HollowSchema;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.Map;

public class HollowBlobHeaderWriter {
    /**
     * Write the header to the data output stream
     * @param header the blob header
     * @param dos the data output stream to write the blov header
     * @throws IOException if the blob header could not be written
     */
    public void writeHeader(HollowBlobHeader header, DataOutputStream dos) throws IOException {
        /// save 4 bytes to indicate FastBlob version header.  This will be changed to indicate backwards incompatibility.
        dos.writeInt(HollowBlobHeader.HOLLOW_BLOB_VERSION_HEADER);

        /// Write randomized tag data -- every state gets a random 64-bit tag.
        /// When attempting to apply a delta, the originating state's random 64-bit tag is compared against the current 64-bit tag.
        /// This prevents deltas from being applied to incorrect states.
        dos.writeLong(header.getOriginRandomizedTag());
        dos.writeLong(header.getDestinationRandomizedTag());

        /// write the schemas contained in this blob to the stream in the pre v2.2.0 backwards compatibility envelope
        ByteArrayOutputStream schemasStream = new ByteArrayOutputStream();
        VarInt.writeVInt(schemasStream, header.getSchemas().size());
        for(HollowSchema schema : header.getSchemas())
            schema.writeTo(schemasStream);
        byte[] schemasData = schemasStream.toByteArray();

        VarInt.writeVInt(dos, schemasData.length + 1); // plus one byte for new backwards compatibility envelope.
        dos.write(schemasData);

        ///backwards compatibility -- new data can be added here by first indicating number of bytes used, will be skipped by existing readers.
        VarInt.writeVInt(dos, 0);

        /// write the header tags -- intended to include input source data versions
        dos.writeShort(header.getHeaderTags().size());

        for (Map.Entry<String, String> headerTag : header.getHeaderTags().entrySet()) {
            dos.writeUTF(headerTag.getKey());
            dos.writeUTF(headerTag.getValue());
        }
    }

}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.read.engine.object;

import static com.netflix.hollow.core.HollowConstants.ORDINAL_NONE;

import com.netflix.hollow.core.memory.ByteData;
import com.netflix.hollow.core.memory.HollowUnsafeHandle;
import com.netflix.hollow.core.memory.encoding.HashCodes;
import com.netflix.hollow.core.memory.encoding.VarInt;
import com.netflix.hollow.core.memory.encoding.ZigZag;
import com.netflix.hollow.core.schema.HollowObjectSchema;
import com.netflix.hollow.core.schema.HollowSchema;
import com.netflix.hollow.core.write.HollowObjectWriteRecord;
import com.netflix.hollow.tools.checksum.HollowChecksum;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.Collections;
import java.util.List;

class HollowObjectTypeReadStateShard {

    private volatile HollowObjectTypeDataElements currentDataVolatile;

    private final HollowObjectSchema schema;

    HollowObjectTypeReadStateShard(HollowObjectSchema schema) {
        this.schema = schema;
    }

    public boolean isNull(int ordinal, int fieldIndex) {
        HollowObjectTypeDataElements currentData;
        long fixedLengthValue;

        do {
            currentData = this.currentDataVolatile;

            long bitOffset = fieldOffset(currentData, ordinal, fieldIndex);
            int numBitsForField = currentData.bitsPerField[fieldIndex];

            fixedLengthValue = numBitsForField <= 56 ?
                    currentData.fixedLengthData.getElementValue(bitOffset, numBitsForField)
                    : currentData.fixedLengthData.getLargeElementValue(bitOffset, numBitsForField);
        } while(readWasUnsafe(currentData));

        switch(schema.getFieldType(fieldIndex)) {
        case BYTES:
        case STRING:
            int numBits = currentData.bitsPerField[fieldIndex];
            return (fixedLengthValue & (1 << (numBits - 1))) != 0;
        case FLOAT:
            return (int)fixedLengthValue == HollowObjectWriteRecord.NULL_FLOAT_BITS;
        case DOUBLE:
            return fixedLengthValue == HollowObjectWriteRecord.NULL_DOUBLE_BITS;
        default:
            return fixedLengthValue == currentData.nullValueForField[fieldIndex];
        }
    }

    public int readOrdinal(int ordinal, int fieldIndex) {
        HollowObjectTypeDataElements currentData;
        long refOrdinal;

        do {
            currentData = this.currentDataVolatile;
            refOrdinal = readFixedLengthFieldValue(currentData, ordinal, fieldIndex);
        } while(readWasUnsafe(currentData));

        if(refOrdinal == currentData.nullValueForField[fieldIndex])
            return ORDINAL_NONE;
        return (int)refOrdinal;
    }

    public int readInt(int ordinal, int fieldIndex) {
        HollowObjectTypeDataElements currentData;
        long value;

        do {
            currentData = this.currentDataVolatile;
            value = readFixedLengthFieldValue(currentData, ordinal, fieldIndex);
        } while(readWasUnsafe(currentData));

        if(value == currentData.nullValueForField[fieldIndex])
            return Integer.MIN_VALUE;
        return ZigZag.decodeInt((int)value);
    }

    public float readFloat(int ordinal, int fieldIndex) {
        HollowObjectTypeDataElements currentData;
        int value;

        do {
            currentData = this.currentDataVolatile;
            value = (int)readFixedLengthFieldValue(currentData, ordinal, fieldIndex);
        } while(readWasUnsafe(currentData));

        if(value == HollowObjectWriteRecord.NULL_FLOAT_BITS)
            return Float.NaN;
        return Float.intBitsToFloat(value);
    }

    public double readDouble(int ordinal, int fieldIndex) {
        HollowObjectTypeDataElements currentData;
        long value;

        do {
            currentData = this.currentDataVolatile;
            long bitOffset = fieldOffset(currentData, ordinal, fieldIndex);
            value = currentData.fixedLengthData.getLargeElementValue(bitOffset, 64, -1L);
        } while(readWasUnsafe(currentData));

        if(value == HollowObjectWriteRecord.NULL_DOUBLE_BITS)
            return Double.NaN;
        return Double.longBitsToDouble(value);
    }

    public long readLong(int ordinal, int fieldIndex) {
        HollowObjectTypeDataElements currentData;
        long value;

        do {
            currentData = this.currentDataVolatile;
            long bitOffset = fieldOffset(currentData, ordinal, fieldIndex);
            int numBitsForField = currentData.bitsPerField[fieldIndex];
            value = currentData.fixedLengthData.getLargeElementValue(bitOffset, numBitsForField);
        } while(readWasUnsafe(currentData));

        if(value == currentData.nullValueForField[fieldIndex])
            return Long.MIN_VALUE;
        return ZigZag.decodeLong(value);
    }

    public Boolean readBoolean(int ordinal, int fieldIndex) {
        HollowObjectTypeDataElements currentData;
        long value;

        do {
            currentData = this.currentDataVolatile;
            value = readFixedLengthFieldValue(currentData, ordinal, fieldIndex);
        } while(readWasUnsafe(currentData));

        if(value == currentData.nullValueForField[fieldIndex])
            return null;
        return value == 1 ? Boolean.TRUE : Boolean.FALSE;
    }

    private long readFixedLengthFieldValue(HollowObjectTypeDataElements currentData, int ordinal, int fieldIndex) {
        long bitOffset = fieldOffset(currentData, ordinal, fieldIndex);
        int numBitsForField = currentData.bitsPerField[fieldIndex];

        long value = currentData.fixedLengthData.getElementValue(bitOffset, numBitsForField);

        return value;
    }

    public byte[] readBytes(int ordinal, int fieldIndex) {
        HollowObjectTypeDataElements currentData;
        byte[] result;

        do {
            int numBitsForField;
            long endByte;
            long startByte;

            do {
                currentData = this.currentDataVolatile;

                numBitsForField = currentData.bitsPerField[fieldIndex];
                long currentBitOffset = fieldOffset(currentData, ordinal, fieldIndex);
                endByte = currentData.fixedLengthData.getElementValue(currentBitOffset, numBitsForField);
                startByte = ordinal != 0 ? currentData.fixedLengthData.getElementValue(currentBitOffset - currentData.bitsPerRecord, numBitsForField) : 0;
            } while(readWasUnsafe(currentData));

            if((endByte & (1L << numBitsForField - 1)) != 0)
                return null;

            startByte &= (1L << numBitsForField - 1) - 1;

            int length = (int)(endByte - startByte);
            result = new byte[length];
            for(int i=0;i<length;i++)
                result[i] = currentData.varLengthData[fieldIndex].get(startByte + i);

        } while(readWasUnsafe(currentData));

        return result;
    }

    public String readString(int ordinal, int fieldIndex) {
        HollowObjectTypeDataElements currentData;
        String result;

        do {
            int numBitsForField;
            long endByte;
            long startByte;

            do {
                currentData = this.currentDataVolatile;

                numBitsForField = currentData.bitsPerField[fieldIndex];
                long currentBitOffset = fieldOffset(currentData, ordinal, fieldIndex);
                endByte = currentData.fixedLengthData.getElementValue(currentBitOffset, numBitsForField);
                startByte = ordinal != 0 ? currentData.fixedLengthData.getElementValue(currentBitOffset - currentData.bitsPerRecord, numBitsForField) : 0;
            } while(readWasUnsafe(currentData));

            if((endByte & (1L << numBitsForField - 1)) != 0)
                return null;

            startByte &= (1L << numBitsForField - 1) - 1;

            int length = (int)(endByte - startByte);

            result = readString(currentData.varLengthData[fieldIndex], startByte, length);
        } while(readWasUnsafe(currentData));

        return result;
    }

    public boolean isStringFieldEqual(int ordinal, int fieldIndex, String testValue) {
        HollowObjectTypeDataElements currentData;
        boolean result;

        do {
            int numBitsForField;
            long endByte;
            long startByte;

            do {
                currentData = this.currentDataVolatile;

                numBitsForField = currentData.bitsPerField[fieldIndex];

                long currentBitOffset = fieldOffset(currentData, ordinal, fieldIndex);
                endByte = currentData.fixedLengthData.getElementValue(currentBitOffset, numBitsForField);
                startByte = ordinal != 0 ? currentData.fixedLengthData.getElementValue(currentBitOffset - currentData.bitsPerRecord, numBitsForField) : 0;
            } while(readWasUnsafe(currentData));

            if((endByte & (1L << numBitsForField - 1)) != 0)
                return testValue == null;
            if(testValue == null)
                return false;

            startByte &= (1L << numBitsForField - 1) - 1;

            int length = (int)(endByte - startByte);

            result = testStringEquality(currentData.varLengthData[fieldIndex], startByte, length, testValue);
        } while(readWasUnsafe(currentData));

        return result;
    }

    public int findVarLengthFieldHashCode(int ordinal, int fieldIndex) {
        HollowObjectTypeDataElements currentData;
        int hashCode;
        do {
            int numBitsForField;
            long endByte;
            long startByte;

            do {
                currentData = this.currentDataVolatile;

                numBitsForField = currentData.bitsPerField[fieldIndex];
                long currentBitOffset = fieldOffset(currentData, ordinal, fieldIndex);
                endByte = currentData.fixedLengthData.getElementValue(currentBitOffset, numBitsForField);
                startByte = ordinal != 0 ? currentData.fixedLengthData.getElementValue(currentBitOffset - currentData.bitsPerRecord, numBitsForField) : 0;
            } while(readWasUnsafe(currentData));

            if((endByte & (1L << numBitsForField - 1)) != 0)
                return -1;

            startByte &= (1L << numBitsForField - 1) - 1;

            int length = (int)(endByte - startByte);

            hashCode = HashCodes.hashCode(currentData.varLengthData[fieldIndex], startByte, length);
        } while(readWasUnsafe(currentData));

        return hashCode;
    }

    /**
     * Warning:  Not thread-safe.  Should only be called within the update thread.
     */
    public int bitsRequiredForField(String fieldName) {
        int fieldIndex = schema.getPosition(fieldName);
        return fieldIndex == -1 ? 0 : currentDataVolatile.bitsPerField[fieldIndex];
    }

    private long fieldOffset(HollowObjectTypeDataElements currentData, int ordinal, int fieldIndex) {
        return ((long)currentData.bitsPerRecord * ordinal) + currentData.bitOffsetPerField[fieldIndex];
    }

    /**
     * Decode a String as a series of VarInts, one per character.<p>
     *
     * @param str
     * @param out
     * @return
     */
    private static final ThreadLocal<char[]> chararr = new ThreadLocal<char[]>();

    private String readString(ByteData data, long position, int length) {
        long endPosition = position + length;

        char chararr[] = getCharArray();

        if(length > chararr.length)
            chararr = new char[length];

        int count = 0;

        while(position < endPosition) {
            int c = VarInt.readVInt(data, position);
            chararr[count++] = (char)c;
            position += VarInt.sizeOfVInt(c);
        }

        // The number of chars may be fewer than the number of bytes in the serialized data
        return new String(chararr, 0, count);
    }

    private boolean testStringEquality(ByteData data, long position, int length, String testValue) {
        if(length < testValue.length()) // can't check exact length here; the length argument is in bytes, which is equal to or greater than the number of characters.
            return false;

        long endPosition = position + length;

        int count = 0;

        while(position < endPosition && count < testValue.length()) {
            int c = VarInt.readVInt(data, position);
            if(testValue.charAt(count++) != (char)c)
                return false;
            position += VarInt.sizeOfVInt(c);
        }

        // The number of chars may be fewer than the number of bytes in the serialized data
        return position == endPosition && count == testValue.length();
    }

    private char[] getCharArray() {
        char ch[] = chararr.get();
        if(ch == null) {
            ch = new char[100];
            chararr.set(ch);
        }
        return ch;
    }

    void invalidate() {
        setCurrentData(null);
    }

    HollowObjectTypeDataElements currentDataElements() {
        return currentDataVolatile;
    }

    private boolean readWasUnsafe(HollowObjectTypeDataElements data) {
        // Use a load (acquire) fence to constrain the compiler reordering prior plain loads so
        // that they cannot "float down" below the volatile load of currentDataVolatile.
        // This ensures data is checked against currentData *after* optimistic calculations
        // have been performed on data.
        //
        // Note: the Java Memory Model allows for the reordering of plain loads and stores
        // before a volatile load (those plain loads and stores can "float down" below the
        // volatile load), but forbids the reordering of plain loads after a volatile load
        // (those plain loads are not allowed to "float above" the volatile load).
        // Similar reordering also applies to plain loads and stores and volatile stores.
        // In effect the ordering of volatile loads and stores is retained and plain loads
        // and stores can be shuffled around and grouped together, which increases
        // optimization opportunities.
        // This is why locks can be coarsened; plain loads and stores may enter the lock region
        // from above (float down the acquire) or below (float above the release) but existing
        // loads and stores may not exit (a "lock roach motel" and why there is almost universal
        // misunderstanding of, and many misguided attempts to optimize, the infamous double
        // checked locking idiom).
        //
        // Note: the fence provides stronger ordering guarantees than a corresponding non-plain
        // load or store since the former affects all prior or subsequent loads and stores,
        // whereas the latter is scoped to the particular load or store.
        //
        // For more details see http://gee.cs.oswego.edu/dl/html/j9mm.html
        HollowUnsafeHandle.getUnsafe().loadFence();
        return data != currentDataVolatile;
    }

    void setCurrentData(HollowObjectTypeDataElements data) {
        this.currentDataVolatile = data;
    }

    protected void applyToChecksum(HollowChecksum checksum, HollowSchema withSchema, BitSet populatedOrdinals, int shardNumber, int numShards) {
        if(!(withSchema instanceof HollowObjectSchema))
            throw new IllegalArgumentException("HollowObjectTypeReadState can only calculate checksum with a HollowObjectSchema: " + schema.getName());

        HollowObjectSchema commonSchema = schema.findCommonSchema((HollowObjectSchema)withSchema);

        List<String> commonFieldNames = new ArrayList<String>();
        for(int i=0;i<commonSchema.numFields();i++)
            commonFieldNames.add(commonSchema.getFieldName(i));
        Collections.sort(commonFieldNames);

        int fieldIndexes[] = new int[commonFieldNames.size()];
        for(int i=0;i<commonFieldNames.size();i++) {
            fieldIndexes[i] = schema.getPosition(commonFieldNames.get(i));
        }

        HollowObjectTypeDataElements currentData = currentDataVolatile;
        int ordinal = populatedOrdinals.nextSetBit(0);
        while(ordinal != ORDINAL_NONE) {
            if((ordinal & (numShards - 1)) == shardNumber) {
                int shardOrdinal = ordinal / numShards;
                checksum.applyInt(ordinal);
                for(int i=0;i<fieldIndexes.length;i++) {
                    int fieldIdx = fieldIndexes[i];
                    if(!schema.getFieldType(fieldIdx).isVariableLength()) {
                        long bitOffset = fieldOffset(currentData, shardOrdinal, fieldIdx);
                        int numBitsForField = currentData.bitsPerField[fieldIdx];
                        long fixedLengthValue = numBitsForField <= 56 ?
                                currentData.fixedLengthData.getElementValue(bitOffset, numBitsForField)
                                : currentData.fixedLengthData.getLargeElementValue(bitOffset, numBitsForField);

                        if(fixedLengthValue == currentData.nullValueForField[fieldIdx])
                            checksum.applyInt(Integer.MAX_VALUE);
                        else
                            checksum.applyLong(fixedLengthValue);
                    } else {
                        checksum.applyInt(findVarLengthFieldHashCode(shardOrdinal, fieldIdx));
                    }
                }
            }

            ordinal = populatedOrdinals.nextSetBit(ordinal + 1);
        }
    }

    public long getApproximateHeapFootprintInBytes() {
        HollowObjectTypeDataElements currentData = currentDataVolatile;
        long bitsPerFixedLengthData = (long)currentData.bitsPerRecord * (currentData.maxOrdinal + 1);

        long requiredBytes = bitsPerFixedLengthData / 8;

        for(int i=0;i<currentData.varLengthData.length;i++) {
            if(currentData.varLengthData[i] != null)
                requiredBytes += currentData.varLengthData[i].size();
        }

        return requiredBytes;
    }

    public long getApproximateHoleCostInBytes(BitSet populatedOrdinals, int shardNumber, int numShards) {
        HollowObjectTypeDataElements currentData = currentDataVolatile;
        long holeBits = 0;

        int holeOrdinal = populatedOrdinals.nextClearBit(0);
        while(holeOrdinal <= currentData.maxOrdinal) {
            if((holeOrdinal & (numShards - 1)) == shardNumber)
                holeBits += currentData.bitsPerRecord;

            holeOrdinal = populatedOrdinals.nextClearBit(holeOrdinal + 1);
        }

        return holeBits / 8;
    }

}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.api.metrics;

import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
import com.netflix.hollow.core.read.engine.HollowTypeReadState;
import java.util.Collection;
import java.util.HashMap;

public abstract class HollowMetrics {

    private HashMap<String, Long> typeHeapFootprint = new HashMap<>();
    private HashMap<String, Integer> typePopulatedOrdinals = new HashMap<>();
    private long currentVersion;
    private long totalHeapFootprint = 0L;
    private int totalPopulatedOrdinals = 0;

    protected void update(long version) {
        setCurrentVersion(version);
    }

    protected void update(HollowReadStateEngine hollowReadStateEngine, long version) {
        setCurrentVersion(version);
        calculateTypeMetrics(hollowReadStateEngine);
    }

    /**
     * Calculates the memory heap footprint and populated ordinals per type and total
     * @param hollowReadStateEngine
     */
    void calculateTypeMetrics(HollowReadStateEngine hollowReadStateEngine) {
        Collection<HollowTypeReadState> typeStates = hollowReadStateEngine.getTypeStates();
        if (typeStates == null)
            return;

        totalHeapFootprint = 0L;
        totalPopulatedOrdinals = 0;
        for (HollowTypeReadState typeState : typeStates) {
            long heapCost = typeState.getApproximateHeapFootprintInBytes();
            totalHeapFootprint += heapCost;

            int populatedOrdinals = typeState.getPopulatedOrdinals().cardinality();
            totalPopulatedOrdinals += populatedOrdinals;
            String type = typeState.getSchema().getName();
            typeHeapFootprint.put(type, heapCost);
            typePopulatedOrdinals.put(type, populatedOrdinals);
        }
    }

    public HashMap<String, Long> getTypeHeapFootprint() {
        return typeHeapFootprint;
    }

    public HashMap<String, Integer> getTypePopulatedOrdinals() {
        return typePopulatedOrdinals;
    }

    public long getCurrentVersion() {
        return this.currentVersion;
    }

    public long getTotalHeapFootprint() {
        return this.totalHeapFootprint;
    }

    public long getTotalPopulatedOrdinals() {
        return this.totalPopulatedOrdinals;
    }

    public void setCurrentVersion(long version) { this.currentVersion = version; }
}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.tools.combine;

import com.netflix.hollow.core.index.HollowPrimaryKeyIndex;
import com.netflix.hollow.core.index.key.PrimaryKey;
import com.netflix.hollow.core.memory.ByteArrayOrdinalMap;
import com.netflix.hollow.core.memory.ByteDataBuffer;
import com.netflix.hollow.core.memory.pool.WastefulRecycler;
import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
import com.netflix.hollow.core.read.engine.HollowTypeReadState;
import com.netflix.hollow.core.read.engine.PopulatedOrdinalListener;
import com.netflix.hollow.core.schema.HollowMapSchema;
import com.netflix.hollow.core.schema.HollowObjectSchema;
import com.netflix.hollow.core.schema.HollowSchema;
import com.netflix.hollow.core.schema.HollowSchema.SchemaType;
import com.netflix.hollow.core.schema.HollowSchemaSorter;
import com.netflix.hollow.core.schema.HollowSetSchema;
import com.netflix.hollow.core.util.HollowWriteStateCreator;
import com.netflix.hollow.core.util.SimultaneousExecutor;
import com.netflix.hollow.core.write.HollowHashableWriteRecord;
import com.netflix.hollow.core.write.HollowHashableWriteRecord.HashBehavior;
import com.netflix.hollow.core.write.HollowTypeWriteState;
import com.netflix.hollow.core.write.HollowWriteRecord;
import com.netflix.hollow.core.write.HollowWriteStateEngine;
import com.netflix.hollow.core.write.copy.HollowRecordCopier;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;

/**
 * The HollowCombiner is used to copy data from one or more copies of hollow datasets (a {@link HollowReadStateEngine}) into a single hollow dataset (a {@link HollowWriteStateEngine}).
 *

 * By default, a HollowCombiner will copy all records from each provided {@link HollowReadStateEngine} into the destination.
 *

 * A {@link HollowCombinerCopyDirector} can be provided, which will specify which specific ordinals to include/exclude while copying.
 *

 * A set of {@link PrimaryKey} can be provided, which will ensure that no duplicate records, as defined by any of the provided keys, will be added to the destination state.
 *
 * @author dkoszewnik
 *
 */
public class HollowCombiner {

    private final HollowReadStateEngine[] inputs;
    private final OrdinalRemapper[] ordinalRemappers;

    private final HollowWriteStateEngine output;
    private final Set<String> typeNamesWithDefinedHashCodes;
    private final Set<String> ignoredTypes;

    private final HollowCombinerCopyDirector copyDirector;

    private List<PrimaryKey> primaryKeys;

    private final ThreadLocal<Map<String, HollowCombinerCopier>> copiersPerType;
    private final Map<String, ByteArrayOrdinalMap> hashOrderIndependentOrdinalMaps;

    /**
     * @param inputs the set of {@link HollowReadStateEngine} to combine data from.
     */
    public HollowCombiner(HollowReadStateEngine... inputs) {
        this(HollowWriteStateCreator.createWithSchemas(validateInputs(inputs)[0].getSchemas()), inputs);
    }

    static HollowReadStateEngine[] validateInputs(HollowReadStateEngine... inputs) {
        Objects.requireNonNull(inputs);
        if (inputs.length == 0) {
            throw new IllegalArgumentException("No input read state engines");
        }
        return inputs;
    }

    /**
     * @param director a {@link HollowCombinerCopyDirector} which will specify which specific records to copy from the input(s).
     * @param inputs the set of {@link HollowReadStateEngine} to combine data from.
     */
    public HollowCombiner(HollowCombinerCopyDirector director, HollowReadStateEngine... inputs) {
        this(director, HollowWriteStateCreator.createWithSchemas(inputs[0].getSchemas()), inputs);
    }

    /**
     * @param output the {@link HollowWriteStateEngine} to use as the destination.
     * @param inputs the set of {@link HollowReadStateEngine} to combine data from.
     */
    public HollowCombiner(HollowWriteStateEngine output, HollowReadStateEngine... inputs) {
        this(HollowCombinerCopyDirector.DEFAULT_DIRECTOR, output, inputs);
    }

    /**
     * @param copyDirector a {@link HollowCombinerCopyDirector} which will specify which specific records to copy from the input(s).
     * @param output the {@link HollowWriteStateEngine} to use as the destination.
     * @param inputs the set of {@link HollowReadStateEngine} to combine data from.
     */
    public HollowCombiner(HollowCombinerCopyDirector copyDirector, HollowWriteStateEngine output, HollowReadStateEngine... inputs) {
        Objects.requireNonNull(copyDirector);
        Objects.requireNonNull(output);

        this.inputs = validateInputs(inputs);
        this.output = output;

        this.typeNamesWithDefinedHashCodes = getAllTypesWithDefinedHashCodes();
        this.ordinalRemappers = new OrdinalRemapper[inputs.length];
        this.copiersPerType = new ThreadLocal<>();
        this.hashOrderIndependentOrdinalMaps = new HashMap<>();
        this.ignoredTypes = new HashSet<>();
        this.copyDirector = copyDirector;

        initializePrimaryKeys();
    }

    private Set<String> getAllTypesWithDefinedHashCodes() {
        Set<String> unionOfTypesWithDefinedHashCodes = new HashSet<>();
        for(HollowReadStateEngine input : inputs) {
            unionOfTypesWithDefinedHashCodes.addAll(input.getTypesWithDefinedHashCodes());
        }
        return unionOfTypesWithDefinedHashCodes;
    }

    /**
     * When provided, a set of {@link PrimaryKey} will ensure that no duplicate records are added to the destination state.
     *
     * If multiple records exist in the inputs matching a single value for any of the supplied primary keys, then only one such record
     * will be copied to the destination.  The specific record which is copied will be the record from the input which was supplied first in the constructor
     * of this HollowCombiner.
     *
     * Further, if any record references another record which was omitted because it would have been duplicate based on this rule, then that reference is
     * remapped in the destination state to the matching record which was chosen to be included.
     *
     * @param newKeys the new primary keys
     */
    public void setPrimaryKeys(PrimaryKey... newKeys) {
        Objects.requireNonNull(newKeys);
        if (newKeys.length == 0) {
            return;
        }

        if (inputs.length == 1) {
            return;
        }

        /// deduplicate new keys with existing keys
        //process existing ones first
        Map<String, PrimaryKey> keysByType = new HashMap<>();
        for (PrimaryKey primaryKey : primaryKeys) {
            keysByType.put(primaryKey.getType(), primaryKey);
        }

        // allow override
        for (PrimaryKey primaryKey : newKeys) {
            keysByType.put(primaryKey.getType(), primaryKey);
        }

        this.primaryKeys = sortPrimaryKeys(new ArrayList<>(keysByType.values()));
    }

    public List<PrimaryKey> getPrimaryKeys() {
        return this.primaryKeys;
    }

    private void initializePrimaryKeys() {
        if (inputs.length == 1) {
            this.primaryKeys = new ArrayList<>();
            return;
        }

        List<PrimaryKey> keys = new ArrayList<>();
        for (HollowSchema schema : output.getSchemas()) {
            if (schema.getSchemaType() == SchemaType.OBJECT && !ignoredTypes.contains(schema.getName())) {
                PrimaryKey pk = ((HollowObjectSchema) schema).getPrimaryKey();
                if (pk != null)
                    keys.add(pk);
            }
        }

        this.primaryKeys = sortPrimaryKeys(keys);
    }

    private List<PrimaryKey> sortPrimaryKeys(List<PrimaryKey> primaryKeys) {
        final List<HollowSchema> dependencyOrderedSchemas = HollowSchemaSorter.dependencyOrderedSchemaList(output.getSchemas());
        primaryKeys.sort(new Comparator<PrimaryKey>() {
            public int compare(PrimaryKey o1, PrimaryKey o2) {
                return schemaDependencyIdx(o1) - schemaDependencyIdx(o2);
            }

            private int schemaDependencyIdx(PrimaryKey key) {
                for (int i = 0; i < dependencyOrderedSchemas.size(); i++) {
                    if (dependencyOrderedSchemas.get(i).getName().equals(key.getType()))
                        return i;
                }
                throw new IllegalArgumentException("Primary key defined for non-existent type: " + key.getType());
            }
        });

        return primaryKeys;
    }

    /**
     * Specify a set of types not to copy.  Be careful: if any included types reference any of the specified types,
     * behavior is undefined.
     *
     * @param typeNames the type names to be ignored
     */
    public void addIgnoredTypes(String... typeNames) {
        for(String typeName : typeNames)
            ignoredTypes.add(typeName);
    }

    /**
     * Perform the combine operation.
     */
    public void combine() {
        SimultaneousExecutor executor = new SimultaneousExecutor(getClass(), "combine");
        final int numThreads = executor.getCorePoolSize();

        createOrdinalRemappers();
        createHashOrderIndependentOrdinalMaps();

        final Set<String> processedTypes = new HashSet<>();
        final Set<PrimaryKey> processedPrimaryKeys = new HashSet<>();
        final Set<PrimaryKey> selectedPrimaryKeys = new HashSet<>();

        while(processedTypes.size() < output.getOrderedTypeStates().size()){

            /// find the next primary keys
            for(PrimaryKey key : primaryKeys) {
                if (!processedPrimaryKeys.contains(key) && !ignoredTypes.contains(key.getType())) {
                    if(!isAnySelectedPrimaryKeyADependencyOf(key.getType(), selectedPrimaryKeys)) {
                        selectedPrimaryKeys.add(key);
                    }
                }
            }

            final Set<String> typesToProcessThisIteration = new HashSet<>();
            final Map<String, HollowPrimaryKeyIndex[]> primaryKeyIndexes = new HashMap<>();
            final HollowCombinerExcludePrimaryKeysCopyDirector primaryKeyCopyDirector = new HollowCombinerExcludePrimaryKeysCopyDirector(copyDirector);

            for(HollowSchema schema : output.getSchemas()) {
                if(!processedTypes.contains(schema.getName()) && !ignoredTypes.contains(schema.getName())) {
                    if(selectedPrimaryKeys.isEmpty() || isAnySelectedPrimaryKeyDependentOn(schema.getName(), selectedPrimaryKeys)) {
                        for(PrimaryKey pk : selectedPrimaryKeys) {
                            if(pk.getType().equals(schema.getName())) {
                                HollowPrimaryKeyIndex[] indexes = new HollowPrimaryKeyIndex[inputs.length];
                                for(int i=0;i<indexes.length;i++) {
                                    if(inputs[i].getTypeState(pk.getType()) != null)
                                        indexes[i] = new HollowPrimaryKeyIndex(inputs[i], pk);
                                }

                                for(int i=0;i<indexes.length;i++) {
                                    HollowTypeReadState typeState = inputs[i].getTypeState(pk.getType());
                                    if(typeState != null) {
                                        BitSet populatedOrdinals = typeState.getListener(PopulatedOrdinalListener.class).getPopulatedOrdinals();

                                        int ordinal = populatedOrdinals.nextSetBit(0);
                                        while(ordinal != -1) {
                                            if(primaryKeyCopyDirector.shouldCopy(typeState, ordinal)) {
                                                Object[] recordKey = indexes[i].getRecordKey(ordinal);

                                                for(int j=i+1;j<indexes.length;j++) {
                                                    primaryKeyCopyDirector.excludeKey(indexes[j], recordKey);
                                                }
                                            }

                                            ordinal = populatedOrdinals.nextSetBit(ordinal + 1);
                                        }
                                    }
                                }

                                primaryKeyIndexes.put(pk.getType(), indexes);
                            }
                        }

                        typesToProcessThisIteration.add(schema.getName());
                    }
                }
            }

            if(typesToProcessThisIteration.isEmpty())
                break;

            for(int i=0;i<numThreads;i++) {
                final int threadNumber = i;
                executor.execute(() -> {
                    for(int i1 =0; i1 <inputs.length; i1++) {
                        HollowCombinerCopyDirector copyDirector = selectedPrimaryKeys.isEmpty()
                                ? HollowCombiner.this.copyDirector
                                : primaryKeyCopyDirector;

                        HollowReadStateEngine inputEngine = inputs[i1];
                        OrdinalRemapper ordinalRemapper = selectedPrimaryKeys.isEmpty()
                                ? ordinalRemappers[i1]
                                : new HollowCombinerPrimaryKeyOrdinalRemapper(ordinalRemappers, primaryKeyIndexes, i1);

                        Map<String, HollowCombinerCopier> copierMap = new HashMap<>();
                        List<HollowCombinerCopier> copierList = new ArrayList<>();

                        for(String typeName : typesToProcessThisIteration) {
                            HollowTypeReadState readState = inputEngine.getTypeState(typeName);
                            HollowTypeWriteState writeState = output.getTypeState(typeName);
                            if (readState != null && writeState != null) {
                                HollowCombinerCopier copier = new HollowCombinerCopier(readState, writeState, ordinalRemapper);
                                copierList.add(copier);
                                copierMap.put(typeName, copier);
                            }
                        }

                        for(String typeName : processedTypes) {
                            HollowTypeReadState readState = inputEngine.getTypeState(typeName);
                            HollowTypeWriteState writeState = output.getTypeState(typeName);
                            if (readState != null && writeState != null) {
                                HollowCombinerCopier copier = new HollowCombinerCopier(readState, writeState, ordinalRemappers[i1]);
                                copierMap.put(typeName, copier);
                            }
                        }

                        copiersPerType.set(copierMap);

                        int currentOrdinal = threadNumber;

                        while(!copierList.isEmpty()) {
                            copyOrdinalForAllStates(currentOrdinal, copierList, ordinalRemapper, copyDirector);

                            currentOrdinal += numThreads;
                        }

                    }
                });
            }

            try {
                executor.awaitSuccessfulCompletionOfCurrentTasks();
            } catch(Throwable th) {
                throw new RuntimeException(th);
            }

            processedTypes.addAll(typesToProcessThisIteration);
            processedPrimaryKeys.addAll(selectedPrimaryKeys);
            selectedPrimaryKeys.clear();
        }

        executor.shutdown();
    }

    private boolean isAnySelectedPrimaryKeyADependencyOf(String type, Set<PrimaryKey> selectedPrimaryKeys) {
        for(PrimaryKey selectedKey : selectedPrimaryKeys) {
            if(HollowSchemaSorter.typeIsTransitivelyDependent(output, type, selectedKey.getType()))
                return true;
        }
        return false;
    }

    private boolean isAnySelectedPrimaryKeyDependentOn(String type, Set<PrimaryKey> selectedPrimaryKeys) {
        for(PrimaryKey selectedKey : selectedPrimaryKeys) {
            if(HollowSchemaSorter.typeIsTransitivelyDependent(output, selectedKey.getType(), type))
                return true;
        }
        return false;
    }

    /**
     * @return the destination {@link HollowWriteStateEngine}
     */
    public HollowWriteStateEngine getCombinedStateEngine() {
        return output;
    }

    void copyOrdinalForAllStates(int currentOrdinal, List<HollowCombinerCopier> copiers,
            OrdinalRemapper ordinalRemapper, HollowCombinerCopyDirector copyDirector) {
        Iterator<HollowCombinerCopier> iter = copiers.iterator();
        while(iter.hasNext()) {
            HollowCombinerCopier copier = iter.next();
            HollowTypeReadState readTypeState = copier.getReadTypeState();

            if(currentOrdinal <= readTypeState.maxOrdinal()) {
                if(copyDirector.shouldCopy(readTypeState, currentOrdinal))
                    copier.copy(currentOrdinal);
            } else {
                iter.remove();
            }
        }
    }

    int copyOrdinal(String typeName, int currentOrdinal) {
        HollowCombinerCopier hollowCombinerCopier = copiersPerType.get().get(typeName);
        return hollowCombinerCopier == null
                ? currentOrdinal
                : hollowCombinerCopier.copy(currentOrdinal);
    }

    private OrdinalRemapper[] createOrdinalRemappers() {
        for(int i=0;i<ordinalRemappers.length;i++)
            ordinalRemappers[i] = new HollowCombinerOrdinalRemapper(this, inputs[i]);

        return ordinalRemappers;
    }

    private void createHashOrderIndependentOrdinalMaps() {
        for(HollowSchema schema : output.getSchemas()) {
            if(isDefinedHashCode(schema)) {
                hashOrderIndependentOrdinalMaps.put(schema.getName(), new ByteArrayOrdinalMap());
            }
        }
    }

    private boolean isDefinedHashCode(HollowSchema schema) {
        if(schema instanceof HollowSetSchema)
            return typeNamesWithDefinedHashCodes.contains(((HollowSetSchema)schema).getElementType());
        if(schema instanceof HollowMapSchema)
            return typeNamesWithDefinedHashCodes.contains(((HollowMapSchema)schema).getKeyType());
        return false;
    }

    private class HollowCombinerCopier {
        private final HollowRecordCopier copier;
        private final BitSet populatedOrdinals;
        private final HollowTypeWriteState writeState;
        private final OrdinalRemapper ordinalRemapper;
        private final ByteArrayOrdinalMap hashOrderIndependentOrdinalMap;
        private final ByteDataBuffer scratch;

        HollowCombinerCopier(HollowTypeReadState readState, HollowTypeWriteState writeState, OrdinalRemapper ordinalRemapper) {
            this.copier = HollowRecordCopier.createCopier(readState, writeState.getSchema(), ordinalRemapper, isDefinedHashCode(readState.getSchema()));
            this.populatedOrdinals = readState.getListener(PopulatedOrdinalListener.class).getPopulatedOrdinals();
            this.writeState = writeState;
            this.ordinalRemapper = ordinalRemapper;
            this.hashOrderIndependentOrdinalMap = hashOrderIndependentOrdinalMaps.get(readState.getSchema().getName());
            this.scratch = hashOrderIndependentOrdinalMap != null ? new ByteDataBuffer(WastefulRecycler.SMALL_ARRAY_RECYCLER) : null;
        }

        int copy(int ordinal) {
            if(isOrdinalPopulated(ordinal)) {
                if(!ordinalRemapper.ordinalIsMapped(getType(), ordinal)) {
                    HollowWriteRecord rec = copier.copy(ordinal);

                    if(hashOrderIndependentOrdinalMap == null) {
                        int outputOrdinal = writeState.add(rec);
                        ordinalRemapper.remapOrdinal(getType(), ordinal, outputOrdinal);
                        return outputOrdinal;
                    } else {
                        scratch.reset();
                        ((HollowHashableWriteRecord)rec).writeDataTo(scratch, HashBehavior.IGNORED_HASHES);
                        int outputOrdinal = hashOrderIndependentOrdinalMap.get(scratch);
                        if(outputOrdinal != -1)
                            return outputOrdinal;

                        synchronized(hashOrderIndependentOrdinalMap) {
                            outputOrdinal = hashOrderIndependentOrdinalMap.get(scratch);
                            if(outputOrdinal != -1)
                                return outputOrdinal;

                            outputOrdinal = writeState.add(rec);
                            ordinalRemapper.remapOrdinal(getType(), ordinal, outputOrdinal);
                            hashOrderIndependentOrdinalMap.put(scratch, outputOrdinal);
                        }
                    }
                }

                return ordinalRemapper.getMappedOrdinal(getType(), ordinal);
            }
            return -1;
        }

        boolean isOrdinalPopulated(int ordinal) {
            return populatedOrdinals.get(ordinal);
        }

        String getType() {
            return copier.getReadTypeState().getSchema().getName();
        }

        HollowTypeReadState getReadTypeState() {
            return copier.getReadTypeState();
        }
    }
}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.write;

import com.netflix.hollow.core.memory.ByteDataBuffer;
import com.netflix.hollow.core.memory.encoding.VarInt;
import com.netflix.hollow.core.memory.encoding.ZigZag;
import com.netflix.hollow.core.memory.pool.WastefulRecycler;
import com.netflix.hollow.core.schema.HollowObjectSchema;
import com.netflix.hollow.core.schema.HollowObjectSchema.FieldType;

public class HollowObjectWriteRecord implements HollowWriteRecord {

    private final HollowObjectSchema schema;

    private final ByteDataBuffer fieldData[];
    private final boolean isNonNull[];

    public HollowObjectWriteRecord(HollowObjectSchema schema) {
        this.schema = schema;
        this.fieldData = new ByteDataBuffer[schema.numFields()];
        this.isNonNull = new boolean[schema.numFields()];
        for (int i = 0; i < fieldData.length; i++) {
            fieldData[i] = new ByteDataBuffer(WastefulRecycler.SMALL_ARRAY_RECYCLER);
        }
    }

    public HollowObjectSchema getSchema() {
        return schema;
    }

    /**
     * Concatenates all fields, in order, to the ByteDataBuffer supplied.  This concatenation is the
     * verbatim serialized representation in the FastBlob.
     *
     * @param buf the data buffer to write data to
     */
    public void writeDataTo(ByteDataBuffer buf) {
        for (int i = 0; i < fieldData.length; i++) {
            writeField(buf, i);
        }
    }

    public void writeDataTo(ByteDataBuffer buf, HollowObjectSchema translate) {
        for(int i=0; i < translate.numFields(); i++) {
            int fieldIndex = schema.getPosition(translate.getFieldName(i));

            if(fieldIndex != -1) {
                writeField(buf, fieldIndex);
            } else {
                writeNull(buf, translate.getFieldType(i));
            }
        }
    }

    private void writeField(ByteDataBuffer buf, int fieldIndex) {
        if (isNonNull[fieldIndex]) {
            if (getSchema().getFieldType(fieldIndex).isVariableLength())
                VarInt.writeVInt(buf, (int)fieldData[fieldIndex].length());
            fieldData[fieldIndex].copyTo(buf);
        } else {
            writeNull(buf, schema.getFieldType(fieldIndex));
        }
    }

    /**
     * Reset the ByteDataBuffers for each field.
     */
    public void reset() {
        for (int i = 0; i < fieldData.length; i++) {
            isNonNull[i] = false;
        }
    }

    public void setNull(String fieldName) {
        int fieldIndex = getSchema().getPosition(fieldName);

        ByteDataBuffer fieldBuffer = getFieldBuffer(fieldIndex);
        FieldType fieldType = getSchema().getFieldType(fieldIndex);

        writeNull(fieldBuffer, fieldType);
    }

    public void setInt(String fieldName, int value) {
        if(value == Integer.MIN_VALUE) {
            setNull(fieldName);
        } else {
            int fieldIndex = getSchema().getPosition(fieldName);

            validateFieldType(fieldIndex, fieldName, FieldType.INT);

            ByteDataBuffer buf = getFieldBuffer(fieldIndex);

            // zig zag encoding
            VarInt.writeVInt(buf, ZigZag.encodeInt(value));
        }
    }

    public void setLong(String fieldName, long value) {
        if(value == Long.MIN_VALUE) {
            setNull(fieldName);
        } else {
            int fieldIndex = getSchema().getPosition(fieldName);

            validateFieldType(fieldIndex, fieldName, FieldType.LONG);

            ByteDataBuffer buf = getFieldBuffer(fieldIndex);

            // zig zag encoding
            VarInt.writeVLong(buf, ZigZag.encodeLong(value));
        }
    }

    public void setFloat(String fieldName, float value) {
        int fieldIndex = getSchema().getPosition(fieldName);

        validateFieldType(fieldIndex, fieldName, FieldType.FLOAT);

        ByteDataBuffer buf = getFieldBuffer(fieldIndex);

        int intBits = Float.floatToIntBits(value);
        writeFixedLengthInt(buf, intBits);
    }

    public void setDouble(String fieldName, double value) {
        int fieldIndex = getSchema().getPosition(fieldName);

        validateFieldType(fieldIndex, fieldName, FieldType.DOUBLE);

        ByteDataBuffer buf = getFieldBuffer(fieldIndex);

        long longBits = Double.doubleToLongBits(value);
        writeFixedLengthLong(buf, longBits);
    }

    public void setBoolean(String fieldName, boolean value) {
        int fieldIndex = getSchema().getPosition(fieldName);

        validateFieldType(fieldIndex, fieldName, FieldType.BOOLEAN);

        ByteDataBuffer buf = getFieldBuffer(fieldIndex);

        buf.write(value ? (byte) 1 : (byte) 0);
    }

    public void setBytes(String fieldName, byte[] value) {
        if(value == null)  return;

        int fieldIndex = getSchema().getPosition(fieldName);

        validateFieldType(fieldIndex, fieldName, FieldType.BYTES);

        ByteDataBuffer buf = getFieldBuffer(fieldIndex);

        for (int i = 0; i < value.length; i++) {
            buf.write(value[i]);
        }
    }

    public void setString(String fieldName, String value) {
        if(value == null)  return;

        int fieldIndex = getSchema().getPosition(fieldName);

        validateFieldType(fieldIndex, fieldName, FieldType.STRING);

        ByteDataBuffer buf = getFieldBuffer(fieldIndex);

        for(int i=0;i<value.length();i++) {
            VarInt.writeVInt(buf, value.charAt(i));
        }
    }

    public void setReference(String fieldName, int ordinal) {
        int fieldIndex = getSchema().getPosition(fieldName);

        validateFieldType(fieldIndex, fieldName, FieldType.REFERENCE);

        ByteDataBuffer buf = getFieldBuffer(fieldIndex);

        VarInt.writeVInt(buf, ordinal);
    }

    private void writeNull(ByteDataBuffer buf, FieldType fieldType) {
        if(fieldType == FieldType.FLOAT) {
            writeNullFloat(buf);
        } else if(fieldType == FieldType.DOUBLE) {
            writeNullDouble(buf);
        } else {
            VarInt.writeVNull(buf);
        }
    }

    /**
     * Returns the buffer which should be used to serialize the data for the field at the given position in the schema.<p>
     *
     * This is used by the FastBlobFrameworkSerializer when writing the data for a specific field.
     *
     * @param field
     * @return
     */
    private ByteDataBuffer getFieldBuffer(int fieldPosition) {
        isNonNull[fieldPosition] = true;
        fieldData[fieldPosition].reset();
        return fieldData[fieldPosition];
    }

    public static final int NULL_FLOAT_BITS = Float.floatToIntBits(Float.NaN) + 1;
    public static final long NULL_DOUBLE_BITS = Double.doubleToLongBits(Double.NaN) + 1;

    /**
     * Serialize a special 4-byte long sequence indicating a null Float value.
     */
    private static void writeNullFloat(final ByteDataBuffer fieldBuffer) {
        writeFixedLengthInt(fieldBuffer, NULL_FLOAT_BITS);
    }

    /**
     * Write 4 consecutive bytes
     */
    private static void writeFixedLengthInt(ByteDataBuffer fieldBuffer, int intBits) {
        fieldBuffer.write((byte) (intBits >>> 24));
        fieldBuffer.write((byte) (intBits >>> 16));
        fieldBuffer.write((byte) (intBits >>> 8));
        fieldBuffer.write((byte) (intBits));
    }

    /**
     * Serialize a special 8-byte long sequence indicating a null Double value.
     */
    private static void writeNullDouble(ByteDataBuffer fieldBuffer) {
        writeFixedLengthLong(fieldBuffer, NULL_DOUBLE_BITS);
    }

    /**
     * Write 8 consecutive bytes
     */
    private static void writeFixedLengthLong(ByteDataBuffer fieldBuffer, long intBits) {
        fieldBuffer.write((byte) (intBits >>> 56));
        fieldBuffer.write((byte) (intBits >>> 48));
        fieldBuffer.write((byte) (intBits >>> 40));
        fieldBuffer.write((byte) (intBits >>> 32));
        fieldBuffer.write((byte) (intBits >>> 24));
        fieldBuffer.write((byte) (intBits >>> 16));
        fieldBuffer.write((byte) (intBits >>> 8));
        fieldBuffer.write((byte) (intBits));
    }

    private void validateFieldType(int fieldIndex, String fieldName, FieldType attemptedFieldType) {
        if(getSchema().getFieldType(fieldIndex) != attemptedFieldType) {
            throw new IllegalArgumentException("Attempting to serialize " + attemptedFieldType + " in field " + fieldName + ".  Carefully check your schema for type " + getSchema().getName() + ".");
        }
    }
}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.index;

import com.netflix.hollow.core.memory.encoding.HashCodes;
import com.netflix.hollow.core.read.iterator.HollowOrdinalIterator;
import java.util.Spliterator;
import java.util.function.IntConsumer;
import java.util.stream.IntStream;
import java.util.stream.StreamSupport;

/**
 * A HollowHashIndexResult contains the matches for a query to a {@link HollowHashIndex}.
 *
 */
public class HollowHashIndexResult {

    private final HollowHashIndex.HollowHashIndexState hashIndexState;
    private final long selectTableStartPointer;
    private final int selectTableSize;
    private final int selectTableBuckets;
    private final int selectBucketMask;

    HollowHashIndexResult(HollowHashIndex.HollowHashIndexState hashIndexState, long selectTableStartPointer, int selectTableSize) {
        this.hashIndexState = hashIndexState;
        this.selectTableStartPointer = selectTableStartPointer;
        this.selectTableSize = selectTableSize;
        this.selectTableBuckets = HashCodes.hashTableSize(selectTableSize);
        this.selectBucketMask = selectTableBuckets - 1;
    }

    /**
     * @return the number of matched records
     */
    public int numResults() {
        return selectTableSize;
    }

    /**
     * @param value the ordinal
     * @return {@code true} if the ordinal is matched, otherwise {@code false}
     */
    public boolean contains(int value) {
        int hash = HashCodes.hashInt(value);
        int bucket = hash & selectBucketMask;

        int selectOrdinal = (int) hashIndexState.getSelectHashArray().getElementValue((selectTableStartPointer + bucket) * hashIndexState.getBitsPerSelectHashEntry(), hashIndexState.getBitsPerSelectHashEntry()) - 1;
        while(selectOrdinal != -1) {
            if(selectOrdinal == value)
                return true;

            bucket = (bucket + 1) & selectBucketMask;
            selectOrdinal = (int) hashIndexState.getSelectHashArray().getElementValue((selectTableStartPointer + bucket) * hashIndexState.getBitsPerSelectHashEntry(), hashIndexState.getBitsPerSelectHashEntry()) - 1;
        }

        return false;
    }

    /**
     * @return A {@link HollowOrdinalIterator} over the matched ordinals.  The ordinals may be used with a generated API or the Generic Object API to inspect
     * the matched records.
     */
    public HollowOrdinalIterator iterator() {
        return new HollowOrdinalIterator() {
            final long endBucket = selectTableStartPointer + selectTableBuckets;
            long currentBucket = selectTableStartPointer;

            @Override
            public int next() {
                while(currentBucket < endBucket) {
                    int selectOrdinal = (int) hashIndexState.getSelectHashArray().getElementValue((currentBucket++) * hashIndexState.getBitsPerSelectHashEntry(), hashIndexState.getBitsPerSelectHashEntry()) - 1;
                    if(selectOrdinal != -1)
                        return selectOrdinal;
                }

                return NO_MORE_ORDINALS;
            }
        };
    }

    /**
     * Returns a stream of matching ordinals.
     * <p>
     * The ordinals may be used with a generated API or the Generic Object API to inspect
     * the matched records.
     *
     * @return an {@code IntStream} of matching ordinals
     */
    public IntStream stream() {
        Spliterator.OfInt si = new Spliterator.OfInt() {
            final long endBucket = selectTableStartPointer + selectTableBuckets;
            long currentBucket = selectTableStartPointer;

            @Override
            public OfInt trySplit() {
                // @@@ Supporting splitting and therefore enable parallelism
                return null;
            }

            @Override
            public boolean tryAdvance(IntConsumer action) {
                while (currentBucket < endBucket) {
                    int selectOrdinal = (int) hashIndexState.getSelectHashArray().getElementValue(
                            (currentBucket++) * hashIndexState.getBitsPerSelectHashEntry(),
                            hashIndexState.getBitsPerSelectHashEntry()) - 1;
                    if (selectOrdinal != -1) {
                        action.accept(selectOrdinal);
                        return true;
                    }
                }
                return false;
            }

            @Override
            public long estimateSize() {
                // @@@
                return 0;
            }

            @Override
            public int characteristics() {
                // @@@ ordinals are distinct?
                return 0;
            }
        };
        return StreamSupport.intStream(si, false);
    }
}