/*
 * Copyright 2013 Rackspace
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

package com.rackspacecloud.blueflood.io.astyanax;

import com.codahale.metrics.Timer;
import com.google.common.collect.LinkedListMultimap;
import com.google.common.collect.Multimap;
import com.google.common.collect.Table;
import com.netflix.astyanax.ColumnListMutation;
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import com.netflix.astyanax.model.ColumnFamily;
import com.netflix.astyanax.serializers.AbstractSerializer;
import com.rackspacecloud.blueflood.cache.CombinedTtlProvider;
import com.rackspacecloud.blueflood.cache.LocatorCache;
import com.rackspacecloud.blueflood.cache.TenantTtlProvider;
import com.rackspacecloud.blueflood.io.CassandraModel;
import com.rackspacecloud.blueflood.io.Instrumentation;
import com.rackspacecloud.blueflood.io.serializers.Serializers;
import com.rackspacecloud.blueflood.io.serializers.astyanax.StringMetadataSerializer;
import com.rackspacecloud.blueflood.rollup.Granularity;
import com.rackspacecloud.blueflood.rollup.SlotKey;
import com.rackspacecloud.blueflood.service.Configuration;
import com.rackspacecloud.blueflood.service.CoreConfig;
import com.rackspacecloud.blueflood.service.SingleRollupWriteContext;
import com.rackspacecloud.blueflood.types.*;
import com.rackspacecloud.blueflood.utils.Clock;
import com.rackspacecloud.blueflood.utils.Util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.*;

public class AstyanaxWriter extends AstyanaxIO {
    private static final Logger log = LoggerFactory.getLogger(AstyanaxWriter.class);
    private static final AstyanaxWriter instance = new AstyanaxWriter();
    private static final Keyspace keyspace = getKeyspace();

    public static AstyanaxWriter getInstance() {
        return instance;
    }

    private static TenantTtlProvider TTL_PROVIDER = CombinedTtlProvider.getInstance();

    private static Granularity DELAYED_METRICS_STORAGE_GRANULARITY =
            Granularity.getRollupGranularity(Configuration.getInstance().getStringProperty(CoreConfig.DELAYED_METRICS_STORAGE_GRANULARITY));

    private static final long MAX_AGE_ALLOWED = Configuration.getInstance().getLongProperty(CoreConfig.ROLLUP_DELAY_MILLIS);

    // insert a full resolution chunk of data. I've assumed that there will not be a lot of overlap (these will all be
    // single column updates).
    public void insertFull(Collection<? extends IMetric> metrics, boolean isRecordingDelayedMetrics, Clock clock) throws ConnectionException {
        Timer.Context ctx = Instrumentation.getWriteTimerContext(CassandraModel.CF_METRICS_FULL_NAME);

        try {
            MutationBatch mutationBatch = keyspace.prepareMutationBatch();
            for (IMetric metric: metrics) {
                final Locator locator = metric.getLocator();

                // key = shard
                // col = locator (acct + entity + check + dimension.metric)
                // value = <nothing>
                if (!LocatorCache.getInstance().isLocatorCurrentInBatchLayer(locator)) {
                    if (mutationBatch != null)
                        insertLocator(locator, mutationBatch);
                    LocatorCache.getInstance().setLocatorCurrentInBatchLayer(locator);
                }

                if (isRecordingDelayedMetrics) {
                    //retaining the same conditional logic that was used to insertLocator(locator, batch) above.
                    if (mutationBatch != null) {
                        insertLocatorIfDelayed(metric, mutationBatch, clock);
                    }
                }

                insertMetric(metric, mutationBatch);

                Instrumentation.markFullResMetricWritten();
            }
            // insert it
            try {
                mutationBatch.execute();
            } catch (ConnectionException e) {
                Instrumentation.markWriteError(e);
                log.error("Connection exception during insertFull", e);
                throw e;
            }
        } finally {
            ctx.stop();
        }
    }

    /**
     * This method inserts the locator into the metric_delayed_locator column family, if the metric is delayed.
     *
     * @param metric
     * @param mutationBatch
     * @param clock
     */
    private void insertLocatorIfDelayed(IMetric metric, MutationBatch mutationBatch, Clock clock) {
        Locator locator = metric.getLocator();

        long delay = clock.now().getMillis() - metric.getCollectionTime();
        if (delay > MAX_AGE_ALLOWED) {

            //track locator for configured granularity level. to re-roll only the delayed locator's for that slot
            int slot = DELAYED_METRICS_STORAGE_GRANULARITY.slot(metric.getCollectionTime());
            if (!LocatorCache.getInstance().isDelayedLocatorForASlotCurrent(slot, locator)) {
                insertDelayedLocator(DELAYED_METRICS_STORAGE_GRANULARITY, slot, locator, mutationBatch);
                LocatorCache.getInstance().setDelayedLocatorForASlotCurrent(slot, locator);
            }
        }
    }

    // numeric only!
    public final void insertLocator(Locator locator, MutationBatch mutationBatch) {
        mutationBatch.withRow(CassandraModel.CF_METRICS_LOCATOR, (long) Util.getShard(locator.toString()))
                .putEmptyColumn(locator, TenantTtlProvider.LOCATOR_TTL);
    }

    // numeric only!
    public final void insertDelayedLocator(Granularity g, int slot, Locator locator, MutationBatch mutationBatch) {
        int shard = Util.getShard(locator.toString());
        mutationBatch.withRow(CassandraModel.CF_METRICS_DELAYED_LOCATOR, SlotKey.of(g, slot, shard))
                .putEmptyColumn(locator, TenantTtlProvider.DELAYED_LOCATOR_TTL);
    }

    private void insertMetric(IMetric metric, MutationBatch mutationBatch) {
        try {
            mutationBatch.withRow(CassandraModel.CF_METRICS_FULL, metric.getLocator())
                        .putColumn(metric.getCollectionTime(),
                                metric.getMetricValue(),
                                Serializers.serializerFor(Object.class),
                                metric.getTtlInSeconds());
        } catch (RuntimeException e) {
            log.error("Error serializing full resolution data", e);
        }
    }

    public void writeMetadataValue(Locator locator, String metaKey, String metaValue) throws ConnectionException {
        Timer.Context ctx = Instrumentation.getWriteTimerContext(CassandraModel.CF_METRICS_METADATA_NAME);
        try {
            keyspace.prepareColumnMutation(CassandraModel.CF_METRICS_METADATA, locator, metaKey)
                    .putValue(metaValue, StringMetadataSerializer.get(), null)
                    .execute();
        } catch (ConnectionException e) {
            Instrumentation.markWriteError(e);
            log.error("Error writing Metadata Value", e);
            throw e;
        } finally {
            ctx.stop();
        }
    }

    public void writeMetadata(Table<Locator, String, String> metaTable) throws ConnectionException {
        ColumnFamily cf = CassandraModel.CF_METRICS_METADATA;
        Timer.Context ctx = Instrumentation.getBatchWriteTimerContext(CassandraModel.CF_METRICS_METADATA_NAME);
        MutationBatch batch = keyspace.prepareMutationBatch();

        try {
            for (Locator locator : metaTable.rowKeySet()) {
                Map<String, String> metaRow = metaTable.row(locator);
                ColumnListMutation<String> mutation = batch.withRow(cf, locator);

                for (Map.Entry<String, String> meta : metaRow.entrySet()) {
                    mutation.putColumn(meta.getKey(), meta.getValue(), StringMetadataSerializer.get(), null);
                }
            }
            try {
                batch.execute();
            } catch (ConnectionException e) {
                Instrumentation.markWriteError(e);
                log.error("Connection exception persisting metadata", e);
                throw e;
            }
        } finally {
            ctx.stop();
        }
    }

    private static Multimap<Locator, IMetric> asMultimap(Collection<IMetric> metrics) {
        Multimap<Locator, IMetric> map = LinkedListMultimap.create();
        for (IMetric metric: metrics)
            map.put(metric.getLocator(), metric);
        return map;
    }

    // generic IMetric insertion. All other metric insertion methods could use this one.
    public void insertMetrics(Collection<IMetric> metrics, ColumnFamily cf, boolean isRecordingDelayedMetrics, Clock clock) throws ConnectionException {
        Timer.Context ctx = Instrumentation.getWriteTimerContext(cf.getName());
        Multimap<Locator, IMetric> map = asMultimap(metrics);
        MutationBatch batch = keyspace.prepareMutationBatch();
        try {
            for (Locator locator : map.keySet()) {
                ColumnListMutation<Long> mutation = batch.withRow(cf, locator);

                for (IMetric metric : map.get(locator)) {

                    mutation.putColumn(
                                metric.getCollectionTime(),
                                metric.getMetricValue(),
                                (AbstractSerializer) (Serializers.serializerFor(metric.getMetricValue().getClass())),
                                metric.getTtlInSeconds());
                    if (cf.getName().equals(CassandraModel.CF_METRICS_PREAGGREGATED_FULL_NAME)) {
                            Instrumentation.markFullResPreaggregatedMetricWritten();
                    }

                    if (isRecordingDelayedMetrics) {
                        //retaining the same conditional logic that was used to perform insertLocator(locator, batch).
                        insertLocatorIfDelayed(metric, batch, clock);
                    }
                }

                if (!LocatorCache.getInstance().isLocatorCurrentInBatchLayer(locator)) {
                    insertLocator(locator, batch);
                    LocatorCache.getInstance().setLocatorCurrentInBatchLayer(locator);
                }
            }
            try {
                batch.execute();
            } catch (ConnectionException e) {
                Instrumentation.markWriteError(e);
                log.error("Connection exception persisting data", e);
                throw e;
            }
        } finally {
            ctx.stop();
        }
    }

    public void insertRollups(List<SingleRollupWriteContext> writeContexts) throws ConnectionException {
        if (writeContexts.size() == 0) {
            return;
        }
        Timer.Context ctx = Instrumentation.getBatchWriteTimerContext(writeContexts.get(0).getDestinationCF().getName());
        MutationBatch mb = keyspace.prepareMutationBatch();
        for (SingleRollupWriteContext writeContext : writeContexts) {
            Rollup rollup = writeContext.getRollup();
            int ttl = (int)TTL_PROVIDER.getTTL(
                    writeContext.getLocator().getTenantId(),
                    writeContext.getGranularity(),
                    writeContext.getRollup().getRollupType()).get().toSeconds();
            AbstractSerializer serializer = Serializers.serializerFor(rollup.getClass());
            try {
                mb.withRow(writeContext.getDestinationCF(), writeContext.getLocator())
                        .putColumn(writeContext.getTimestamp(),
                                rollup,
                                serializer,
                                ttl);
            } catch (RuntimeException ex) {
                // let's not let stupidness prevent the rest of this put.
                log.warn(String.format("Cannot save %s", writeContext.getLocator().toString()), ex);
            }
        }
        try {
            mb.execute();
        } catch (ConnectionException e) {
            Instrumentation.markWriteError(e);
            log.error("Error writing rollup batch", e);
            throw e;
        } finally {
            ctx.stop();
        }
    }
}


package com.rackspacecloud.blueflood.io.datastax;

import com.codahale.metrics.Meter;
import com.codahale.metrics.Timer;
import com.datastax.driver.core.BatchStatement;
import com.datastax.driver.core.BoundStatement;
import com.datastax.driver.core.ResultSetFuture;
import com.datastax.driver.core.Session;
import com.datastax.driver.core.Statement;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Table;
import com.rackspacecloud.blueflood.cache.LocatorCache;
import com.rackspacecloud.blueflood.cache.MetadataCache;
import com.rackspacecloud.blueflood.exceptions.CacheException;
import com.rackspacecloud.blueflood.io.*;
import com.rackspacecloud.blueflood.outputs.formats.MetricData;
import com.rackspacecloud.blueflood.rollup.Granularity;
import com.rackspacecloud.blueflood.service.Configuration;
import com.rackspacecloud.blueflood.service.CoreConfig;
import com.rackspacecloud.blueflood.service.SingleRollupWriteContext;
import com.rackspacecloud.blueflood.types.*;
import com.rackspacecloud.blueflood.utils.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.*;

/**
 * This class deals with aspects of reading/writing metrics which are common across all column families
 * using Datastax driver
 */
public abstract class DAbstractMetricsRW extends AbstractMetricsRW {

    private static final Logger LOG = LoggerFactory.getLogger(DAbstractMetricsRW.class);

    private static final long MAX_AGE_ALLOWED = Configuration.getInstance().getLongProperty(CoreConfig.ROLLUP_DELAY_MILLIS);

    private static final Granularity DELAYED_METRICS_STORAGE_GRANULARITY =
            Granularity.getRollupGranularity(Configuration.getInstance().getStringProperty(CoreConfig.DELAYED_METRICS_STORAGE_GRANULARITY));

    protected final DLocatorIO locatorIO;
    protected final DDelayedLocatorIO delayedLocatorIO;
    protected final boolean isBatchIngestEnabled;

    /**
     * Constructor
     * @param locatorIO
     * @param isRecordingDelayedMetrics
     */
    protected DAbstractMetricsRW(DLocatorIO locatorIO, DDelayedLocatorIO delayedLocatorIO,
                                 boolean isRecordingDelayedMetrics,
                                 Clock clock) {
        this.isRecordingDelayedMetrics = isRecordingDelayedMetrics;
        this.isBatchIngestEnabled = false;
        this.locatorIO = locatorIO;
        this.delayedLocatorIO = delayedLocatorIO;
        this.clock = clock;
    }

    /**
     * Constructor
     * @param locatorIO
     * @param isRecordingDelayedMetrics
     */
    @VisibleForTesting
    protected DAbstractMetricsRW(DLocatorIO locatorIO, DDelayedLocatorIO delayedLocatorIO,
                                 boolean isRecordingDelayedMetrics,
                                 boolean isBatchIngestEnabled, Clock clock) {
        this.isRecordingDelayedMetrics = isRecordingDelayedMetrics;
        this.isBatchIngestEnabled = isBatchIngestEnabled;
        this.locatorIO = locatorIO;
        this.delayedLocatorIO = delayedLocatorIO;
        this.clock = clock;
    }

    /**
     * Return the appropriate IO object which interacts with the Cassandra database.
     *
     * For Preaggregated Metrics, only rollup type is required.
     * For Basic Numeric Metrics, the granularity is required as metrics_full is handled differently
     * than the other granularities.
     *
     * @param rollupType
     * @param gran
     * @return
     */
    protected abstract DAbstractMetricIO getIO( String rollupType, Granularity gran );

    /**
     * This method inserts a collection of {@link com.rackspacecloud.blueflood.service.SingleRollupWriteContext} objects
     * to the appropriate Cassandra column family.
     *
     * It performs the inserts by executing an UNLOGGED BATCH statement.
     *
     * @param writeContexts
     *
     * @throws IOException
     */
    @Override
    public void insertRollups(List<SingleRollupWriteContext> writeContexts) {

        if (writeContexts.size() == 0) {
            return;
        }

        Timer.Context ctx = Instrumentation.getWriteTimerContext( writeContexts.get( 0 ).getDestinationCF().getName() );
        try {

            BatchStatement batch = new BatchStatement(BatchStatement.Type.UNLOGGED);

            for (SingleRollupWriteContext writeContext : writeContexts) {
                Rollup rollup = writeContext.getRollup();
                Locator locator = writeContext.getLocator();
                Granularity granularity = writeContext.getGranularity();
                int ttl = getTtl(locator, rollup.getRollupType(), granularity);

                // lookup the right writer
                RollupType rollupType = writeContext.getRollup().getRollupType();
                DAbstractMetricIO io = getIO(rollupType.name().toLowerCase(), granularity);

                Statement statement = io.createStatement(locator, writeContext.getTimestamp(), rollup, writeContext.getGranularity(), ttl);
                batch.add(statement);
            }

            Session session = DatastaxIO.getSession();
            session.execute(batch);

        } catch (Exception ex) {
            Instrumentation.markWriteError();
            LOG.error(String.format("error writing locator batch of size %s, granularity %s", writeContexts.size(), writeContexts.get(0).getGranularity()), ex);
        } finally {
            ctx.stop();
        }
    }

    /**
     * Fetches {@link com.rackspacecloud.blueflood.outputs.formats.MetricData} objects for the
     * specified {@link com.rackspacecloud.blueflood.types.Locator} and
     * {@link com.rackspacecloud.blueflood.types.Range} from the specified column family
     *
     * @param locator
     * @param range
     * @param granularity
     * @return
     */
    @Override
    public MetricData getDatapointsForRange(final Locator locator,
                                            Range range,
                                            Granularity granularity) {
        Map<Locator, MetricData> result = getDatapointsForRange(new ArrayList<Locator>() {{
            add(locator);
        }}, range, granularity);
        return result.get(locator);
    }

    /**
     * Fetches {@link com.rackspacecloud.blueflood.outputs.formats.MetricData} objects for the
     * specified {@link com.rackspacecloud.blueflood.types.Locator} and
     * {@link com.rackspacecloud.blueflood.types.Range} from the specified column family
     *
     * This is a helper method used to get basic metrics as well as preaggregated metrics.  Hence, both granularity
     * and columnFamily are required.
     *
     * @param locators
     * @param range
     * @param columnFamily
     * @param granularity
     * @return
     */
    public Map<Locator, MetricData> getDatapointsForRange( List<Locator> locators,
                                                              Range range,
                                                              String columnFamily,
                                                              Granularity granularity ) {

        Timer.Context ctx = Instrumentation.getReadTimerContext(columnFamily);

        try {

            MetadataCache metadataCache = MetadataCache.getInstance();

            // in this loop, we will fire all the executeAsync() of
            // various select statements, the collect all of the
            // ResultSetFutures
            Map<Locator, List<ResultSetFuture>> locatorToFuturesMap = new HashMap<Locator, List<ResultSetFuture>>();
            Map<Locator, DAbstractMetricIO> locatorIOMap = new HashMap<Locator, DAbstractMetricIO>();

            for (Locator locator : locators) {
                try {

                    String rType = metadataCache.get(locator, MetricMetadata.ROLLUP_TYPE.name().toLowerCase());

                    DAbstractMetricIO io = getIO( rType, granularity );

                    // put everything in a map of locator -> io so
                    // we can use em up later
                    locatorIOMap.put( locator, io );

                    // do the query
                    List<ResultSetFuture> selectFutures = io.selectForLocatorAndRange( columnFamily, locator, range );

                    // add all ResultSetFutures for a particular locator together
                    List<ResultSetFuture> existing = locatorToFuturesMap.get(locator);
                    if (existing == null) {
                        existing = new ArrayList<ResultSetFuture>();
                        locatorToFuturesMap.put(locator, existing);
                    }
                    existing.addAll(selectFutures);

                } catch (CacheException ex) {
                    Instrumentation.markReadError();
                    LOG.error(String.format("Error looking up locator %s in cache", locator), ex);
                }
            }
            return resultSetsToMetricData(locatorToFuturesMap, locatorIOMap, columnFamily, range);
        }
        finally {

            ctx.stop();
        }
    }

    /**
     * Fetches a {@link com.rackspacecloud.blueflood.types.Points} object for a
     * particular locator and rollupType from the specified column family and
     * range
     *
     * @param locator
     * @param rollupType
     * @param range
     * @param columnFamilyName
     * @return
     */
    @Override
    public Points getDataToRollup(final Locator locator,
                                  RollupType rollupType,
                                  Range range,
                                  String columnFamilyName) throws IOException {
        Timer.Context ctx = Instrumentation.getReadTimerContext(columnFamilyName);
        try {
            // read the rollup object from the proper IO class
            DAbstractMetricIO io = getIO( rollupType.name().toLowerCase(), CassandraModel.getGranularity( columnFamilyName ) );

            Table<Locator, Long, Object> locatorTimestampRollup = io.getRollupsForLocator( locator, columnFamilyName, range );

            Points points = new Points();
            for (Table.Cell<Locator, Long, Object> cell : locatorTimestampRollup.cellSet()) {
                points.add( createPoint( cell.getColumnKey(), cell.getValue()));
            }
            return points;
        } catch( Exception e ) {

            Instrumentation.markReadError();
            LOG.error( String.format( "Unable to read locator=%s rolluptype=%s columnFamilyName=%s for rollup",
                    locator, rollupType.name(), columnFamilyName ), e );

            throw new IOException( e );
        }
        finally {
            ctx.stop();
        }
    }

    /**
     * Converts a list of {@link com.datastax.driver.core.ResultSetFuture} for each
     * {@link com.rackspacecloud.blueflood.types.Locator} to
     * {@link com.rackspacecloud.blueflood.outputs.formats.MetricData} object.
     *
     * @param resultSets
     * @param locatorIO
     * @param columnFamily
     * @return
     */
    protected Map<Locator, MetricData> resultSetsToMetricData(Map<Locator, List<ResultSetFuture>> resultSets,
                                                              Map<Locator, DAbstractMetricIO> locatorIO,
                                                              String columnFamily,
                                                              Range range) {

        MetadataCache metadataCache = MetadataCache.getInstance();

        // iterate through all ResultSetFuture
        Map<Locator, MetricData> locatorMetricDataMap = new HashMap<Locator, MetricData>();
        for (Map.Entry<Locator, List<ResultSetFuture>> entry : resultSets.entrySet() ) {
            Locator locator = entry.getKey();
            List<ResultSetFuture> futures = entry.getValue();

            DAbstractMetricIO io = locatorIO.get(locator);

            // get ResultSets to a Table of locator, timestamp, rollup
            Table<Locator, Long, Object> locatorTimestampRollup = io.toLocatorTimestampValue(futures, locator, columnFamily, range);

            Map<Long, Object> tsRollupMap = locatorTimestampRollup.row( locator );

            // convert to Points and MetricData
            Points points = convertToPoints( tsRollupMap );

            // create MetricData
            MetricData metricData = new MetricData( points, metadataCache.getUnitString( locator ) );
            locatorMetricDataMap.put( locator, metricData );
        }
        return locatorMetricDataMap;
    }

    /**
     * Given a map of timestamps -> Rollups/SimpleNumbers, return RollupType, or null, if not a Rollup.
     *
     * @param tsRollupMap
     * @return
     */
    private RollupType getRollupType( Map<Long, Object> tsRollupMap ) {

        if( tsRollupMap.isEmpty() )
            return null;
        else {

            Object value = tsRollupMap.values().iterator().next();
            return value instanceof Rollup ? ( (Rollup) value ).getRollupType() : null;
        }
    }

    /**
     * Determines if the metric is considered delayed or not
     *
     * @param metric
     * @return
     */
    protected boolean isDelayed(IMetric metric) {
        long delay = clock.now().getMillis() - metric.getCollectionTime();
        return delay > MAX_AGE_ALLOWED;
    }

    /**
     * Returns the delayed slot for a metric
     *
     * @param metric
     * @return
     */
    protected int getDelayedSlot(IMetric metric) {
        //track locator for configured granularity level. to re-roll only the delayed locator's for that slot
        return DELAYED_METRICS_STORAGE_GRANULARITY.slot(metric.getCollectionTime());
    }

    /**
     * This method inserts the locator into the metric_delayed_locator column family, if the metric is delayed.
     *
     * @param metric
     * @throws IOException
     */
    protected void insertLocatorIfDelayed(IMetric metric) throws IOException {
        Locator locator = metric.getLocator();
        if ( isDelayed(metric) ) {
            int slot = getDelayedSlot(metric);
            if (!LocatorCache.getInstance().isDelayedLocatorForASlotCurrent(slot, locator)) {
                delayedLocatorIO.insertLocator(DELAYED_METRICS_STORAGE_GRANULARITY, slot, locator);
                LocatorCache.getInstance().setDelayedLocatorForASlotCurrent(slot, locator);
            }
        }
    }

    /**
     * Returns a BoundStatement if a metric needs to be inserted to the metrics_delayed_locator
     * Column Family. Returns null otherwise.
     *
     * @param metric
     * @return
     */
    protected BoundStatement getBoundStatementForMetricIfDelayed(IMetric metric) {
        Locator locator = metric.getLocator();
        if ( isDelayed(metric) ) {
            int slot = getDelayedSlot(metric);
            if (!LocatorCache.getInstance().isDelayedLocatorForASlotCurrent(slot, locator)) {
                LocatorCache.getInstance().setDelayedLocatorForASlotCurrent(slot, locator);
                return delayedLocatorIO.getBoundStatementForLocator(DELAYED_METRICS_STORAGE_GRANULARITY, slot, locator);
            }
        }
        return null;
    }
}


package com.rackspacecloud.blueflood.io;

import com.codahale.metrics.Histogram;
import com.codahale.metrics.Timer;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.rackspacecloud.blueflood.service.Configuration;
import com.rackspacecloud.blueflood.service.ElasticIOConfig;
import com.rackspacecloud.blueflood.types.Locator;
import com.rackspacecloud.blueflood.types.Token;
import com.rackspacecloud.blueflood.utils.GlobPattern;
import com.rackspacecloud.blueflood.utils.Metrics;
import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.*;

import static java.util.stream.Collectors.joining;

/**
 * {@link TokenDiscoveryIO} implementation using elasticsearch
 */
public class ElasticTokensIO implements TokenDiscoveryIO {

    public static final String ES_DOCUMENT_TYPE = "tokens";

    protected final Histogram batchHistogram = Metrics.histogram(getClass(), "Batch Sizes");
    protected final Timer writeTimer = Metrics.timer(getClass(), "Write Duration");

    protected final Timer esMetricNamesQueryTimer = Metrics.timer(getClass(), "ES Metric Names Query Duration");

    private static final Logger log = LoggerFactory.getLogger(ElasticTokensIO.class);

    public static String ELASTICSEARCH_TOKEN_INDEX_NAME_WRITE = Configuration.getInstance().getStringProperty(ElasticIOConfig.ELASTICSEARCH_TOKEN_INDEX_NAME_WRITE);
    public static String ELASTICSEARCH_TOKEN_INDEX_NAME_READ = Configuration.getInstance().getStringProperty(ElasticIOConfig.ELASTICSEARCH_TOKEN_INDEX_NAME_READ);

    public ElasticsearchRestHelper elasticsearchRestHelper;

    public ElasticTokensIO() {
        this.elasticsearchRestHelper = ElasticsearchRestHelper.getInstance();
    }

    @Override
    public void insertDiscovery(Token token) throws IOException {
        List<Token> batch = new ArrayList<>();
        batch.add(token);
        insertDiscovery(batch);
    }

    @Override
    public void insertDiscovery(List<Token> tokens) throws IOException {
        batchHistogram.update(tokens.size());
        if (tokens.size() == 0) return;

        Timer.Context ctx = writeTimer.time();
        try {
            elasticsearchRestHelper.indexTokens(tokens);
        } finally {
            ctx.stop();
        }
    }

    @Override
    public List<MetricName> getMetricNames(String tenantId, String query) throws Exception {
        return searchESByIndexes(tenantId, query, getIndexesToSearch());
    }

    /**
     * This method queries elasticsearch for a given glob query and returns list of {@link MetricName}'s.
     *
     * @param tenantId
     * @param query is glob
     * @param indexes
     * @return
     */
    private List<MetricName> searchESByIndexes(String tenantId, String query, String[] indexes) {
        if (StringUtils.isEmpty(query)) return new ArrayList<>();

        Timer.Context timerCtx = esMetricNamesQueryTimer.time();
        try {
            String response = elasticsearchRestHelper.fetchTokenDocuments(indexes, tenantId, query);

            List<MetricName> metricNames = getMetricNames(response);
            Set<MetricName> uniqueMetricNames = new HashSet<>(metricNames);
            return new ArrayList<>(uniqueMetricNames);
        } catch (IOException e) {
            log.error("IOException: Elasticsearch token query failed for tenantId {}. {}", tenantId, e.getMessage());
            throw new RuntimeException(String.format("searchESByIndexes failed with message: %s", e.getMessage()), e);
        } catch (Exception e) {
            log.error("Exception: Elasticsearch token query failed for tenantId {}. {}", tenantId, e.getMessage());
            throw new RuntimeException(String.format("searchESByIndexes failed with message: %s", e.getMessage()), e);
        } finally {
            timerCtx.stop();
        }
    }

    private List<MetricName> getMetricNames(String response) throws IOException {
        ObjectMapper mapper = new ObjectMapper();
        JsonNode root = mapper.readTree(response);

        Iterator<JsonNode> iter = root.get("hits").get("hits").elements();

        List<MetricName> metricNames = new ArrayList<>();
        while(iter.hasNext()){
            JsonNode source = iter.next().get("_source");

            String parent = source.get(ESFieldLabel.parent.toString()).asText();
            String token = source.get(ESFieldLabel.token.toString()).asText();
            boolean isCompleteName = source.get(ESFieldLabel.isLeaf.toString()).asBoolean();

            StringBuilder metricName = new StringBuilder(parent);
            if (metricName.length() > 0) {
                metricName.append(Locator.METRIC_TOKEN_SEPARATOR);
            }
            metricName.append(token);

            metricNames.add(new MetricName(metricName.toString(), isCompleteName));
        }
        return metricNames;
    }

    /**
     * For a given glob, gives regex for {@code Locator.METRIC_TOKEN_SEPARATOR} separated tokens
     *
     * For example:
     *      globPattern of foo.*.* would produce a regex  foo\.[^.]+\.[^.]+
     *      globPattern of foo.b*.* would produce a regex foo\.b[^.]*\.[^.]+
     *
     * @param globPattern
     * @return
     */
    protected String getRegexToHandleTokens(GlobPattern globPattern) {
        String[] queryRegexParts = globPattern.compiled().toString().split("\\\\.");

        return Arrays.stream(queryRegexParts)
                     .map(this::convertRegexToCaptureUptoNextToken)
                     .collect(joining(Locator.METRIC_TOKEN_SEPARATOR_REGEX));
    }

    private String convertRegexToCaptureUptoNextToken(String queryRegex) {

        if (queryRegex.equals(".*"))
            return queryRegex.replaceAll("\\.\\*", "[^.]+");
        else
            return queryRegex.replaceAll("\\.\\*", "[^.]*");
    }

    protected String[] getIndexesToSearch() {
        return new String[] {ELASTICSEARCH_TOKEN_INDEX_NAME_READ};
    }
}


/*
 * Copyright 2013 Rackspace
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */
package com.rackspacecloud.blueflood.io.astyanax;

import com.codahale.metrics.Timer;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.HashBasedTable;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Table;
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.connectionpool.OperationResult;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import com.netflix.astyanax.connectionpool.exceptions.NotFoundException;
import com.netflix.astyanax.model.*;
import com.netflix.astyanax.serializers.AbstractSerializer;
import com.netflix.astyanax.shallows.EmptyColumnList;
import com.netflix.astyanax.util.RangeBuilder;
import com.rackspacecloud.blueflood.cache.MetadataCache;
import com.rackspacecloud.blueflood.io.CassandraModel;
import com.rackspacecloud.blueflood.io.Instrumentation;
import com.rackspacecloud.blueflood.io.serializers.Serializers;
import com.rackspacecloud.blueflood.io.serializers.astyanax.StringMetadataSerializer;
import com.rackspacecloud.blueflood.outputs.formats.MetricData;
import com.rackspacecloud.blueflood.rollup.Granularity;
import com.rackspacecloud.blueflood.types.*;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.*;

public class AstyanaxReader extends AstyanaxIO {
    private static final Logger log = LoggerFactory.getLogger(AstyanaxReader.class);
    private static final MetadataCache metaCache = MetadataCache.getInstance();
    private static final AstyanaxReader INSTANCE = new AstyanaxReader();
    private static final String rollupTypeCacheKey = MetricMetadata.ROLLUP_TYPE.toString().toLowerCase();
    private static final Keyspace keyspace = getKeyspace();

    public static AstyanaxReader getInstance() {
        return INSTANCE;
    }

    /**
     * Method that returns all metadata for a given locator as a map.
     *
     * @param locator  locator name
     * @return Map of metadata for that locator
     * @throws RuntimeException(com.netflix.astyanax.connectionpool.exceptions.ConnectionException)
     */
    public Map<String, String> getMetadataValues(Locator locator) {
        Timer.Context ctx = Instrumentation.getReadTimerContext(CassandraModel.CF_METRICS_METADATA_NAME);
        try {
            final ColumnList<String> results = keyspace.prepareQuery(CassandraModel.CF_METRICS_METADATA)
                    .getKey(locator)
                    .execute().getResult();
            return new HashMap<String, String>(){{
                for (Column<String> result : results) {
                    put(result.getName(), result.getValue(StringMetadataSerializer.get()));
                }
            }};
        } catch (NotFoundException ex) {
            Instrumentation.markNotFound(CassandraModel.CF_METRICS_METADATA_NAME);
            return null;
        } catch (ConnectionException e) {
            log.error("Error reading metadata value", e);
            Instrumentation.markReadError(e);
            throw new RuntimeException(e);
        } finally {
            ctx.stop();
        }
    }

    public Table<Locator, String, String> getMetadataValues(Set<Locator> locators) {
        ColumnFamily CF = CassandraModel.CF_METRICS_METADATA;
        boolean isBatch = locators.size() > 1;
        Table<Locator, String, String> metaTable = HashBasedTable.create();

        Timer.Context ctx = isBatch ? Instrumentation.getBatchReadTimerContext(CF.getName()) : Instrumentation.getReadTimerContext(CF.getName());
        try {
            // We don't paginate this call. So we should make sure the number of reads is tolerable.
            // TODO: Think about paginating this call.
            OperationResult<Rows<Locator, String>> query = keyspace
                    .prepareQuery(CF)
                    .getKeySlice(locators)
                    .execute();

            for (Row<Locator, String> row : query.getResult()) {
                ColumnList<String> columns = row.getColumns();
                for (Column<String> column : columns) {
                    String metaValue = column.getValue(StringMetadataSerializer.get());
                    String metaKey = column.getName();
                    metaTable.put(row.getKey(), metaKey, metaValue);
                }
            }
        } catch (ConnectionException e) {
            if (e instanceof NotFoundException) { // TODO: Not really sure what happens when one of the keys is not found.
                Instrumentation.markNotFound(CF.getName());
            } else {
                if (isBatch) { Instrumentation.markBatchReadError(e); }
                else { Instrumentation.markReadError(e); }
            }
            log.error((isBatch ? "Batch " : "") + " read query failed for column family " + CF.getName() + " for locators: " + StringUtils.join(locators, ","), e);
        } finally {
            ctx.stop();
        }

        return metaTable;
    }

    private ColumnList<Long> getColumnsFromDB(final Locator locator, ColumnFamily<Locator, Long> srcCF, Range range) {
        List<Locator> locators = new LinkedList<Locator>(){{ add(locator); }};
        ColumnList<Long> columns = getColumnsFromDB(locators, srcCF, range).get(locator);
        return columns == null ? new EmptyColumnList<Long>() : columns;
    }

    protected Map<Locator, ColumnList<Long>> getColumnsFromDB(List<Locator> locators, ColumnFamily<Locator, Long> CF,
                                                            Range range) {
        if (range.getStart() > range.getStop()) {
            throw new RuntimeException(String.format("Invalid rollup range: ", range.toString()));
        }
        boolean isBatch = locators.size() != 1;

        final Map<Locator, ColumnList<Long>> columns = new HashMap<Locator, ColumnList<Long>>();
        final RangeBuilder rangeBuilder = new RangeBuilder().setStart(range.getStart()).setEnd(range.getStop());

        Timer.Context ctx = isBatch ? Instrumentation.getBatchReadTimerContext(CF.getName()) : Instrumentation.getReadTimerContext(CF.getName());
        try {
            // We don't paginate this call. So we should make sure the number of reads is tolerable.
            // TODO: Think about paginating this call.
            OperationResult<Rows<Locator, Long>> query = keyspace
                    .prepareQuery(CF)
                    .getKeySlice(locators)
                    .withColumnRange(rangeBuilder.build())
                    .execute();
            for (Row<Locator, Long> row : query.getResult()) {
                columns.put(row.getKey(), row.getColumns());
            }

        } catch (ConnectionException e) {
            if (e instanceof NotFoundException) { // TODO: Not really sure what happens when one of the keys is not found.
                Instrumentation.markNotFound(CF.getName());
            } else {
                if (isBatch) { Instrumentation.markBatchReadError(e); }
                else { Instrumentation.markReadError(e); }
            }
            log.error((isBatch ? "Batch " : "") + " read query failed for column family " + CF.getName() + " for locators: " + StringUtils.join(locators, ","), e);
        } finally {
            ctx.stop();
        }

        return columns;
    }

    // todo: this could be the basis for every rollup read method.
    // todo: A better interface may be to pass the serializer in instead of the class type.
    public <T extends Rollup> Points<T> getDataToRoll(Class<T> type, final Locator locator, Range range, ColumnFamily<Locator, Long> cf) throws IOException {
        AbstractSerializer serializer = Serializers.serializerFor(type);
        // special cases. :( the problem here is that the normal full res serializer returns Number instances instead of
        // SimpleNumber instances.
        // todo: this logic will only become more complicated. It needs to be in its own method and the serializer needs
        // to be known before we ever get to this method (see above comment).
        if (cf == CassandraModel.CF_METRICS_FULL) {
            serializer = Serializers.simpleNumberSerializer;
        } else if ( cf == CassandraModel.CF_METRICS_PREAGGREGATED_FULL) {
            // consider a method for this.  getSerializer(CF, TYPE);
            if (type.equals(BluefloodTimerRollup.class)) {
                serializer = Serializers.timerRollupInstance;
            } else if (type.equals(BluefloodSetRollup.class)) {
                serializer = Serializers.setRollupInstance;
            } else if (type.equals(BluefloodGaugeRollup.class)) {
                serializer = Serializers.gaugeRollupInstance;
            } else if (type.equals(BluefloodCounterRollup.class)) {
                serializer = Serializers.counterRollupInstance;
            }
            else {
                serializer = Serializers.simpleNumberSerializer;
            }
        }

        ColumnList<Long> cols = getColumnsFromDB(locator, cf, range);
        Points<T> points = new Points<T>();
        try {
            for (Column<Long> col : cols) {
                points.add(new Points.Point<T>(col.getName(), (T)col.getValue(serializer)));
            }

            // we only want to count the number of points we
            // get when we're querying the metrics_full
            // we don't do for aggregated or other granularities
            if ( cf == CassandraModel.CF_METRICS_FULL ) {
                Instrumentation.getRawPointsIn5MinHistogram().update(points.getPoints().size());
            }
        } catch (RuntimeException ex) {
            log.error("Problem deserializing data for " + locator + " (" + range + ") from " + cf.getName(), ex);
            throw new IOException(ex);
        }
        return points;
    }

    /**
     * Get data points for a particular {@link Locator}, {@link Range}, and
     * {@link Granularity}.
     *
     * This method is eventually called from Rollup process.
     *
     * @param locator
     * @param range
     * @param gran
     * @return a {@link MetricData} containing the data points requested
     */
    public MetricData getDatapointsForRange(Locator locator, Range range, Granularity gran) {
        RollupType rollupType = RollupType.BF_BASIC;
        String rollupTypeStr = metaCache.safeGet(locator, rollupTypeCacheKey);
        if ( rollupTypeStr != null ) {
            rollupType = RollupType.fromString(rollupTypeStr);
        }

        return getNumericMetricDataForRange(locator, range, gran, rollupType);
    }

    /**
     * Get data points for multiple {@link Locator}, for the specified {@link Range} and
     * {@link Granularity}.
     *
     * This method is eventually called from all the output handlers for query requests.
     *
     * @param locators
     * @param range
     * @param gran
     * @return
     */
    public Map<Locator, MetricData> getDatapointsForRange(List<Locator> locators, Range range, Granularity gran) {
        ListMultimap<ColumnFamily, Locator> locatorsByCF =
                ArrayListMultimap.create();
        Map<Locator, MetricData> results = new HashMap<Locator, MetricData>();
        for (Locator locator : locators) {
            try {
                RollupType rollupType = RollupType.fromString(
                        metaCache.get(locator, MetricMetadata.ROLLUP_TYPE.name().toLowerCase()));

                ColumnFamily cf = CassandraModel.getColumnFamily(rollupType, gran);
                List<Locator> locs = locatorsByCF.get(cf);
                locs.add(locator);
            } catch (Exception e) {
                // pass for now. need metric to figure this stuff out.
                log.error(String.format("error getting datapoints for locator %s, range %s, granularity %s", locator, range.toString(), gran.toString()), e);
            }
        }

        for (ColumnFamily CF : locatorsByCF.keySet()) {
            List<Locator> locs = locatorsByCF.get(CF);
            results.putAll(getNumericDataForRangeLocatorList(range, gran, CF, locs));
        }

        return results;
    }

    private Map<Locator, MetricData> getNumericDataForRangeLocatorList(Range range, Granularity gran, ColumnFamily CF, List<Locator> locs) {
        Map<Locator, ColumnList<Long>> metrics = getColumnsFromDB(locs, CF, range);
        Map<Locator, MetricData> results = new HashMap<Locator, MetricData>();

        // transform columns to MetricData
        for (Locator loc : metrics.keySet()) {
            MetricData data = transformColumnsToMetricData(loc, metrics.get(loc), gran);
            if (data != null && data.getData() != null) {
                results.put(loc, data);
            }
        }

        return results;
    }

    private MetricData getNumericMetricDataForRange(Locator locator, Range range, Granularity gran, RollupType rollupType) {
        ColumnFamily<Locator, Long> CF = CassandraModel.getColumnFamily(rollupType, gran);
        Points points = new Points();
        ColumnList<Long> results = getColumnsFromDB(locator, CF, range);

        // todo: this will not work when we cannot derive data type from granularity. we will need to know what kind of
        // data we are asking for and use a specific reader method.
        AbstractSerializer serializer = Serializers.serializerFor(RollupType.classOf(rollupType, gran));

        for (Column<Long> column : results) {
            try {
                points.add(pointFromColumn(column, serializer));
            } catch (RuntimeException ex) {
                log.error("Problem deserializing data for " + locator + " (" + range + ") from " + CF.getName(), ex);
            }
        }

        return new MetricData(points, metaCache.getUnitString(locator));
    }

    private MetricData transformColumnsToMetricData(Locator locator, ColumnList<Long> columns,
                                                    Granularity gran) {
        try {
            RollupType rollupType = RollupType.fromString(metaCache.get(locator, rollupTypeCacheKey));
            String unit = metaCache.getUnitString(locator);
            Points points = getPointsFromColumns(columns, rollupType, gran);
            MetricData data = new MetricData(points, unit);
            return data;
        } catch (Exception e) {
            return null;
        }
    }

    private Points getPointsFromColumns(ColumnList<Long> columnList, RollupType rollupType,
                                        Granularity gran) {
        Points points = new Points();

        AbstractSerializer serializer = serializerFor(rollupType, gran);
        for (Column<Long> column : columnList) {
            points.add(pointFromColumn(column, serializer));
        }

        return points;
    }

    private Points.Point pointFromColumn(Column<Long> column, AbstractSerializer serializer) {
        if (serializer instanceof Serializers.RawSerializer) {
            return new Points.Point(column.getName(), new SimpleNumber(column.getValue(serializer)));
        }
        else
            // this works for EVERYTHING except SimpleNumber.
            return new Points.Point(column.getName(), column.getValue(serializer));
    }
}


package com.rackspacecloud.blueflood.io.astyanax;

import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import com.rackspacecloud.blueflood.io.AbstractMetricsRW;
import com.rackspacecloud.blueflood.io.CassandraModel;
import com.rackspacecloud.blueflood.outputs.formats.MetricData;
import com.rackspacecloud.blueflood.rollup.Granularity;
import com.rackspacecloud.blueflood.service.SingleRollupWriteContext;
import com.rackspacecloud.blueflood.types.*;
import com.rackspacecloud.blueflood.utils.Clock;
import com.rackspacecloud.blueflood.utils.DefaultClockImpl;

import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Map;

/**
 * This class deals with reading/writing metrics to the metrics_{granularity} & metrics_string column families
 * using Astyanax driver
 */
public class ABasicMetricsRW extends AbstractMetricsRW {

    public ABasicMetricsRW(boolean isRecordingDelayedMetrics, Clock clock) {
        this.isRecordingDelayedMetrics = isRecordingDelayedMetrics;
        this.clock = clock;
    }

    @Override
    public void insertMetrics( Collection<IMetric> metrics ) throws IOException {

        try {
            AstyanaxWriter.getInstance().insertFull(metrics, isRecordingDelayedMetrics, clock);
        } catch (ConnectionException e) {
            throw new IOException(e);
        }
    }

    /**
     * This method inserts a collection of {@link com.rackspacecloud.blueflood.types.IMetric} rolled up
     * objects to the appropriate Cassandra column family
     *
     * @throws IOException
     */
    @Override
    public void insertRollups(List<SingleRollupWriteContext> writeContexts) throws IOException {

        try {
            AstyanaxWriter.getInstance().insertRollups(writeContexts);
        } catch (ConnectionException ex) {
            throw new IOException(ex);
        }
    }

    /**
     * Fetches {@link com.rackspacecloud.blueflood.outputs.formats.MetricData} objects for the
     * specified {@link com.rackspacecloud.blueflood.types.Locator} and
     * {@link com.rackspacecloud.blueflood.types.Range} from the specified column family
     *
     * @param locator
     * @param range
     * @param gran
     * @return
     */
    @Override
    public MetricData getDatapointsForRange(Locator locator, Range range, Granularity gran) {

        return AstyanaxReader.getInstance().getDatapointsForRange( locator, range, gran );
    }

    /**
     * Fetches {@link com.rackspacecloud.blueflood.outputs.formats.MetricData} objects for the
     * specified list of {@link com.rackspacecloud.blueflood.types.Locator} and
     * {@link com.rackspacecloud.blueflood.types.Range} from the specified column family
     *
     * @param locators
     * @param range
     * @param gran
     * @return
     */
    @Override
    public Map<Locator, MetricData> getDatapointsForRange(List<Locator> locators, Range range, Granularity gran) {

        return AstyanaxReader.getInstance().getDatapointsForRange( locators, range, gran );
    }

    /**
     * Fetches a {@link com.rackspacecloud.blueflood.types.Points} object for a
     * particular locator and rollupType from the specified column family and
     * range
     *
     * @param locator
     * @param rollupType
     * @param range
     * @param columnFamilyName

     * @return
     */
    @Override
    public Points<BasicRollup> getDataToRollup(final Locator locator,
                                               RollupType rollupType,
                                               Range range,
                                               String columnFamilyName) throws IOException {

        return AstyanaxReader.getInstance().getDataToRoll( BasicRollup.class,
                locator,
                range,
                CassandraModel.getColumnFamily( columnFamilyName ) );
    }
}


/*
 * Copyright 2013 Rackspace
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

package com.rackspacecloud.blueflood.io.serializers;

import com.google.common.annotations.VisibleForTesting;
import com.netflix.astyanax.serializers.AbstractSerializer;
import com.rackspacecloud.blueflood.exceptions.SerializationException;
import com.rackspacecloud.blueflood.io.serializers.metrics.*;
import com.rackspacecloud.blueflood.types.*;

import java.nio.ByteBuffer;

public class Serializers {
    // NumericSerializer can be used with Rollup and full resolution metrics.

    public static final SimpleNumberSerializer simpleNumberSerializer = new SimpleNumberSerializer();
    private static RawSerializer fullInstance = new RawSerializer();
    private static BasicRollupSerializer basicRollupInstance = new BasicRollupSerializer();
    public static TimerRollupSerializer timerRollupInstance = new TimerRollupSerializer();
    public static SetRollupSerializer setRollupInstance = new SetRollupSerializer();
    public static GaugeRollupSerializer gaugeRollupInstance = new GaugeRollupSerializer();
    public static CounterRollupSerializer counterRollupInstance = new CounterRollupSerializer();

    static class Type {
        static final byte B_ROLLUP = (byte)'r';
        static final byte B_FLOAT_AS_DOUBLE = (byte)'f';
        static final byte B_ROLLUP_STAT = (byte)'t';
        static final byte B_COUNTER = (byte)'C';
        static final byte B_TIMER = (byte)'T';
        static final byte B_SET = (byte)'S';
        static final byte B_GAUGE = (byte)'G';
    }

    /** return a serializer for a specific type */
    public static <T> AbstractSerializer<T> serializerFor(Class<T> type) {
        if (type == null)
            throw new RuntimeException("serializable type cannot be null",
                    new SerializationException("serializable type cannot be null"));
        else if (type.equals(String.class))
            throw new RuntimeException("We don't serialize strings anymore",
                    new SerializationException("We don't serialize strings anymore"));

        if (type.equals(BasicRollup.class))
            return (AbstractSerializer<T>) basicRollupInstance;
        else if (type.equals(BluefloodTimerRollup.class))
            return (AbstractSerializer<T>)timerRollupInstance;
        else if (type.equals(BluefloodCounterRollup.class))
            return (AbstractSerializer<T>) counterRollupInstance;
        else if (type.equals(BluefloodGaugeRollup.class))
            return (AbstractSerializer<T>)gaugeRollupInstance;
        else if (type.equals(BluefloodSetRollup.class))
            return (AbstractSerializer<T>)setRollupInstance;
        else if (type.equals(SimpleNumber.class))
            return (AbstractSerializer<T>)fullInstance;
        else if (type.equals(Integer.class))
            return (AbstractSerializer<T>)fullInstance;
        else if (type.equals(Long.class))
            return (AbstractSerializer<T>)fullInstance;
        else if (type.equals(Double.class))
            return (AbstractSerializer<T>)fullInstance;
        else if (type.equals(Float.class))
                return (AbstractSerializer<T>)fullInstance;
        else if (type.equals(byte[].class))
            return (AbstractSerializer<T>)fullInstance;
        else if (type.equals(Object.class))
            return (AbstractSerializer<T>)fullInstance;
        else
            return (AbstractSerializer<T>)fullInstance;
    }

    public static class RawSerializer extends AbstractSerializer<Object> {

        private static RawSerDes serDes = new RawSerDes();

        // prevent people from instantiating this class
        private RawSerializer() {
        }

        @Override
        public ByteBuffer toByteBuffer(Object obj) {
            return serDes.serialize(obj);
        }

        @Override
        public Object fromByteBuffer(ByteBuffer byteBuffer) {
            return serDes.deserialize(byteBuffer);
        }
    }

    // composes a raw serializer.
    private static class SimpleNumberSerializer extends AbstractSerializer<SimpleNumber> {

        private static final RawSerializer rawSerde = new RawSerializer();

        // prevent people from instantiating this class
        private SimpleNumberSerializer() {
        }

        @Override
        public ByteBuffer toByteBuffer(SimpleNumber obj) {
            return rawSerde.toByteBuffer(obj);
        }

        @Override
        public SimpleNumber fromByteBuffer(ByteBuffer byteBuffer) {
            return new SimpleNumber(rawSerde.fromByteBuffer(byteBuffer));
        }
    }

    public static class BasicRollupSerializer extends AbstractSerializer<BasicRollup> {
        private BasicRollupSerDes serDes = new BasicRollupSerDes();

        // prevent people from instantiating this class
        private BasicRollupSerializer() {
        }

        @Override
        public ByteBuffer toByteBuffer(BasicRollup basicRollup) {
            return serDes.serialize(basicRollup);
        }

        @Override
        public BasicRollup fromByteBuffer(ByteBuffer byteBuffer) {
            return serDes.deserialize(byteBuffer);
        }
    }

    public static class TimerRollupSerializer extends AbstractSerializer<BluefloodTimerRollup> {

        private static TimerRollupSerDes serDes = new TimerRollupSerDes();

        // prevent people from instantiating this class
        private TimerRollupSerializer() {
        }

        @Override
        public ByteBuffer toByteBuffer(BluefloodTimerRollup timerRollup) {
            return serDes.serialize(timerRollup);
        }

        @VisibleForTesting
        public ByteBuffer toByteBufferWithV1Serialization(BluefloodTimerRollup timerRollup) {
            return serDes.serializeV1(timerRollup);
        }

        @Override
        public BluefloodTimerRollup fromByteBuffer(ByteBuffer byteBuffer) {
            return serDes.deserialize(byteBuffer);
        }
    }

    public static class SetRollupSerializer extends AbstractSerializer<BluefloodSetRollup> {

        private static SetSerDes serDes = new SetSerDes();

        // prevent people from instantiating this class
        private SetRollupSerializer() {
        }

        @Override
        public ByteBuffer toByteBuffer(BluefloodSetRollup setRollup) {
            return serDes.serialize(setRollup);
        }

        @Override
        public BluefloodSetRollup fromByteBuffer(ByteBuffer byteBuffer) {
            return serDes.deserialize(byteBuffer);
        }
    }

    public static class GaugeRollupSerializer extends AbstractSerializer<BluefloodGaugeRollup> {

        private static GaugeSerDes serDes = new GaugeSerDes();

        // prevent people from instantiating this class
        private GaugeRollupSerializer() {
        }

        @Override
        public ByteBuffer toByteBuffer(BluefloodGaugeRollup gaugeRollup) {
           return serDes.serialize(gaugeRollup);
        }

        @Override
        public BluefloodGaugeRollup fromByteBuffer(ByteBuffer byteBuffer) {
            return serDes.deserialize(byteBuffer);
        }
    }

    // for now let's try to get away with a single serializer for all single value rollups. We'll still encode specific
    // types so we can differentiate.
    public static class CounterRollupSerializer extends AbstractSerializer<BluefloodCounterRollup> {

        private static CounterSerDes serDes = new CounterSerDes();

        // prevent people from instantiating this class
        private CounterRollupSerializer() {
        }

        @Override
        public ByteBuffer toByteBuffer(BluefloodCounterRollup counterRollup) {
            return serDes.serialize(counterRollup);
        }

        @Override
        public BluefloodCounterRollup fromByteBuffer(ByteBuffer byteBuffer) {
            return serDes.deserialize(byteBuffer);
        }
    }

    // prevent people from instantiating this class
    private Serializers() {
    }
}


package com.rackspacecloud.blueflood.http;

import com.google.common.base.Strings;
import io.netty.handler.codec.http.*;

/**
 * A small utility class to be used by our Ingest and Query handlers to check for
 * valid media-type related headers.
 */
public class MediaTypeChecker {

    private static final String MEDIA_TYPE_APPLICATION_JSON = "application/json";
    private static final String ACCEPT_ALL = "*/*";

    /**
     * Checks the Content-Type header to see if clients specify the right
     * media type
     * @param headers
     * @return
     */
    public boolean isContentTypeValid(HttpHeaders headers) {

        String contentType = headers.get(HttpHeaders.Names.CONTENT_TYPE);

        // if we get no Content-Type or we get application/json, then it's valid
        // any other, it's invalid
        return (Strings.isNullOrEmpty(contentType) ||
                contentType.toLowerCase().contains(MEDIA_TYPE_APPLICATION_JSON));
    }

    /**
     * Checks the Accept header to see if clients accept the correct
     * media type
     * @param headers
     * @return
     */
    public boolean isAcceptValid(HttpHeaders headers) {
        String accept = headers.get(HttpHeaders.Names.ACCEPT);

        // if we get no Accept (which means */*), or */*,
        // or application/json, then it's valid
        return ( Strings.isNullOrEmpty(accept) ||
                accept.contains(ACCEPT_ALL) ||
                accept.toLowerCase().contains(MEDIA_TYPE_APPLICATION_JSON));
    }
}


/*
 * Copyright 2013 Rackspace
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

package com.rackspacecloud.blueflood.io;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import com.rackspacecloud.blueflood.types.IMetric;

import com.codahale.metrics.Timer;

import org.apache.http.HttpStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.*;

public class ElasticIO extends AbstractElasticIO {

    private static final Logger log = LoggerFactory.getLogger(ElasticIO.class);

    public ElasticIO() {
        this.elasticsearchRestHelper = ElasticsearchRestHelper.getInstance();
    }

    public void insertDiscovery(IMetric metric) throws IOException {
        List<IMetric> batch = new ArrayList<>();
        batch.add(metric);
        insertDiscovery(batch);
    }

    // REST call to index into ES
    public void insertDiscovery(List<IMetric> batch) throws IOException {
        batchHistogram.update(batch.size());
        if (batch.size() == 0) {
            log.debug("ElasticIO: batch size for insertDiscovery is zero, so skip calling Elasticsearch ingest.");
            return;
        }

        Timer.Context ctx = writeTimer.time();
        try {
            for (Object obj : batch) {
                if (!(obj instanceof IMetric)) {
                    classCastExceptionMeter.mark();
                    continue;
                }
            }

            elasticsearchRestHelper.indexMetrics(batch);
        } finally {
            ctx.stop();
        }
    }

    @VisibleForTesting
    public void setINDEX_NAME_WRITE (String indexNameWrite) {
        ELASTICSEARCH_INDEX_NAME_WRITE = indexNameWrite;
    }

    @VisibleForTesting
    public void setINDEX_NAME_READ (String indexNameRead) {
        ELASTICSEARCH_INDEX_NAME_READ = indexNameRead;
    }

    @Override
    protected List<SearchResult> dedupResults(List<SearchResult> results) {
        HashMap<String, SearchResult> dedupedResults = new HashMap<>();
        for (SearchResult result : results)
            dedupedResults.put(result.getMetricName(), result);
        return Lists.newArrayList(dedupedResults.values());
    }
}


/*
 * Copyright 2014 Rackspace
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

package com.rackspacecloud.blueflood.inputs.formats;

import com.google.common.annotations.VisibleForTesting;
import com.google.gson.Gson;
import com.google.gson.JsonElement;
import com.rackspacecloud.blueflood.exceptions.InvalidDataException;
import com.rackspacecloud.blueflood.inputs.constraints.EpochRange;
import com.rackspacecloud.blueflood.inputs.constraints.EpochRangeLimits;
import com.rackspacecloud.blueflood.io.Instrumentation;
import com.rackspacecloud.blueflood.outputs.formats.ErrorResponse;
import com.rackspacecloud.blueflood.service.Configuration;
import com.rackspacecloud.blueflood.service.CoreConfig;
import com.rackspacecloud.blueflood.types.*;
import org.hibernate.validator.constraints.NotEmpty;

import javax.validation.*;
import javax.validation.constraints.AssertTrue;
import java.util.*;

// Using nested classes for now. Expect this to be cleaned up.
public class AggregatedPayload {

    private static final long TRACKER_DELAYED_METRICS_MILLIS = Configuration.getInstance().getLongProperty(CoreConfig.TRACKER_DELAYED_METRICS_MILLIS);
    private static final long MAX_AGE_ALLOWED = Configuration.getInstance().getLongProperty(CoreConfig.ROLLUP_DELAY_MILLIS);
    private static final long SHORT_DELAY = Configuration.getInstance().getLongProperty(CoreConfig.SHORT_DELAY_METRICS_ROLLUP_DELAY_MILLIS);

    @NotEmpty
    private String tenantId;

    @EpochRange(maxPast = EpochRangeLimits.BEFORE_CURRENT_TIME_MS,
            maxFuture = EpochRangeLimits.AFTER_CURRENT_TIME_MS,
            message = "Out of bounds. Cannot be more than ${maxPast.getValue()} milliseconds into the past. Cannot be more than ${maxFuture.getValue()} milliseconds into the future")
    private long timestamp; // millis since epoch.

    // this field is optional
    @org.hibernate.validator.constraints.Range(min=0, max=Long.MAX_VALUE)
    private long flushInterval = 0;

    @Valid
    private BluefloodGauge[] gauges;

    @Valid
    private BluefloodCounter[] counters;

    @Valid
    private BluefloodTimer[] timers;

    @Valid
    private BluefloodSet[] sets;

    private final List<ErrorResponse.ErrorData> validationErrors;

    private static final ValidatorFactory factory = Validation.buildDefaultValidatorFactory();
    protected static final Validator validator = factory.getValidator();

    public AggregatedPayload() {
        this.validationErrors = new java.util.ArrayList<ErrorResponse.ErrorData>();
    }

    public static AggregatedPayload create(JsonElement json) {
        AggregatedPayload payload = new Gson().fromJson(json, AggregatedPayload.class);
        validate(payload);

        return payload;
    }

    public static AggregatedPayload create(String json) {

        AggregatedPayload payload = new Gson().fromJson(json, AggregatedPayload.class);
        validate(payload);

        return payload;
    }

    private static void validate(AggregatedPayload payload) {
        if (payload == null) {
            throw new InvalidDataException("Invalid request body");
        }

        Set<ConstraintViolation<AggregatedPayload>> constraintViolations = validator.validate(payload);
        for (ConstraintViolation<AggregatedPayload> constraintViolation : constraintViolations) {
            String source = constraintViolation.getPropertyPath().toString().equalsIgnoreCase("valid") ? "" :
                    constraintViolation.getPropertyPath().toString();

            String metricName = "";

            Object leafBean = constraintViolation.getLeafBean();
            if (leafBean instanceof BluefloodGauge) {
                metricName = ((BluefloodGauge) leafBean).getName();
            } else if (leafBean instanceof BluefloodCounter) {
                metricName = ((BluefloodCounter) leafBean).getName();
            } else if (leafBean instanceof BluefloodTimer) {
                metricName = ((BluefloodTimer) leafBean).getName();
            } else if (leafBean instanceof BluefloodSet) {
                metricName = ((BluefloodSet) leafBean).getName();
            }

            payload.validationErrors.add(new ErrorResponse.ErrorData(payload.getTenantId(), metricName,
                    source, constraintViolation.getMessage(), payload.getTimestamp()));
        }
    }

    @VisibleForTesting
    public AggregatedPayload(String tenantId, long timestamp, long flushInterval, BluefloodGauge[] gauges,
                             BluefloodCounter[] counters, BluefloodTimer[] timers, BluefloodSet[] sets) {

        this();
        this.tenantId = tenantId;
        this.timestamp = timestamp;
        this.flushInterval = flushInterval;
        this.gauges = gauges;
        this.counters = counters;
        this.timers = timers;
        this.sets = sets;
    }

    public String toString() {
        return String.format("%s (%d)", tenantId, timestamp);
    }

    public String getTenantId() { return tenantId; }

    // seconds since epoch.
    public long getTimestamp() { return timestamp; }

    public long getFlushIntervalMillis() { return flushInterval; }

    public Collection<BluefloodGauge> getGauges() { return safeAsList(gauges); }
    public Collection<BluefloodCounter> getCounters() { return safeAsList(counters); }
    public Collection<BluefloodTimer> getTimers() { return safeAsList(timers); }
    public Collection<BluefloodSet> getSets() { return safeAsList(sets); }

    public List<ErrorResponse.ErrorData> getValidationErrors() {
        return this.validationErrors;
    }

    /**
     * This method is invoked by the validator automatically
     */
    @AssertTrue(message="At least one of the aggregated metrics(gauges, counters, timers, sets) are expected")
    private boolean isValid() {
        boolean isGaugePresent = gauges != null && gauges.length > 0;
        boolean isCounterPresent = counters != null && counters.length > 0;
        boolean isTimerPresent = timers != null && timers.length > 0;
        boolean isSetPresent = sets != null && sets.length > 0;

        return (isGaugePresent || isCounterPresent || isTimerPresent || isSetPresent);
    }

    /**
     * Determines if the metrics represented in this {@link AggregatedPayload}
     * has collection timestamp that we consider "late" or "delayed".
     *
     * @param ingestTime
     * @return
     */
    public boolean hasDelayedMetrics(long ingestTime) {
        return getDelayTime(ingestTime) > TRACKER_DELAYED_METRICS_MILLIS;
    }

    /**
     * Calculates how many milliseconds is the collection timestamp of
     * this {@link AggregatedPayload} delayed by.
     *
     * @param ingestTime  ingest timestamp in milliseconds
     * @return
     */
    public long getDelayTime(long ingestTime) {
        return ingestTime - timestamp;
    }

    /**
     * Marks/instruments our internal metrics that we have received
     * short or delayed metrics
     *
     * @param ingestTime
     * @return
     */
    public void markDelayMetricsReceived(long ingestTime) {
        long delay = getDelayTime(ingestTime);
        if ( delay > MAX_AGE_ALLOWED ) {
            if ( delay <= SHORT_DELAY ) {
                Instrumentation.markMetricsWithShortDelayReceived();
            } else {
                Instrumentation.markMetricsWithLongDelayReceived();
            }
        }
    }

    public List<String> getAllMetricNames() {
        List<String> metricNames = new java.util.ArrayList<String>();
        if ( gauges != null && gauges.length > 0) {
            for (int index=0; index<gauges.length; index++) {
                metricNames.add(gauges[index].getName());
            }
        }

        if ( counters != null && counters.length > 0) {
            for (int index=0; index<counters.length; index++) {
                metricNames.add(counters[index].getName());
            }
        }

        if ( timers != null && timers.length > 0) {
            for (int index=0; index<timers.length; index++) {
                metricNames.add(timers[index].getName());
            }
        }

        if ( sets != null && sets.length > 0) {
            for (int index = 0; index < sets.length; index++) {
                metricNames.add(sets[index].getName());
            }
        }

        return metricNames;
    }

    //@SafeVarargs (1.7 only doge)
    public static <T> List<T> safeAsList(final T... a) {
        if (a == null)
            return new java.util.ArrayList<T>();
        else
            return new ArrayList<T>(a);
    }

    private static class ArrayList<E> extends AbstractList<E> {
        private final E[] array;

        public ArrayList(E[] array) {
            this.array = array;
        }

        @Override
        public E get(int index) {
            if (index < 0 || index > array.length-1)
                throw new ArrayIndexOutOfBoundsException("Invalid array offset: " + index);
            return array[index];
        }

        @Override
        public int size() {
            return array.length;
        }
    }
}


/*
 * Copyright 2014 Rackspace
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.

 * Original author: gdusbabek
 * Modified by: chinmay
 */

package com.rackspacecloud.blueflood.CloudFilesBackfiller.download;

import com.codahale.metrics.Timer;
import com.rackspacecloud.blueflood.utils.Metrics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;

public class DownloadService {
    private static final Logger log = LoggerFactory.getLogger(DownloadService.class);
    private static final int MAX_FILES = 5;
    private static final int MAX_UNEXPECTED_ERRORS = 5;
    private final File downloadDir;
    private final Thread thread;
    private final Lock downloadLock = new ReentrantLock(true);
    private FileManager fileManager = null; // todo: should be final.
    private static Timer waitingTimer = Metrics.timer(DownloadService.class, "Delay introduced because cloud files were not consumed/merged");

    private boolean running = false;
    private boolean terminated = false;

    private int unexpectedErrors = 0;

    public DownloadService(final File downloadDir) {
        this.downloadDir = downloadDir;
        this.running = false;
        this.thread = new Thread("Download Service") {
            public void run() {
                while (!terminated) {
                    // if there are > 1 tmp files in the dir, just wait.
                    FilenameFilter filter = new FilenameFilter() {
                        public boolean accept(File dir, String name) {
                            return name.endsWith(".json.gz.tmp");
                        }
                    };
                    while (downloadDir.listFiles(filter).length > 1) {
                        try {
                            sleep(200L);
                        } catch (InterruptedException ex) {
                            Thread.interrupted();
                        }
                    }
                    doCheck();
                    try {
                        sleep(1000);
                    } catch (InterruptedException e) {
                        Thread.interrupted();
                    }
                }
                log.debug("Download service thread stopping");
            }
        };
        this.thread.start();
    }

    public void setFileManager(FileManager newFileManager) {
        downloadLock.lock();
        try {
            fileManager = newFileManager;
        } finally {
            downloadLock.unlock();
        }
    }

    public synchronized void start() throws IOException {
        if (terminated)
            throw new IOException("Download service has been terminated. It cannot be restarted.");
        if (running)
            throw new IOException("Download service is already running.");
        if (!downloadDir.exists())
            throw new IOException("Download directory does not exist");

        log.info("Resuming downloads");
        running = true;
        thread.interrupt();
    }

    public synchronized void stop() {
        log.info("Stopping download service");
        running = false;
        thread.interrupt();
    }

    public synchronized void terminate(boolean waitFor) {
        terminated = true;
        stop();
        log.info("Terminating download service");
        thread.interrupt();
        if (waitFor) {
            log.info("Wating for download service termination");
            while (thread.isInterrupted() || thread.isAlive()) {
                try { Thread.sleep(100); } catch (Exception ex) {}
            }
            log.info("Download service terminated");
        }
    }

    // gets run by the thread.
    private void doCheck() {
        if (!running) return;
        if (fileManager == null) return;

        if (unexpectedErrors > MAX_UNEXPECTED_ERRORS) {
            log.info("Terminating because of errors");
            terminate(false);
            return;
        }

        Timer.Context waitTimerContext = waitingTimer.time();
        // Possible infinite thread sleep? This will make sure we fire downloading only when are the files are consumed/merged
        while (downloadDir.listFiles().length != 0) {
            log.debug("Waiting for files in download directory to clear up. Sleeping for 1 min. If you see this persistently, it means the downloaded files are not getting merged properly/timely");
            try { Thread.sleep(60000); } catch (Exception ex) {}
        }
        waitTimerContext.stop();
        if (downloadLock.tryLock()) {
            try {
                if (fileManager.hasNewFiles()) {
                    fileManager.downloadNewFiles(downloadDir);
                }
            } catch (Throwable unexpected) {
                unexpectedErrors += 1;
                log.error("UNEXPECTED; WILL TRY TO RECOVER");
                log.error(unexpected.getMessage(), unexpected);
                // sleep for a minute?
                if (Thread.interrupted()) {
                    try {
                        thread.sleep(60000);
                    } catch (Exception ex) {
                        log.error(ex.getMessage(), ex);
                    }
                }
            } finally {
                downloadLock.unlock();
            }
        } else {
            log.debug("Download in progress");
        }
    }
}


/*
 * Copyright (c) 2016 Rackspace.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * 	http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.rackspacecloud.blueflood.io.datastax;

import com.codahale.metrics.Timer;
import com.datastax.driver.core.*;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Multimap;
import com.rackspacecloud.blueflood.cache.LocatorCache;
import com.rackspacecloud.blueflood.exceptions.InvalidDataException;
import com.rackspacecloud.blueflood.io.*;
import com.rackspacecloud.blueflood.outputs.formats.MetricData;
import com.rackspacecloud.blueflood.rollup.Granularity;
import com.rackspacecloud.blueflood.types.*;
import com.rackspacecloud.blueflood.utils.*;
import com.rackspacecloud.blueflood.utils.Metrics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.*;

/**
 * This class deals with reading/writing metrics to the metrics_preaggregated_* column families
 * using Datastax driver
 */
public class DPreaggregatedMetricsRW extends DAbstractMetricsRW implements PreaggregatedRW {

    private static final Logger LOG = LoggerFactory.getLogger(DPreaggregatedMetricsRW.class);
    private static final Timer waitResultsTimer = Metrics.timer(DPreaggregatedMetricsRW.class, "PreAggr Metrics Wait Write Results");

    private final DCounterIO counterIO = new DCounterIO();
    private final DGagueIO gaugeIO = new DGagueIO();
    private final DSetIO setIO = new DSetIO();
    private final DTimerIO timerIO = new DTimerIO();

    // a map of RollupType to its IO class that knows
    // how to read/write that particular type of rollup
    private final Map<RollupType, DAbstractMetricIO> rollupTypeToIO =
            new HashMap<RollupType, DAbstractMetricIO>();

    /**
     * Constructor
     * @param locatorIO
     * @param isRecordingDelayedMetrics
     */
    public DPreaggregatedMetricsRW(DLocatorIO locatorIO, DDelayedLocatorIO delayedLocatorIO,
                                   boolean isRecordingDelayedMetrics,
                                   Clock clock) {
        super(locatorIO, delayedLocatorIO, isRecordingDelayedMetrics, clock);
        rollupTypeToIO.put(RollupType.COUNTER, counterIO);
        rollupTypeToIO.put(RollupType.GAUGE, gaugeIO);
        rollupTypeToIO.put(RollupType.SET, setIO);
        rollupTypeToIO.put(RollupType.TIMER, timerIO);
    }

    /**
     * Constructor
     * @param locatorIO
     * @param isRecordingDelayedMetrics
     */
    @VisibleForTesting
    public DPreaggregatedMetricsRW(DLocatorIO locatorIO, DDelayedLocatorIO delayedLocatorIO,
                                   boolean isRecordingDelayedMetrics,
                                   boolean isBatchIngestEnabled, Clock clock) {
        super(locatorIO, delayedLocatorIO, isRecordingDelayedMetrics, isBatchIngestEnabled, clock);
        rollupTypeToIO.put(RollupType.COUNTER, counterIO);
        rollupTypeToIO.put(RollupType.GAUGE, gaugeIO);
        rollupTypeToIO.put(RollupType.SET, setIO);
        rollupTypeToIO.put(RollupType.TIMER, timerIO);
    }

    /**
     * Inserts a collection of metrics to the metrics_preaggregated_full column family
     *
     * @param metrics
     * @throws IOException
     */
    @Override
    public void insertMetrics(Collection<IMetric> metrics) throws IOException {
        insertMetrics(metrics, Granularity.FULL);
    }

    /**
     * Inserts a collection of rolled up metrics to the metrics_preaggregated_{granularity} column family.
     * Only our tests should call this method. Services should call either insertMetrics(Collection metrics)
     * or insertRollups()
     *
     * @param metrics
     * @throws IOException
     */
    @VisibleForTesting
    @Override
    public void insertMetrics(Collection<IMetric> metrics,
                              Granularity granularity) throws IOException {

        Timer.Context ctx = Instrumentation.getWriteTimerContext(
                CassandraModel.getPreaggregatedColumnFamilyName(granularity));
        try {
            Multimap<Locator, IMetric> map = asMultimap(metrics);

            if ( isBatchIngestEnabled ) {
                insertMetricsInBatch(map, granularity);
            } else {
                insertMetricsIndividually(map, granularity);
            }
        } finally {
            ctx.stop();
        }
    }

    /**
     * Fetches {@link com.rackspacecloud.blueflood.outputs.formats.MetricData} objects for the
     * specified list of {@link com.rackspacecloud.blueflood.types.Locator} and
     * {@link com.rackspacecloud.blueflood.types.Range} from the specified column family
     *
     * @param locators
     * @param range
     * @param granularity
     * @return
     */
    @Override
    public Map<Locator, MetricData> getDatapointsForRange(List<Locator> locators,
                                                          Range range,
                                                          Granularity granularity) /*throws IOException*/ {

        String columnFamily = CassandraModel.getPreaggregatedColumnFamilyName(granularity);

        return getDatapointsForRange( locators, range, columnFamily, granularity );
    }

    /**
     * Return the appropriate IO object which interacts with the Cassandra database.
     *
     * For Preaggregated Metrics, only rollup type is required.
     * For Basic Numeric Metrics, the granularity is required as metrics_full is handled differently
     * than the other granularities.
     *
     * @param rollupType
     * @param granularity
     * @return
     */
    @Override
    public DAbstractMetricIO getIO( String rollupType, Granularity granularity ) {

        // find out the rollupType for this locator
        RollupType rType = RollupType.fromString( rollupType );

        // get the right PreaggregatedIO class that can process
        // this rollupType
        DAbstractMetricIO io = rollupTypeToIO.get( rType );

        if (io == null) {
            throw new InvalidDataException( String.format("getIO: unsupported rollupType=%s", rollupType));
        }

        return io;
    }

    private void insertMetricsIndividually(Multimap<Locator, IMetric> map, Granularity granularity)
        throws IOException
    {
        Map<ResultSetFuture, Locator> futureLocatorMap = new HashMap<ResultSetFuture, Locator>();
        for (Locator locator : map.keySet()) {
            for (IMetric metric : map.get(locator)) {
                RollupType rollupType = metric.getRollupType();

                // lookup the right io object
                DAbstractMetricIO io = rollupTypeToIO.get(rollupType);
                if ( io == null ) {
                    throw new InvalidDataException(
                            String.format("insertMetrics(locator=%s, granularity=%s): unsupported preaggregated rollupType=%s",
                                    locator, granularity, rollupType.name()));
                }

                if (!(metric.getMetricValue() instanceof Rollup)) {
                    throw new InvalidDataException(
                            String.format("insertMetrics(locator=%s, granularity=%s): metric value %s is not type Rollup",
                                    locator, granularity, metric.getMetricValue().getClass().getSimpleName())
                    );
                }
                ResultSetFuture future = io.putAsync(locator, metric.getCollectionTime(),
                        (Rollup) metric.getMetricValue(),
                        granularity, metric.getTtlInSeconds());

                futureLocatorMap.put(future, locator);
                if (granularity == Granularity.FULL) {
                    Instrumentation.markFullResPreaggregatedMetricWritten();
                }

                if ( !LocatorCache.getInstance().isLocatorCurrentInBatchLayer(locator) ) {
                    locatorIO.insertLocator(locator);
                    LocatorCache.getInstance().setLocatorCurrentInBatchLayer(locator);
                }  else {
                    LOG.trace("insertMetrics(): not inserting locator " + locator);
                }

                if (isRecordingDelayedMetrics) {
                    insertLocatorIfDelayed(metric);
                }

            }
        }

        for (ResultSetFuture future : futureLocatorMap.keySet()) {
            try {
                future.getUninterruptibly().all();
            } catch (Exception ex) {
                Instrumentation.markWriteError();
                LOG.error(String.format("error writing preaggregated metric for locator %s, granularity %s",
                        futureLocatorMap.get(future), granularity), ex);
            }
        }
    }

    private void insertMetricsInBatch(Multimap<Locator, IMetric> map, Granularity granularity) {
        BatchStatement batch = new BatchStatement(BatchStatement.Type.UNLOGGED);

        for (Locator locator : map.keySet()) {
            for (IMetric metric : map.get(locator)) {
                RollupType rollupType = metric.getRollupType();

                DAbstractMetricIO io = rollupTypeToIO.get(rollupType);
                BoundStatement boundStatement = io.getBoundStatementForMetric(metric, granularity);
                batch.add(boundStatement);

                if (granularity == Granularity.FULL) {
                    Instrumentation.markFullResPreaggregatedMetricWritten();
                }

                if( !LocatorCache.getInstance().isLocatorCurrentInBatchLayer(locator) ) {
                    LocatorCache.getInstance().setLocatorCurrentInBatchLayer(locator);
                    batch.add(locatorIO.getBoundStatementForLocator( locator ));
                }

                // if we are recording delayed metrics, we may need to do an
                // extra insert
                if ( isRecordingDelayedMetrics ) {
                    BoundStatement bs = getBoundStatementForMetricIfDelayed(metric);
                    if ( bs != null ) {
                        batch.add(bs);
                    }
                }
            }
        }
        LOG.trace(String.format("insert preaggregated batch statement size=%d", batch.size()));

        try {
            DatastaxIO.getSession().execute(batch);
        } catch ( Exception ex ) {
            Instrumentation.markWriteError();
            LOG.error(String.format("error writing batch of %d preaggregated metrics", batch.size()), ex );
        }
    }
}


package io.arivera.oss.embedded.rabbitmq.util;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.net.ServerSocket;
import javax.net.ServerSocketFactory;

public class RandomPortSupplier {

  private static final Logger LOGGER = LoggerFactory.getLogger(RandomPortSupplier.class);

  private final ServerSocketFactory severSocketFactory;

  public RandomPortSupplier() {
    this(ServerSocketFactory.getDefault());
  }

  public RandomPortSupplier(ServerSocketFactory severSocketFactory) {
    this.severSocketFactory = severSocketFactory;
  }

  /**
   * @return an available port assigned at random by the OS.
   *
   * @throws IllegalStateException if the port cannot be determined.
   */
  public int get() throws IllegalStateException {
    ServerSocket socket = null;
    try {
      socket = this.severSocketFactory.createServerSocket(0);
      socket.setReuseAddress(false);
      return socket.getLocalPort();
    } catch (IOException e) {
      throw new IllegalStateException("Could not determine random port to assign.", e);
    } finally {
      if (socket != null) {
        try {
          socket.close();
        } catch (IOException e) {
          LOGGER.debug("Couldn't close socket that was temporarily opened to determine random port.", e);
        }
      }
    }
  }
}


package io.arivera.oss.embedded.rabbitmq.extract;

import io.arivera.oss.embedded.rabbitmq.EmbeddedRabbitMqConfig;

public class ExtractorFactory {

  private EmbeddedRabbitMqConfig config;

  public ExtractorFactory(EmbeddedRabbitMqConfig config) {
    this.config = config;
  }

  /**
   * Returns an Extractor instance appropriate based on the given configuration.
   */
  public Extractor getNewInstance() {
    Extractor extractor = new BasicExtractor(config);
    if (config.shouldCachedDownload()) {
      extractor = new CachedExtractor(extractor, config);
    }
    return extractor;
  }

}