package com.rackspacecloud.blueflood.tools.ops;

import com.google.common.collect.Sets;
import com.netflix.astyanax.AstyanaxContext;
import com.netflix.astyanax.ColumnListMutation;
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.RowCallback;
import com.netflix.astyanax.connectionpool.NodeDiscoveryType;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import com.netflix.astyanax.connectionpool.impl.ConnectionPoolConfigurationImpl;
import com.netflix.astyanax.connectionpool.impl.CountingConnectionPoolMonitor;
import com.netflix.astyanax.connectionpool.impl.FixedRetryBackoffStrategy;
import com.netflix.astyanax.impl.AstyanaxConfigurationImpl;
import com.netflix.astyanax.model.ByteBufferRange;
import com.netflix.astyanax.model.Column;
import com.netflix.astyanax.model.ColumnFamily;
import com.netflix.astyanax.model.ColumnList;
import com.netflix.astyanax.model.Row;
import com.netflix.astyanax.model.Rows;
import com.netflix.astyanax.retry.RetryNTimes;
import com.netflix.astyanax.thrift.ThriftFamilyFactory;
import com.netflix.astyanax.util.RangeBuilder;
import com.rackspacecloud.blueflood.io.CassandraModel;
import com.rackspacecloud.blueflood.types.Locator;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;

import javax.xml.bind.DatatypeConverter;
import java.io.PrintStream;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;

public class Migration {

    private static final Options cliOptions = new Options();
    private static final double VERIFY_PERCENT = 0.005f; // half of one percent.

    private static final String SRC = "src";
    private static final String DST = "dst";
    private static final String FROM = "from";
    private static final String TO = "to";
    private static final String COLUMN_FAMILY = "cf";
    private static final String SKIP = "skip";
    private static final String LIMIT = "limit";
    private static final String TTL = "ttl";
    private static final String WRITE_THREADS = "writethreads";
    private static final String READ_THREADS = "readthreads";
    private static final String BATCH_SIZE = "batchsize";
    private static final String VERIFY = "verify";
    private static final String DISCOVER = "discover";
    private static final String RATE = "rate";

    private static final PrintStream out = System.out;

    static {
        cliOptions.addOption(OptionBuilder.isRequired().hasArg(true).withValueSeparator(',').withDescription("[required] Source cassandra cluster (host:port:keyspace).").create(SRC));
        cliOptions.addOption(OptionBuilder.isRequired().hasArg(true).withValueSeparator(',').withDescription("[required] Destination cassandra cluster (host:port:keyspace).").create(DST));
        cliOptions.addOption(OptionBuilder.hasArg().withDescription("[optional] ISO 6801 datetime (or millis since epoch) of when to start migrating data. defaults to one year ago.").create(FROM));
        cliOptions.addOption(OptionBuilder.hasArg().withDescription("[optional] ISO 6801 datetime (or millis since epoch) Datetime of when to stop migrating data. defaults to right now.").create(TO));
        cliOptions.addOption(OptionBuilder.isRequired().hasArg().withValueSeparator(',').withDescription("[required] Which column family to migrate").create(COLUMN_FAMILY));
        cliOptions.addOption(OptionBuilder.hasArg().withDescription("[optional] Number of keys to skip before processing. default=0.").create(SKIP));
        cliOptions.addOption(OptionBuilder.hasArg().withDescription("[optional] Maximum number of keys to process. default=MAX_INT.").create(LIMIT));
        cliOptions.addOption(OptionBuilder.hasArg().withDescription("[optional] ttl in seconds for new data. default=5x the default for the column family.").create(TTL));
        cliOptions.addOption(OptionBuilder.hasArg().withDescription("[optional] number of read threads to use. default=1").create(READ_THREADS));
        cliOptions.addOption(OptionBuilder.hasArg().withDescription("[optional] number of write threads to use. default=1").create(WRITE_THREADS));
        cliOptions.addOption(OptionBuilder.hasArg().withDescription("[optional] number of rows to read per query. default=100").create(BATCH_SIZE));
        cliOptions.addOption(OptionBuilder.withDescription("[optional] verify a sampling 0.5% of data copied").create(VERIFY));
        cliOptions.addOption(OptionBuilder.withDescription("[optional] discover and query other cassandra nodes").create(DISCOVER));
        cliOptions.addOption(OptionBuilder.hasArg().withDescription("[optional] maximum number of columns per/second to transfer. default=500").create(RATE));
    }

    private static long nowInSeconds() {
        return System.currentTimeMillis() / 1000;
    }

    public static void main(String args[]) {
        nullRouteAllLog4j();

        Map<String, Object> options = parseOptions(args);

        final int readThreads = (Integer)options.get(READ_THREADS);
        final int writeThreads = (Integer)options.get(WRITE_THREADS);
        final int keyLimit = (Integer)options.get(LIMIT);
        final int batchSize = (Integer)options.get(BATCH_SIZE);
        final int skip = (Integer)options.get(SKIP);
        final int ttl = (Integer)options.get(TTL);
        final int rate = (Integer)options.get(RATE);
        NodeDiscoveryType discovery = (NodeDiscoveryType)options.get(DISCOVER);

        // connect to src cluster.
        String[] srcParts = options.get(SRC).toString().split(":", -1);
        final AstyanaxContext<Keyspace> srcContext = connect(srcParts[0], Integer.parseInt(srcParts[1]), srcParts[2], readThreads, discovery);
        final Keyspace srcKeyspace = srcContext.getEntity();

        // connect to dst cluster.
        String[] dstParts = options.get(DST).toString().split(":", -1);
        final AstyanaxContext<Keyspace> dstContext = connect(dstParts[0], Integer.parseInt(dstParts[1]), dstParts[2], writeThreads, discovery);
        final Keyspace dstKeyspace = dstContext.getEntity();

        final AtomicLong columnsTransferred = new AtomicLong(0);
        final long startClockTime = nowInSeconds();

        // establish column range.
        final ByteBufferRange range = new RangeBuilder()
                .setStart((Long) options.get(FROM))
                .setEnd((Long) options.get(TO)).build();

        // create a threadpool that will write stuff into the destination.
        final ThreadPoolExecutor destWriteExecutor = new ThreadPoolExecutor(writeThreads, writeThreads,
                0L, TimeUnit.MILLISECONDS,
                new LinkedBlockingQueue<Runnable>());

        // this threadpool ensures single-threaded output statements.
        final ThreadPoolExecutor postExecutor = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>());

        // this threadpool performs verifications.
        final ThreadPoolExecutor verifyExecutor = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>());

        // keep track of the number of keys that have been copied.
        final AtomicInteger processedKeys = new AtomicInteger(0);

        // keep track of the number of keys that have been iterated over (includes skips).
        final AtomicInteger iteratedKeys = new AtomicInteger(0);
        final AtomicBoolean skipping = new AtomicBoolean(true);

        // sentinal that indicates it is time to stop doing everything.
        final AtomicBoolean stopAll = new AtomicBoolean(false);

        final AtomicLong heartbeat = new AtomicLong(System.currentTimeMillis());
        final boolean verify = (Boolean)options.get(VERIFY);
        final Random random = new Random(System.nanoTime());

        // indicate what's going to happen.
        out.println(String.format("Will process roughly %d keys from %s to %s for dates %s to %s",
                keyLimit,
                options.get(SRC),
                options.get(DST),
                new Date((Long)options.get(FROM)),
                new Date((Long)options.get(TO))));
        if (skip > 0) {
            out.println("Be patient while I skip " + skip + " keys");
        }

        try {
            final ColumnFamily<Locator, Long> columnFamily = (ColumnFamily<Locator, Long>)options.get(COLUMN_FAMILY);

            // when we skip, we'd like the batch size to be larger. that way, if the user specified a small batch size,
            // we don't spend a lot of time iterating through nothing. this should reduce the number of rount trips to
            // one per thread.
            int realizedBatchSize = batchSize;
            if (skip > 0 && iteratedKeys.get() < skip) {
                realizedBatchSize = skip / readThreads;
            }
            realizedBatchSize = Math.min(5000, realizedBatchSize); // no too big though.

            // we have no way of knowing when we've processed all rows (no callbacks or anthying); this thread makes
            // sure that work is being done. when it sees a 30s period of nothing, it shuts things down.
            new Thread("Stagnation") {
                public void run() {
                    while (!destWriteExecutor.isShutdown()) {
                        if (System.currentTimeMillis() - heartbeat.get() > 30*1000) {
                            if (!skipping.get()) {
                                out.println("It looks like we're done");
                                destWriteExecutor.shutdown();
                                postExecutor.shutdown();
                                verifyExecutor.shutdown();
                                srcContext.shutdown();
                                dstContext.shutdown();
                                break;
                            }
                        }
                        try { sleep(1000L); } catch (Exception ex) {}
                    }
                }
            }.start();

            // get all the data.
            srcKeyspace.prepareQuery(columnFamily)
                    .getAllRows()
                    .setRowLimit(realizedBatchSize)
                    .setRepeatLastToken(false)
                    .withColumnRange(range)
                    .setConcurrencyLevel(readThreads)
                    .executeWithCallback(new RowCallback<Locator, Long>() {

                        @Override
                        public void success(Rows<Locator, Long> rows) {
                            if (skipping.get()) {
                                out.println(String.format("skipping... " + iteratedKeys.get()));
                            }
                            for (Locator locator : rows.getKeys()) {

                                // short circuit quit if wee need to.
                                if (stopAll.get()) break;

                                // do skipping if needed.
                                int overallKey = iteratedKeys.incrementAndGet();
                                if (overallKey < skip) {
                                    continue;
                                }
                                skipping.set(false);

                                // shut things down when we reach the key limit.
                                if (processedKeys.get() >= keyLimit && !stopAll.get()) {
                                    out.println("Reached key limit.");
                                    stopAll.set(true);
                                    destWriteExecutor.shutdownNow();
                                    verifyExecutor.shutdownNow();
                                    postExecutor.submit(new Runnable() {
                                        public void run() {
                                            srcContext.shutdown();
                                            dstContext.shutdown();
                                        }
                                    });
                                    postExecutor.shutdown();

                                    break;
                                }

                                final Locator locatorCapture = locator;
                                final Row<Locator, Long> row = rows.getRow(locator);

                                // send copy commands to the write thread pool.
                                destWriteExecutor.submit(new Runnable() {
                                    public void run() {
                                        // back out if we've processed our quota of rows.
                                        if (processedKeys.get() >= keyLimit) {
                                            return;
                                        }

                                        // copy the column.
                                        MutationBatch batch = dstKeyspace.prepareMutationBatch();
                                        ColumnListMutation<Long> mutation = batch.withRow(columnFamily, locatorCapture);

                                        assert ttl != 0;
                                        long colCount = 0;
                                        for (Column<Long> c : row.getColumns()) {
                                            mutation.putColumn(c.getName(), c.getByteBufferValue(), ttl);
                                            colCount += 1;
                                        }
                                        columnsTransferred.addAndGet(colCount);

                                        // save it, submit a log message to be shown later.
                                        try {
                                            batch.execute();
                                            if (verify && random.nextFloat() < VERIFY_PERCENT) {
                                                verifyExecutor.submit(new Runnable() {public void run() {
                                                    try {
                                                        ColumnList<Long> srcData = srcKeyspace.prepareQuery(columnFamily).getKey(locatorCapture)
                                                                .withColumnRange(range)
                                                                .execute()
                                                                .getResult();
                                                        ColumnList<Long> dstData = dstKeyspace.prepareQuery(columnFamily).getKey(locatorCapture)
                                                                .withColumnRange(range)
                                                                .execute()
                                                                .getResult();

                                                        checkSameResults(srcData, dstData);
                                                        postExecutor.submit(new Runnable() {public void run() {
                                                            out.println(String.format("verified %s", locatorCapture.toString()));
                                                        }});
                                                    } catch (ConnectionException ex) {
                                                        stopAll.set(true);
                                                        out.println("There was an error verifying data: " + ex.getMessage());
                                                        ex.printStackTrace(out);
                                                    } catch (Exception ex) {
                                                        stopAll.set(true);
                                                        out.println(ex.getMessage() + " " + locatorCapture.toString());
                                                    }
                                                }});
                                            }

                                            final long fColCount = colCount;
                                            postExecutor.submit(new Runnable() {
                                                public void run() {
                                                    int rowIteration = processedKeys.incrementAndGet();
                                                    long colsPerSecond = columnsTransferred.get() / Math.max(1, (nowInSeconds() - startClockTime));
                                                    out.println(String.format("%d copied %d for %s (%d m/s), %d", rowIteration, fColCount, locatorCapture.toString(), colsPerSecond, columnsTransferred.get()));
                                                    heartbeat.set(System.currentTimeMillis());
                                                }
                                            });

                                            // possibly throttle if we've sent a lot of columns.
                                            while (columnsTransferred.get() / (nowInSeconds() - startClockTime) > rate) {
                                                try { Thread.sleep(200); } catch (Exception ex) {}
                                            }
                                        }
                                        catch (ConnectionException ex) {
                                            stopAll.set(true);
                                            out.println("There was an error A: " + ex.getMessage());
                                            ex.printStackTrace(out);
                                        }
                                    }
                                });
                            }
                        }

                        @Override
                        public boolean failure(ConnectionException ex) {
                            // if we were stopped, then it was either done cleanly, or the error was already logged.
                            if (!stopAll.get()) {
                                stopAll.set(true);
                                out.println("There was an error iterating rows: " + ex.getMessage());
                                ex.printStackTrace(out);
                            }
                            return false;
                        }
                    });
        } catch (ConnectionException ex) {
            if (!stopAll.get()) {
                stopAll.set(true);
                out.println("There was an error C: " + ex.getMessage());
                ex.printStackTrace(out);
            }
        }
    }

    private static void checkSameResults(ColumnList<Long> x, ColumnList<Long> y) throws Exception {
        if (x.size() != y.size()) {
            throw new Exception("source and destination column lengths do not match");
        }
        if (Sets.difference(new HashSet<Long>(x.getColumnNames()), new HashSet<Long>(y.getColumnNames())).size() != 0) {
            throw new Exception("source and destination did not contain the same column names");
        }

        for (int i = 0; i < x.size(); i++) {
            byte[] bx = x.getColumnByIndex(i).getByteArrayValue();
            byte[] by = y.getColumnByIndex(i).getByteArrayValue();
            if (bx.length != by.length) {
                throw new Exception("source and destination column values did not match for column " + i);
            }
            // only examine every third byte.
            for (int j = 0; j < bx.length; j+=3) {
                if (bx[j] != by[j]) {
                    throw new Exception("source and destination column values did not match for column " + i);
                }
            }
        }
    }

    private static void nullRouteAllLog4j() {
        List<Logger> loggers = Collections.<Logger>list(LogManager.getCurrentLoggers());
        loggers.add(LogManager.getRootLogger());
        for ( Logger logger : loggers ) {
            logger.setLevel(Level.OFF);
        }
    }

    private static AstyanaxContext<Keyspace> connect(String host, int port, String keyspace, int threads, NodeDiscoveryType discovery) {
        AstyanaxContext<Keyspace> context = new AstyanaxContext.Builder()
                        .forKeyspace(keyspace)
                .withAstyanaxConfiguration(new AstyanaxConfigurationImpl()
                    .setDiscoveryType(discovery)
                    .setRetryPolicy(new RetryNTimes(10)))

                .withConnectionPoolConfiguration(new ConnectionPoolConfigurationImpl(host + ":" + keyspace)
                        .setMaxConns(threads * 2)
                        .setSeeds(host)
                        .setPort(port)
                        .setRetryBackoffStrategy(new FixedRetryBackoffStrategy(1000, 1000)))
                .withConnectionPoolMonitor(new CountingConnectionPoolMonitor())
                .buildKeyspace(ThriftFamilyFactory.getInstance());
        context.start();
        return context;
    }

    // construct a well-formed options map. There should be no guesswork/checking for null after this point. All defaults
    // should be populated.
    private static Map<String, Object> parseOptions(String[] args) {
        final GnuParser parser = new GnuParser();
        final Map<String, Object> options = new HashMap<String, Object>();
        try {
            final long now = System.currentTimeMillis();
            CommandLine line = parser.parse(cliOptions, args);

            options.put(SRC, line.getOptionValue(SRC));
            options.put(DST, line.getOptionValue(DST));

            // default range is one year ago until now.
            options.put(FROM, line.hasOption(FROM) ? parseDateTime(line.getOptionValue(FROM)) : now-(365L*24L*60L*60L*1000L));
            options.put(TO, line.hasOption(TO) ? parseDateTime(line.getOptionValue(TO)) : now);

            options.put(LIMIT, line.hasOption(LIMIT) ? Integer.parseInt(line.getOptionValue(LIMIT)) : Integer.MAX_VALUE);
            options.put(SKIP, line.hasOption(SKIP) ? Integer.parseInt(line.getOptionValue(SKIP)) : 0);
            options.put(BATCH_SIZE, line.hasOption(BATCH_SIZE) ? Integer.parseInt(line.getOptionValue(BATCH_SIZE)) : 100);

            // create a mapping of all cf names -> cf.
            // then determine which column family to process.
            Map<String, ColumnFamily<Locator, Long>> nameToCf = new HashMap<String, ColumnFamily<Locator, Long>>() {{
                for (CassandraModel.MetricColumnFamily cf : CassandraModel.getMetricColumnFamilies()) {
                    put(cf.getName(), cf);
                }
            }};
            if (nameToCf.get(line.getOptionValue(COLUMN_FAMILY)) == null) {
                throw new ParseException("Invalid column family");
            }
            CassandraModel.MetricColumnFamily columnFamily = (CassandraModel.MetricColumnFamily)nameToCf.get(line.getOptionValue(COLUMN_FAMILY));
            options.put(COLUMN_FAMILY, columnFamily);

            options.put(TTL, line.hasOption(TTL) ? Integer.parseInt(line.getOptionValue(TTL)) : (int)(5 * columnFamily.getDefaultTTL().toSeconds()));

            options.put(READ_THREADS, line.hasOption(READ_THREADS) ? Integer.parseInt(line.getOptionValue(READ_THREADS)) : 1);
            options.put(WRITE_THREADS, line.hasOption(WRITE_THREADS) ? Integer.parseInt(line.getOptionValue(WRITE_THREADS)) : 1);

            options.put(VERIFY, line.hasOption(VERIFY));

            options.put(DISCOVER, line.hasOption(DISCOVER) ? NodeDiscoveryType.RING_DESCRIBE : NodeDiscoveryType.NONE);
            options.put(RATE, line.hasOption(RATE) ? Integer.parseInt(line.getOptionValue(RATE)) : 500);

        } catch (ParseException ex) {
            HelpFormatter helpFormatter = new HelpFormatter();
            helpFormatter.printHelp("bf-migrate", cliOptions);
            System.exit(-1);
        }

        return options;
    }

    private static long parseDateTime(String s) {
        try {
            return Long.parseLong(s);
        } catch (NumberFormatException ex) {
            // convert from a ISO 6801 date String.
            return DatatypeConverter.parseDateTime(s).getTime().getTime();
        }
    }
}


/*
 * Copyright (c) 2016 Rackspace.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * 	http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.rackspacecloud.blueflood.io.datastax;

import com.datastax.driver.core.BoundStatement;
import com.datastax.driver.core.PreparedStatement;
import com.rackspacecloud.blueflood.exceptions.InvalidDataException;
import com.rackspacecloud.blueflood.io.serializers.metrics.TimerRollupSerDes;
import com.rackspacecloud.blueflood.rollup.Granularity;
import com.rackspacecloud.blueflood.types.BluefloodCounterRollup;
import com.rackspacecloud.blueflood.types.BluefloodTimerRollup;
import com.rackspacecloud.blueflood.types.IMetric;
import com.rackspacecloud.blueflood.types.Rollup;

import java.nio.ByteBuffer;

/**
 * This class holds the utility methods to read/write Timer metrics
 * using Datastax driver.
 */
public class DTimerIO extends DAbstractMetricIO {

    private TimerRollupSerDes serDes = new TimerRollupSerDes();

    /**
     * Provides a way for the sub class to get a {@link java.nio.ByteBuffer}
     * representation of a certain Rollup object.
     *
     * @param value
     * @return
     */
    @Override
    protected ByteBuffer toByteBuffer(Object value) {
        if ( ! (value instanceof BluefloodTimerRollup) ) {
            throw new IllegalArgumentException("toByteBuffer(): expecting BluefloodTimerRollup class but got " + value.getClass().getSimpleName());
        }
        BluefloodTimerRollup TimerRollup = (BluefloodTimerRollup) value;
        return serDes.serialize(TimerRollup);
    }

    /**
     * Provides a way for the sub class to construct the right Rollup
     * object from a {@link java.nio.ByteBuffer}
     *
     * @param byteBuffer
     * @return
     */
    @Override
    protected BluefloodTimerRollup fromByteBuffer(ByteBuffer byteBuffer) {
        return serDes.deserialize(byteBuffer);
    }

    /**
     * Retrieves the {@link BoundStatement} for a particular counter metric
     * @param metric
     * @param granularity
     * @return
     */
    @Override
    protected BoundStatement getBoundStatementForMetric(IMetric metric, Granularity granularity) {
        Object metricValue = metric.getMetricValue();
        if (!(metricValue instanceof BluefloodTimerRollup)) {
            throw new InvalidDataException(
                    String.format("getBoundStatementForMetric(locator=%s, granularity=%s): metric value %s is not type BluefloodTimerRollup",
                            metric.getLocator(), granularity, metric.getMetricValue().getClass().getSimpleName())
            );
        }
        PreparedStatement statement = metricsCFPreparedStatements.preaggrGranToInsertStatement.get(granularity);
        return statement.bind(
                metric.getLocator().toString(),
                metric.getCollectionTime(),
                serDes.serialize( (BluefloodTimerRollup) metricValue ),
                metric.getTtlInSeconds() );
    }
}


/*
 * Copyright (c) 2016 Rackspace.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * 	http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.rackspacecloud.blueflood.io.astyanax;

import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import com.netflix.astyanax.model.ColumnFamily;
import com.rackspacecloud.blueflood.io.AbstractMetricsRW;
import com.rackspacecloud.blueflood.io.CassandraModel;
import com.rackspacecloud.blueflood.io.PreaggregatedRW;
import com.rackspacecloud.blueflood.outputs.formats.MetricData;
import com.rackspacecloud.blueflood.rollup.Granularity;
import com.rackspacecloud.blueflood.service.SingleRollupWriteContext;
import com.rackspacecloud.blueflood.types.*;
import com.rackspacecloud.blueflood.utils.Clock;

import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Map;

/**
 * This class deals with reading/writing metrics to the metrics_preaggregated_* column families
 * using Astyanax driver
 */
public class APreaggregatedMetricsRW extends AbstractMetricsRW implements PreaggregatedRW{

    public APreaggregatedMetricsRW(boolean isRecordingDelayedMetrics, Clock clock) {
        this.isRecordingDelayedMetrics = isRecordingDelayedMetrics;
        this.clock = clock;
    }

    /**
     * Inserts a collection of metrics to the metrics_preaggregated_full column family
     *
     * @param metrics
     * @throws IOException
     */
    @Override
    public void insertMetrics(Collection<IMetric> metrics) throws IOException {
        insertMetrics(metrics, Granularity.FULL);
    }

    /**
     * Inserts a collection of metrics to the correct column family based on
     * the specified granularity
     *
     * @param metrics
     * @param granularity
     * @throws IOException
     */
    @Override
    public void insertMetrics(Collection<IMetric> metrics, Granularity granularity) throws IOException {
        try {
            AstyanaxWriter.getInstance().insertMetrics(metrics, CassandraModel.getPreaggregatedColumnFamily(granularity), isRecordingDelayedMetrics, clock);
        } catch (ConnectionException ex) {
            throw new IOException(ex);
        }
    }

    @Override
    public void insertRollups(List<SingleRollupWriteContext> writeContexts) throws IOException {
        try {
            AstyanaxWriter.getInstance().insertRollups(writeContexts);
        } catch (ConnectionException ex) {
            throw new IOException(ex);
        }
    }

    /**
     * Fetches {@link com.rackspacecloud.blueflood.outputs.formats.MetricData} objects for the
     * specified {@link com.rackspacecloud.blueflood.types.Locator} and
     * {@link com.rackspacecloud.blueflood.types.Range} from the specified column family
     *
     * @param locator
     * @param range
     * @param gran
     * @return
     */
    @Override
    public MetricData getDatapointsForRange(final Locator locator,
                                            Range range,
                                            Granularity gran) {
        return AstyanaxReader.getInstance().getDatapointsForRange(locator, range, gran);
    }

    /**
     * Fetches {@link com.rackspacecloud.blueflood.outputs.formats.MetricData} objects for the
     * specified list of {@link com.rackspacecloud.blueflood.types.Locator} and
     * {@link com.rackspacecloud.blueflood.types.Range} from the specified column family
     *
     * @param locators
     * @param range
     * @param gran
     * @return
     */
    @Override
    public Map<Locator, MetricData> getDatapointsForRange(List<Locator> locators,
                                                          Range range,
                                                          Granularity gran) {
        return AstyanaxReader.getInstance().getDatapointsForRange(locators, range, gran);
    }

    /**
     * Fetches a {@link com.rackspacecloud.blueflood.types.Points} object for a
     * particular locator and rollupType from the specified column family and
     * range
     *
     * @param locator
     * @param rollupType
     * @param range
     * @param columnFamilyName
     * @param <T> the type of Rollup object
     * @return
     */
    @Override
    public <T extends Rollup> Points<T> getDataToRollup(final Locator locator,
                                                        RollupType rollupType,
                                                        Range range,
                                                        String columnFamilyName) throws IOException {
        ColumnFamily cf = CassandraModel.getColumnFamily(columnFamilyName);
        // a quick n dirty hack, this code will go away someday
        Granularity granularity = CassandraModel.getGranularity(cf);
        Class<? extends Rollup> rollupClass = RollupType.classOf(rollupType, granularity);
        return AstyanaxReader.getInstance().getDataToRoll(rollupClass, locator, range, cf);
    }
}


package com.rackspacecloud.blueflood.io.datastax;

import com.datastax.driver.core.BoundStatement;
import com.datastax.driver.core.ResultSetFuture;
import com.rackspacecloud.blueflood.io.serializers.metrics.RawSerDes;
import com.rackspacecloud.blueflood.rollup.Granularity;
import com.rackspacecloud.blueflood.types.IMetric;

import java.nio.ByteBuffer;

/**
 * This class holds the utility methods to read/write simple numbers
 * using Datastax driver. This is mostly used during Ingest and metrics
 * are inserted to the metrics_full Column Family
 */
public class DSimpleNumberIO extends DAbstractMetricIO {

    private RawSerDes serDes = new RawSerDes();

    @Override
    protected ByteBuffer toByteBuffer( Object value ) {

        return serDes.serialize( value );
    }

    @Override
    protected Object fromByteBuffer( ByteBuffer byteBuffer ) {

        return serDes.deserialize( byteBuffer );
    }

    /**
     * Inserts a raw metric (not its rolled up value) to the proper
     * column family
     *
     * @param metric
     * @return
     */
    public ResultSetFuture insertRawAsync( IMetric metric ) {

        BoundStatement bound = getBoundStatementForMetric(metric);
        return session.executeAsync(bound);

    }

    /**
     * Constructs a {@link BoundStatement} for a particular metric.
     * The statement could be part of a BatchStatement or it could
     * be executed as is.
     *
     * @param metric
     * @return
     */
    public BoundStatement getBoundStatementForMetric(IMetric metric) {
        return metricsCFPreparedStatements.insertToMetricsBasicFullStatement.bind(
                metric.getLocator().toString(),
                metric.getCollectionTime(),
                serDes.serialize( metric.getMetricValue() ),
                metric.getTtlInSeconds() );
    }

    @Override
    protected BoundStatement getBoundStatementForMetric(IMetric metric, Granularity granularity) {
        // we ignore granularity from the abstract parent because
        // this class is only used to write to metrics_full
        return getBoundStatementForMetric(metric);
    }
}


/*
 * Copyright 2013 Rackspace
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

package com.rackspacecloud.blueflood.service;

import com.codahale.metrics.Histogram;
import com.codahale.metrics.Timer;
import com.google.common.annotations.VisibleForTesting;
import com.rackspacecloud.blueflood.io.IOContainer;
import com.rackspacecloud.blueflood.rollup.Granularity;
import com.rackspacecloud.blueflood.rollup.SlotKey;
import com.rackspacecloud.blueflood.types.Locator;
import com.rackspacecloud.blueflood.types.Range;
import com.rackspacecloud.blueflood.utils.Metrics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadPoolExecutor;

/**
 * fetches locators for a given slot and feeds a worker queue with rollup work. When those are all done notifies the
 * RollupService that slot can be removed from running.
  */
class LocatorFetchRunnable implements Runnable {
    private static final Logger log = LoggerFactory.getLogger(LocatorFetchRunnable.class);
    private static final int LOCATOR_WAIT_FOR_ALL_SECS = 1000;

    private ExecutorService rollupReadExecutor;
    private ThreadPoolExecutor rollupWriteExecutor;
    private SlotKey parentSlotKey;
    private ScheduleContext scheduleCtx;
    private long serverTime;
    private static final Timer rollupLocatorExecuteTimer = Metrics.timer(RollupService.class, "Locate and Schedule Rollups for Slot");
    private static final Histogram locatorsPerShard = Metrics.histogram(RollupService.class, "Locators Per Shard");
    private static final Histogram locatorsPerShardForReroll = Metrics.histogram(RollupService.class, "Locators Per Shard for re-rolls");

    private static boolean RECORD_DELAYED_METRICS =
            Configuration.getInstance().getBooleanProperty(CoreConfig.RECORD_DELAYED_METRICS);

    private static Granularity DELAYED_METRICS_STORAGE_GRANULARITY =
            Granularity.getRollupGranularity(Configuration.getInstance().getStringProperty(CoreConfig.DELAYED_METRICS_STORAGE_GRANULARITY));

    private static Granularity DELAYED_METRICS_REROLL_GRANULARITY =
            Granularity.getRollupGranularity(Configuration.getInstance().getStringProperty(CoreConfig.DELAYED_METRICS_REROLL_GRANULARITY));

    private Range parentRange;

    LocatorFetchRunnable(ScheduleContext scheduleCtx,
                         SlotKey destSlotKey,
                         ExecutorService rollupReadExecutor,
                         ThreadPoolExecutor rollupWriteExecutor) {

        initialize(scheduleCtx, destSlotKey, rollupReadExecutor,
                rollupWriteExecutor);
    }

    @VisibleForTesting
    public void initialize(ScheduleContext scheduleCtx,
                           SlotKey destSlotKey,
                           ExecutorService rollupReadExecutor,
                           ThreadPoolExecutor rollupWriteExecutor) {

        this.rollupReadExecutor = rollupReadExecutor;
        this.rollupWriteExecutor = rollupWriteExecutor;
        this.parentSlotKey = destSlotKey;
        this.scheduleCtx = scheduleCtx;
        this.serverTime = scheduleCtx.getCurrentTimeMillis();
        this.parentRange = getGranularity().deriveRange(getParentSlot(), serverTime);
    }

    protected Granularity getGranularity() { return parentSlotKey.getGranularity(); }
    protected int getParentSlot() { return parentSlotKey.getSlot(); }
    protected int getShard() { return parentSlotKey.getShard(); }

    public void run() {
        final Timer.Context timerCtx = rollupLocatorExecuteTimer.time();

        try {
            getGranularity().finer();
        } catch (Exception ex) {
            log.error("No finer granularity available than " + getGranularity());
            return;
        }

        if (log.isTraceEnabled())
            log.trace("Getting locators for {} {} @ {}", new Object[]{parentSlotKey, parentRange.toString(), scheduleCtx.getCurrentTimeMillis()});
        // todo: I can see this set becoming a memory hog.  There might be a better way of doing this.
        long waitStart = System.currentTimeMillis();
        int rollCount = 0;

        final RollupExecutionContext executionContext = createRollupExecutionContext();
        final RollupBatchWriter rollupBatchWriter = createRollupBatchWriter(executionContext);

        boolean isReroll = scheduleCtx.isReroll(parentSlotKey);

        Set<Locator> locators = getLocators(executionContext, isReroll,
                DELAYED_METRICS_REROLL_GRANULARITY, DELAYED_METRICS_STORAGE_GRANULARITY);

        log.info(String.format("Number of locators getting rolled up for slotkey: [%s] are %s; isReroll: %s", parentSlotKey, locators.size(), isReroll));

        if (log.isTraceEnabled())
            log.trace("locators retrieved: {}", locators.size());

        for (Locator locator : locators) {
            rollCount = processLocator(rollCount, executionContext, rollupBatchWriter, locator);
        }

        // now wait until ctx is drained. someone needs to be notified.
        drainExecutionContext(waitStart, rollCount, executionContext, rollupBatchWriter);

        timerCtx.stop();
    }

    /**
     * This methods gets locators to rollup a slot.
     *
     * During re-rolls, If the granularity of the slot is finer(or equal) to the configured
     * DELAYED_METRICS_REROLL_GRANULARITY, we re-roll only the delayed locator's corresponding
     * to that slot. if not, we re-roll the locators of the entire shard.
     *
     * @param executionContext
     * @param isReroll
     * @param delayedMetricsRerollGranularity
     * @param delayedMetricsStorageGranularity
     * @return
     */
    protected Set<Locator> getLocators(RollupExecutionContext executionContext,
                                       boolean isReroll,
                                       Granularity delayedMetricsRerollGranularity,
                                       Granularity delayedMetricsStorageGranularity) {
        Set<Locator> locators = new HashSet<Locator>();

        //if delayed metric tracking is enabled, if its re-roll, if slot granularity is no coarser than
        // DELAYED_METRICS_REROLL_GRANULARITY, get delayed locators
        if (RECORD_DELAYED_METRICS &&
                isReroll &&
                !getGranularity().isCoarser(delayedMetricsRerollGranularity)) {

            if (getGranularity().isCoarser(delayedMetricsStorageGranularity)) {

                // For example, if we are re-rolling a 60m slot, and we store delayed metrics at 20m, we need to
                // grab delayed metrics for 3 * 20m slots corresponding to the 60m slot.
                for (SlotKey slotKey: parentSlotKey.getChildrenKeys(delayedMetricsStorageGranularity)) {
                    locators.addAll(getDelayedLocators(executionContext, slotKey));
                }

            } else {
                locators = getDelayedLocators(executionContext,
                        parentSlotKey.extrapolate(delayedMetricsStorageGranularity));
            }

        } else {
            locators = getLocators(executionContext);
        }
        return locators;
    }

    protected RollupExecutionContext createRollupExecutionContext() {
        return new RollupExecutionContext(Thread.currentThread());
    }

    protected RollupBatchWriter createRollupBatchWriter(RollupExecutionContext executionContext) {
        return new RollupBatchWriter(rollupWriteExecutor, executionContext);
    }

    public void drainExecutionContext(long waitStart, int rollCount, RollupExecutionContext executionContext, RollupBatchWriter rollupBatchWriter) {
        log.debug("Waiting for rollups to finish for " + parentSlotKey);
        while (!executionContext.doneReading() || !executionContext.doneWriting()) {
            if (executionContext.doneReading()) {
                rollupBatchWriter.drainBatch(); // gets any remaining rollups enqueued for put. should be no-op after being called once
            }
            try {
                waitForRollups();
            } catch (InterruptedException ex) {
                if (log.isTraceEnabled())
                    log.trace("Woken wile waiting for rollups to coalesce for {} {}", parentSlotKey);
            } finally {
                String verb = executionContext.doneReading() ? "writing" : "reading";
                log.debug("Still waiting for rollups to finish {} for {} {}", new Object[] {verb, parentSlotKey, System.currentTimeMillis() - waitStart });
            }
        }
        if (log.isDebugEnabled())
            log.debug("Finished {} rollups for (gran,slot,shard) {} in {}", new Object[] {rollCount, parentSlotKey, System.currentTimeMillis() - waitStart});

        finishExecution(waitStart, executionContext);
    }

    protected void waitForRollups() throws InterruptedException {
        Thread.currentThread().sleep(LOCATOR_WAIT_FOR_ALL_SECS * 1000);
    }

    public void finishExecution(long waitStart, RollupExecutionContext executionContext) {
        if (executionContext.wasSuccessful()) {
            this.scheduleCtx.clearFromRunning(parentSlotKey);
            log.info("Successful completion of rollups for (gran,slot,shard) {} in {} ms",
                    new Object[]{parentSlotKey, System.currentTimeMillis() - waitStart});
        } else {
            log.error("Performing BasicRollups for {} failed", parentSlotKey);
            this.scheduleCtx.pushBackToScheduled(parentSlotKey, false);
        }
    }

    public int processLocator(int rollCount, RollupExecutionContext executionContext, RollupBatchWriter rollupBatchWriter, Locator locator) {
        if (log.isTraceEnabled())
            log.trace("Rolling up (check,metric,dimension) {} for (gran,slot,shard) {}", locator, parentSlotKey);
        try {
            executeRollupForLocator(executionContext, rollupBatchWriter, locator);
            rollCount += 1;
        } catch (Throwable any) {
            // continue on, but log the problem so that we can fix things later.
            executionContext.markUnsuccessful(any);
            executionContext.decrementReadCounter();
            log.error(String.format(
                            "BasicRollup failed for %s, locator %s, at %d",
                            parentSlotKey, locator, serverTime),
                    any);
        }

        return rollCount;
    }

    public void executeRollupForLocator(RollupExecutionContext executionContext, RollupBatchWriter rollupBatchWriter, Locator locator) {
        executionContext.incrementReadCounter();
        final SingleRollupReadContext singleRollupReadContext = new SingleRollupReadContext(locator, parentRange, getGranularity());
        RollupRunnable rollupRunnable = new RollupRunnable(executionContext, singleRollupReadContext, rollupBatchWriter);
        rollupReadExecutor.execute(rollupRunnable);
    }

    public Set<Locator> getDelayedLocators(RollupExecutionContext executionContext, SlotKey slotkey) {
        Set<Locator> locators = new HashSet<Locator>();

        try {
            // get a list of all delayed locators to rollup for a slot key.
            locators.addAll(IOContainer.fromConfig().getDelayedLocatorIO().getLocators(slotkey));
            locatorsPerShardForReroll.update(locators.size());
        } catch (Throwable e) {
            log.error("Failed reading delayed locators for slot: " + getParentSlot(), e);
            executionContext.markUnsuccessful(e);
        }

        return locators;
    }

    public Set<Locator> getLocators(RollupExecutionContext executionContext) {
        Set<Locator> locators = new HashSet<Locator>();

        try {
            // get a list of all locators to rollup for a shard
            locators.addAll(IOContainer.fromConfig().getLocatorIO().getLocators(getShard()));

            if (scheduleCtx.isReroll(parentSlotKey)) {
                locatorsPerShardForReroll.update(locators.size());
            } else {
                locatorsPerShard.update(locators.size());
            }
        } catch (Throwable e) {
            log.error("Failed reading locators for slot: " + getParentSlot(), e);
            executionContext.markUnsuccessful(e);
        }
        return locators;
    }
}


package com.rackspacecloud.blueflood.io;

import com.codahale.metrics.Histogram;
import com.codahale.metrics.Meter;
import com.codahale.metrics.Timer;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.rackspacecloud.blueflood.service.Configuration;
import com.rackspacecloud.blueflood.service.ElasticIOConfig;
import com.rackspacecloud.blueflood.utils.GlobPattern;
import com.rackspacecloud.blueflood.utils.Metrics;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;

import static com.rackspacecloud.blueflood.types.Locator.METRIC_TOKEN_SEPARATOR_REGEX;
import static java.util.stream.Collectors.toSet;

public abstract class AbstractElasticIO implements DiscoveryIO {
    private static final Logger log = LoggerFactory.getLogger(AbstractElasticIO.class);

    public ElasticsearchRestHelper elasticsearchRestHelper;
    protected static final String ELASTICSEARCH_DOCUMENT_TYPE = "metrics";

    protected final Timer searchTimer = Metrics.timer(getClass(), "Search Duration");
    protected final Timer esMetricNamesQueryTimer = Metrics.timer(getClass(), "ES Metric Names Query Duration");
    protected final Timer writeTimer = Metrics.timer(getClass(), "Write Duration");
    protected final Histogram batchHistogram = Metrics.histogram(getClass(), "Batch Sizes");
    protected Meter classCastExceptionMeter = Metrics.meter(getClass(), "Failed Cast to IMetric");
    protected Histogram queryBatchHistogram = Metrics.histogram(getClass(), "Query Batch Size");
    private final Histogram searchResultsSizeHistogram = Metrics.histogram(getClass(), "Metrics search results size");

    public static String ELASTICSEARCH_INDEX_NAME_WRITE = Configuration.getInstance().getStringProperty(ElasticIOConfig.ELASTICSEARCH_INDEX_NAME_WRITE);
    public static String ELASTICSEARCH_INDEX_NAME_READ = Configuration.getInstance().getStringProperty(ElasticIOConfig.ELASTICSEARCH_INDEX_NAME_READ);

    //grabs chars until the next "." which is basically a token
    protected static final String REGEX_TO_GRAB_SINGLE_TOKEN = "[^.]*";

    private String queryToFetchMetricNamesFromElasticsearchFormat = "{\n" +
            "  \"size\": 0,\n" +
            "  \"query\": {\n" +
            "    \"bool\": {\n" +
            "      \"must\": [\n" +
            "        {\n" +
            "          \"term\": {\n" +
            "            \"tenantId\": \"%s\"\n" +
            "          }\n" +
            "        },\n" +
            "        {\n" +
            "          \"regexp\": {\n" +
            "            \"metric_name\": {\n" +
            "              \"value\": \"%s\"\n" +
            "            }\n" +
            "          }\n" +
            "        }\n" +
            "      ]\n" +
            "    }\n" +
            "  },\n" +
            "  \"aggs\": {\n" +
            "    \"metric_name_tokens\": {\n" +
            "      \"terms\": {\n" +
            "        \"field\": \"metric_name\",\n" +
            "        \"include\": \"%s\",\n" +
            "        \"execution_hint\": \"map\",\n" +
            "        \"size\": 0\n" +
            "      }\n" +
            "    }\n" +
            "  }\n" +
            "}";

    public List<SearchResult> search(String tenant, String query) throws Exception {
        return search(tenant, Arrays.asList(query));
    }

    public List<SearchResult> search(String tenant, List<String> queries) throws Exception {
        return searchUsingRestApi(tenant, queries);
    }

    private List<SearchResult> searchUsingRestApi(String tenant, List<String> queries) {
        List<SearchResult> results = new ArrayList<>();
        Timer.Context multiSearchCtx = searchTimer.time();
        int hitsCount;

        try {
            queryBatchHistogram.update(queries.size());
            String response = elasticsearchRestHelper.fetch(
                    ELASTICSEARCH_INDEX_NAME_READ, ELASTICSEARCH_DOCUMENT_TYPE, tenant, queries);

            List<SearchResult> searchResults = getSearchResults(response);
            hitsCount = searchResults.size();
            results.addAll(searchResults);
        } catch (JsonProcessingException e) {
            log.error("Elasticsearch metrics query failed for tenantId {}. {}", tenant, e.getMessage());
            throw new RuntimeException(String.format("searchUsingRestApi failed with message: %s", e.getMessage()), e);
        } catch (IOException e) {
            log.error("Elasticsearch metrics query failed for tenantId {}. {}", tenant, e.getMessage());
            throw new RuntimeException(String.format("searchUsingRestApi failed with message: %s", e.getMessage()), e);
        } finally {
            multiSearchCtx.stop();
        }

        searchResultsSizeHistogram.update(hitsCount);

        return dedupResults(results);
    }

    private List<SearchResult> getSearchResults(String response) throws IOException {
        ObjectMapper mapper = new ObjectMapper();
        JsonNode root = mapper.readTree(response);

        Iterator<JsonNode> iter = root.get("hits").get("hits").elements();

        List<SearchResult> searchResults = new ArrayList<>();
        while(iter.hasNext()){
            JsonNode source = iter.next().get("_source");
            String metricName = source.get(ESFieldLabel.metric_name.toString()).asText();
            String tenantId = source.get(ESFieldLabel.tenantId.toString()).asText();
            String unit = null;
            if(source.has(ESFieldLabel.unit.toString()))
                unit = source.get(ESFieldLabel.unit.toString()).asText();

            SearchResult result = new SearchResult(tenantId, metricName, unit);
            searchResults.add(result);
        }
        return searchResults;
    }

    /**
     * This method returns a list of {@link MetricName}'s matching the given glob query.
     *
     * for metrics: foo.bar.xxx,
     *              foo.bar.baz.qux,
     *
     * for query=foo.bar.*, returns the below list of metric names
     *
     * new MetricName("foo.bar.xxx", true)   <- From metric foo.bar.xxx
     * new MetricName("foo.bar.baz", false)  <- From metric foo.bar.baz.qux
     *
     * @param tenant
     * @param query is glob representation of hierarchical levels of token. Ex: foo.bar.*
     * @return
     * @throws Exception
     */
    public List<MetricName> getMetricNames(final String tenant, final String query) throws Exception {

        Timer.Context esMetricNamesQueryTimerCtx = esMetricNamesQueryTimer.time();
        String response;

        try {
            String regexString = regexToGrabCurrentAndNextLevel(query);
            // replace one '\' char with two '\\'
            regexString = regexString.replaceAll("\\\\", "\\\\\\\\");
            response = getMetricNamesFromES(tenant, regexString);
        } finally {
            esMetricNamesQueryTimerCtx.stop();
        }

        // For example, if query = foo.bar.*, base level is 3 which is equal to the number of tokens in the query.
        int baseLevel = getTotalTokens(query);
        MetricIndexData metricIndexData = getMetricIndexData(response, baseLevel);

        List<MetricName> metricNames = new ArrayList<>();

        //Metric Names matching query which have next level
        metricNames.addAll(metricIndexData.getMetricNamesWithNextLevel()
                .stream()
                .map(x -> new MetricName(x, false))
                .collect(toSet()));

        //complete metric names matching query
        metricNames.addAll(metricIndexData.getCompleteMetricNamesAtBaseLevel()
                .stream()
                .map(x -> new MetricName(x, true))
                .collect(toSet()));

        return metricNames;
    }

    /**
     * Performs terms aggregation by metric_name which returns doc_count by
     * metric_name index that matches the given regex.
     *
     *  Sample request body:
     *
     *  {
     *      "size": 0,
     *      "query": {
     *          "bool" : {
     *              "must" : [ {
     *                  "term" : {
     *                      "tenantId" : "ratanasv"
     *                  }
     *              }, {
     *                  "regexp" : {
     *                      "metric_name" : {
     *                         "value" : "<regex>"
     *                      }
     *                  }
     *              } ]
     *          }
     *      },
     *      "aggs": {
     *          "metric_name_tokens": {
     *              "terms": {
     *                  "field" : "metric_name",
     *                  "include": "<regex>",
     *                  "execution_hint": "map",
     *                  "size": 0
     *              }
     *          }
     *      }
     *  }
     *
     * The two regex expressions used in the query above would be same, one to filter
     * at query level and another to filter the aggregation buckets.
     *
     * Execution hint of "map" works by using field values directly instead of ordinals
     * in order to aggregate data per-bucket
     *
     * @param tenant
     * @param regexMetricName
     * @return
     */
    private String getMetricNamesFromES(final String tenant, final String regexMetricName) throws IOException {
        String metricNamesFromElasticsearchQueryString = String.format(queryToFetchMetricNamesFromElasticsearchFormat,
                tenant, regexMetricName, regexMetricName);

        return elasticsearchRestHelper.fetchDocuments(
                ELASTICSEARCH_INDEX_NAME_READ,
                ELASTICSEARCH_DOCUMENT_TYPE,
                tenant,
                metricNamesFromElasticsearchQueryString);
    }

    private MetricIndexData getMetricIndexData(final String response, final int baseLevel) throws IOException {
        MetricIndexData metricIndexData = new MetricIndexData(baseLevel);

        ObjectMapper mapper = new ObjectMapper();
        JsonNode root = mapper.readTree(response);

        Iterator<JsonNode> iter = root.get("aggregations").get("metric_name_tokens").get("buckets").elements();

        while(iter.hasNext()){
            JsonNode node = iter.next();
            String key = node.get("key").asText();
            long docCount = node.get("doc_count").asInt();
            metricIndexData.add(key, docCount);
        }
        return metricIndexData;
    }

    private int getTotalTokens(String query) {

        if (StringUtils.isEmpty(query))
            return 0;

        return query.split(METRIC_TOKEN_SEPARATOR_REGEX).length;
    }

    /**
     * Returns regex which could grab metric names from current level to the next level
     * for a given query.
     *
     * (Some exceptions when query has only one level due to the nature of underlying data)
     *
     * for metrics : foo.bar.baz,
     *               foo.bar.baz.qux,
     *
     * for query=foo.bar.*, the regex which this method returns will capture the following metric token paths.
     *
     *  "foo.bar.baz"       <- current level
     *  "foo.bar.baz.qux"   <- next level
     *
     * @param query
     * @return
     */
    protected String regexToGrabCurrentAndNextLevel(final String query) {

        if (StringUtils.isEmpty(query)) {
            throw new IllegalArgumentException("Query(glob) string cannot be null/empty");
        }

        String queryRegex = getRegex(query);
        int totalQueryTokens = getTotalTokens(query);

        if (totalQueryTokens == 1) {

            // get metric names which matches the given query and have a next level,
            // Ex: For metric foo.bar.baz.qux, if query=*, we should get foo.bar. We are not
            // grabbing 0 level as it will give back bar, baz, qux because of the way data is structured.
            String baseRegex = convertRegexToCaptureUptoNextToken(queryRegex);
            return baseRegex + METRIC_TOKEN_SEPARATOR_REGEX + REGEX_TO_GRAB_SINGLE_TOKEN;

        } else {

            String[] queryRegexParts = queryRegex.split("\\\\.");

            String queryRegexUptoPrevLevel = StringUtils.join(queryRegexParts, METRIC_TOKEN_SEPARATOR_REGEX, 0, totalQueryTokens - 1);
            String baseRegex = convertRegexToCaptureUptoNextToken(queryRegexUptoPrevLevel);

            String queryRegexLastLevel = queryRegexParts[totalQueryTokens - 1];
            String lastTokenRegex = convertRegexToCaptureUptoNextToken(queryRegexLastLevel);

            // Ex: For metric foo.bar.baz.qux.xxx, if query=foo.bar.b*, get foo.bar.baz, foo.bar.baz.qux
            // In this case baseRegex = "foo.bar", lastTokenRegex = "b[^.]*"' and the final
            // regex is foo\.bar\.b[^.]*(\.[^.]*){0,1}
            return baseRegex +
                    METRIC_TOKEN_SEPARATOR_REGEX + lastTokenRegex +
                        "(" +
                    METRIC_TOKEN_SEPARATOR_REGEX + REGEX_TO_GRAB_SINGLE_TOKEN +
                        ")"  + "{0,1}";
        }
    }

    private String convertRegexToCaptureUptoNextToken(String queryRegex) {
        return queryRegex.replaceAll("\\.\\*", REGEX_TO_GRAB_SINGLE_TOKEN);
    }

    private String getRegex(String glob) {
        GlobPattern pattern = new GlobPattern(glob);
        return pattern.compiled().toString();
    }

    protected abstract List<SearchResult> dedupResults(List<SearchResult> results);
}


/*
 * Copyright 2014 Rackspace
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

package com.rackspacecloud.blueflood.outputs.cloudfiles;

import com.codahale.metrics.Gauge;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.rackspacecloud.blueflood.eventemitter.RollupEvent;
import com.rackspacecloud.blueflood.service.CloudfilesConfig;
import com.rackspacecloud.blueflood.service.Configuration;
import com.rackspacecloud.blueflood.utils.Metrics;
import org.jclouds.rest.AuthorizationException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collections;
import java.util.LinkedList;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;

public class StorageManager {
    private final Configuration config = Configuration.getInstance();
    private final File bufferDir;
    private final int maxBufferAge;
    private final int maxBufferSize;
    private static final int UPLOAD_RETRY_INTERVAL = 30000;
    private final BlockingQueue<RollupFile> done = new LinkedBlockingQueue<RollupFile>();
    private RollupFile current;
    private Thread uploaderThread;
    private DoneFileUploader fileUploader;

    private Meter fileCreationMeter = Metrics.meter(StorageManager.class, "Rollup Files Created");
    private Meter rollupEventsSeen = Metrics.meter(StorageManager.class, "Rollup Events Received");
    private Meter uploadExceptionMeter = Metrics.meter(StorageManager.class, "Rollup Remote Upload Exception");
    private Meter rollupWriteFailures = Metrics.meter(StorageManager.class, "Rollup Event Local Write Failures");
    private Gauge<Integer> uploadQueueDepthGauge;

    private static final Logger log = LoggerFactory.getLogger(StorageManager.class);

    public StorageManager() throws IOException {
        this.maxBufferAge = config.getIntegerProperty(CloudfilesConfig.CLOUDFILES_MAX_BUFFER_AGE);
        this.maxBufferSize = config.getIntegerProperty(CloudfilesConfig.CLOUDFILES_MAX_BUFFER_SIZE);
        this.bufferDir = new File(config.getStringProperty(CloudfilesConfig.CLOUDFILES_BUFFER_DIR));
        this.uploadQueueDepthGauge = new Gauge<Integer>() {
            @Override
            public Integer getValue() {
                return done.size();
            }
        };

        Metrics.getRegistry().register(MetricRegistry.name(StorageManager.class, "Upload Queue Depth"), this.uploadQueueDepthGauge);

        if (!bufferDir.isDirectory()) {
            throw new IOException("Specified BUFFER_DIR is not a directory: " + bufferDir.getAbsolutePath());
        }

        File[] bufferFiles = bufferDir.listFiles(RollupFile.fileFilter);
        LinkedList<RollupFile> rollupFileList = new LinkedList<RollupFile>();

        // Build a list of all buffer files in the directory
        for (File bufferFile : bufferFiles) {
            rollupFileList.add(new RollupFile(bufferFile));
        }

        Collections.sort(rollupFileList);

        // Take the newest metric file as the "current" one, or make a new one if there are none
        if (!rollupFileList.isEmpty()) {
            current = rollupFileList.removeLast();
        } else {
            current = RollupFile.buildRollupFile(bufferDir);
        }

        // Queue the rest for upload
        done.addAll(rollupFileList);
    }

    /**
     * Start background storage management and uploading tasks.
     */
    public synchronized void start() {
        if (uploaderThread != null) {
            throw new RuntimeException("StorageManager is already started");
        }

        fileUploader = new DoneFileUploader();
        uploaderThread = new Thread(fileUploader, "StorageManager uploader");
        uploaderThread.start();
    }

    /**
     * Stop background storage management.
     * @throws IOException
     */
    public synchronized void stop() throws IOException {
        if (uploaderThread == null) {
            throw new RuntimeException("Not running");
        }

        uploaderThread.interrupt();
        uploaderThread = null;
        fileUploader.shutdown();
    }

    public synchronized void store(RollupEvent... events) throws IOException {
        if (current.getAge() > maxBufferAge) {
            log.info("buffer file reached age limit, rotating: {}", current.getName());
            rotateCurrent();
        } else if (current.getSize() > maxBufferSize) {
            log.info("buffer file reached size limit, rotating: {}", current.getName());
            rotateCurrent();
        }

        for (RollupEvent event : events) {
            rollupEventsSeen.mark();
            try {
                current.append(event);
            } catch (Exception e) {
                rollupWriteFailures.mark();
                log.error("Could not locally persist rollupEvent, throwing away.", event, e);
            }
        }
    }

    private synchronized void rotateCurrent() throws IOException {
        current.close();
        done.add(current);
        current = RollupFile.buildRollupFile(bufferDir);
        fileCreationMeter.mark();
    }

    private class DoneFileUploader implements Runnable {
        private CloudFilesPublisher publisher;
        private Gzipper gzipper = new Gzipper();

        public DoneFileUploader() {
            resetPublisher();
        }

        @Override
        public void run() {
            /**
             * Process from the pending file queue until we are interrupted. The queue's take method will throw an
             * InterruptedException when this thread is interrupted, however it will clear the thread's interrupted flag
             * in doing so. When we catch an InterruptedException we restore the thread's interrupted state before
             * returning in case anyone up the call stack is curious as to what happened. See:
             * http://www.ibm.com/developerworks/java/library/j-jtp05236/index.html
             */
            try {
                while (true) {
                    uploadAndDeleteFile(done.take());
                }
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
            }
        }

        private void resetPublisher() {
            if (this.publisher != null) {
                try {
                    this.publisher.close();
                } catch (IOException e) {
                    log.warn("Error closing down existing publisher", e);
                }
            }
            this.publisher = new CloudFilesPublisher();
        }

        private void shutdown() throws IOException {
            this.publisher.close();
        }

        private synchronized void uploadAndDeleteFile(RollupFile file) throws InterruptedException {
            while (true) {
                try {
                    InputStream fileStream = file.asReadStream();
                    publisher.publish(file.getRemoteName() + ".gz", gzipper.gzip(fileStream));
                    file.delete();
                    break;
                } catch (FileNotFoundException e) {
                    log.error("File could not be found to be deleted.", e);
                    break; // assume file is already gone, so just break
                } catch (IllegalAccessException e) {
                    log.error("File exists but could not be deleted.", e);
                    break; // prevent getting stuck in a loop of re-uploading the file indefinitely
                } catch (IOException e) {
                    log.error("error reading or removing metric file, ignoring", e);
                    uploadExceptionMeter.mark();
                    break;
                } catch (AuthorizationException e) {
                    log.error("Authorization error uploading metric file. let's make a new publisher", e);
                    uploadExceptionMeter.mark();
                    resetPublisher();
                } catch (RuntimeException e) {
                    /**
                     * These are *probably* jclouds exceptions, but they make it very hard to know.
                     */
                    log.error("Error uploading RollupFile", e);
                    uploadExceptionMeter.mark();
                }

                Thread.sleep(UPLOAD_RETRY_INTERVAL);
            }

            log.info("uploaded and removed metric file {}", file.getName());
        }
    }
}


/*
 * Copyright 2013 Rackspace
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

package com.rackspacecloud.blueflood.types;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.Map;

public class BasicRollup extends BaseRollup implements IBaseRollup {
    private static final Logger log = LoggerFactory.getLogger(BasicRollup.class);

    private double sum;

    public BasicRollup() {
        super();
        this.sum = 0;
    }

    @Override
    public boolean equals(Object other) {
        if (!(other instanceof BasicRollup)) {
            return false;
        }

        BasicRollup otherBasicRollup = (BasicRollup)other;

        return super.equals( otherBasicRollup )
                && sum == otherBasicRollup.getSum();
    }

    public double getSum() {
        return sum;
    }

    public String toString() {
        return String.format("cnt:%d, avg:%s, var:%s, min:%s, max:%s, sum: %s", getCount(), getAverage(), getVariance(),
                getMinValue(), getMaxValue(), sum);
    }

    public void setSum(double s) {

        sum = s;
    }

    // merge simple numbers with this rollup.
    protected void computeFromSimpleMetrics(Points<SimpleNumber> input) throws IOException {

        super.computeFromSimpleMetrics( input );

        if (input.isEmpty()) {
            return;
        }

        Map<Long, Points.Point<SimpleNumber>> points = input.getPoints();
        for (Map.Entry<Long, Points.Point<SimpleNumber>> item : points.entrySet()) {
            SimpleNumber numericMetric = item.getValue().getData();
            sum += numericMetric.getValue().doubleValue();
        }
    }

    // merge rollups into this rollup.
    protected void computeFromRollups(Points<BasicRollup> input) throws IOException {

        computeFromRollupsHelper( input );

        if (input.isEmpty()) {
            return;
        }

        // See this and get mind blown:
        // http://stackoverflow.com/questions/18907262/bounded-wildcard-related-compiler-error
        Map<Long, ? extends Points.Point<? extends BasicRollup>> points = input.getPoints();

        for (Map.Entry<Long, ? extends Points.Point<? extends BasicRollup>> item : points.entrySet()) {
            BasicRollup rollup = item.getValue().getData();
            if (!(rollup instanceof BasicRollup)) {
                throw new IOException("Cannot create BasicRollup from type " + rollup.getClass().getName());
            }
            BasicRollup basicRollup = (BasicRollup) rollup;
            sum += basicRollup.getSum();
        }
    }

    public static BasicRollup buildRollupFromRawSamples(Points<SimpleNumber> input) throws IOException {
        final BasicRollup basicRollup = new BasicRollup();
        basicRollup.computeFromSimpleMetrics(input);

        return basicRollup;
    }

    public static BasicRollup buildRollupFromRollups(Points<BasicRollup> input) throws IOException {
        final BasicRollup basicRollup = new BasicRollup();
        basicRollup.computeFromRollups(input);
        return basicRollup;
    }

    @Override
    public RollupType getRollupType() {
        return RollupType.BF_BASIC;
    }
}


package com.rackspacecloud.blueflood.cache;

import com.codahale.metrics.Gauge;
import com.codahale.metrics.MetricRegistry;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.rackspacecloud.blueflood.types.Locator;
import com.rackspacecloud.blueflood.utils.Metrics;

import java.util.concurrent.TimeUnit;

/**
 * This class is used to cache locator's that were written recently to our persistence layers by the available writers.
 * All the writers, check the cache to see if it is written recently(isCurrent) before writing them again.
 *
 * Different writers that we have:
 *
 *  {@link com.rackspacecloud.blueflood.inputs.processors.BatchWriter} This writes to cassandra
 *  {@link com.rackspacecloud.blueflood.inputs.processors.DiscoveryWriter} This supports metric discovery (/metric/search)
 *  {@link com.rackspacecloud.blueflood.inputs.processors.TokenDiscoveryWriter} This support metric tokens discovery (metric_name/search)
 *
 * Each writer maintains its own indicator in {@link LocatorCacheEntry} to indicate whether a locator is current. This
 * is useful in cases, where persisting a locator with one writer is successful but not with other writers.
 *
 */
public class LocatorCache {

    // this collection is used to reduce the number of locators that get written.
    // Simply, if a locator has been seen within the last 10 minutes, don't bother.
    private final Cache<String, LocatorCacheEntry> insertedLocators;

    // this collection is used to reduce the number of delayed locators that get
    // written per slot. Simply, if a locator has been seen for a slot, don't bother.
    private final Cache<String, Boolean> insertedDelayedLocators;

    private static LocatorCache instance = new LocatorCache(10, TimeUnit.MINUTES,
                                                            3, TimeUnit.DAYS);

    static {
        Metrics.getRegistry().register(MetricRegistry.name(LocatorCache.class, "Current Locators Count"),
                new Gauge<Long>() {
                    @Override
                    public Long getValue() {
                        return instance.getCurrentLocatorCount();
                    }
                });

        Metrics.getRegistry().register(MetricRegistry.name(LocatorCache.class, "Current Delayed Locators Count"),
                new Gauge<Long>() {
                    @Override
                    public Long getValue() {
                        return instance.getCurrentDelayedLocatorCount();
                    }
                });
    }

    public static LocatorCache getInstance() {
        return instance;
    }

    protected LocatorCache(long expireAfterAccessDuration, TimeUnit expireAfterAccessTimeUnit,
                           long expireAfterWriteDuration, TimeUnit expireAfterWriteTimeUnit) {

        insertedLocators =
                CacheBuilder.newBuilder()
                        .expireAfterAccess(expireAfterAccessDuration, expireAfterAccessTimeUnit)
                        .expireAfterWrite(expireAfterWriteDuration, expireAfterWriteTimeUnit)
                        .concurrencyLevel(16)
                        .build();

        insertedDelayedLocators =
                CacheBuilder.newBuilder()
                        .expireAfterAccess(expireAfterAccessDuration, expireAfterAccessTimeUnit)
                        .concurrencyLevel(16)
                        .build();
    }

    @VisibleForTesting
    public static LocatorCache getInstance(long expireAfterAccessDuration, TimeUnit expireAfterAccessTimeUnit,
                                           long expireAfterWriteDuration, TimeUnit expireAfterWriteTimeUnit) {

        return new LocatorCache(expireAfterAccessDuration, expireAfterAccessTimeUnit,
                                expireAfterWriteDuration, expireAfterWriteTimeUnit);
    }

    public long getCurrentLocatorCount() {
        return insertedLocators.size();
    }

    public long getCurrentDelayedLocatorCount() {
        return insertedDelayedLocators.size();
    }

    /**
     * Checks if Locator is recently inserted in the batch layer
     *
     * @param loc
     * @return
     */
    public synchronized boolean isLocatorCurrentInBatchLayer(Locator loc) {
        LocatorCacheEntry entry = insertedLocators.getIfPresent(loc.toString());
        return entry != null && entry.isBatchCurrent();
    }

    /**
     * Checks if Locator is recently inserted in the discovery layer
     *
     * @param loc
     * @return
     */
    public synchronized boolean isLocatorCurrentInDiscoveryLayer(Locator loc) {
        LocatorCacheEntry entry = insertedLocators.getIfPresent(loc.toString());
        return entry != null && entry.isDiscoveryCurrent();
    }

    /**
     * Checks if Locator is recently inserted in the token discovery layer
     *
     * @param loc
     * @return
     */
    public synchronized boolean isLocatorCurrentInTokenDiscoveryLayer(Locator loc) {
        LocatorCacheEntry entry = insertedLocators.getIfPresent(loc.toString());
        return entry != null && entry.isTokenDiscoveryCurrent();
    }

    /**
     * Check if the delayed locator is recently inserted for a given slot
     *
     * @param slot
     * @param locator
     * @return
     */
    public synchronized boolean isDelayedLocatorForASlotCurrent(int slot, Locator locator) {
        return insertedDelayedLocators.getIfPresent(getLocatorSlotKey(slot, locator)) != null;
    }

    private String getLocatorSlotKey(int slot, Locator locator) {
        return slot + "," + locator.toString();
    }

    private LocatorCacheEntry getOrCreateInsertedLocatorEntry(Locator loc) {
        LocatorCacheEntry entry = insertedLocators.getIfPresent(loc.toString());

        if(entry == null) {
            entry = new LocatorCacheEntry();
            insertedLocators.put(loc.toString(), entry);
        }

        return entry;
    }

    /**
     * Marks the Locator as recently inserted in the batch layer
     * @param loc
     */
    public synchronized void setLocatorCurrentInBatchLayer(Locator loc) {
        getOrCreateInsertedLocatorEntry(loc).setBatchCurrent();
    }

    /**
     * Marks the Locator as recently inserted in the discovery layer
     * @param loc
     */
    public synchronized void setLocatorCurrentInDiscoveryLayer(Locator loc) {
        getOrCreateInsertedLocatorEntry(loc).setDiscoveryCurrent();
    }

    /**
     * Marks the Locator as recently inserted in the token discovery layer
     * @param loc
     */
    public synchronized void setLocatorCurrentInTokenDiscoveryLayer(Locator loc) {
        getOrCreateInsertedLocatorEntry(loc).setTokenDiscoveryCurrent();
    }

    /**
     * Marks the delayed locator as recently inserted for a given slot
     * @param slot
     * @param locator
     */
    public synchronized void setDelayedLocatorForASlotCurrent(int slot, Locator locator) {
        insertedDelayedLocators.put(getLocatorSlotKey(slot, locator), Boolean.TRUE);
    }

    @VisibleForTesting
    public synchronized void resetCache() {
        insertedLocators.invalidateAll();
        insertedDelayedLocators.invalidateAll();
    }

    @VisibleForTesting
    public synchronized void resetInsertedLocatorsCache() {
        insertedLocators.invalidateAll();
    }

    /**
     * Cache entry which defines where the locator has been inserted into during the caching period.
     */
    private class LocatorCacheEntry {
        private boolean discoveryCurrent = false;
        private boolean batchCurrent = false;
        private boolean tokenDiscoveryCurrent = false;

        void setDiscoveryCurrent() {
            this.discoveryCurrent = true;
        }

        void setBatchCurrent() {
            this.batchCurrent = true;
        }

        void setTokenDiscoveryCurrent() {
            this.tokenDiscoveryCurrent = true;
        }

        boolean isDiscoveryCurrent() {
            return discoveryCurrent;
        }

        boolean isBatchCurrent() {
            return batchCurrent;
        }

        boolean isTokenDiscoveryCurrent() {
            return tokenDiscoveryCurrent;
        }
    }

}


/*
 * Copyright (c) 2016 Rackspace.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * 	http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.rackspacecloud.blueflood.io.datastax;

import com.datastax.driver.core.BoundStatement;
import com.datastax.driver.core.PreparedStatement;
import com.rackspacecloud.blueflood.exceptions.InvalidDataException;
import com.rackspacecloud.blueflood.io.serializers.metrics.SetSerDes;
import com.rackspacecloud.blueflood.rollup.Granularity;
import com.rackspacecloud.blueflood.types.BluefloodCounterRollup;
import com.rackspacecloud.blueflood.types.BluefloodSetRollup;
import com.rackspacecloud.blueflood.types.IMetric;
import com.rackspacecloud.blueflood.types.Rollup;

import java.nio.ByteBuffer;

/**
 * This class holds the utility methods to read/write set metrics
 * using Datastax driver.
 */
public class DSetIO extends DAbstractMetricIO {

    private SetSerDes serDes = new SetSerDes();

    /**
     * Provides a way for the sub class to get a {@link java.nio.ByteBuffer}
     * representation of a certain Rollup object.
     *
     * @param value
     * @return
     */
    @Override
    protected ByteBuffer toByteBuffer(Object value) {
        if ( ! (value instanceof BluefloodSetRollup) ) {
            throw new IllegalArgumentException("toByteBuffer(): expecting BluefloodSetRollup class but got " + value.getClass().getSimpleName());
        }
        BluefloodSetRollup setRollup = (BluefloodSetRollup) value;
        return serDes.serialize(setRollup);
    }

    /**
     * Provides a way for the sub class to construct the right Rollup
     * object from a {@link java.nio.ByteBuffer}
     *
     * @param byteBuffer
     * @return
     */
    @Override
    protected BluefloodSetRollup fromByteBuffer(ByteBuffer byteBuffer) {
        return serDes.deserialize(byteBuffer);
    }

    /**
     * Retrieves the {@link BoundStatement} for a particular set metric
     * @param metric
     * @param granularity
     * @return
     */
    @Override
    protected BoundStatement getBoundStatementForMetric(IMetric metric, Granularity granularity) {
        Object metricValue = metric.getMetricValue();
        if (!(metricValue instanceof BluefloodSetRollup)) {
            throw new InvalidDataException(
                    String.format("getBoundStatementForMetric(locator=%s, granularity=%s): metric value %s is not type BluefloodSetRollup",
                            metric.getLocator(), granularity, metric.getMetricValue().getClass().getSimpleName())
            );
        }
        PreparedStatement statement = metricsCFPreparedStatements.preaggrGranToInsertStatement.get(granularity);
        return statement.bind(
                metric.getLocator().toString(),
                metric.getCollectionTime(),
                serDes.serialize( (BluefloodSetRollup) metricValue ),
                metric.getTtlInSeconds() );
    }
}


/*
 * Copyright 2013 Rackspace
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

package com.rackspacecloud.blueflood.types;

import com.rackspacecloud.blueflood.exceptions.GranularityException;
import com.rackspacecloud.blueflood.rollup.Granularity;

import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;

// typed pair tuple.
public class Range {

    public final long start;
    public final long stop;

    public Range(long l, long r) {
        if (l >= r) {
            throw new IllegalArgumentException("start cannot be greater than end");
        }
        start = l;
        stop = r;
    }

    public long getStart() {
        return start;
    }

    public long getStop() {
        return stop;
    }

    @Override
    public int hashCode() {
        return (int)(start * 3 + stop * 7);
    }

    @Override
    public boolean equals(Object obj) {
        if (!(obj instanceof Range))
            return false;
        else {
            Range other = (Range)obj;
            return other.start == start && other.stop == stop;
        }
    }

    @Override
    public String toString() {
        return String.format("%d:%d (%d)", start, stop, stop-start);
    }

    /**
     * given a start and stop time, return an iterator over ranges in *this* granularity that should be rolled up into
     * single points in the next coarser granularity.
     *
     * Here is an example:  Given start/end (s,e), we need to return all the ranges in Y that correspond to rollup
     * periods A,B,C.  This involves returning ranges prior to (s) and after (e) that map to parts of (A) and (C) in
     * the coarser rollup.
     *
     * X [       A      |       B      |       C      |       D      ]
     * Y [    |    | s  |    |    |    |  e |    |    |    |    |    ]\
     *
     * @param startMillis
     * @param stopMillis
     * @return
     */
    public static Iterable<Range> getRangesToRollup(Granularity g, final long startMillis,
                                                    final long stopMillis) throws GranularityException {
        final long snappedStartMillis = g.coarser().snapMillis(startMillis);
        final long snappedStopMillis = g.coarser().snapMillis(stopMillis + g.coarser().milliseconds());

        return new IntervalRangeIterator(g, snappedStartMillis, snappedStopMillis);
    }

    /**
     * Returns a mapping of ranges in the coarser granularity to the sub-ranges in finer granularity
     *
     * Here is an example: Given start/end (s,e), we need to return mapping between ranges in Y that will be mapped to
     * a single range in X. From the example above, it will be mapping from A to all the sub-ranges in Y that get rolled
     * to a single point in A
     * @param g Coarser Granularity
     * @param range Range to be mapped
     * @return
     * @throws GranularityException
     */
    public static Map<Range, Iterable<Range>> mapFinerRanges(Granularity g, Range range) throws GranularityException {

        if(range.getStart() >= range.getStop())
            throw new IllegalArgumentException("start cannot be greater than end. Start: " + range.getStart() + " Stop:" + range.getStop());

        final long snappedStartMillis = g.snapMillis(range.getStart());
        final long snappedStopMillis = g.snapMillis(range.getStop() + g.milliseconds());
        HashMap<Range, Iterable<Range>> rangeMap = new HashMap<Range, Iterable<Range>>();
        long tempStartMillis = snappedStartMillis;
        int numberOfMillis = g.milliseconds();

        while (tempStartMillis <= (snappedStopMillis - numberOfMillis)) {
            Range slotRange = new Range(tempStartMillis, tempStartMillis + numberOfMillis);
            rangeMap.put(slotRange, new IntervalRangeIterator(g.finer(), slotRange.start, slotRange.stop));
            tempStartMillis = tempStartMillis + numberOfMillis;
        }

        return rangeMap;
    }

    /** return the Ranges for an interval at this granularity
     * @param from start time
     * @param to end time
     * @return Range[]
     */
    public static Iterable<Range> rangesForInterval(Granularity g, final long from, final long to) {
        if (g == Granularity.FULL) {
            return Arrays.asList(new Range(from, to));
        }

        final long snappedStartMillis = g.snapMillis(from);
        final long snappedStopMillis = g.snapMillis(to + g.milliseconds());

        return new IntervalRangeIterator(g, snappedStartMillis, snappedStopMillis);
    }

    /** iterate over Ranges in an interval at a Granularity */
    private static class IntervalRangeIterator implements Iterable<Range> {

        final Granularity granularity;
        final long start;
        final long stop;

        IntervalRangeIterator(Granularity g, long start, long stop) {
            granularity = g;
            this.start = start;
            this.stop = Math.min(stop, System.currentTimeMillis());
        }

        public Iterator<Range> iterator() {
            return new Iterator<Range>() {
                long pos = start;
                public boolean hasNext() {
                    return pos < stop;
                }

                public Range next() {
                    Range res = null;
                    if (pos + granularity.milliseconds() > stop) {
                        res = new Range(pos, stop - 1);
                        pos = stop;
                    } else {
                        long end = granularity.snapMillis(pos + granularity.milliseconds()) - 1;
                        res = new Range(pos, end);
                        pos = end + 1;
                    }
                    return res;
                }

                public void remove() { throw new RuntimeException("Not supported"); }
            };
        };
    }
}