package com.rackspacecloud.blueflood.io.datastax;

import com.codahale.metrics.Timer;
import com.datastax.driver.core.BatchStatement;
import com.datastax.driver.core.BoundStatement;
import com.datastax.driver.core.ResultSet;
import com.datastax.driver.core.ResultSetFuture;
import com.google.common.annotations.VisibleForTesting;
import com.rackspacecloud.blueflood.cache.LocatorCache;
import com.rackspacecloud.blueflood.io.*;
import com.rackspacecloud.blueflood.outputs.formats.MetricData;
import com.rackspacecloud.blueflood.rollup.Granularity;
import com.rackspacecloud.blueflood.types.*;
import com.rackspacecloud.blueflood.utils.Clock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.*;

/**
 * This class deals with reading/writing metrics to the basic metrics_* column families
 * using Datastax driver
 */
public class DBasicMetricsRW extends DAbstractMetricsRW {

    private static final Logger LOG = LoggerFactory.getLogger(DBasicMetricsRW.class);

    private DSimpleNumberIO simpleNumberIO = new DSimpleNumberIO();
    private final DBasicNumericIO basicIO = new DBasicNumericIO();

    /**
     * Constructor
     * @param isRecordingDelayedMetrics if true, delayed metrics are recorded in metrics_delayed_locator
     * @param isBatchIngestEnabled  if true, metrics are inserted using batch statement
     */
    @VisibleForTesting
    public DBasicMetricsRW(DLocatorIO locatorIO, DDelayedLocatorIO delayedLocatorIO,
                           boolean isRecordingDelayedMetrics,
                           boolean isBatchIngestEnabled, Clock clock) {
        super(locatorIO, delayedLocatorIO, isRecordingDelayedMetrics, isBatchIngestEnabled, clock);
    }

    /**
     * Constructor. Using this, batch Ingest is NOT enabled.
     * @param isRecordingDelayedMetrics
     */
    @VisibleForTesting
    public DBasicMetricsRW(DLocatorIO locatorIO, DDelayedLocatorIO delayedLocatorIO,
                           boolean isRecordingDelayedMetrics,
                           Clock clock) {
        super(locatorIO, delayedLocatorIO, isRecordingDelayedMetrics, false, clock);
    }

    /**
     * This method inserts a collection of {@link com.rackspacecloud.blueflood.types.IMetric} objects
     * to the appropriate Cassandra column family. Effectively, this method is only called to write
     * the raw metrics received during Ingest requests. Another method, insertRollups, is used by
     * the Rollup processes to write rolled up metrics.
     *
     * @param metrics
     *
     * @throws IOException
     */
    @Override
    public void insertMetrics( Collection<IMetric> metrics ) throws IOException {

        Timer.Context ctx = Instrumentation.getWriteTimerContext( CassandraModel.CF_METRICS_FULL_NAME );
        try {
            if ( isBatchIngestEnabled ) {
                insertMetricsInBatch(metrics);
            } else {
                insertMetricsIndividually(metrics);
            }
        }
        finally {
            ctx.stop();
        }
    }

    /**
     * Fetches {@link com.rackspacecloud.blueflood.outputs.formats.MetricData} objects for the
     * specified list of {@link com.rackspacecloud.blueflood.types.Locator} and
     * {@link com.rackspacecloud.blueflood.types.Range} from the specified column family
     *
     * @param locators
     * @param range
     * @param gran
     * @return
     */
    @Override
    public Map<Locator, MetricData> getDatapointsForRange( List<Locator> locators, Range range, Granularity gran ) {

        Map<Locator, MetricData> metrics = new HashMap<Locator, MetricData>();

        String columnFamily = CassandraModel.getBasicColumnFamilyName( gran );

        metrics.putAll( super.getDatapointsForRange( locators, range, columnFamily, gran));

        return metrics;
    }

    /**
     * Return the appropriate IO object which interacts with the Cassandra database.
     *
     * For Preaggregated Metrics, only rollup type is required.
     * For Basic Numeric Metrics, the granularity is required as metrics_full is handled differently
     * than the other granularities.
     *
     * @param rollupType
     * @param granularity
     * @return
     */
    @Override
    public DAbstractMetricIO getIO( String rollupType, Granularity granularity ) {

      if( granularity == Granularity.FULL )
        return simpleNumberIO;
      else
        return basicIO;
    }

    private void insertMetricsIndividually(Collection<IMetric> metrics) throws IOException {

        Map<Locator, ResultSetFuture> futures = new HashMap<Locator, ResultSetFuture>();
        for( IMetric metric : metrics ) {

            Locator locator = metric.getLocator();

            if( !LocatorCache.getInstance().isLocatorCurrentInBatchLayer(locator) ) {

                LocatorCache.getInstance().setLocatorCurrentInBatchLayer(locator);
                locatorIO.insertLocator( locator );
            }

            if (isRecordingDelayedMetrics) {
                insertLocatorIfDelayed(metric);
            }

            futures.put(locator, simpleNumberIO.insertRawAsync(metric));

            Instrumentation.markFullResMetricWritten();
        }

        for( Map.Entry<Locator, ResultSetFuture> f : futures.entrySet() ) {

            try {
                ResultSet result = f.getValue().getUninterruptibly();

                LOG.trace( "result.size=" + result.all().size() );
            }
            catch ( Exception e ) {
                Instrumentation.markWriteError();
                LOG.error(String.format("error writing metric for locator %s",
                        f.getKey()), e );
            }
        }
    }

    /**
     * Inserts a collection of metrics in a batch using an unlogged
     * {@link BatchStatement}
     *
     * @param metrics
     * @return
     */
    private void insertMetricsInBatch(Collection<IMetric> metrics) throws IOException {

        BatchStatement batch = new BatchStatement(BatchStatement.Type.UNLOGGED);

        for (IMetric metric : metrics) {
            BoundStatement bound = simpleNumberIO.getBoundStatementForMetric(metric);
            batch.add(bound);
            Instrumentation.markFullResMetricWritten();

            Locator locator = metric.getLocator();
            if( !LocatorCache.getInstance().isLocatorCurrentInBatchLayer(locator) ) {
                LocatorCache.getInstance().setLocatorCurrentInBatchLayer(locator);
                batch.add(locatorIO.getBoundStatementForLocator( locator ));
            }

            // if we are recording delayed metrics, we may need to do an
            // extra insert
            if ( isRecordingDelayedMetrics ) {
                BoundStatement bs = getBoundStatementForMetricIfDelayed(metric);
                if ( bs != null ) {
                    batch.add(bs);
                }
            }
        }
        LOG.trace(String.format("insert batch statement size=%d", batch.size()));

        try {
            DatastaxIO.getSession().execute(batch);
        } catch ( Exception ex ) {
            Instrumentation.markWriteError();
            LOG.error(String.format("error writing batch of %d metrics", batch.size()), ex );
        }
    }
}


/*
The MIT License (MIT)
Copyright (c) 2013 Naoyuki Kanezawa

Permission is hereby granted, free of charge, to any person obtaining a copy of this software and as
sociated documentation files (the "Software"), to deal in the Software without restriction, includin
g without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subj
ect to the following conditions:

The above copyright notice and this permission notice shall be included in all copies or substantial
 portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NO
T LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABI
LITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WIT
H THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

(The MIT License)

Copyright (c) 2011 Guillermo Rauch <guillermo@learnboost.com>

Permission is hereby granted, free of charge, to any person obtaining a copy of this software and as
sociated documentation files (the 'Software'), to deal in the Software without restriction, includin
g without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subj
ect to the following conditions:

The above copyright notice and this permission notice shall be included in all copies or substantial
 portions of the Software.

THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NO
T LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABI
LITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WIT
H THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

/**
 * This is a modified version of the EventEmitter found as part of nkzawa's engine.io-client library
 * https://github.com/nkzawa/engine.io-client.java
 *
 * The only difference is that it uses generics (T... args) rather than (Object... args)
 */
package com.rackspacecloud.blueflood.eventemitter;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Future;

/**
 * The event emitter which is ported from the JavaScript module. This class is thread-safe.
 *
 * @see https://github.com/component/emitter
 */
public class Emitter<T> {

    private ConcurrentMap<String, ConcurrentLinkedQueue<Listener>> callbacks
            = new ConcurrentHashMap<String, ConcurrentLinkedQueue<Listener>>();

    private ConcurrentMap<Listener, Listener> onceCallbacks = new ConcurrentHashMap<Listener, Listener>();

    /**
     * Listens on the event.
     * @param event event name.
     * @param fn
     * @return a reference to this object.
     */
    public Emitter on(String event, Listener fn) {
        ConcurrentLinkedQueue<Listener> callbacks = this.callbacks.get(event);
        if (callbacks == null) {
            callbacks = new ConcurrentLinkedQueue<Listener>();
            ConcurrentLinkedQueue<Listener> _callbacks = this.callbacks.putIfAbsent(event, callbacks);
            if (_callbacks != null) {
                callbacks = _callbacks;
            }
        }
        callbacks.add(fn);
        return this;
    }

    /**
     * Adds a one time listener for the event.
     *
     * @param event an event name.
     * @param fn
     * @return a reference to this object.
     */
    public Emitter once(final String event, final Listener<T> fn) {
        Listener on = new Listener<T>() {
            @Override
            public void call(T... args) {
                Emitter.this.off(event, this);
                fn.call(args);
            }
        };

        this.onceCallbacks.put(fn, on);
        this.on(event, on);
        return this;
    }

    /**
     * Removes all registered listeners.
     *
     * @return a reference to this object.
     */
    public Emitter off() {
        this.callbacks.clear();
        this.onceCallbacks.clear();
        return this;
    }

    /**
     * Removes all listeners of the specified event.
     *
     * @param event an event name.
     * @return a reference to this object.
     */
    public Emitter off(String event) {
        ConcurrentLinkedQueue<Listener> callbacks = this.callbacks.remove(event);
        if (callbacks != null) {
            for (Listener fn : callbacks) {
                this.onceCallbacks.remove(fn);
            }
        }
        return this;
    }

    /**
     * Removes the listener.
     *
     * @param event an event name.
     * @param fn
     * @return a reference to this object.
     */
    public Emitter off(String event, Listener fn) {
        ConcurrentLinkedQueue<Listener> callbacks = this.callbacks.get(event);
        if (callbacks != null) {
            Listener off = this.onceCallbacks.remove(fn);
            callbacks.remove(off != null ? off : fn);
        }
        return this;
    }

    /**
     * Executes each of listeners with the given args.
     *
     * @param event an event name.
     * @param args
     * @return a reference to this object.
     */
    public Future emit(String event, T... args) {
        ConcurrentLinkedQueue<Listener> callbacks = this.callbacks.get(event);
        if (callbacks != null) {
            callbacks = new ConcurrentLinkedQueue<Listener>(callbacks);
            for (Listener fn : callbacks) {
                fn.call(args);
            }
        }
        return null;
    }

    /**
     * Returns a list of listeners for the specified event.
     *
     * @param event an event name.
     * @return a reference to this object.
     */
    public List<Listener> listeners(String event) {
        ConcurrentLinkedQueue<Listener> callbacks = this.callbacks.get(event);
        return callbacks != null ?
                new ArrayList<Listener>(callbacks) : new ArrayList<Listener>();
    }

    /**
     * Check if this emitter has listeners for the specified event.
     *
     * @param event an event name.
     * @return a reference to this object.
     */
    public boolean hasListeners(String event) {
        return !this.listeners(event).isEmpty();
    }

    public static interface Listener<T> {
        public void call(T... args);
    }
}


/*
 * Copyright 2014 Rackspace
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

package com.rackspacecloud.blueflood.outputs.cloudfiles;

import com.rackspacecloud.blueflood.eventemitter.RollupEvent;
import com.rackspacecloud.blueflood.outputs.serializers.RollupEventSerializer;
import com.rackspacecloud.blueflood.service.CloudfilesConfig;
import com.rackspacecloud.blueflood.service.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.text.SimpleDateFormat;
import java.util.Date;

// Not thread-safe
public class RollupFile implements Comparable {
    private static final Logger log = LoggerFactory.getLogger(RollupFile.class);

    private final File file;
    private FileOutputStream outputStream;
    private long timestamp;
    private RollupEventSerializer serializer = new RollupEventSerializer();
    public static final FileFilter fileFilter = new RollupFileFilter();

    public RollupFile(File file) {
        this.file = file;
        this.timestamp = parseTimestamp(file.getName());
    }

    /**
     * Retrieve the timestamp associated with this file.
     *
     * @return The timestamp associated with the file.
     */
    public long getTimestamp() {
        return timestamp;
    }

    /**
     * Get the name of the buffer file.
     *
     * @return The name of the buffer file.
     */
    public String getName() {
        return file.getName();
    }

    /**
     * Get the remote file name.
     *
     * @return The path to the remote file.
     */
    public String getRemoteName() {
        Date time = new Date(timestamp);
        String formattedTime = new SimpleDateFormat("yyyyMMdd_").format(time);
        return formattedTime + System.currentTimeMillis() + "_" + Configuration.getInstance().getStringProperty(CloudfilesConfig.CLOUDFILES_HOST_UNIQUE_IDENTIFIER);
    }

    /**
     * Get the age of the rollup file in milliseconds.
     *
     * @return The age of the rollup file in milliseconds.
     */
    public long getAge() {
        return System.currentTimeMillis() - timestamp;
    }

    /**
     * Get the size of the rollup file in bytes.
     *
     * @return The size of the rollup file in bytes.
     * @throws IOException
     */
    public long getSize() throws IOException {
        ensureOpen();
        return outputStream.getChannel().size();
    }

    /**
     * Retrieve an InputStream which reads from this rollup file.
     *
     * @return An InputStream of data from the file.
     * @throws FileNotFoundException
     */
    public InputStream asReadStream() throws FileNotFoundException {
        return new FileInputStream(file);
    }

    /**
     * Serialize a Rollup Event and append it to the file.
     *
     * @param rollup The rollup to append.
     * @throws IOException
     */
    public void append(RollupEvent rollup) throws IOException {
        ensureOpen();
        outputStream.write(serializer.toBytes(rollup));
        outputStream.write('\n');
        outputStream.flush();
    }

    public void close() throws IOException {
        if (outputStream != null) {
            outputStream.close();
            outputStream = null;
        }
    }

    public void delete() throws FileNotFoundException, IllegalAccessException {
        if (!file.delete()) {
            if (!file.exists()) {
                throw new FileNotFoundException("File does not exist");
            }
            throw new IllegalAccessException("File exists but could not be deleted. File is probably locked.");
        }
    }

    public int compareTo(Object other) {
        return new Long(getTimestamp()).compareTo(((RollupFile) other).getTimestamp());
    }

    private void ensureOpen() throws FileNotFoundException {
        if (outputStream == null) {
            log.info("opening buffer file for writing: {}", file.getName());
            outputStream = new FileOutputStream(file, true);
        }
    }

    /**
     * Parse the timestamp from a filename.
     *
     * @param fileName The file name to parse.
     * @return The timestamp contained in the file name.
     * @throws NumberFormatException
     */
    private static long parseTimestamp(String fileName) throws NumberFormatException {
        String numberPart = fileName.substring(0, fileName.length() - 5);
        return Long.parseLong(numberPart);
    }

    /**
     * Build a new RollupFile using the current time as the timestamp. The file won't be created until it is actually
     * written to for the first time.
     *
     * @param bufferDir The directory in which to create the file.
     * @return The new RollupFile.
     */
    public static RollupFile buildRollupFile(File bufferDir) {
        return new RollupFile(new File(bufferDir, System.currentTimeMillis() + ".json"));
    }

    private static class RollupFileFilter implements FileFilter {
        public boolean accept(File f) {
            String fileName = f.getName();

            if (!f.isFile()) {
                return false;
            }

            if (!fileName.endsWith(".json")) {
                log.info("skipping non-JSON file: {}", fileName);
                return false;
            }

            try {
                parseTimestamp(fileName);
            } catch (NumberFormatException e) {
                log.info("skipping malformatted filename: {}", fileName);
                return false;
            }
            return true;
        }
    }
}


/*
 * Copyright 2013-2015 Rackspace
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

package com.rackspacecloud.blueflood.inputs.handlers;

import com.codahale.metrics.Counter;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.ListenableFuture;
import com.rackspacecloud.blueflood.cache.MetadataCache;
import com.rackspacecloud.blueflood.concurrent.ThreadPoolBuilder;
import com.rackspacecloud.blueflood.http.DefaultHandler;
import com.rackspacecloud.blueflood.http.QueryStringDecoderAndRouter;
import com.rackspacecloud.blueflood.http.RouteMatcher;
import com.rackspacecloud.blueflood.inputs.processors.*;
import com.rackspacecloud.blueflood.io.EventsIO;
import com.rackspacecloud.blueflood.service.*;
import com.rackspacecloud.blueflood.tracker.Tracker;
import com.rackspacecloud.blueflood.types.IMetric;
import com.rackspacecloud.blueflood.types.MetricsCollection;
import com.rackspacecloud.blueflood.utils.Metrics;
import com.rackspacecloud.blueflood.utils.ModuleLoader;
import com.rackspacecloud.blueflood.utils.TimeValue;
import io.netty.bootstrap.ServerBootstrap;
import io.netty.channel.*;
import io.netty.channel.group.ChannelGroup;
import io.netty.channel.group.DefaultChannelGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import io.netty.handler.codec.http.*;
import io.netty.handler.logging.LogLevel;
import io.netty.handler.logging.LoggingHandler;
import io.netty.handler.timeout.IdleStateHandler;
import io.netty.util.concurrent.GlobalEventExecutor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.net.InetSocketAddress;
import java.util.List;
import java.util.concurrent.TimeUnit;

public class HttpMetricsIngestionServer {
    private static final Logger log = LoggerFactory.getLogger(HttpMetricsIngestionServer.class);
    private static TimeValue DEFAULT_TIMEOUT = new TimeValue(5, TimeUnit.SECONDS);
    private int httpIngestPort;
    private String httpIngestHost;
    private Processor processor;
    private HttpEventsIngestionHandler httpEventsIngestionHandler;
    private final int httpMaxContentLength;

    private TimeValue timeout;

    private EventLoopGroup acceptorGroup;
    private EventLoopGroup workerGroup;
    private ChannelGroup allOpenChannels = new DefaultChannelGroup("allOpenChannels", GlobalEventExecutor.INSTANCE);

    private int HTTP_CONNECTION_READ_IDLE_TIME_SECONDS =
            Configuration.getInstance().getIntegerProperty(HttpConfig.HTTP_CONNECTION_READ_IDLE_TIME_SECONDS);

    public static boolean EXP_TOKEN_SEARCH_IMPROVEMENTS =
            Configuration.getInstance().getBooleanProperty(CoreConfig.ENABLE_TOKEN_SEARCH_IMPROVEMENTS);

    private static final boolean ENABLE_PER_TENANT_METRICS =
            Configuration.getInstance().getBooleanProperty(CoreConfig.ENABLE_PER_TENANT_METRICS);

    /**
     * This is HTTP Header name is inserted by RouterMatcher based on the
     * router configuration in startServer()
     */
    public static final String TENANT_ID_HEADER = "tenantId";

    /**
     * Constructor. Instantiate Metrics Ingest server
     * @param context
     */
    public HttpMetricsIngestionServer(ScheduleContext context) {
        this.httpIngestPort = Configuration.getInstance().getIntegerProperty(HttpConfig.HTTP_INGESTION_PORT);
        this.httpIngestHost = Configuration.getInstance().getStringProperty(HttpConfig.HTTP_INGESTION_HOST);
        this.timeout = DEFAULT_TIMEOUT; //TODO: make configurable
        this.processor = new Processor(context, timeout);
        this.httpMaxContentLength = Configuration.getInstance().getIntegerProperty(HttpConfig.HTTP_MAX_CONTENT_LENGTH);

        int acceptThreads = Configuration.getInstance().getIntegerProperty(HttpConfig.MAX_WRITE_ACCEPT_THREADS);
        int workerThreads = Configuration.getInstance().getIntegerProperty(HttpConfig.MAX_WRITE_WORKER_THREADS);
        acceptorGroup = new NioEventLoopGroup(acceptThreads); // acceptor threads
        workerGroup = new NioEventLoopGroup(workerThreads);   // client connections threads
    }

    /**
     * Starts the Ingest server
     *
     * @throws InterruptedException
     */
    public void startServer() throws InterruptedException {

        RouteMatcher router = new RouteMatcher();
        router.get("/v1.0", new DefaultHandler());
        router.post("/v1.0/multitenant/experimental/metrics",
                new HttpMultitenantMetricsIngestionHandler(processor, timeout, ENABLE_PER_TENANT_METRICS));
        router.post("/v1.0/:tenantId/experimental/metrics",
                new HttpMetricsIngestionHandler(processor, timeout, ENABLE_PER_TENANT_METRICS));
        router.post("/v1.0/:tenantId/experimental/metrics/statsd",
                new HttpAggregatedIngestionHandler(processor, timeout, ENABLE_PER_TENANT_METRICS));

        router.get("/v2.0", new DefaultHandler());
        router.post("/v2.0/:tenantId/ingest/multi",
                new HttpMultitenantMetricsIngestionHandler(processor, timeout, ENABLE_PER_TENANT_METRICS));
        router.post("/v2.0/:tenantId/ingest",
                new HttpMetricsIngestionHandler(processor, timeout, ENABLE_PER_TENANT_METRICS));
        router.post("/v2.0/:tenantId/ingest/aggregated",
                new HttpAggregatedIngestionHandler(processor, timeout, ENABLE_PER_TENANT_METRICS));
        router.post("/v2.0/:tenantId/ingest/aggregated/multi",
                new HttpAggregatedMultiIngestionHandler(processor, timeout, ENABLE_PER_TENANT_METRICS));
        router.post("/v2.0/:tenantId/events", getHttpEventsIngestionHandler());
        final RouteMatcher finalRouter = router;

        log.info("Starting metrics listener HTTP server on port {}", httpIngestPort);
        ServerBootstrap server = new ServerBootstrap();
        server.group(acceptorGroup, workerGroup)
                .channel(NioServerSocketChannel.class)
                .childHandler(new ChannelInitializer<SocketChannel>() {
                    @Override
                    public void initChannel(SocketChannel channel) throws Exception {
                        setupPipeline(channel, finalRouter);
                    }
                });

        Channel channel = server.bind(new InetSocketAddress(httpIngestHost, httpIngestPort)).sync().channel();
        allOpenChannels.add(channel);

        //register the tracker MBean for JMX/jolokia
        log.info("Registering tracker service");
        Tracker.getInstance().register();

        log.info("Token search improvements enabled: " + EXP_TOKEN_SEARCH_IMPROVEMENTS);
    }

    private void setupPipeline(SocketChannel channel, RouteMatcher router) {
        final ChannelPipeline pipeline = channel.pipeline();

        pipeline.addLast("logging", new LoggingHandler(LogLevel.TRACE)); //duplex handler
        pipeline.addLast("idleStateHandler", new IdleStateHandler(HTTP_CONNECTION_READ_IDLE_TIME_SECONDS, 0, 0)); //duplex handler
        pipeline.addLast("eventHandler", new UserDefinedEventHandler()); //duplex handler
        pipeline.addLast("encoder", new HttpResponseEncoder()); //outbound handler
        pipeline.addLast("decoder", new HttpRequestDecoder() { //inbound handler

            // if something bad happens during the decode, assume the client send bad data. return a 400.
            @Override
            public void exceptionCaught(ChannelHandlerContext ctx, Throwable thr) throws Exception {
                log.debug("Exception during decoding", thr);
                try {
                    if (ctx.channel().isWritable()) {
                        log.debug("request decoder error " + thr.getCause() + " on channel " + ctx.channel().toString());
                        ctx.channel().write(
                                new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.BAD_REQUEST))
                                .addListener(ChannelFutureListener.CLOSE);
                    } else {
                        log.debug("channel " + ctx.channel().toString() + " is no longer writeable, not sending 400 response back to client");
                    }
                } catch (Exception ex) {
                    // If we are getting exception trying to write,
                    // don't propagate to caller. It may cause this
                    // method to be called again and will produce
                    // stack overflow. So just log it here.
                    log.debug("Can't write to channel " + ctx.channel().toString(), ex);
                }
            }
        });
        pipeline.addLast("inflater", new HttpContentDecompressor()); //inbound handler
        pipeline.addLast("chunkaggregator", new HttpObjectAggregator(httpMaxContentLength)); //inbound handler
        pipeline.addLast("respdecoder", new HttpResponseDecoder()); //inbound handler
        pipeline.addLast("handler", new QueryStringDecoderAndRouter(router)); //inbound handler
    }

    private HttpEventsIngestionHandler getHttpEventsIngestionHandler() {
        if (this.httpEventsIngestionHandler == null) {
            this.httpEventsIngestionHandler = new HttpEventsIngestionHandler((EventsIO) ModuleLoader.getInstance(EventsIO.class, CoreConfig.EVENTS_MODULES));
        }
        return this.httpEventsIngestionHandler;
    }

    @VisibleForTesting
    public void setHttpEventsIngestionHandler(HttpEventsIngestionHandler httpEventsIngestionHandler) {
        this.httpEventsIngestionHandler = httpEventsIngestionHandler;
    }

    static class Processor {
        private static int BATCH_SIZE = Configuration.getInstance().getIntegerProperty(CoreConfig.METRIC_BATCH_SIZE);
        private static int WRITE_THREADS =
            Configuration.getInstance().getIntegerProperty(CoreConfig.METRICS_BATCH_WRITER_THREADS); // metrics will be batched into this many partitions.

        private final TypeAndUnitProcessor typeAndUnitProcessor;
        private final RollupTypeCacher rollupTypeCacher;
        private final DiscoveryWriter discoveryWriter;
        private final TokenDiscoveryWriter tokenDiscoveryWriter;
        private final BatchWriter batchWriter;
        private IncomingMetricMetadataAnalyzer metricMetadataAnalyzer =
            new IncomingMetricMetadataAnalyzer(MetadataCache.getInstance());
        private int HTTP_MAX_TYPE_UNIT_PROCESSOR_THREADS =
            Configuration.getInstance().getIntegerProperty(HttpConfig.HTTP_MAX_TYPE_UNIT_PROCESSOR_THREADS);
        private final Counter bufferedMetrics = Metrics.counter(HttpMetricsIngestionHandler.class, "Buffered Metrics");
        private final TimeValue timeout;

        Processor(ScheduleContext context, TimeValue timeout) {
            this.timeout = timeout;

            typeAndUnitProcessor = new TypeAndUnitProcessor(
                new ThreadPoolBuilder()
                    .withName("Metric type and unit processing")
                    .withCorePoolSize(HTTP_MAX_TYPE_UNIT_PROCESSOR_THREADS)
                    .withMaxPoolSize(HTTP_MAX_TYPE_UNIT_PROCESSOR_THREADS)
                    .build(),
                    metricMetadataAnalyzer);
            typeAndUnitProcessor.withLogger(log);

            batchWriter = new BatchWriter(
                new ThreadPoolBuilder()
                        .withName("Metric Batch Writing")
                        .withCorePoolSize(WRITE_THREADS)
                        .withMaxPoolSize(WRITE_THREADS)
                        .withSynchronousQueue()
                        .build(),
                timeout,
                bufferedMetrics,
                context
            );
            batchWriter.withLogger(log);

            discoveryWriter =
                new DiscoveryWriter(new ThreadPoolBuilder()
                    .withName("Metric Discovery Writing")
                    .withCorePoolSize(Configuration.getInstance().getIntegerProperty(CoreConfig.DISCOVERY_WRITER_MIN_THREADS))
                    .withMaxPoolSize(Configuration.getInstance().getIntegerProperty(CoreConfig.DISCOVERY_WRITER_MAX_THREADS))
                    .withUnboundedQueue()
                    .build());
            discoveryWriter.withLogger(log);

            tokenDiscoveryWriter =
                new TokenDiscoveryWriter(
                    new ThreadPoolBuilder()
                        .withName("Metric Token Discovery Writing")
                        .withCorePoolSize(Configuration.getInstance().getIntegerProperty(CoreConfig.TOKEN_DISCOVERY_WRITER_MIN_THREADS))
                        .withMaxPoolSize(Configuration.getInstance().getIntegerProperty(CoreConfig.TOKEN_DISCOVERY_WRITER_MAX_THREADS))
                        .withUnboundedQueue()
                        .build());
            tokenDiscoveryWriter.withLogger(log);

            // RollupRunnable keeps a static one of these. It would be nice if we could register it and share.
            MetadataCache rollupTypeCache = MetadataCache.createLoadingCacheInstance(
                    new TimeValue(48, TimeUnit.HOURS),
                    Configuration.getInstance().getIntegerProperty(CoreConfig.MAX_ROLLUP_READ_THREADS));
            rollupTypeCacher = new RollupTypeCacher(
                    new ThreadPoolBuilder().withName("Rollup type persistence").build(),
                    rollupTypeCache);
            rollupTypeCacher.withLogger(log);

        }

        ListenableFuture<List<Boolean>> apply(MetricsCollection collection) throws Exception {
            typeAndUnitProcessor.apply(collection);
            rollupTypeCacher.apply(collection);
            List<List<IMetric>> batches = collection.splitMetricsIntoBatches(BATCH_SIZE);
            discoveryWriter.apply(batches);

            if (EXP_TOKEN_SEARCH_IMPROVEMENTS)
                tokenDiscoveryWriter.processTokens(batches);

            return batchWriter.apply(batches);
        }
    }

    @VisibleForTesting
    public void shutdownServer() {
        try {
            allOpenChannels.close().await(5, TimeUnit.SECONDS);
        } catch (InterruptedException e) {
            // Pass
        }
        acceptorGroup.shutdownGracefully();
        workerGroup.shutdownGracefully();
    }
}


/*
 * Copyright 2013 Rackspace
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

package com.rackspacecloud.blueflood.service;

import com.codahale.metrics.Meter;
import com.codahale.metrics.Timer;
import com.rackspacecloud.blueflood.cache.MetadataCache;
import com.rackspacecloud.blueflood.exceptions.GranularityException;
import com.rackspacecloud.blueflood.io.AbstractMetricsRW;
import com.rackspacecloud.blueflood.io.CassandraModel;
import com.rackspacecloud.blueflood.io.CassandraModel.MetricColumnFamily;
import com.rackspacecloud.blueflood.rollup.Granularity;
import com.rackspacecloud.blueflood.types.*;
import com.rackspacecloud.blueflood.utils.Metrics;
import com.rackspacecloud.blueflood.utils.RollupUtils;
import com.rackspacecloud.blueflood.utils.TimeValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.rackspacecloud.blueflood.eventemitter.RollupEventEmitter;
import com.rackspacecloud.blueflood.eventemitter.RollupEvent;

import java.util.*;
import java.util.concurrent.TimeUnit;

/** rolls up data into one data point, inserts that data point. */
public class RollupRunnable implements Runnable {
    private static final Logger LOG = LoggerFactory.getLogger(RollupRunnable.class);

    protected final SingleRollupReadContext singleRollupReadContext;
    protected static final MetadataCache metadataCache = MetadataCache.getInstance();
    protected static final MetadataCache rollupTypeCache = MetadataCache.createLoadingCacheInstance(
            new TimeValue(48, TimeUnit.HOURS), // todo: need a good default expiration here.
            Configuration.getInstance().getIntegerProperty(CoreConfig.MAX_ROLLUP_READ_THREADS));

    protected final RollupExecutionContext executionContext;
    protected final RollupBatchWriter rollupBatchWriter;
    protected final long startWait;

    private static final Timer calcTimer = Metrics.timer(RollupRunnable.class, "Read And Calculate Rollup");
    private static final Meter noPointsToCalculateRollup = Metrics.meter(RollupRunnable.class, "No points to calculate rollup");
    private static HashMap<Granularity, Meter> granToMeters = new HashMap<Granularity, Meter>();

    static {
        for (Granularity rollupGranularity : Granularity.rollupGranularities()) {
            granToMeters.put(rollupGranularity, Metrics.meter(RollupRunnable.class, String.format("%s Rollup", rollupGranularity.shortName())));
        }
    }

    public RollupRunnable(RollupExecutionContext executionContext, SingleRollupReadContext singleRollupReadContext, RollupBatchWriter rollupBatchWriter) {
        this.executionContext = executionContext;
        this.singleRollupReadContext = singleRollupReadContext;
        this.rollupBatchWriter = rollupBatchWriter;
        startWait = System.currentTimeMillis();
    }

    public void run() {
        // done waiting.
        singleRollupReadContext.getWaitHist().update(System.currentTimeMillis() - startWait);

        Granularity srcGran;
        try {
            srcGran = singleRollupReadContext.getRollupGranularity().finer();
        } catch (GranularityException ex) {
            executionContext.decrementReadCounter();
            return; // no work to be done.
        }

        if (LOG.isTraceEnabled()) {
            LOG.trace("Executing rollup from {} for {} {}", new Object[]{
                    srcGran.shortName(),
                    singleRollupReadContext.getRange().toString(),
                    singleRollupReadContext.getLocator()});
        }

        // start timing this action.
        Timer.Context timerContext = singleRollupReadContext.getExecuteTimer().time();

        try {
            Timer.Context calcrollupContext = calcTimer.time();
            granToMeters.get(srcGran.coarser()).mark();

            // Read data and compute rollup
            Points input;
            Rollup rollup = null;
            Locator rollupLocator = singleRollupReadContext.getLocator();
            RollupType rollupType = RollupType.fromString((String) rollupTypeCache.get(
                    rollupLocator, MetricMetadata.ROLLUP_TYPE.name().toLowerCase()));

            // RollupType   | Class                          | Column Family
            // -------------| ------------------------------ |--------------
            // COUNTER      | BluefloodCounterRollup         | metrics_preaggr_{gran}
            // TIMER        | BluefloodTimerRollup           | metrics_preaggr_{gran}
            // SET          | BluefloodSetRollup             | metrics_preaggr_{gran}
            // GAUGE        | BluefloodGaugeRollup           | metrics_preaggr_{gran}
            // BF_BASIC     | BasicRollup (if gran != full)  | metrics_{gran}
            //              | SimpleNumber (if gran == full) | metrics_full

            Class<? extends Rollup> rollupClass = RollupType.classOf(rollupType, srcGran.coarser());
            MetricColumnFamily srcCF = CassandraModel.getColumnFamily(rollupClass, srcGran);
            Granularity dstGran = srcGran.coarser();
            MetricColumnFamily dstCF = CassandraModel.getColumnFamily(rollupClass, dstGran);

            // first, get the points.
            AbstractMetricsRW metricsRW;
            try {
                metricsRW = RollupUtils.getMetricsRWForRollupType(rollupType);

                input = metricsRW.getDataToRollup(
                        singleRollupReadContext.getLocator(),
                        rollupType,
                        singleRollupReadContext.getRange(),
                        srcCF.getName());

                if (input.isEmpty()) {
                    LOG.debug(String.format("No points rollup for locator %s", singleRollupReadContext.getLocator()));
                    noPointsToCalculateRollup.mark();
                    return;
                }

                // next, compute the rollup.
                rollup =  RollupRunnable.getRollupComputer(rollupType, srcGran).compute(input);
            } finally {
                calcrollupContext.stop();
            }
            // now enqueue the new rollup for writing.
            rollupBatchWriter.enqueueRollupForWrite(new SingleRollupWriteContext(rollup, singleRollupReadContext, dstCF));

            RollupService.lastRollupTime.set(System.currentTimeMillis());
            //Emit a rollup event to event emitter
            RollupEventEmitter.getInstance().emit(RollupEventEmitter.ROLLUP_EVENT_NAME,
                    new RollupEvent(singleRollupReadContext.getLocator(), rollup,
                            metadataCache.getUnitString(singleRollupReadContext.getLocator()),
                            singleRollupReadContext.getRollupGranularity().name(),
                            singleRollupReadContext.getRange().getStart()));
        } catch (Exception e) {
            LOG.error("Rollup failed; Locator: {}, Source Granularity: {}, For period: {}", new Object[]{
                    singleRollupReadContext.getLocator(),
                    srcGran.name(),
                    singleRollupReadContext.getRange().toString(),
                    e});
        } finally {
            executionContext.decrementReadCounter();
            timerContext.stop();
        }
    }

    // determine which DataType to use for serialization.
    public static Rollup.Type getRollupComputer(RollupType srcType, Granularity srcGran) {
        switch (srcType) {
            case COUNTER:
                return Rollup.CounterFromCounter;
            case TIMER:
                return Rollup.TimerFromTimer;
            case GAUGE:
                return Rollup.GaugeFromGauge;
            case BF_BASIC:
                return srcGran == Granularity.FULL ? Rollup.BasicFromRaw : Rollup.BasicFromBasic;
            case SET:
                return Rollup.SetFromSet;
            default:
                break;
        }
        throw new IllegalArgumentException(String.format("Cannot compute rollups for %s from %s", srcType.name(), srcGran.shortName()));
    }
}


/*
 * Copyright (c) 2016 Rackspace.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * 	http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.rackspacecloud.blueflood.io;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
import com.rackspacecloud.blueflood.io.astyanax.*;
import com.rackspacecloud.blueflood.io.datastax.*;
import com.rackspacecloud.blueflood.service.Configuration;
import com.rackspacecloud.blueflood.service.CoreConfig;
import com.rackspacecloud.blueflood.utils.DefaultClockImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * This class holds the necessary IO classes for a particular driver type.
 */
public class IOContainer {

    // These are numerous IO factories designed to read/write data from/to
    // individual Column Family. Instead of creating factory classes for
    // each one, I elected to take a short-cut and create one for all
    // of them

    private static final Configuration configuration = Configuration.getInstance();
    private static final Logger LOG = LoggerFactory.getLogger(IOContainer.class);
    private static IOContainer FROM_CONFIG_INSTANCE = null;

    private ShardStateIO shardStateIO;
    private MetadataIO metadataIO;
    private LocatorIO locatorIO;
    private DelayedLocatorIO delayedLocatorIO;
    private AbstractMetricsRW preAggregatedMetricsRW;
    private AbstractMetricsRW basicMetricsRW;

    // more IO classes to follow

    /**
     * Returns an instance of this class based on what configuration says our
     * driver should be.
     *
     * @return IOContainer
     */
    public static synchronized IOContainer fromConfig() {
        if ( FROM_CONFIG_INSTANCE == null ) {

            String driver = configuration.getStringProperty(CoreConfig.CASSANDRA_DRIVER);
            LOG.info(String.format("Using driver %s", driver));

            boolean isRecordingDelayedMetrics = configuration.getBooleanProperty(CoreConfig.RECORD_DELAYED_METRICS);
            LOG.info(String.format("Recording delayed metrics: %s", isRecordingDelayedMetrics));

            boolean isDtxIngestBatchEnabled = configuration.getBooleanProperty(CoreConfig.ENABLE_DTX_INGEST_BATCH);
            LOG.info(String.format("Datastax Ingest batch enabled: %s", isDtxIngestBatchEnabled));

            FROM_CONFIG_INSTANCE = new IOContainer(DriverType.getDriverType(driver),
                                            isRecordingDelayedMetrics,
                                            isDtxIngestBatchEnabled);
        }

        return FROM_CONFIG_INSTANCE;
    }

    @VisibleForTesting
    static synchronized void resetInstance() {
        FROM_CONFIG_INSTANCE = null;
    }

    /**
     * Construct an instance of this class based on the specified
     * {@link com.rackspacecloud.blueflood.io.IOContainer.DriverType}
     *
     * @param driver
     * @param isRecordingDelayedMetrics
     */
    private IOContainer(DriverType driver,
                        boolean isRecordingDelayedMetrics,
                        boolean isDtxIngestBatchEnabled) {

        if ( driver == DriverType.DATASTAX ) {

            metadataIO = new DMetadataIO();
            shardStateIO = new DShardStateIO();
            locatorIO = new DLocatorIO();
            delayedLocatorIO = new DDelayedLocatorIO();
            basicMetricsRW = new DBasicMetricsRW((DLocatorIO)locatorIO, (DDelayedLocatorIO) delayedLocatorIO,
                    isRecordingDelayedMetrics, isDtxIngestBatchEnabled, new DefaultClockImpl());
            preAggregatedMetricsRW = new DPreaggregatedMetricsRW((DLocatorIO)locatorIO, (DDelayedLocatorIO) delayedLocatorIO,
                    isRecordingDelayedMetrics, isDtxIngestBatchEnabled, new DefaultClockImpl());

        } else {

            metadataIO = new AMetadataIO();
            shardStateIO = new AShardStateIO();
            locatorIO = new ALocatorIO();
            delayedLocatorIO = new ADelayedLocatorIO();
            basicMetricsRW = new ABasicMetricsRW(isRecordingDelayedMetrics, new DefaultClockImpl());
            preAggregatedMetricsRW = new APreaggregatedMetricsRW(isRecordingDelayedMetrics, new DefaultClockImpl());
        }
    }

    /**
     * @return a class for reading/writing ShardState
     */
    public ShardStateIO getShardStateIO() {
        return shardStateIO;
    }

    /**
     * @return a class for reading/writing Metadata
     */
    public MetadataIO getMetadataIO() {
        return metadataIO;
    }

    /**
     * @return a class for reading/writing Locators
     */
    public LocatorIO getLocatorIO() {
        return locatorIO;
    }

    /**
     * @return a class for reading/writing delayed locators
     */
    public DelayedLocatorIO getDelayedLocatorIO() {
        return delayedLocatorIO;
    }

    /**
     * @return a class for reading/writing pre-aggregated metrics
     */
    public AbstractMetricsRW getPreAggregatedMetricsRW() {
        return preAggregatedMetricsRW;
    }

    /**
     * Sets the {@link com.rackspacecloud.blueflood.io.MetricsRW}
     *
     * @param preAggregatedMetricsRW
     */
    public void setPreAggregatedMetricsRW(AbstractMetricsRW preAggregatedMetricsRW) {
        this.preAggregatedMetricsRW = preAggregatedMetricsRW;
    }

    /**
     * @return a class for reading/writing basic metrics
     */
    public AbstractMetricsRW getBasicMetricsRW() {
        return basicMetricsRW;
    }

    /**
     *
     * @param basicMetricsRW
     */
    public void setBasicMetricsRW(AbstractMetricsRW basicMetricsRW) {
        this.basicMetricsRW = basicMetricsRW;
    }

    /**
     * An enumeration of the possible driver type
     */
    public static enum DriverType {
        ASTYANAX("astyanax"),
        DATASTAX("datastax");

        private DriverType(String driver) {
            name = driver;
        }

        private String name;

        public String getName() {
            return name;
        }

        /**
         * From a string value indicating driver type, return the corresponding
         * {@link com.rackspacecloud.blueflood.io.IOContainer.DriverType} value.
         * The typical use of this method is one where a driver type string is read
         * from a configuration file.
         *
         * @param driver
         * @return
         */
        public static DriverType getDriverType(String driver) {
            // if driver is explicitly datastax, then return DATASTAX
            // anything else, return ASTYANAX
            if ( ! Strings.isNullOrEmpty(driver) && DriverType.valueOf(driver.toUpperCase()) == DATASTAX ) {
                return DATASTAX;
            }
            return ASTYANAX;
        }
    }
}


/*
 * Copyright 2014 Rackspace
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

package com.rackspacecloud.blueflood.cache;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Optional;
import com.google.common.collect.ImmutableTable;
import com.rackspacecloud.blueflood.rollup.Granularity;
import com.rackspacecloud.blueflood.service.Configuration;
import com.rackspacecloud.blueflood.service.TtlConfig;
import com.rackspacecloud.blueflood.types.RollupType;
import com.rackspacecloud.blueflood.utils.TimeValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.concurrent.TimeUnit;

public class ConfigTtlProvider implements TenantTtlProvider {
    private static final Logger log = LoggerFactory.getLogger(ConfigTtlProvider.class);

    private final ImmutableTable<Granularity, RollupType, TimeValue> ttlMapper;
    private static final ConfigTtlProvider INSTANCE = new ConfigTtlProvider();
    private static final boolean ARE_TTLS_FORCED = Configuration.getInstance().getBooleanProperty(TtlConfig.ARE_TTLS_FORCED);
    private static final TimeValue TTL_CONFIG_FOR_INGESTION = new TimeValue(Configuration.getInstance().getIntegerProperty(TtlConfig.TTL_CONFIG_CONST), TimeUnit.DAYS);

    public static ConfigTtlProvider getInstance() {
        return INSTANCE;
    }

    @VisibleForTesting
    ConfigTtlProvider() {
        final Configuration config = Configuration.getInstance();

        ImmutableTable.Builder<Granularity, RollupType, TimeValue> ttlMapBuilder =
                new ImmutableTable.Builder<Granularity, RollupType, TimeValue>();

        // Basic rollups
        put(ttlMapBuilder, config, Granularity.FULL, RollupType.BF_BASIC, TtlConfig.RAW_METRICS_TTL);
        put(ttlMapBuilder, config, Granularity.MIN_5, RollupType.BF_BASIC, TtlConfig.BASIC_ROLLUPS_MIN5);
        put(ttlMapBuilder, config, Granularity.MIN_20, RollupType.BF_BASIC, TtlConfig.BASIC_ROLLUPS_MIN20);
        put(ttlMapBuilder, config, Granularity.MIN_60, RollupType.BF_BASIC, TtlConfig.BASIC_ROLLUPS_MIN60);
        put(ttlMapBuilder, config, Granularity.MIN_240, RollupType.BF_BASIC, TtlConfig.BASIC_ROLLUPS_MIN240);
        put(ttlMapBuilder, config, Granularity.MIN_1440, RollupType.BF_BASIC, TtlConfig.BASIC_ROLLUPS_MIN1440);

        /* Pre-aggregated rollups */

        // Set rollups
        put(ttlMapBuilder, config, Granularity.FULL, RollupType.SET, TtlConfig.SET_ROLLUPS_FULL);
        put(ttlMapBuilder, config, Granularity.MIN_5, RollupType.SET, TtlConfig.SET_ROLLUPS_MIN5);
        put(ttlMapBuilder, config, Granularity.MIN_20, RollupType.SET, TtlConfig.SET_ROLLUPS_MIN20);
        put(ttlMapBuilder, config, Granularity.MIN_60, RollupType.SET, TtlConfig.SET_ROLLUPS_MIN60);
        put(ttlMapBuilder, config, Granularity.MIN_240, RollupType.SET, TtlConfig.SET_ROLLUPS_MIN240);
        put(ttlMapBuilder, config, Granularity.MIN_1440, RollupType.SET, TtlConfig.SET_ROLLUPS_MIN1440);
        // Timer rollups
        put(ttlMapBuilder, config, Granularity.FULL, RollupType.TIMER, TtlConfig.TIMER_ROLLUPS_FULL);
        put(ttlMapBuilder, config, Granularity.MIN_5, RollupType.TIMER, TtlConfig.TIMER_ROLLUPS_MIN5);
        put(ttlMapBuilder, config, Granularity.MIN_20, RollupType.TIMER, TtlConfig.TIMER_ROLLUPS_MIN20);
        put(ttlMapBuilder, config, Granularity.MIN_60, RollupType.TIMER, TtlConfig.TIMER_ROLLUPS_MIN60);
        put(ttlMapBuilder, config, Granularity.MIN_240, RollupType.TIMER, TtlConfig.TIMER_ROLLUPS_MIN240);
        put(ttlMapBuilder, config, Granularity.MIN_1440, RollupType.TIMER, TtlConfig.TIMER_ROLLUPS_MIN1440);
        // Gauge rollups
        put(ttlMapBuilder, config, Granularity.FULL, RollupType.GAUGE, TtlConfig.GAUGE_ROLLUPS_FULL);
        put(ttlMapBuilder, config, Granularity.MIN_5, RollupType.GAUGE, TtlConfig.GAUGE_ROLLUPS_MIN5);
        put(ttlMapBuilder, config, Granularity.MIN_20, RollupType.GAUGE, TtlConfig.GAUGE_ROLLUPS_MIN20);
        put(ttlMapBuilder, config, Granularity.MIN_60, RollupType.GAUGE, TtlConfig.GAUGE_ROLLUPS_MIN60);
        put(ttlMapBuilder, config, Granularity.MIN_240, RollupType.GAUGE, TtlConfig.GAUGE_ROLLUPS_MIN240);
        put(ttlMapBuilder, config, Granularity.MIN_1440, RollupType.GAUGE, TtlConfig.GAUGE_ROLLUPS_MIN1440);
        this.ttlMapper = ttlMapBuilder.build();
    }

    /**
     * Helper function to build the ttl mapping. Only insert to the mapping if the value is a valid date.
     * @param ttlMapBuilder
     * @param config
     * @param gran
     * @param rollupType
     * @param configKey
     * @return true if the insertion is successful, false otherwise.
     */
    private boolean put(
        ImmutableTable.Builder<Granularity, RollupType, TimeValue> ttlMapBuilder,
        Configuration config,
        Granularity gran,
        RollupType rollupType,
        TtlConfig configKey) {
        int value;
        try {
            value = config.getIntegerProperty(configKey);
            if (value < 0) return false;
        } catch (NumberFormatException ex) {
            log.trace(String.format("No valid TTL config set for granularity: %s, rollup type: %s",
                    gran.name(), rollupType.name()), ex);
            return false;
        }
        ttlMapBuilder.put(gran, rollupType, new TimeValue(value, TimeUnit.DAYS));
        return true;
    }

    @Override
    public Optional<TimeValue> getTTL(String tenantId, Granularity gran, RollupType rollupType) {
        final TimeValue ttl = ttlMapper.get(gran, rollupType);

        if (ttl == null) {
            log.trace("No valid TTL entry for granularity: {}, rollup type: {}" +
                      " in config. Resorting to safe TTL values.",
                       gran.name(), rollupType.name());
            return Optional.absent();
        }

        return Optional.of(ttl);
    }

    public TimeValue getConfigTTLForIngestion() {
        return TTL_CONFIG_FOR_INGESTION;
    }

    public boolean areTTLsForced() {
        return ARE_TTLS_FORCED;
    }
}


/*
 * Copyright 2016 Rackspace
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

package com.rackspacecloud.blueflood.types;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.Map;

/**
 * The sub-metrics used by {@link com.rackspacecloud.blueflood.types.BasicRollup} and
 * {@link com.rackspacecloud.blueflood.types.BluefloodGaugeRollup}.
 * Common sub-metrics are:
 *

 *
average

 *
max

 *
min

 *
sum

 *

 */
public abstract class BaseRollup implements Rollup {
    private static final Logger log = LoggerFactory.getLogger( BaseRollup.class );
    public static final int NUM_STATS = 4;

    private Average average;
    private Variance variance;
    private MinValue minValue;
    private MaxValue maxValue;
    private long count;

    public BaseRollup() {
        this.average = new Average();
        this.variance = new Variance();
        this.minValue = new MinValue();
        this.maxValue = new MaxValue();
        this.count = 0;
    }

    @Override
    public boolean equals(Object other) {
        if (!(other instanceof BaseRollup)) {
            return false;
        }

        BaseRollup otherBaseRollup = (BaseRollup)other;

        return (this.count == otherBaseRollup.getCount())
                && average.equals(otherBaseRollup.getAverage())
                && variance.equals(otherBaseRollup.getVariance())
                && minValue.equals(otherBaseRollup.getMinValue())
                && maxValue.equals(otherBaseRollup.getMaxValue());
    }

    public Average getAverage() {
        return this.average;
    }

    public Variance getVariance() {
        return this.variance;
    }

    public MinValue getMinValue() {
        return this.minValue;
    }

    public MaxValue getMaxValue() {
        return this.maxValue;
    }

    public long getCount() {
        return this.count;
    }

    public String toString() {
        return String.format("cnt:%d, avg:%s, var:%s, min:%s, max:%s", count, average, variance, minValue, maxValue);
    }

    public void setCount(long count) {
        this.count = count;
    }

    public void setMin(MinValue min) {
        this.minValue = min;
    }

    public void setMin(Number min) {
        AbstractRollupStat.set(this.minValue, min);
    }

    public void setMax(MaxValue max) {
        this.maxValue = max;
    }

    public void setMax(Number max) {
        AbstractRollupStat.set(this.maxValue, max);
    }

    public void setVariance(Variance var) {
        this.variance = var;
    }

    public void setVariance(Number var) {
        AbstractRollupStat.set(this.variance, var);
    }

    public void setAverage(Average avg) {
        this.average = avg;
    }

    public void setAverage(Number avg) {
        AbstractRollupStat.set(this.average, avg);
    }

    // merge simple numbers with this rollup.
    protected void computeFromSimpleMetrics(Points<SimpleNumber> input) throws IOException {
        if (input == null) {
            throw new IOException("Null input to create rollup from");
        }

        if (input.isEmpty()) {
            return;
        }

        Map<Long, Points.Point<SimpleNumber>> points = input.getPoints();
        for (Map.Entry<Long, Points.Point<SimpleNumber>> item : points.entrySet()) {
            this.count += 1;
            SimpleNumber numericMetric = item.getValue().getData();
            average.handleFullResMetric(numericMetric.getValue());
            variance.handleFullResMetric(numericMetric.getValue());
            minValue.handleFullResMetric(numericMetric.getValue());
            maxValue.handleFullResMetric(numericMetric.getValue());
        }
    }

    // merge rollups into this rollup.
    protected void computeFromRollupsHelper(Points<? extends IBaseRollup> input) throws IOException {
        if (input == null) {
            throw new IOException("Null input to create rollup from");
        }

        if (input.isEmpty()) {
            return;
        }

        // See this and get mind blown:
        // http://stackoverflow.com/questions/18907262/bounded-wildcard-related-compiler-error
        Map<Long, ? extends Points.Point<? extends IBaseRollup>> points = input.getPoints();

        for (Map.Entry<Long, ? extends Points.Point<? extends IBaseRollup>> item : points.entrySet()) {
            IBaseRollup rollup = item.getValue().getData();
            if (!(rollup instanceof BaseRollup)) {
                throw new IOException("Cannot create BaseRollup from type " + rollup.getClass().getName());
            }
            IBaseRollup baseRollup = (IBaseRollup) rollup;
            this.count += baseRollup.getCount();
            average.handleRollupMetric(baseRollup);
            variance.handleRollupMetric(baseRollup);
            minValue.handleRollupMetric(baseRollup);
            maxValue.handleRollupMetric(baseRollup);
        }
    }

    @Override
    public Boolean hasData() {
        return getCount() > 0;
    }

    public abstract RollupType getRollupType();
}


/*
 * Copyright (c) 2016 Rackspace.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * 	http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.rackspacecloud.blueflood.utils;

import com.rackspacecloud.blueflood.io.AbstractMetricsRW;
import com.rackspacecloud.blueflood.io.IOContainer;
import com.rackspacecloud.blueflood.types.RollupType;

/**
 * A utility class containing helper methods related to rollups
 */
public class RollupUtils {

    /**
     * For a particular rollupType, determine what is the proper class to do
     * read/write with and return that
     *
     * @param rollupType
     * @return
     */
    public static AbstractMetricsRW getMetricsRWForRollupType(RollupType rollupType) {
        if (rollupType == null || rollupType == RollupType.BF_BASIC ) {
            return IOContainer.fromConfig().getBasicMetricsRW();
        } else {
            return IOContainer.fromConfig().getPreAggregatedMetricsRW();
        }
    }
}


package com.rackspacecloud.blueflood.io.datastax;

import com.codahale.metrics.Timer;
import com.datastax.driver.core.*;
import com.datastax.driver.core.querybuilder.Insert;
import com.datastax.driver.core.querybuilder.QueryBuilder;
import com.datastax.driver.core.querybuilder.Select;
import com.rackspacecloud.blueflood.cache.TenantTtlProvider;
import com.rackspacecloud.blueflood.io.CassandraModel;
import com.rackspacecloud.blueflood.io.DelayedLocatorIO;
import com.rackspacecloud.blueflood.io.Instrumentation;
import com.rackspacecloud.blueflood.rollup.Granularity;
import com.rackspacecloud.blueflood.rollup.SlotKey;
import com.rackspacecloud.blueflood.types.Locator;
import com.rackspacecloud.blueflood.utils.Util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;

import static com.datastax.driver.core.querybuilder.QueryBuilder.*;

public class DDelayedLocatorIO implements DelayedLocatorIO {

    private static final Logger LOG = LoggerFactory.getLogger(DDelayedLocatorIO.class);

    private static final String KEY = "key";
    private static final String COLUMN1 = "column1";
    private static final String VALUE = "value";

    private final PreparedStatement getValue;
    private final PreparedStatement putValue;

    public DDelayedLocatorIO() {

        // create a generic select statement for retrieving from metrics_delayed_locator
        Select.Where select = QueryBuilder
                .select()
                .all()
                .from( CassandraModel.CF_METRICS_DELAYED_LOCATOR_NAME )
                .where( eq ( KEY, bindMarker() ));
        getValue = DatastaxIO.getSession().prepare( select );

        // create a generic insert statement for inserting into metrics_delayed_locator
        Insert insert = QueryBuilder.insertInto( CassandraModel.CF_METRICS_DELAYED_LOCATOR_NAME)
                .using(ttl(TenantTtlProvider.DELAYED_LOCATOR_TTL))
                .value(KEY, bindMarker())
                .value(COLUMN1, bindMarker())
                .value(VALUE, bindMarker());
        putValue = DatastaxIO.getSession()
                .prepare(insert)
                .setConsistencyLevel( ConsistencyLevel.LOCAL_ONE );
    }

    @Override
    public void insertLocator(Granularity g, int slot, Locator locator) throws IOException {
        Session session = DatastaxIO.getSession();

        Timer.Context timer = Instrumentation.getWriteTimerContext(CassandraModel.CF_METRICS_DELAYED_LOCATOR_NAME);
        try {
            // bound values and execute
            BoundStatement bs = getBoundStatementForLocator(g, slot, locator);
            session.execute(bs);
        } finally {
            timer.stop();
        }
    }

    // package protect so other classes in this package can call it
    BoundStatement getBoundStatementForLocator(Granularity granularity, int slot, Locator locator) {
        int shard = Util.getShard(locator.toString());
        return putValue.bind(SlotKey.of(granularity, slot, shard).toString(), locator.toString(), "");
    }

    @Override
    public Collection<Locator> getLocators(SlotKey slotKey) throws IOException {

        Timer.Context ctx = Instrumentation.getReadTimerContext(CassandraModel.CF_METRICS_DELAYED_LOCATOR_NAME);
        Session session = DatastaxIO.getSession();

        Collection<Locator> locators = new ArrayList<Locator>();

        try {
            // bind value
            BoundStatement bs = getValue.bind(slotKey.toString());
            List<Row> results = session.execute(bs).all();
            for ( Row row : results ) {
                locators.add(Locator.createLocatorFromDbKey(row.getString(COLUMN1)));
            }

            // return results
            if (locators.size() == 0) {
                Instrumentation.markNotFound(CassandraModel.CF_METRICS_DELAYED_LOCATOR_NAME);
                return Collections.emptySet();
            }
            else {
                return locators;
            }
        } finally {
            ctx.stop();
        }
    }
}


/*
 * Copyright 2013 Rackspace
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

package com.rackspacecloud.blueflood.tools.jmx;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;

public class OutputFormatter implements Comparable<OutputFormatter> {
    private static final String GAP = "  ";

    private final String host;
    private final String[] results;

    public OutputFormatter(HostAndPort hostInfo, String[] results) {
        this.host = hostInfo.getHost();
        this.results = results;
    }

    public int compareTo(OutputFormatter o) {
        return host.compareTo(o.host);
    }

    // compute the maximum width for each field across a collection of formatters.
    public static int [] computeMaximums(String[] headers, OutputFormatter... outputs) {
        int[] max = new int[headers.length];
        for (int i = 0; i < headers.length; i++)
            max[i] = headers[i].length();
        for (OutputFormatter output : outputs) {
            max[0] = Math.max(output.host.length(), max[0]);
            for (int i = 1; i < headers.length; i++)
                max[i] = Math.max(output.results[i-1].length(), max[i]);
        }
        return max;
    }

    // formats a header row after maximums have been established.
    public static String formatHeader(int[] maximums, String[] headers) {
        StringBuilder sb = new StringBuilder();
        for (int i = 0; i < headers.length; i++)
            sb = sb.append(formatIn(headers[i], maximums[i], false)).append(GAP);
        return sb.toString();
    }

    // formats results and sets formattedStrings.
    public static String[] format(int[] maximums, OutputFormatter... outputs) {
        String[] formattedStrings = new String[outputs.length];
        int pos = 0;
        for (OutputFormatter output : outputs) {
            StringBuilder sb = new StringBuilder();
            sb = sb.append(formatIn(output.host, maximums[0], false));
            for (int i = 0; i < output.results.length; i++)
                sb = sb.append(GAP).append(formatIn(output.results[i], maximums[i+1], true));
            formattedStrings[pos++] = sb.toString();
        }
        return formattedStrings;
    }

    private static String formatIn(String s, int spaces, boolean rightAlign) {
        while (s.length() < spaces) {
            if (rightAlign)
                s = " " + s;
            else
                s += " ";
        }
        return s;
    }

    public static Collection<OutputFormatter> sort(Collection<OutputFormatter> src) {
        List<OutputFormatter> sortedList = new ArrayList<OutputFormatter>(src);
        Collections.sort(sortedList);
        return sortedList;
    }

}