/*
 * Copyright 2013 Rackspace
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

package com.rackspacecloud.blueflood.rollup;

import com.rackspacecloud.blueflood.cache.CombinedTtlProvider;
import com.rackspacecloud.blueflood.exceptions.GranularityException;
import com.rackspacecloud.blueflood.service.Configuration;
import com.rackspacecloud.blueflood.service.CoreConfig;
import com.rackspacecloud.blueflood.types.Range;
import com.rackspacecloud.blueflood.utils.Clock;
import com.rackspacecloud.blueflood.utils.DefaultClockImpl;

import java.util.Calendar;

/**
 * 1440m    [ not enough space to show the relationship, but there would be 6 units of the 240m ranges in 1 1440m range.
 * 240m     [                                               |                                               |        ...
 * 60m      [           |           |           |           |           |           |           |           |        ...
 * 20m      [   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |...
 * 5m       [||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||...
 * full     [ granularity to the second, but ranges are partitioned the same as in 5m.                               ...
 */
public final class Granularity {
    private static final int GET_BY_POINTS_ASSUME_INTERVAL = Configuration.getInstance().getIntegerProperty(CoreConfig.GET_BY_POINTS_ASSUME_INTERVAL);
    private static final String GET_BY_POINTS_SELECTION_ALGORITHM = Configuration.getInstance().getStringProperty(CoreConfig.GET_BY_POINTS_GRANULARITY_SELECTION);
    private static final Clock DEFAULT_TTL_COMPARISON_SOURCE = new DefaultClockImpl();
    private static int INDEX_COUNTER = 0;
    private static final int BASE_SLOTS_PER_GRANULARITY = 4032; // needs to be a multiple of the GCF of 4, 12, 48, 288.
    public static final int MILLISECONDS_IN_SLOT = 300000;
    private static final int SECS_PER_DAY = 86400;

    public static final Granularity FULL = new Granularity("metrics_full", 300000, BASE_SLOTS_PER_GRANULARITY, "full");
    public static final Granularity MIN_5 = new Granularity("metrics_5m", 300000, BASE_SLOTS_PER_GRANULARITY, "5m");
    public static final Granularity MIN_20 = new Granularity("metrics_20m", 1200000, (BASE_SLOTS_PER_GRANULARITY / 4), "20m");
    public static final Granularity MIN_60 = new Granularity("metrics_60m", 3600000, (BASE_SLOTS_PER_GRANULARITY / 12), "60m");
    public static final Granularity MIN_240 = new Granularity("metrics_240m", 14400000, (BASE_SLOTS_PER_GRANULARITY / 48), "240m");
    public static final Granularity MIN_1440 = new Granularity("metrics_1440m", 86400000, (BASE_SLOTS_PER_GRANULARITY / 288), "1440m");

    private static final Granularity[] granularities = new Granularity[] { FULL, MIN_5, MIN_20, MIN_60, MIN_240, MIN_1440 }; // order is important.

    public static final Granularity LAST = MIN_1440;

    private static final Granularity[] rollupGranularities = new Granularity[] { MIN_5, MIN_20, MIN_60, MIN_240, MIN_1440 }; // order is important.

    public static final int MAX_NUM_SLOTS = FULL.numSlots() + MIN_5.numSlots() + MIN_20.numSlots() + MIN_60.numSlots() + MIN_240.numSlots() + MIN_1440.numSlots();

    private static CombinedTtlProvider TTL_PROVIDER;

    // simple counter for all instances, since there will be very few.
    private final int index;

    // name of column family where rollups are kept.
    private final String cf;

    // like cf, but shorter.
    private final String shortName;

    // number of milliseconds in one slot of this rollup.
    private final int milliseconds;

    // number of slots for this granularity.  This number decreases as granularity is more coarse.  Also, the number of
    // minutes indicated by a single slot increases as the number of slots goes down.
    private final int numSlots;

    private Granularity(String cf, int milliseconds, int numSlots, String shortName) {
        index = INDEX_COUNTER++;
        this.cf = cf;
        this.milliseconds = milliseconds;
        this.numSlots = numSlots;
        this.shortName = shortName;
    }

    // name->column_family
    public String name() { return cf; }

    // name->tenant ttl key.
    public String shortName() { return shortName; }

    /** @return the number of seconds in one slot range. */
    public int milliseconds() { return milliseconds; }

    public int numSlots() { return numSlots; }

    // returns the next coarser granularity.
    // FULL -> 5m -> 20m -> 60m -> 240m -> 1440m -> explosion.
    public Granularity coarser() throws GranularityException {
        if (this == LAST) throw new GranularityException("Nothing coarser than " + name());
        return granularities[index + 1];
    }

    // opposite of coarser().
    public Granularity finer() throws GranularityException {
        if (this == FULL) throw new GranularityException("Nothing finer than " + name());
        return granularities[index - 1];
    }

    public boolean isCoarser(Granularity other) {
        return indexOf(this) > indexOf(other);
    }

    private int indexOf(Granularity gran) {
        for (int i = 0; i < granularities.length; i++) {
            if (gran == granularities[i]) {
                return i;
            }
        }

        throw new RuntimeException("Granularity " + gran.toString() + " not present in granularities list.");
    }

    /**
     * Gets the floor multiple of number of milliseconds in this granularity
     * @param millis
     * @return
     */
    public long snapMillis(long millis) {
        if (this == FULL) return millis;
        else return (millis / milliseconds) * milliseconds;
    }

    /**
     * At full granularity, a slot is 300 continuous seconds.  The duration of a single slot goes up (way up) as
     * granularity is lost.  At the same time the number of slots decreases.
     * @param millis
     * @return
     */
    public int slot(long millis) {
        // the actual slot is
        int fullSlot = millisToSlot(millis);
        return (numSlots * fullSlot) / BASE_SLOTS_PER_GRANULARITY;
    }

    /**
     * returns the slot for the current granularity based on a supplied slot from the granularity one resolution finer
     * i.e, slot 144 for a 5m is == slot 36 of 20m (because 144 / (20m/5m)), slot 12 at 60m, slot 3 at 240m, etc
     * @param finerSlot
     * @return
     */
    public int slotFromFinerSlot(int finerSlot) throws GranularityException {
        return (finerSlot * numSlots()) / this.finer().numSlots();
    }

    /**
     * We need to derive ranges (actual times) from slots (which are fixed integers that wrap) when we discover a late
     * slot. These ranges can be derived from a reference point (which is usually something like now).
     * @param slot
     * @param referenceMillis
     * @return
     */
    public Range deriveRange(int slot, long referenceMillis) {
        // referenceMillis refers to the current time in reference to the range we want to generate from the supplied
        // slot. This implies that the range we wish to return is before slot(reference).  allow for slot wrapping.
        referenceMillis = snapMillis(referenceMillis);
        int refSlot = slot(referenceMillis);
        int slotDiff = slot > refSlot ? (numSlots() - slot + refSlot) : (refSlot - slot);
        long rangeStart = referenceMillis - slotDiff * milliseconds();
        return new Range(rangeStart, rangeStart + milliseconds() - 1);
    }

    /**
     * Return granularity that maps most closely to requested number of points,
     * using the algorithm specified in the
     * {@code GET_BY_POINTS_SELECTION_ALGORITHM} config value. See
     * {@link #granularityFromPointsInInterval(String, long, long, int, String, long)}.
     *
     * @param from beginning of interval (millis)
     * @param to end of interval (millis)
     * @param points count of desired data points
     * @return
     */
    public static Granularity granularityFromPointsInInterval(String tenantid, long from, long to, int points) {
        return granularityFromPointsInInterval(tenantid, from, to, points, GET_BY_POINTS_SELECTION_ALGORITHM, GET_BY_POINTS_ASSUME_INTERVAL, DEFAULT_TTL_COMPARISON_SOURCE);
    }
    /**
     * {@code ttlComparisonClock} defaults to {@link #DEFAULT_TTL_COMPARISON_SOURCE}.
     *
     * @see #granularityFromPointsInInterval(String, long, long, int, String, long, Clock)
     */
    public static Granularity granularityFromPointsInInterval(String tenantid, long from, long to, int points, String algorithm, long assumedIntervalMillis) {
        return granularityFromPointsInInterval(tenantid, from, to, points, algorithm, assumedIntervalMillis, DEFAULT_TTL_COMPARISON_SOURCE);
    }
    /**
     * Return granularity that maps most closely to requested number of points based on
     * provided selection algorithm
     *
     * @param from beginning of interval (millis)
     * @param to end of interval (millis)
     * @param points count of desired data points
     * @param algorithm the algorithm to use. Valid values are
     *                  {@code "GEOMETRIC"}, {@code "LINEAR"}, and
     *                  {@code "LESSTHANEQUAL"}. Any other value is treated as
     *                  {@code "GEOMETRIC"}.
     * @param assumedIntervalMillis FULL resolution is tricky because we don't
     *                              know the period of check in question.
     *                              Assume the minimum period and go from there.
     *                              NOTE: This value and description was from a
     *                              previous version of the software, and the
     *                              original author is no longer available to
     *                              clarify. The exact purpose of this
     *                              parameter is not yet fully understood.
     * @param ttlComparisonClock The GEOMETRIC algorithm checks the specified
     *                           interval against a TTL. Normally, the TTL is
     *                           measured relative to the current point in
     *                           time. For testing purposes, an alternate time
     *                           source can be provided, to measure the TTL
     *                           relative to a different point in time.
     * @return
     */
    public static Granularity granularityFromPointsInInterval(String tenantid, long from, long to, int points, String algorithm, long assumedIntervalMillis, Clock ttlComparisonClock) {
        if (from >= to) {
            throw new RuntimeException("Invalid interval specified for fromPointsInInterval");
        }

        double requestedDuration = to - from;

        if (algorithm.startsWith("GEOMETRIC"))
            return granularityFromPointsGeometric(tenantid, from, to, requestedDuration, points, assumedIntervalMillis, ttlComparisonClock);
        else if (algorithm.startsWith("LINEAR"))
            return granularityFromPointsLinear(requestedDuration, points, assumedIntervalMillis);
        else if (algorithm.startsWith("LESSTHANEQUAL"))
            return granularityFromPointsLessThanEqual(requestedDuration, points, assumedIntervalMillis);

        return granularityFromPointsGeometric(tenantid, from, to, requestedDuration, points, assumedIntervalMillis, ttlComparisonClock);
    }

    /**
     * Find the granularity in the interval that will yield a number of data points that are
     * closest to the requested points but <= requested points.
     *
     * @param requestedDuration
     * @param points
     * @return
     */
    private static Granularity granularityFromPointsLessThanEqual(double requestedDuration, int points, long assumedIntervalMillis) {
        Granularity gran = granularityFromPointsLinear(requestedDuration, points, assumedIntervalMillis);

        if (requestedDuration / gran.milliseconds() > points) {
            try {
                gran = gran.coarser();
            } catch (GranularityException e) { /* do nothing, already at 1440m */ }
        }

        return gran;
    }

    /**
     * Find the granularity in the interval that will yield a number of data points that are close to $points
     * in terms of linear distance.
     *
     * @param requestedDuration
     * @param points
     * @return
     */
    private static Granularity granularityFromPointsLinear(double requestedDuration, int points, long assumedIntervalMillis) {
        int closest = Integer.MAX_VALUE;
        int diff = 0;
        Granularity gran = null;

        for (Granularity g : Granularity.granularities()) {
            if (g == Granularity.FULL)
                diff = (int)Math.abs(points - (requestedDuration / assumedIntervalMillis));
            else
                diff = (int)Math.abs(points - (requestedDuration /g.milliseconds()));
            if (diff < closest) {
                closest = diff;
                gran = g;
            } else {
                break;
            }
        }

        return gran;
    }

    /**
     *
     * Look for the granularity that would generate the density of data points closest to the value desired. For
     * example, if 500 points were requested, it is better to return 1000 points (2x more than were requested)
     * than it is to return 100 points (5x less than were requested). Our objective is to generate reasonable
     * looking graphs.
     *
     * @param requestedDuration (milliseconds)
     */
    private static Granularity granularityFromPointsGeometric(String tenantid, long from, long to, double requestedDuration, int requestedPoints, long assumedIntervalMillis, Clock ttlComparisonSource) {
        double minimumPositivePointRatio = Double.MAX_VALUE;
        Granularity gran = null;
        if (TTL_PROVIDER == null) {
            TTL_PROVIDER = CombinedTtlProvider.getInstance();
        }

        for (Granularity g : Granularity.granularities()) {
            long ttl = TTL_PROVIDER.getFinalTTL(tenantid, g);

            if (from < ttlComparisonSource.now().getMillis() - ttl) {
                continue;
            }

            // FULL resolution is tricky because we don't know the period of check in question. Assume the minimum
            // period and go from there.
            long period = (g == Granularity.FULL) ? assumedIntervalMillis : g.milliseconds();
            double providablePoints = requestedDuration / period;
            double positiveRatio;

            // Generate a ratio >= 1 of either (points requested / points provided by this granularity) or the inverse.
            // Think of it as an "absolute ratio". Our goal is to minimize this ratio.
            if (providablePoints > requestedPoints) {
                positiveRatio = providablePoints / requestedPoints;
            } else {
                positiveRatio = requestedPoints / providablePoints;
            }

            if (positiveRatio < minimumPositivePointRatio) {
                minimumPositivePointRatio = positiveRatio;
                gran = g;
            } else {
                break;
            }
        }

        if (gran == null) {
            gran = Granularity.LAST;
        }

        return gran;
    }

    /** calculate the full/5m slot based on 4032 slots of 300000 milliseconds per slot. */
    static int millisToSlot(long millis) {
        return (int)((millis % (BASE_SLOTS_PER_GRANULARITY * MILLISECONDS_IN_SLOT)) / MILLISECONDS_IN_SLOT);
    }

    @Override
    public int hashCode() {
        return name().hashCode();
    }

    @Override
    public boolean equals(Object obj) {
        if (!(obj instanceof Granularity)) return false;
        else return obj == this;
    }

    public static Granularity[] granularities() { return granularities; }

    public static Granularity[] rollupGranularities() { return rollupGranularities; }

    public static Granularity fromString(String s) {
        for (Granularity g : granularities)
            if (g.name().equals(s) || g.shortName().equals(s))
                return g;
        return null;
    }

    public static Granularity getRollupGranularity(String s) {
        for (Granularity g : rollupGranularities)
            if (g.name().equals(s) || g.shortName().equals(s))
                return g;
        return null;
    }

    @Override
    public String toString() {
        return name();
    }
}


/*
 * Copyright 2013 Rackspace
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

package com.rackspacecloud.blueflood.service;

import com.codahale.metrics.*;
import com.codahale.metrics.Timer;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.rackspacecloud.blueflood.concurrent.InstrumentedThreadPoolExecutor;
import com.rackspacecloud.blueflood.concurrent.ThreadPoolBuilder;
import com.rackspacecloud.blueflood.rollup.Granularity;
import com.rackspacecloud.blueflood.rollup.SlotKey;
import com.rackspacecloud.blueflood.tools.jmx.JmxBooleanGauge;
import com.rackspacecloud.blueflood.utils.Metrics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.management.MBeanServer;
import javax.management.ObjectName;
import java.lang.management.ManagementFactory;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicLong;

public class RollupService implements Runnable, RollupServiceMBean {

    private static final Logger log = LoggerFactory.getLogger(RollupService.class);

    private final Timer polltimer = Metrics.timer(RollupService.class, "Poll Timer");
    private final Meter rejectedSlotChecks = Metrics.meter(RollupService.class, "Rejected Slot Checks");

    private final long rollupDelayMillis;
    private final long rollupDelayForMetricsWithShortDelay;
    private final long rollupWaitForMetricsWithLongDelay;

    private transient Thread thread;
    private final ScheduleContext context;
    private final ShardStateManager shardStateManager;
    private final ThreadPoolExecutor locatorFetchExecutors;
    private final ThreadPoolExecutor rollupReadExecutors;
    private final ThreadPoolExecutor rollupWriteExecutors;

    private long pollerPeriod;
    private final long configRefreshInterval;

    private long lastSlotCheckFinishedAt = 0L;

    private boolean active = true;
    private boolean keepingServerTime = true;

    private Gauge activeGauge;
    private Gauge inflightRollupGauge;
    private Gauge pollerPeriodGauge;
    private Gauge serverTimeGauge;
    private Gauge rollupConcurrencyGauge;
    private Gauge scheduledSlotCheckGauge;
    private Gauge secondsSinceLastSlotCheckGauge;
    private Gauge queuedRollupGauge;
    private Gauge slotCheckConcurrencyGauge;
    private Gauge recentlyScheduledShardGauge;
    private Gauge managedShardGauge;

    protected static final AtomicLong lastRollupTime = new AtomicLong(System.currentTimeMillis());
    private static final Gauge<Long> timeSinceLastRollupGauge;

    static {
        timeSinceLastRollupGauge = new Gauge<Long>() {
            @Override
            public Long getValue() {
                return System.currentTimeMillis() - lastRollupTime.get();
            }
        };
        Metrics.getRegistry().register(MetricRegistry.name(RollupService.class, "Milliseconds Since Last Rollup"), timeSinceLastRollupGauge);
    }

    public RollupService(ScheduleContext context) {

        pollerPeriod = Configuration.getInstance().getIntegerProperty(CoreConfig.SCHEDULE_POLL_PERIOD);
        configRefreshInterval = Configuration.getInstance().getIntegerProperty(CoreConfig.CONFIG_REFRESH_PERIOD);

        this.context = context;
        this.shardStateManager = context.getShardStateManager();

        // NOTE: higher locatorFetchConcurrency means that the queue used in rollupReadExecutors needs to be correspondingly
        // higher.
        Configuration config = Configuration.getInstance();
        rollupDelayMillis = config.getLongProperty(CoreConfig.ROLLUP_DELAY_MILLIS);
        rollupDelayForMetricsWithShortDelay = config.getLongProperty(CoreConfig.SHORT_DELAY_METRICS_ROLLUP_DELAY_MILLIS);
        rollupWaitForMetricsWithLongDelay = config.getLongProperty(CoreConfig.LONG_DELAY_METRICS_ROLLUP_WAIT_MILLIS);

        log.info(String.format("Delay configs -> ROLLUP_DELAY_MILLIS: [%d] SHORT_DELAY_METRICS_ROLLUP_DELAY_MILLIS: [%d] " +
                "LONG_DELAY_METRICS_ROLLUP_WAIT_MILLIS: [%d]", rollupDelayMillis, rollupDelayForMetricsWithShortDelay,
                rollupWaitForMetricsWithLongDelay));

        ThreadFactory locatorFetchThreadFactory = new ThreadFactoryBuilder().setNameFormat("locator-fetcher-%d").build();
        final int locatorFetchConcurrency = config.getIntegerProperty(CoreConfig.MAX_LOCATOR_FETCH_THREADS);
        ThreadPoolExecutor _locatorFetchExecutors = new ThreadPoolExecutor(
                locatorFetchConcurrency, locatorFetchConcurrency,
                30, TimeUnit.SECONDS,
                new ArrayBlockingQueue<Runnable>(locatorFetchConcurrency * 5),
                locatorFetchThreadFactory,
                new RejectedExecutionHandler() {
                    public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
                        // in this case, we want to throw a RejectedExecutionException so that the slot can be removed
                        // from the running queue.
                        throw new RejectedExecutionException("Threadpool is saturated. unable to service this slot.");
                    }
                }
        ) {
            @Override
            protected void afterExecute(Runnable r, Throwable t) {
                lastSlotCheckFinishedAt = RollupService.this.context.getCurrentTimeMillis();
                super.afterExecute(r, t);
            }
        };

        // unbounded work queue.
        ThreadFactory rollupReaderThreadFactory = new ThreadFactoryBuilder().setNameFormat("rollup-reader-%d").build();
        final BlockingQueue<Runnable> rollupReadQueue = new LinkedBlockingQueue<Runnable>();
        ThreadPoolExecutor _rollupReadExecutors = new ThreadPoolExecutor(
                // "RollupReadsThreadpool",
                config.getIntegerProperty(CoreConfig.MAX_ROLLUP_READ_THREADS),
                config.getIntegerProperty(CoreConfig.MAX_ROLLUP_READ_THREADS),
                30, TimeUnit.SECONDS,
                rollupReadQueue,
                rollupReaderThreadFactory,
                new ThreadPoolExecutor.AbortPolicy()
        );

        ThreadFactory rollupWriterThreadFactory = new ThreadFactoryBuilder().setNameFormat("rollup-writer-%d").build();
        final BlockingQueue<Runnable> rollupWriteQueue = new LinkedBlockingQueue<Runnable>();
        ThreadPoolExecutor _rollupWriteExecutors = new ThreadPoolExecutor(
                // "RollupWritesThreadpool",
                config.getIntegerProperty(CoreConfig.MAX_ROLLUP_WRITE_THREADS),
                config.getIntegerProperty(CoreConfig.MAX_ROLLUP_WRITE_THREADS),
                30, TimeUnit.SECONDS,
                rollupWriteQueue,
                rollupWriterThreadFactory,
                new ThreadPoolExecutor.AbortPolicy()
        );

        initializeGauges();

        locatorFetchExecutors = _locatorFetchExecutors;
        InstrumentedThreadPoolExecutor.instrument(locatorFetchExecutors, "LocatorFetchThreadPool");

        rollupReadExecutors = _rollupReadExecutors;
        InstrumentedThreadPoolExecutor.instrument(rollupReadExecutors, "RollupReadsThreadpool");

        rollupWriteExecutors = _rollupWriteExecutors;
        InstrumentedThreadPoolExecutor.instrument(rollupWriteExecutors, "RollupWritesThreadpool");
    }
    @VisibleForTesting
    public RollupService(ScheduleContext context,
                         ShardStateManager shardStateManager,
                         ThreadPoolExecutor locatorFetchExecutors,
                         ThreadPoolExecutor rollupReadExecutors,
                         ThreadPoolExecutor rollupWriteExecutors,
                         long rollupDelayMillis,
                         long rollupDelayForMetricsWithShortDelay,
                         long rollupWaitForMetricsWithLongDelay,
                         long pollerPeriod,
                         long configRefreshInterval) {

        this.context = context;
        this.shardStateManager = shardStateManager;
        this.locatorFetchExecutors = locatorFetchExecutors;
        this.rollupReadExecutors = rollupReadExecutors;
        this.rollupWriteExecutors = rollupWriteExecutors;
        this.rollupDelayMillis = rollupDelayMillis;
        this.rollupDelayForMetricsWithShortDelay = rollupDelayForMetricsWithShortDelay;
        this.rollupWaitForMetricsWithLongDelay = rollupWaitForMetricsWithLongDelay;
        this.pollerPeriod = pollerPeriod;
        this.configRefreshInterval = configRefreshInterval;

        initializeGauges();
    }

    public void initializeGauges() {
        try {
            final MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
            final String name = String.format("com.rackspacecloud.blueflood.service:type=%s", getClass().getSimpleName());
            final ObjectName nameObj = new ObjectName(name);
            mbs.registerMBean(this, nameObj);

            MetricRegistry reg = Metrics.getRegistry();
            activeGauge = reg.register(MetricRegistry.name(RollupService.class, "Active"),
                    new JmxBooleanGauge(nameObj, "Active"));
            inflightRollupGauge = reg.register(MetricRegistry.name(RollupService.class, "In Flight Rollup Count"),
                    new JmxAttributeGauge(nameObj, "InFlightRollupCount"));

            pollerPeriodGauge = reg.register(MetricRegistry.name(RollupService.class, "Poller Period"),
                    new JmxAttributeGauge(nameObj, "PollerPeriod"));
            queuedRollupGauge = reg.register(MetricRegistry.name(RollupService.class, "Queued Rollup Count"),
                    new JmxAttributeGauge(nameObj, "QueuedRollupCount"));
            rollupConcurrencyGauge = reg.register(MetricRegistry.name(RollupService.class, "Rollup Concurrency"),
                    new JmxAttributeGauge(nameObj, "RollupConcurrency"));
            scheduledSlotCheckGauge = reg.register(MetricRegistry.name(RollupService.class, "Scheduled Slot Check"),
                    new JmxAttributeGauge(nameObj, "ScheduledSlotCheckCount"));
            secondsSinceLastSlotCheckGauge = reg.register(MetricRegistry.name(RollupService.class, "Seconds Since Last Slot Check"),
                    new JmxAttributeGauge(nameObj, "SecondsSinceLastSlotCheck"));
            serverTimeGauge = reg.register(MetricRegistry.name(RollupService.class, "Server Time"),
                    new JmxAttributeGauge(nameObj, "ServerTime"));
            slotCheckConcurrencyGauge = reg.register(MetricRegistry.name(RollupService.class, "Slot Check Concurrency"),
                    new JmxAttributeGauge(nameObj, "SlotCheckConcurrency"));

            recentlyScheduledShardGauge = reg.register(MetricRegistry.name(RollupService.class, "Recently Scheduled Shards"),
                    new Gauge<Integer>() {
                        @Override
                        public Integer getValue() {
                            return getRecentlyScheduledShards().size();
                        }
                    });

            managedShardGauge = reg.register(MetricRegistry.name(RollupService.class, "Managed Shards"),
                    new Gauge<Integer>() {
                        @Override
                        public Integer getValue() {
                            return getManagedShards().size();
                        }
                    });

        } catch (Exception exc) {
            log.error("Unable to register mbean for " + getClass().getSimpleName(), exc);
        }
    }

    public void forcePoll() {
        thread.interrupt();
    }

    final void poll() {
        Timer.Context timer = polltimer.time();
        // schedule for rollup anything that has not been updated in ROLLUP_DELAY_SECS
        context.scheduleEligibleSlots(rollupDelayMillis, rollupDelayForMetricsWithShortDelay, rollupWaitForMetricsWithLongDelay);
        timer.stop();
    }

    public void run() {
        thread = Thread.currentThread();

        while (shouldKeepRunning()) {
            long startRun = System.currentTimeMillis();

            poll();

            // if there are schedules slots, run what we can.
            boolean rejected = false;
            while (context.hasScheduled() && !rejected && active) {
                final SlotKey slotKey = context.getNextScheduled();
                if (slotKey == null) { continue; }
                try {
                    UpdateStamp stamp = shardStateManager.getUpdateStamp(slotKey);
                    long currentTimeMillis = context.getCurrentTimeMillis();
                    boolean isReroll = context.isReroll(slotKey);
                    log.info("Scheduling slotKey {} @ {} last collection time: {} last rollup time: {} isReroll: {}",
                            new Object[]{slotKey, currentTimeMillis, stamp.getTimestamp(),
                                    stamp.getLastRollupTimestamp(), isReroll});

                    locatorFetchExecutors.execute(new LocatorFetchRunnable(context, slotKey, rollupReadExecutors,
                            rollupWriteExecutors));

                } catch (RejectedExecutionException ex) {
                    // puts it back at the top of the list of scheduled slots.  When this happens it means that
                    // there is too much rollup work to do. if the CPU cores are not tapped out, it means you don't
                    // have enough threads allocated to processing rollups or slot checks.
                    rejectedSlotChecks.mark();
                    context.pushBackToScheduled(slotKey, true);
                    rejected = true;
                }
            }
            long endRun = System.currentTimeMillis();
            if (endRun - startRun > pollerPeriod)
                log.error("It took longer than {} to poll for rollups.", pollerPeriod);
            else
                try {
                    thread.sleep(Math.max(0, pollerPeriod - endRun + startRun));
                } catch (Exception ex) {
                    log.debug("RollupService poller woke up");
                }
        }
    }

    private boolean keepRunning = true;
    @VisibleForTesting
    boolean shouldKeepRunning() {
        return keepRunning;
    }
    @VisibleForTesting
    void setShouldKeepRunning(boolean value) {
        keepRunning = value;
    }

    //
    // JMX exposure
    //

    // set the server time in millis.
    public synchronized void setServerTime(long millis) {
        log.info("Manually setting server time to {}  {}", millis, new java.util.Date(millis));
        context.setCurrentTimeMillis(millis);
    }

    // get the server time in seconds.
    public synchronized long getServerTime() { return context.getCurrentTimeMillis(); }

    public synchronized void setKeepingServerTime(boolean b) { keepingServerTime = b; }

    public synchronized boolean getKeepingServerTime() { return keepingServerTime; }

    public synchronized void setPollerPeriod(long l) {
        // todo: alter the design so that you don't have to keep a thread reference around. one way to do this is to
        // override the function in the caller (where the thread is accessible).
        pollerPeriod = l;
        if (thread != null)
            thread.interrupt();
    }

    public synchronized long getPollerPeriod() { return pollerPeriod; }

    public synchronized int getScheduledSlotCheckCount() { return context.getScheduledCount(); }

    public synchronized int getSecondsSinceLastSlotCheck() {
        return (int)((context.getCurrentTimeMillis() - lastSlotCheckFinishedAt) / 1000);
    }

    public synchronized int getSlotCheckConcurrency() {
        return locatorFetchExecutors.getMaximumPoolSize();
    }

    public synchronized void setSlotCheckConcurrency(int i) {
        locatorFetchExecutors.setCorePoolSize(i);
        locatorFetchExecutors.setMaximumPoolSize(i);
    }

    public synchronized int getRollupConcurrency() {
        return rollupReadExecutors.getMaximumPoolSize();
    }

    public synchronized void setRollupConcurrency(int i) {
        rollupReadExecutors.setCorePoolSize(i);
        rollupReadExecutors.setMaximumPoolSize(i);
    }

    public synchronized int getQueuedRollupCount() { return rollupReadExecutors.getQueue().size(); }
    public synchronized int getInFlightRollupCount() { return rollupReadExecutors.getActiveCount(); }

    public synchronized boolean getActive() { return active; }

    public synchronized void setActive(boolean b) {
        active = b;
        if (active && thread != null)
            thread.interrupt();
    }

    /**
     * Add a shard to be managed (via JMX)
     *
     * @param shard shard to be added
     */
    public void addShard(Integer shard) {
        if (!shardStateManager.getManagedShards().contains(shard))
            context.addShard(shard);
    }

    /**
     * Remove a shard from being managed (via JMX)
     *
     * @param shard shard to be removed
     */
    public void removeShard(Integer shard) {
        if (shardStateManager.getManagedShards().contains(shard))
            context.removeShard(shard);
    }

    /**
     * Get list of managed shards (via JMX)
     *
     * @return list of managed shards (unmodifiable collection)
     */
    public Collection<Integer> getManagedShards() {
        return new TreeSet<Integer>(shardStateManager.getManagedShards());
    }

    public synchronized Collection<Integer> getRecentlyScheduledShards() {
        // note: already sorted when it comes from the context.
        return context.getRecentlyScheduledShards();
    }

    public synchronized Collection<String> getOldestUnrolledSlotPerGranularity(int shard) {
        final Set<String> results = new HashSet<String>();

        for (Granularity g : Granularity.rollupGranularities()) {
            final Map<Integer, UpdateStamp> stateTimestamps = context.getSlotStamps(g, shard);
            if (stateTimestamps == null || stateTimestamps.isEmpty()) {
                continue;
            }

            // Iterate through the map of slot to UpdateStamp and find the oldest one
            SlotState minSlot = new SlotState().withTimestamp(System.currentTimeMillis());
            boolean add = false;
            for (Map.Entry<Integer, UpdateStamp> entry : stateTimestamps.entrySet()) {
                final UpdateStamp stamp = entry.getValue();
                if (stamp.getState() != UpdateStamp.State.Rolled && stamp.getTimestamp() < minSlot.getTimestamp()) {
                    minSlot = new SlotState(g, entry.getKey(), stamp.getState()).withTimestamp(stamp.getTimestamp());
                    add = true;
                }
            }

            if (add) {
                results.add(minSlot.toString());
            }
        }

        return results;
    }
}


package com.rackspacecloud.blueflood.inputs.processors;

import com.codahale.metrics.Meter;
import com.google.common.util.concurrent.ListenableFuture;
import com.rackspacecloud.blueflood.cache.LocatorCache;
import com.rackspacecloud.blueflood.cache.TokenCache;
import com.rackspacecloud.blueflood.concurrent.FunctionWithThreadPool;
import com.rackspacecloud.blueflood.io.TokenDiscoveryIO;
import com.rackspacecloud.blueflood.service.Configuration;
import com.rackspacecloud.blueflood.service.CoreConfig;
import com.rackspacecloud.blueflood.types.IMetric;
import com.rackspacecloud.blueflood.types.Locator;
import com.rackspacecloud.blueflood.types.Token;
import com.rackspacecloud.blueflood.utils.Metrics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.*;
import java.util.concurrent.ThreadPoolExecutor;

import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toSet;

/**
 * A separate discovery writer for tokens. This class is modelled after {@link DiscoveryWriter}
 */
public class TokenDiscoveryWriter extends FunctionWithThreadPool<List<List<IMetric>>, Void> {

    private final List<TokenDiscoveryIO> tokenDiscoveryIOs = new ArrayList<>();
    private final Map<Class<? extends TokenDiscoveryIO>, Meter> writeErrorMeters = new HashMap<>();
    private static final Logger log = LoggerFactory.getLogger(TokenDiscoveryWriter.class);

    public TokenDiscoveryWriter(ThreadPoolExecutor executor) {
        super(executor);
        registerIOModules();
    }

    public void registerIO(TokenDiscoveryIO io) {
        tokenDiscoveryIOs.add(io);
        writeErrorMeters.put(io.getClass(),
                             Metrics.meter(io.getClass(), "TokenDiscoveryWriter Write Errors")
        );
    }

    public void registerIOModules() {
        List<String> modules = Configuration.getInstance().getListProperty(CoreConfig.TOKEN_DISCOVERY_MODULES);

        ClassLoader classLoader = TokenDiscoveryIO.class.getClassLoader();
        for (String module : modules) {
            log.info("Loading token discovery module " + module);
            try {
                Class discoveryClass = classLoader.loadClass(module);
                TokenDiscoveryIO discoveryIOModule = (TokenDiscoveryIO) discoveryClass.newInstance();
                log.info("Registering token discovery module " + module);
                registerIO(discoveryIOModule);
            } catch (InstantiationException e) {
                log.error("Unable to create instance of token discovery class for: " + module, e);
            } catch (IllegalAccessException e) {
                log.error("Error starting token discovery module: " + module, e);
            } catch (ClassNotFoundException e) {
                log.error("Unable to locate token discovery module: " + module, e);
            } catch (RuntimeException e) {
                log.error("Error starting token discovery module: " + module, e);
            } catch (Throwable e) {
                log.error("Error starting token discovery module: " + module, e);
            }
        }
    }

    /**
     * For all of these {@link Locator}'s, get a unique list of {@link Token}'s which are not in token cache
     *
     * @param locators
     * @return
     */
    static Set<Token> getUniqueTokens(final List<Locator> locators) {

        return Token.getUniqueTokens(locators.stream())
                    .filter(token -> !TokenCache.getInstance().isTokenCurrent(token))
                    .collect(toSet());
    }

    /**
     * Get all {@link Locator}'s corresponding to the metrics which are not current.
     *
     * @param input
     * @return
     */
    static List<Locator> getLocators(final List<List<IMetric>> input) {

        //converting list of lists of metrics to flat list of locators that are not current.
        return input.stream()
                    .flatMap(List::stream)
                    .map(IMetric::getLocator)
                    .filter(locator -> !LocatorCache.getInstance().isLocatorCurrentInTokenDiscoveryLayer(locator))
                    .collect(toList());
    }

    /**
     * For a given batch of metrics, insert unique tokens using {@link TokenDiscoveryIO}.
     *
     * This methods has the following steps.
     * 1) For a batch of metrics, get all locators, which are not current by calling {@link #getLocators(List)}.
     * 2) For all these locators, get unique tokens, which are not current, by calling {@link #getUniqueTokens(List)}
     * 3) insert tokens
     * 4) After successful insertion, update both {@link LocatorCache} and {@link TokenCache}
     *
     * @param input
     * @return
     */
    public ListenableFuture<Boolean> processTokens(final List<List<IMetric>> input) {

        return getThreadPool().submit(() -> {
            boolean success = true;

            List<Locator> locators = getLocators(input);

            List<Token> tokens = new ArrayList<>();
            tokens.addAll(getUniqueTokens(locators));

            if (tokens.size() > 0) {
                for (TokenDiscoveryIO io : tokenDiscoveryIOs) {
                    try {
                        io.insertDiscovery(tokens);
                    } catch (Exception ex) {
                        getLogger().error(ex.getMessage(), ex);
                        writeErrorMeters.get(io.getClass()).mark();
                        success = false;
                    }
                }
            }

            if (success && tokens.size() > 0) {

                tokens.stream()
                      .filter(token -> !token.isLeaf()) //do not cache leaf nodes
                      .forEach(token -> {

                          //updating token cache
                          TokenCache.getInstance().setTokenCurrent(token);
                      });

                locators.stream()
                        .forEach(locator -> {

                            //updating locator cache
                            LocatorCache.getInstance().setLocatorCurrentInTokenDiscoveryLayer(locator);
                        });
            }

            return success;
        });
    }

    @Override
    public Void apply(List<List<IMetric>> input) throws Exception {
        processTokens(input);
        return null;
    }
}


package com.rackspacecloud.blueflood.concurrent;

import com.codahale.metrics.Gauge;
import com.codahale.metrics.MetricRegistry;
import com.rackspacecloud.blueflood.utils.Metrics;

import java.util.concurrent.ThreadPoolExecutor;

public class InstrumentedThreadPoolExecutor {
    /**
     * Given a {@link ThreadPoolExecutor}, attach various {@link Gauge}s against its monitoring
     * properties.
     * @param executor {@link ThreadPoolExecutor} to monitor.
     * @param threadPoolName a unique name for this thread pool.
     */
    public static void instrument(final ThreadPoolExecutor executor, String threadPoolName) {
        MetricRegistry registry = Metrics.getRegistry();
        registry.register(name(threadPoolName, "queue-size"), new Gauge<Integer>() {
            @Override
            public Integer getValue() {
                return executor.getQueue().size();
            }
        });
        registry.register(name(threadPoolName, "queue-max"), new Gauge<Integer>() {
            @Override
            public Integer getValue() {
                return executor.getQueue().size() + executor.getQueue().remainingCapacity();
            }
        });
        registry.register(name(threadPoolName, "threadpool-active"), new Gauge<Integer>() {
            @Override
            public Integer getValue() {
                return executor.getActiveCount();
            }
        });
        registry.register(name(threadPoolName, "threadpool-max"), new Gauge<Integer>() {
            @Override
            public Integer getValue() {
                return executor.getMaximumPoolSize();
            }
        });
    }

    static String name(String threadPoolName, String suffix) {
        return MetricRegistry.name(InstrumentedThreadPoolExecutor.class, threadPoolName, suffix);
    }
}


/*
 * Copyright (c) 2016 Rackspace.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * 	http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.rackspacecloud.blueflood.io;

import com.rackspacecloud.blueflood.service.Configuration;
import com.rackspacecloud.blueflood.service.CoreConfig;

import java.net.InetSocketAddress;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;

/**
 * This is a helper class that deals with reading/computing configuration
 * related to IO classes.
 */
public class IOConfig {

    private static IOConfig INSTANCE = new IOConfig();

    public static IOConfig singleton() { return INSTANCE; }

    private final Configuration config = Configuration.getInstance();

    /**
     * Retrieves the set of unique Cassandra hosts from configuration file.
     * If a single host appears multiple times in the configuration, only one will
     * be listed.
     *
     * @return set of unique Cassandra hosts
     */
    public Set<String> getUniqueHosts() {
        Set<String> uniqueHosts = new HashSet<String>();
        Collections.addAll(uniqueHosts, config.getStringProperty(CoreConfig.CASSANDRA_HOSTS).split(","));
        return uniqueHosts;
    }

    /**
     * Retrieves a set of unique Cassandra hosts with Binary Transport protocol
     * enabled from configuration file. This may or may not be different than
     * {@link #getUniqueHosts()}
     *
     * @return set of unique Cassandra hosts with binary transport protocol enabled
     */
    protected Set<String> getUniqueBinaryTransportHosts() {
        Set<String> uniqueHosts = new HashSet<String>();
        Collections.addAll(uniqueHosts, config.getStringProperty(CoreConfig.CASSANDRA_BINXPORT_HOSTS).split(","));
        return uniqueHosts;
    }

    /**
     * Retrieves the datacenter location as known by Cassandra.
     * @return name of the datacenter
     */
    public String getDatacenterName() {
        return config.getStringProperty(CoreConfig.CASSANDRA_LOCAL_DATACENTER_NAME);
    }

    /**
     * Retrieves a set of unique Cassandra hosts with Binary Transport protocol
     * enabled from configuration file as a set of {@link java.net.InetSocketAddress}
     * objects.
     *
     * @return set of InetSocketAddress objects containing unique Cassandra hosts
     * with binary transport protocol enabled
     */
    public Set<InetSocketAddress> getUniqueBinaryTransportHostsAsInetSocketAddresses() {
        Set<String> hosts = getUniqueBinaryTransportHosts();

        Set<InetSocketAddress> inetAddresses = new HashSet<InetSocketAddress>();
        for (String host: hosts) {
            String[] parts = host.split(":");
            InetSocketAddress inetSocketAddress;
            if ( parts.length == 1 ) {
                inetSocketAddress = new InetSocketAddress(parts[0], config.getIntegerProperty(CoreConfig.CASSANDRA_BINXPORT_PORT));
            } else {
                inetSocketAddress = new InetSocketAddress(parts[0], Integer.parseInt(parts[1]));
            }
            inetAddresses.add(inetSocketAddress);
        }
        return inetAddresses;
    }

    /**
     * Calculates the number of max connections per Cassandra hosts
     *
     * @param numHosts the number of Cassandra hosts in the cluster
     * @return the number of max connections
     */
    public int getMaxConnPerHost(int numHosts) {
        int maxConns = config.getIntegerProperty(CoreConfig.MAX_CASSANDRA_CONNECTIONS);
        return maxConns / numHosts + (maxConns % numHosts == 0 ? 0 : 1);
    }

    public int getDatastaxCoreConnectionsPerHost() {
        return config.getIntegerProperty(CoreConfig.DATASTAX_CORE_CONNECTIONS_PER_HOST);
    }

    public int getDatastaxMaxConnectionsPerHost() {
        return config.getIntegerProperty(CoreConfig.DATASTAX_MAX_CONNECTIONS_PER_HOST);
    }

    public int getDatastaxMaxRequestsPerConnection() {
        return config.getIntegerProperty(CoreConfig.DATASTAX_MAX_REQUESTS_PER_CONNECTION);
    }

    /**
     * @return the Cassandra request timeout
     */
    public int getRequestTimeout() {
        return config.getIntegerProperty(CoreConfig.CASSANDRA_REQUEST_TIMEOUT);
    }

    public int getReadTimeoutMaxRetries() { return config.getIntegerProperty(CoreConfig.DATASTAX_READ_TIMEOUT_MAX_RETRIES); }

    public int getWriteTimeoutMaxRetries() { return config.getIntegerProperty(CoreConfig.DATASTAX_WRITE_TIMEOUT_MAX_RETRIES); }

    public int getUnavailableMaxRetries() { return config.getIntegerProperty(CoreConfig.DATASTAX_UNAVAILABLE_MAX_RETRIES); }

    // prevent people from instantiating directly
    private IOConfig() {
    }
}


/*
 * Copyright 2016 Rackspace
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

package com.rackspacecloud.blueflood.io.serializers.metrics;

import com.codahale.metrics.Histogram;
import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.CodedInputStream;
import com.google.protobuf.CodedOutputStream;
import com.rackspacecloud.blueflood.exceptions.SerializationException;
import com.rackspacecloud.blueflood.io.Constants;
import com.rackspacecloud.blueflood.types.BaseRollup;
import com.rackspacecloud.blueflood.utils.Metrics;

import java.io.IOException;

/**
 * The serialization and deserialization methods for sub-metrics used by
 * {@link com.rackspacecloud.blueflood.types.BasicRollup} and {@link com.rackspacecloud.blueflood.types.BluefloodGaugeRollup}.
 * Common sub-metrics are:
 *

 *
average

 *
max

 *
min

 *
sum

 *

 */
public class BaseRollupSerDes extends AbstractSerDes {

    /**
     * Our internal metric to count the number of rollups
     */
    protected static Histogram rollupSize = Metrics.histogram( BasicRollupSerDes.class, "Rollup Metric Size" );

    protected int sizeOfBaseRollup( BaseRollup baseRollup ) {
        int sz = sizeOfSize();

        sz += CodedOutputStream.computeRawVarint64Size( baseRollup.getCount() );
        if ( baseRollup.getCount() > 0 ) {
            sz += averageStatDeSer.sizeOf( baseRollup.getAverage() );
            sz += varianceStatDeSer.sizeOf( baseRollup.getVariance() );
            sz += minStatDeSer.sizeOf( baseRollup.getMinValue() );
            sz += maxStatDeSer.sizeOf( baseRollup.getMaxValue() );
        }
        return sz;
    }

    /**
     * Serialize without sum attribute.  This is used by {@link GaugeSerDes}
     *
     * @param baseRollup
     * @param protobufOut
     * @throws IOException
     */
    protected void serializeRollupV1( BaseRollup baseRollup, CodedOutputStream protobufOut ) throws IOException {
        protobufOut.writeRawByte( Constants.VERSION_1_ROLLUP );
        serializeBaseRollupHelper( baseRollup, protobufOut );
    }

    protected void serializeBaseRollupHelper( BaseRollup baseRollup, CodedOutputStream protobufOut ) throws IOException {
        protobufOut.writeRawVarint64( baseRollup.getCount() );          // stat count

        if ( baseRollup.getCount() > 0 ) {
            putRollupStat( baseRollup.getAverage(), protobufOut );
            putRollupStat( baseRollup.getVariance(), protobufOut );
            putRollupStat( baseRollup.getMinValue(), protobufOut );
            putRollupStat( baseRollup.getMaxValue(), protobufOut );
        }
    }

    protected void deserializeBaseRollup( BaseRollup baseRollup, CodedInputStream in, byte version ) throws IOException {

        final long count = in.readRawVarint64();
        baseRollup.setCount(count);

        if (count <= 0) {
            return;
        }

        for ( int i = 0; i < BaseRollup.NUM_STATS; i++ ) {
            byte statType = in.readRawByte();
            switch ( statType ) {
                case Constants.AVERAGE:
                    averageStatDeSer.deserialize( baseRollup.getAverage(), in );
                    break;
                case Constants.VARIANCE:
                    varianceStatDeSer.deserialize( baseRollup.getVariance(), in );
                    break;
                case Constants.MIN:
                    minStatDeSer.deserialize( baseRollup.getMinValue(), in );
                    break;
                case Constants.MAX:
                    maxStatDeSer.deserialize( baseRollup.getMaxValue(), in );
                    break;
                default:
                    throw new SerializationException( "invalid stat " + (int) version + " type: " + (int) statType );
            }

        }
    }
}


package com.rackspacecloud.blueflood.io;

import com.google.common.annotations.VisibleForTesting;
import com.rackspacecloud.blueflood.service.Configuration;
import com.rackspacecloud.blueflood.service.CoreConfig;
import com.rackspacecloud.blueflood.service.ElasticIOConfig;
import com.rackspacecloud.blueflood.types.*;
import com.rackspacecloud.blueflood.utils.GlobPattern;
import org.apache.commons.lang.StringUtils;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.*;
import org.apache.http.entity.ContentType;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicHeader;
import org.apache.http.nio.entity.NStringEntity;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.*;

import static java.util.stream.Collectors.joining;

public class ElasticsearchRestHelper {
    private static final Logger logger = LoggerFactory.getLogger(ElasticsearchRestHelper.class);
    private final CloseableHttpClient closeableHttpClient;
    private PoolingHttpClientConnectionManager pool;
    private String[] baseUrlArray;
    private int baseUrlIndex;
    private int numberOfElasticsearchEndpoints;
    private int MAX_CALL_COUNT = 10;
    private int MAX_RESULT_LIMIT = Configuration.getInstance().getIntegerProperty(CoreConfig.MAX_DISCOVERY_RESULT_SIZE);

    public static ElasticsearchRestHelper getInstance() {
        return new ElasticsearchRestHelper();
    }

    private ElasticsearchRestHelper(){
        logger.info("Creating a new instance of ElasticsearchRestHelper...");
        Configuration config = Configuration.getInstance();
        String[] endpoints = config.getStringProperty(ElasticIOConfig.ELASTICSEARCH_HOST_FOR_REST_CLIENT).split(",");

        initializeBaseUrlCollection(endpoints);

        this.pool = new PoolingHttpClientConnectionManager();
        int maxThreadsPerRoute = config.getIntegerProperty(ElasticIOConfig.ELASTICSEARCH_HTTP_CLIENT_THREADS_PER_ROUTE);
        pool.setDefaultMaxPerRoute(maxThreadsPerRoute);
        pool.setMaxTotal(endpoints.length * maxThreadsPerRoute);
        this.closeableHttpClient = HttpClients.custom().setConnectionManager(pool).build();
    }

    private String getNextBaseUrl(){
        return String.format("http://%s", baseUrlArray[getNextIndexSync()]);
    }

    private synchronized int getNextIndexSync(){
        if(baseUrlIndex >= numberOfElasticsearchEndpoints) baseUrlIndex = 0;
        return baseUrlIndex++;
    }

    private void initializeBaseUrlCollection(String[] endpoints) {
        baseUrlArray = new String[endpoints.length];
        numberOfElasticsearchEndpoints = endpoints.length;

        for(int i = 0; i < numberOfElasticsearchEndpoints; i++){
            baseUrlArray[i] = endpoints[i].trim();
        }

        baseUrlIndex = 0;
    }

    public String fetchEvents(String tenantId, Map<String, List<String>> query) throws IOException {
        String tempUrl = String.format("%s/%s/%s/_search?routing=%s&size=%d", getNextBaseUrl(),
                EventElasticSearchIO.EVENT_INDEX, EventElasticSearchIO.ES_TYPE, tenantId, MAX_RESULT_LIMIT);

        String queryDslString = getDslString(tenantId, query);
        HttpEntity httpEntity = new NStringEntity(queryDslString, ContentType.APPLICATION_JSON);

        Queue<String> callQ = new LinkedList<>();
        callQ.add(tempUrl);
        int callCount = 0;
        while(!callQ.isEmpty() && callCount < MAX_CALL_COUNT) {
            callCount++;
            String url = callQ.remove();
            logger.info("Using url [{}]", url);

            HttpPost httpPost = new HttpPost(url);
            httpPost.setHeaders(getHeaders());

            CloseableHttpResponse response = null;

            try {
                httpPost.setEntity(httpEntity);
                response = closeableHttpClient.execute(httpPost);
                String str = EntityUtils.toString(response.getEntity());
                EntityUtils.consume(response.getEntity());
                return str;
            } catch (Exception e) {
                if(response == null){
                    logger.error("fetchEvents failed with message: {}", e.getMessage());
                    url = String.format("%s/%s/%s/_search?routing=%s&size=%d", getNextBaseUrl(),
                            EventElasticSearchIO.EVENT_INDEX, EventElasticSearchIO.ES_TYPE,
                            tenantId, MAX_RESULT_LIMIT);
                    callQ.add(url);
                }
                else {
                    logger.error("fetchEvents failed with status code: {} and exception message: {}",
                            response.getStatusLine().getStatusCode(), e.getMessage());
                }
            } finally {
                if(response != null) {
                    response.close();
                }
            }
        }

        return "";
    }

    private String getDslString(String tenantId, Map<String, List<String>> query) {
        String tenantIdQ = getTermQueryString(ESFieldLabel.tenantId.toString(), tenantId);

        if(query == null){
            return "{\"query\":{\"bool\" : {\"must\": [" + tenantIdQ + "]}}}";
        }

        String tagsValue = extractFieldFromQuery(Event.FieldLabels.tags.toString(), query);
        String untilValue = extractFieldFromQuery(Event.untilParameterName, query);
        String fromValue = extractFieldFromQuery(Event.fromParameterName, query);

        String tagsQString = "";

        if (StringUtils.isNotEmpty(tagsValue))
            tagsQString = getTermQueryString(Event.FieldLabels.tags.toString(), tagsValue);

        String rangeQueryString;

        if (StringUtils.isNotEmpty(untilValue) && StringUtils.isNotEmpty(fromValue)) {
            rangeQueryString = String.format("{\"range\":{\"when\":{\"from\":%d,\"to\":%d}}}",
                    Long.parseLong(fromValue), Long.parseLong(untilValue));
        } else if (StringUtils.isNotEmpty(untilValue)) {
            rangeQueryString = String.format("{\"range\":{\"when\":{\"to\":%d}}}", Long.parseLong(untilValue));
        } else if (StringUtils.isNotEmpty(fromValue)) {
            rangeQueryString = String.format("{\"range\":{\"when\":{\"from\":%d}}}", Long.parseLong(fromValue));
        } else {
            logger.info("In query DSL, both 'from' and 'to' parameters are empty.");
            rangeQueryString = "";
        }

        StringBuilder sb = new StringBuilder(tenantIdQ);

        if(StringUtils.isNotEmpty(tagsQString)) sb.append("," + tagsQString);
        if(StringUtils.isNotEmpty(rangeQueryString)) sb.append("," + rangeQueryString);

        List<String> strings = new ArrayList<>();
        strings.add(sb.toString());

        String dslString = getBoolQueryString(getMustQueryString(strings));

        return dslString;
    }

    private String extractFieldFromQuery(String name, Map<String, List<String>> query) {
        String result = "";
        if (query.containsKey(name)) {
            List<String> temp = query.get(name);
            if(temp == null || temp.size() == 0) return result;
            result = temp.get(0);
        }
        return result;
    }

    public String fetch(String indexName, String documentType, String tenantId, List<String> queries) throws IOException {
        String queryDslString = getQueryDslString(tenantId, queries);
        return fetchDocuments(indexName, documentType, tenantId, queryDslString);
    }

    public String fetchTokenDocuments(String[] indices, String tenantId, String query) throws IOException {
        String queryDslString = getTokenQueryDslString(tenantId, query);
        String multiIndexString = String.join(",", indices);

        String temp = String.format("%s/_search?routing=%s&size=%d", multiIndexString, tenantId, MAX_RESULT_LIMIT);
        String urlFormat = "%s/" + temp;

        return fetchDocs(queryDslString, urlFormat);
    }

    public String fetchDocuments(String indexName, String documentType, String tenantId, String queryDslString) throws IOException {
        //Example URL: localhost:9200/metric_metadata/metrics/_search?routing=123456&size=100
        String temp = String.format("%s/%s/_search?routing=%s&size=%d",
                indexName, documentType, tenantId, MAX_RESULT_LIMIT);
        String urlFormat = "%s/" + temp;

        return fetchDocs(queryDslString, urlFormat);
    }

    private String fetchDocs(String queryDslString, String urlFormat) throws IOException {
        String tempUrl = String.format(urlFormat, getNextBaseUrl());

        Queue<String> callQ = new LinkedList<>();
        callQ.add(tempUrl);
        int callCount = 0;
        while(!callQ.isEmpty() && callCount < MAX_CALL_COUNT) {
            callCount++;
            String url = callQ.remove();

            logger.debug("Using url [{}]", url);
            HttpPost httpPost = new HttpPost(url);
            httpPost.setHeaders(getHeaders());
            HttpEntity httpEntity = new NStringEntity(queryDslString, ContentType.APPLICATION_JSON);
            httpPost.setEntity(httpEntity);

            CloseableHttpResponse response = null;

            try {
                response = closeableHttpClient.execute(httpPost);
                HttpEntity entity = response.getEntity();
                String str = EntityUtils.toString(entity);
                EntityUtils.consume(entity);
                return str;
            } catch (Exception e) {
                if(response == null){
                    logger.error("fetchDocs failed with message: {}", e.getMessage());
                    url = String.format(urlFormat, getNextBaseUrl());
                    callQ.add(url);
                }
                else {
                    logger.error("fetch failed with status code: {} and exception message: {}",
                            response.getStatusLine().getStatusCode(), e.getMessage());
                }
            } finally {
                if(response != null) {
                    response.close();
                }
            }
        }

        return "";
    }

    private String getQueryDslString(String tenantId, List<String> queries){
        List<String> mustStrings = new ArrayList<>();
        String tenantIdQString = getTermQueryString(ESFieldLabel.tenantId.toString(), tenantId);
        mustStrings.add(tenantIdQString);
        String mustValueString = getMustValueString(mustStrings);

        List<String> shouldStrings = new ArrayList<>();
        for(String query : queries) {
            String metricNameQString;
            GlobPattern pattern = new GlobPattern(query);

            if (pattern.hasWildcard()) {
                String compiledString = pattern.compiled().toString();
                // replace one '\' char with two '\\'
                compiledString = compiledString.replaceAll("\\\\", "\\\\\\\\");
                metricNameQString = getRegexpQueryString(ESFieldLabel.metric_name.toString(), compiledString);
            }
            else {
                metricNameQString = getTermQueryString(ESFieldLabel.metric_name.toString(), query);
            }
            shouldStrings.add(metricNameQString);
        }

        String shouldValueString = getShouldValueString(shouldStrings);

        String dslString = getBoolQueryString(mustValueString, shouldValueString);
        return dslString;
    }

    /**
     * Builds ES query to grab tokens corresponding to the given query glob.
     * For a given query foo.bar.*, we would like to grab all the tokens with
     * parent as foo.bar
     *
     * Sample ES query for a query glob = foo.bar.*:
     *
     *      "query": {
     *          "bool" : {
     *              "must" : [
     *                  { "term": {  "tenantId": "<tenantId>" }},
     *                  { "term": {  "parent": "foo.bar" }}
     *              ]
     *          }
     *      }
     *
     * @param tenantId
     * @param query
     * @return
     */
    private String getTokenQueryDslString(String tenantId, String query) {
        String[] queryTokens = query.split(Locator.METRIC_TOKEN_SEPARATOR_REGEX);
        String lastToken = queryTokens[queryTokens.length - 1];

        /**
         * Builds parent part of the query for the given input query glob tokens.
         * For a given query foo.bar.*, parent part is foo.bar
         *
         * For example:
         *
         *  For query = foo.bar.*
         *          { "term": {  "parent": "foo.bar" }}
         *
         *  For query = foo.*.*
         *          { "regexp": {  "parent": "foo.[^.]+" }}
         *
         *  For query = foo.b*.*
         *          { "regexp": {  "parent": "foo.b[^.]*" }}
         */

        String parentString = getParentString(queryTokens);
        String tenantIdTermQueryString = getTermQueryString(ESFieldLabel.tenantId.name(), tenantId);
        List<String> mustNodes = new ArrayList<>();
        mustNodes.add(tenantIdTermQueryString);
        mustNodes.add(parentString);

        // For example: if query=foo.bar.*, we can just get every token for the parent=foo.bar
        // but if query=foo.bar.b*, we want to add the token part of the query for "b*"
        if (!lastToken.equals("*")) {

            String tokenQString;

            GlobPattern pattern = new GlobPattern(lastToken);

            if (pattern.hasWildcard()) {
                String compiledString = pattern.compiled().toString();
                tokenQString = getRegexpQueryString(ESFieldLabel.token.name(), compiledString);
            } else {
                tokenQString = getTermQueryString(ESFieldLabel.token.name(), lastToken);
            }

            mustNodes.add(tokenQString);
        }

        String mustQueryString = getMustQueryString(mustNodes);
        String boolQueryString = getBoolQueryString(mustQueryString);
        // replace one '\' char with two '\\'
        boolQueryString = boolQueryString.replaceAll("\\\\", "\\\\\\\\");

        return boolQueryString;
    }

    private String getParentString(String[] queryTokens) {
        if (queryTokens.length == 1) return getTermQueryString(ESFieldLabel.parent.name(), "");

        String parent = Arrays.stream(queryTokens)
                .limit(queryTokens.length - 1)
                .collect(joining(Locator.METRIC_TOKEN_SEPARATOR));

        GlobPattern parentGlob = new GlobPattern(parent);

        String parentQueryString;
        if (parentGlob.hasWildcard()) {
            parentQueryString = getRegexpQueryString(ESFieldLabel.parent.name(), getRegexToHandleTokens(parentGlob));
        } else {
            parentQueryString = getTermQueryString(ESFieldLabel.parent.name(), parent);
        }

        return parentQueryString;
    }

    /**
     * For a given glob, gives regex for {@code Locator.METRIC_TOKEN_SEPARATOR} separated tokens
     *
     * For example:
     *      globPattern of foo.*.* would produce a regex  foo\.[^.]+\.[^.]+
     *      globPattern of foo.b*.* would produce a regex foo\.b[^.]*\.[^.]+
     *
     * @param globPattern
     * @return
     */
    protected String getRegexToHandleTokens(GlobPattern globPattern) {
        String[] queryRegexParts = globPattern.compiled().toString().split("\\\\.");

        return Arrays.stream(queryRegexParts)
                .map(this::convertRegexToCaptureUptoNextToken)
                .collect(joining(Locator.METRIC_TOKEN_SEPARATOR_REGEX));
    }

    private String convertRegexToCaptureUptoNextToken(String queryRegex) {

        if (queryRegex.equals(".*"))
            return queryRegex.replaceAll("\\.\\*", "[^.]+");
        else
            return queryRegex.replaceAll("\\.\\*", "[^.]*");
    }

    public void indexMetrics(List<IMetric> metrics) throws IOException {
        String bulkString = bulkStringify(metrics);
        String urlFormat = "%s/_bulk";
        index(urlFormat, bulkString);
    }

    public void indexTokens(List<Token> tokens) throws IOException {
        String bulkString = bulkStringifyTokens(tokens);
        String urlFormat = "%s/_bulk";
        index(urlFormat, bulkString);
    }

    public void indexEvent(Map<String, Object> event) throws IOException {
        String eventString = stringifyEvent(event);
        String temp = String.format("%s/%s?routing=%s",
                EventElasticSearchIO.EVENT_INDEX, EventElasticSearchIO.ES_TYPE,
                event.get(Event.FieldLabels.tenantId.toString()));

        String urlFormat = "%s/" + temp;

        index(urlFormat, eventString);
    }

    private String stringifyEvent(Map<String, Object> event){
        StringBuilder sb = new StringBuilder();

        sb.append(String.format("{\"what\": \"%s\",", event.get(Event.FieldLabels.what.toString())));
        sb.append(String.format("\"when\": \"%d\",", (long) event.get(Event.FieldLabels.when.toString())));
        sb.append(String.format("\"tags\": \"%s\",", event.get(Event.FieldLabels.tags.toString())));
        sb.append(String.format("\"tenantId\": \"%s\",", event.get(Event.FieldLabels.tenantId.toString())));
        sb.append(String.format("\"data\": \"%s\"}", event.get(Event.FieldLabels.data.toString())));

        return sb.toString();
    }

    private String bulkStringifyTokens(List<Token> tokens){
        StringBuilder sb = new StringBuilder();

        for(Token token : tokens){
            sb.append(String.format(
                    "{ \"index\" : { \"_index\" : \"%s\", \"_type\" : \"%s\", \"_id\" : \"%s\", \"routing\" : \"%s\" } }%n",
                    ElasticTokensIO.ELASTICSEARCH_TOKEN_INDEX_NAME_WRITE, ElasticTokensIO.ES_DOCUMENT_TYPE,
                    token.getId(), token.getLocator().getTenantId()));

            sb.append(String.format(
                    "{ \"%s\" : \"%s\", \"%s\" : \"%s\", \"%s\" : \"%s\", \"%s\" : \"%s\" }%n",
                    ESFieldLabel.token.toString(), token.getToken(),
                    ESFieldLabel.parent.toString(), token.getParent(),
                    ESFieldLabel.isLeaf.toString(), token.isLeaf(),
                    ESFieldLabel.tenantId.toString(), token.getLocator().getTenantId()));
        }

        return sb.toString();
    }

    private String bulkStringify(List<IMetric> metrics){
        StringBuilder sb = new StringBuilder();

        for(IMetric metric : metrics){
            Locator locator = metric.getLocator();

            if(locator.getMetricName() == null)
                throw new IllegalArgumentException("trying to insert metric discovery without a metricName");

            sb.append(String.format(
                    "{ \"index\" : { \"_index\" : \"%s\", \"_type\" : \"%s\", \"_id\" : \"%s\", \"routing\" : \"%s\" } }%n",
                    AbstractElasticIO.ELASTICSEARCH_INDEX_NAME_WRITE, AbstractElasticIO.ELASTICSEARCH_DOCUMENT_TYPE,
                    locator.getTenantId() + ":" + locator.getMetricName(), locator.getTenantId()));

            if(metric instanceof Metric && getUnit((Metric) metric) != null){
                sb.append(String.format(
                        "{ \"%s\" : \"%s\", \"%s\" : \"%s\", \"%s\" : \"%s\" }%n",
                        ESFieldLabel.tenantId.toString(), locator.getTenantId(),
                        ESFieldLabel.metric_name.toString(), locator.getMetricName(),
                        ESFieldLabel.unit.toString(), getUnit((Metric) metric)));
            }
            else {
                sb.append(String.format(
                        "{ \"%s\" : \"%s\", \"%s\" : \"%s\" }%n",
                        ESFieldLabel.tenantId.toString(), locator.getTenantId(),
                        ESFieldLabel.metric_name.toString(), locator.getMetricName()));
            }
        }

        return sb.toString();
    }

    private String getUnit(Metric metric) {
        return metric.getUnit();
    }

    //If index() fails for whatever reason, it always throws IOException because indexing failed for Elasticsearch.
    public void index(final String urlFormat, final String bulkString) throws IOException {
        String tempUrl = String.format(urlFormat, getNextBaseUrl());
        HttpEntity entity = new NStringEntity(bulkString, ContentType.APPLICATION_JSON);
        int statusCode = 0;

        /*
        Here I am using Queue to keep a round-robin selection of next base url. If current base URL fails for
        whatever reason (with response == null), then in catch block, I am enqueueing the next base URL so that
        in next iteration call picks up the new URL. If URL works, queue will be empty and loop will break out.
        */
        Queue<String> callQ = new LinkedList<>();
        callQ.add(tempUrl);
        int callCount = 0;
        while(!callQ.isEmpty() && callCount < MAX_CALL_COUNT) {
            callCount++;
            String url = callQ.remove();
            logger.debug("Using url [{}]", url);
            HttpPost httpPost = new HttpPost(url);
            httpPost.setHeaders(getHeaders());
            httpPost.setEntity(entity);

            CloseableHttpResponse response = null;

            try {
                logger.debug("ElasticsearchRestHelper.index Thread name in use: [{}]", Thread.currentThread().getName());
                response = closeableHttpClient.execute(httpPost);

                statusCode = response.getStatusLine().getStatusCode();
                String str = EntityUtils.toString(response.getEntity());
                EntityUtils.consume(response.getEntity());

                if (statusCode != HttpStatus.SC_OK && statusCode != HttpStatus.SC_CREATED) {
                    logger.error("index method failed with status code: {} and error: {}", statusCode, str);
                }
            }
            catch (Exception e) {
                if(response == null){
                    logger.error("index method failed with message: {}", e.getMessage());
                    url = String.format(urlFormat, getNextBaseUrl());
                    callQ.add(url);
                }
                else {
                    logger.error("index method failed with status code: {} and exception message: {}",
                            statusCode, e.getMessage());
                }
            } finally {
                if(response != null) {
                    response.close();
                }
            }
        }

        if(statusCode != HttpStatus.SC_OK && statusCode != HttpStatus.SC_CREATED)
            throw new IOException("Elasticsearch indexing failed with status code: [" + statusCode + "]");
    }

    private String getTermQueryString(String key, String value){
        return String.format("{\"term\":{\"%s\":\"%s\"}}", key, value);
    }

    private String getRegexpQueryString(String key, String value){
        return String.format("{\"regexp\":{\"%s\":\"%s\"}}", key, value);
    }

    private String getMustQueryString(List<String> termStrings){
        return String.format("{\"must\":%s}", getMustValueString(termStrings));
    }

    private String getMustValueString(List<String> termStrings){
        String termsString = String.join(",", termStrings);
        return String.format("[%s]", termsString);
    }

    private String getShouldValueString(List<String> termStrings){
        String termsString = String.join(",", termStrings);
        return String.format("[%s]", termsString);
    }

    private String getBoolQueryString(String mustString){
        return String.format("{\"query\":{\"bool\":%s}}", mustString);
    }

    private String getBoolQueryString(String mustValueString, String shouldValueString){
        return String.format("{\"query\":{\"bool\":{\"must\":%s,\"should\":%s,\"minimum_should_match\": 1}}}",
                mustValueString, shouldValueString);
    }

    private Header[] getHeaders(){
        Map<String, String> headersMap = new HashMap<>();
        headersMap.put("Accept", "application/json");
        headersMap.put("Content-Type", "application/json");

        Header[] headers = new Header[headersMap.size()];
        int i = 0;
        for(String key : headersMap.keySet()){
            headers[i++] = new BasicHeader(key, headersMap.get(key));
        }
        return headers;
    }

    @VisibleForTesting
    public int refreshIndex(String indexName) throws IOException {
        String url = String.format("http://%s/%s/_refresh", baseUrlArray[0], indexName);
        HttpGet httpGet = new HttpGet(url);
        CloseableHttpResponse response = null;
        try {
            response = closeableHttpClient.execute(httpGet);
        } catch (IOException e) {
            logger.error("Refresh for index {} failed with status code: {} and exception message: {}",
                    indexName, response.getStatusLine().getStatusCode(), e.getMessage());
        }
        finally {
            response.close();
        }

        return response.getStatusLine().getStatusCode();
    }
}


package com.rackspacecloud.blueflood.io;

import com.netflix.astyanax.model.ColumnFamily;
import com.netflix.astyanax.serializers.LongSerializer;
import com.netflix.astyanax.serializers.StringSerializer;
import com.rackspacecloud.blueflood.io.serializers.astyanax.LocatorSerializer;
import com.rackspacecloud.blueflood.io.serializers.astyanax.SlotKeySerializer;
import com.rackspacecloud.blueflood.io.serializers.astyanax.SlotStateSerializer;
import com.rackspacecloud.blueflood.rollup.Granularity;
import com.rackspacecloud.blueflood.rollup.SlotKey;
import com.rackspacecloud.blueflood.service.Configuration;
import com.rackspacecloud.blueflood.service.CoreConfig;
import com.rackspacecloud.blueflood.service.SlotState;
import com.rackspacecloud.blueflood.types.*;
import com.rackspacecloud.blueflood.utils.TimeValue;

import java.util.*;
import java.util.concurrent.TimeUnit;

public class CassandraModel {
    public static final String KEYSPACE = Configuration.getInstance().getStringProperty(CoreConfig.ROLLUP_KEYSPACE);
    public static final String QUOTED_KEYSPACE = "\"" + KEYSPACE + "\"";
    public static final String CLUSTER = Configuration.getInstance().getStringProperty(CoreConfig.CLUSTER_NAME);

    /**
     * The following CF_*_NAME are the names of all of our Column Family
     */
    public static final String CF_METRICS_STATE_NAME = "metrics_state";
    public static final String CF_METRICS_METADATA_NAME = "metrics_metadata";
    public static final String CF_METRICS_LOCATOR_NAME = "metrics_locator";
    public static final String CF_METRICS_DELAYED_LOCATOR_NAME = "metrics_delayed_locator";

    public static final String CF_METRICS_FULL_NAME = "metrics_full";
    public static final String CF_METRICS_5M_NAME = "metrics_5m";
    public static final String CF_METRICS_20M_NAME = "metrics_20m";
    public static final String CF_METRICS_60M_NAME = "metrics_60m";
    public static final String CF_METRICS_240M_NAME = "metrics_240m";
    public static final String CF_METRICS_1440M_NAME = "metrics_1440m";

    public static final String CF_METRICS_PREAGGREGATED_FULL_NAME = "metrics_preaggregated_full";
    public static final String CF_METRICS_PREAGGREGATED_5M_NAME = "metrics_preaggregated_5m";
    public static final String CF_METRICS_PREAGGREGATED_20M_NAME = "metrics_preaggregated_20m";
    public static final String CF_METRICS_PREAGGREGATED_60M_NAME = "metrics_preaggregated_60m";
    public static final String CF_METRICS_PREAGGREGATED_240M_NAME = "metrics_preaggregated_240m";
    public static final String CF_METRICS_PREAGGREGATED_1440M_NAME = "metrics_preaggregated_1440m";

    public static final MetricColumnFamily CF_METRICS_FULL = new MetricColumnFamily(CF_METRICS_FULL_NAME, new TimeValue(5, TimeUnit.DAYS));
    public static final MetricColumnFamily CF_METRICS_5M = new MetricColumnFamily(CF_METRICS_5M_NAME, new TimeValue(10, TimeUnit.DAYS));
    public static final MetricColumnFamily CF_METRICS_20M = new MetricColumnFamily(CF_METRICS_20M_NAME, new TimeValue(20, TimeUnit.DAYS));
    public static final MetricColumnFamily CF_METRICS_60M = new MetricColumnFamily(CF_METRICS_60M_NAME, new TimeValue(31 * 5, TimeUnit.DAYS));
    public static final MetricColumnFamily CF_METRICS_240M = new MetricColumnFamily(CF_METRICS_240M_NAME, new TimeValue(60 * 5, TimeUnit.DAYS));
    public static final MetricColumnFamily CF_METRICS_1440M = new MetricColumnFamily(CF_METRICS_1440M_NAME, new TimeValue(365 * 5, TimeUnit.DAYS));

    public static final MetricColumnFamily CF_METRICS_PREAGGREGATED_FULL = new MetricColumnFamily(CF_METRICS_PREAGGREGATED_FULL_NAME, new TimeValue(5, TimeUnit.DAYS));
    public static final MetricColumnFamily CF_METRICS_PREAGGREGATED_5M = new MetricColumnFamily(CF_METRICS_PREAGGREGATED_5M_NAME, new TimeValue(10, TimeUnit.DAYS));
    public static final MetricColumnFamily CF_METRICS_PREAGGREGATED_20M = new MetricColumnFamily(CF_METRICS_PREAGGREGATED_20M_NAME, new TimeValue(20, TimeUnit.DAYS));
    public static final MetricColumnFamily CF_METRICS_PREAGGREGATED_60M = new MetricColumnFamily(CF_METRICS_PREAGGREGATED_60M_NAME, new TimeValue(31 * 5, TimeUnit.DAYS));
    public static final MetricColumnFamily CF_METRICS_PREAGGREGATED_240M = new MetricColumnFamily(CF_METRICS_PREAGGREGATED_240M_NAME, new TimeValue(60 * 5, TimeUnit.DAYS));
    public static final MetricColumnFamily CF_METRICS_PREAGGREGATED_1440M = new MetricColumnFamily(CF_METRICS_PREAGGREGATED_1440M_NAME, new TimeValue(365 * 5, TimeUnit.DAYS));

    public static final ColumnFamily<Locator, String> CF_METRICS_METADATA = new ColumnFamily<Locator, String>(CF_METRICS_METADATA_NAME,
            LocatorSerializer.get(),
            StringSerializer.get());

    public static final ColumnFamily<Long, Locator> CF_METRICS_LOCATOR = new ColumnFamily<Long, Locator>(CF_METRICS_LOCATOR_NAME,
            LongSerializer.get(),
            LocatorSerializer.get());

    public static final ColumnFamily<SlotKey, Locator> CF_METRICS_DELAYED_LOCATOR = new ColumnFamily<SlotKey,
            Locator>(CF_METRICS_DELAYED_LOCATOR_NAME,
            SlotKeySerializer.get(),
            LocatorSerializer.get());

    public static final ColumnFamily<Long, SlotState> CF_METRICS_STATE = new ColumnFamily<Long, SlotState>(CF_METRICS_STATE_NAME,
            LongSerializer.get(),
            SlotStateSerializer.get());

    private static final MetricColumnFamily[] METRIC_COLUMN_FAMILES = new MetricColumnFamily[] {
            CF_METRICS_FULL, CF_METRICS_5M, CF_METRICS_20M, CF_METRICS_60M, CF_METRICS_240M, CF_METRICS_1440M,
            CF_METRICS_PREAGGREGATED_FULL, CF_METRICS_PREAGGREGATED_5M, CF_METRICS_PREAGGREGATED_20M,
            CF_METRICS_PREAGGREGATED_60M, CF_METRICS_PREAGGREGATED_240M, CF_METRICS_PREAGGREGATED_1440M
    };

    private static final ColumnFamily[] BF_SYSTEM_COLUMN_FAMILIES = new ColumnFamily[] {
            CF_METRICS_METADATA, CF_METRICS_LOCATOR, CF_METRICS_DELAYED_LOCATOR, CF_METRICS_STATE
    };

    private static final Collection<ColumnFamily> ALL_COLUMN_FAMILIES;

    private static final ColumnFamilyMapper METRICS_GRAN_TO_CF;
    private static final ColumnFamilyMapper PREAG_GRAN_TO_CF;

    private static final Map<String, ColumnFamily<Locator, Long>> CF_NAME_TO_CF = new HashMap<String, ColumnFamily<Locator, Long>>();
    private static final Map<ColumnFamily<Locator, Long>, Granularity> CF_TO_GRAN;

    static {
        final Map<Granularity, MetricColumnFamily> columnFamilyMap = new HashMap<Granularity, MetricColumnFamily>();
        columnFamilyMap.put(Granularity.FULL, CF_METRICS_FULL);
        columnFamilyMap.put(Granularity.MIN_5, CF_METRICS_5M);
        columnFamilyMap.put(Granularity.MIN_20, CF_METRICS_20M);
        columnFamilyMap.put(Granularity.MIN_60, CF_METRICS_60M);
        columnFamilyMap.put(Granularity.MIN_240, CF_METRICS_240M);
        columnFamilyMap.put(Granularity.MIN_1440, CF_METRICS_1440M);

        final Map<Granularity, MetricColumnFamily> preagCFMap = new HashMap<Granularity, MetricColumnFamily>();
        preagCFMap.put(Granularity.FULL, CF_METRICS_PREAGGREGATED_FULL);
        preagCFMap.put(Granularity.MIN_5, CF_METRICS_PREAGGREGATED_5M);
        preagCFMap.put(Granularity.MIN_20, CF_METRICS_PREAGGREGATED_20M);
        preagCFMap.put(Granularity.MIN_60, CF_METRICS_PREAGGREGATED_60M);
        preagCFMap.put(Granularity.MIN_240, CF_METRICS_PREAGGREGATED_240M);
        preagCFMap.put(Granularity.MIN_1440, CF_METRICS_PREAGGREGATED_1440M);

        Map<ColumnFamily<Locator, Long>, Granularity> cfToGranMap = new HashMap<ColumnFamily<Locator, Long>, Granularity>();
        cfToGranMap.put(CF_METRICS_FULL, Granularity.FULL);
        cfToGranMap.put(CF_METRICS_5M, Granularity.MIN_5);
        cfToGranMap.put(CF_METRICS_20M, Granularity.MIN_20);
        cfToGranMap.put(CF_METRICS_60M, Granularity.MIN_60);
        cfToGranMap.put(CF_METRICS_240M, Granularity.MIN_240);
        cfToGranMap.put(CF_METRICS_1440M, Granularity.MIN_1440);
        cfToGranMap.put(CF_METRICS_PREAGGREGATED_FULL, Granularity.FULL);
        cfToGranMap.put(CF_METRICS_PREAGGREGATED_5M, Granularity.MIN_5);
        cfToGranMap.put(CF_METRICS_PREAGGREGATED_20M, Granularity.MIN_20);
        cfToGranMap.put(CF_METRICS_PREAGGREGATED_60M, Granularity.MIN_60);
        cfToGranMap.put(CF_METRICS_PREAGGREGATED_240M, Granularity.MIN_240);
        cfToGranMap.put(CF_METRICS_PREAGGREGATED_1440M, Granularity.MIN_1440);

        METRICS_GRAN_TO_CF = new ColumnFamilyMapper() {
            @Override
            public MetricColumnFamily get(Granularity gran) {
                return columnFamilyMap.get(gran);
            }
        };
        PREAG_GRAN_TO_CF = new ColumnFamilyMapper() {
            @Override
            public MetricColumnFamily get(Granularity gran) {
                return preagCFMap.get(gran);
            }
        };
        CF_TO_GRAN = Collections.unmodifiableMap(cfToGranMap);
        List<ColumnFamily> cfs = new ArrayList<ColumnFamily>();

        for (ColumnFamily cf : METRIC_COLUMN_FAMILES) {
            cfs.add(cf);
        }

        for (ColumnFamily cf : BF_SYSTEM_COLUMN_FAMILIES) {
            cfs.add(cf);
        }
        ALL_COLUMN_FAMILIES = Collections.unmodifiableList(cfs);

        for ( ColumnFamily cf : ALL_COLUMN_FAMILIES) {
            CF_NAME_TO_CF.put(cf.getName(), cf);
        }
    }

    public static MetricColumnFamily getColumnFamily(Class<? extends Rollup> type, Granularity granularity) {
        if (type.equals(SimpleNumber.class)) {
            return CF_METRICS_FULL;
        } else if (type.equals(BasicRollup.class)) {
            return METRICS_GRAN_TO_CF.get(granularity);
        } else if (type.equals(BluefloodSetRollup.class) || type.equals(BluefloodTimerRollup.class) || type.equals(BluefloodGaugeRollup.class) ||
                type.equals(BluefloodCounterRollup.class) ) {
            return PREAG_GRAN_TO_CF.get(granularity);
        } else {
            throw new RuntimeException("Unsupported rollup type: " + type.getName());
        }
    }

    public static MetricColumnFamily getColumnFamily(RollupType type, Granularity gran) {
        if (type == null) {
            type = RollupType.BF_BASIC;
        }

        return getColumnFamily(RollupType.classOf(type, gran), gran);
    }

    public static MetricColumnFamily getPreaggregatedColumnFamily(Granularity gran) {
        return PREAG_GRAN_TO_CF.get(gran);
    }

    public static String getPreaggregatedColumnFamilyName(Granularity gran) {
        return PREAG_GRAN_TO_CF.get(gran).getName();
    }

    public static MetricColumnFamily getBasicColumnFamily(Granularity gran) {
        return METRICS_GRAN_TO_CF.get(gran);
    }

    public static String getBasicColumnFamilyName(Granularity gran) {
        return METRICS_GRAN_TO_CF.get(gran).getName();
    }

    public static ColumnFamily getColumnFamily(String columnFamilyName) {
        return CF_NAME_TO_CF.get(columnFamilyName);
    }

    // iterate over all column families that store metrics.
    public static Iterable<MetricColumnFamily> getMetricColumnFamilies() {
        return new Iterable<MetricColumnFamily>() {
            @Override
            public Iterator<MetricColumnFamily> iterator() {
                return new Iterator<MetricColumnFamily>() {
                    private int pos = 0;
                    @Override
                    public boolean hasNext() {
                        return pos < METRIC_COLUMN_FAMILES.length;
                    }

                    @Override
                    public MetricColumnFamily next() {
                        return METRIC_COLUMN_FAMILES[pos++];
                    }

                    @Override
                    public void remove() {
                        throw new NoSuchMethodError("Not implemented");
                    }
                };
            }
        };
    }

    public static Granularity getGranularity(ColumnFamily cf) {
        return CF_TO_GRAN.get(cf);
    }

    public static Granularity getGranularity(String cf) {
        return CF_TO_GRAN.get(CF_NAME_TO_CF.get(cf));
    }

    public static Collection<ColumnFamily> getAllColumnFamilies() {
        return ALL_COLUMN_FAMILIES;
    }

    public static Collection<String> getAllColumnFamiliesNames() {

        List<String> names = new ArrayList<String>() {{
                for (ColumnFamily cf : ALL_COLUMN_FAMILIES) {
                    add(cf.getName());
                }
        }};
        return names;
    }

    public static class MetricColumnFamily extends ColumnFamily<Locator, Long>  {
        private final TimeValue ttl;

        public MetricColumnFamily(String name, TimeValue ttl) {
            super(name, LocatorSerializer.get(), LongSerializer.get());
            this.ttl = ttl;
        }

        public TimeValue getDefaultTTL() {
            return ttl;
        }
    }

    // future versions will have get(Granularity, RollupType).
    public interface ColumnFamilyMapper {
        public MetricColumnFamily get(Granularity gran);
    }
}