/*
 * Copyright 2014-2018 Red Hat, Inc. and/or its affiliates
 * and other contributors as indicated by the @author tags.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.hawkular.metrics.core.service;

import static java.time.ZoneOffset.UTC;

import static org.hawkular.metrics.core.service.Functions.isValidTagMap;
import static org.hawkular.metrics.core.service.Functions.makeSafe;
import static org.hawkular.metrics.core.service.Order.ASC;
import static org.hawkular.metrics.model.MetricType.AVAILABILITY;
import static org.hawkular.metrics.model.MetricType.COUNTER;
import static org.hawkular.metrics.model.MetricType.GAUGE;
import static org.hawkular.metrics.model.MetricType.STRING;
import static org.hawkular.metrics.model.Utils.isValidTimeRange;

import static com.google.common.base.Preconditions.checkArgument;

import java.time.Instant;
import java.time.ZonedDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
import java.util.regex.Pattern;

import org.hawkular.metrics.core.service.compress.CompressedPointContainer;
import org.hawkular.metrics.core.service.log.CoreLogger;
import org.hawkular.metrics.core.service.log.CoreLogging;
import org.hawkular.metrics.core.service.tags.ExpressionTagQueryParser;
import org.hawkular.metrics.core.service.tags.SimpleTagQueryParser;
import org.hawkular.metrics.core.service.tags.TagsConverter;
import org.hawkular.metrics.core.service.transformers.DataPointCompressTransformer;
import org.hawkular.metrics.core.service.transformers.DataPointDecompressTransformer;
import org.hawkular.metrics.core.service.transformers.MetricFromDataRowTransformer;
import org.hawkular.metrics.core.service.transformers.MetricIdentifierFromFullDataRowTransformer;
import org.hawkular.metrics.core.service.transformers.MetricsIndexRowTransformer;
import org.hawkular.metrics.core.service.transformers.NumericBucketPointTransformer;
import org.hawkular.metrics.core.service.transformers.SortedMerge;
import org.hawkular.metrics.core.service.transformers.TaggedBucketPointTransformer;
import org.hawkular.metrics.core.service.transformers.TempTableCompressTransformer;
import org.hawkular.metrics.datetime.DateTimeService;
import org.hawkular.metrics.model.AvailabilityBucketPoint;
import org.hawkular.metrics.model.AvailabilityType;
import org.hawkular.metrics.model.BucketPoint;
import org.hawkular.metrics.model.Buckets;
import org.hawkular.metrics.model.DataPoint;
import org.hawkular.metrics.model.Metric;
import org.hawkular.metrics.model.MetricId;
import org.hawkular.metrics.model.MetricType;
import org.hawkular.metrics.model.NamedDataPoint;
import org.hawkular.metrics.model.NumericBucketPoint;
import org.hawkular.metrics.model.Percentile;
import org.hawkular.metrics.model.Retention;
import org.hawkular.metrics.model.TaggedBucketPoint;
import org.hawkular.metrics.model.Tenant;
import org.hawkular.metrics.model.exception.MetricAlreadyExistsException;
import org.hawkular.metrics.model.exception.RuntimeApiError;
import org.hawkular.metrics.model.exception.TenantAlreadyExistsException;
import org.hawkular.metrics.model.param.BucketConfig;
import org.hawkular.metrics.model.param.Tags;
import org.hawkular.metrics.model.param.TimeRange;
import org.hawkular.metrics.sysconfig.Configuration;
import org.hawkular.metrics.sysconfig.ConfigurationService;
import org.joda.time.Duration;

import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
import com.datastax.driver.core.ResultSet;
import com.datastax.driver.core.ResultSetFuture;
import com.datastax.driver.core.Row;
import com.datastax.driver.core.Session;
import com.datastax.driver.core.exceptions.DriverException;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;

import rx.Completable;
import rx.Observable;
import rx.functions.Func1;
import rx.functions.Func6;
import rx.observable.ListenableFutureObservable;
import rx.schedulers.Schedulers;

/**
 * @author John Sanda
 */
public class MetricsServiceImpl implements MetricsService {
    private static final CoreLogger log = CoreLogging.getCoreLogger(MetricsServiceImpl.class);

    private static final long DAY_TO_MILLIS = 24 * 3600 * 1000;
    public static final String SYSTEM_TENANT_ID = makeSafe("sysconfig");

    private static class DataRetentionKey {
        private final MetricId<?> metricId;

        public DataRetentionKey(String tenantId, MetricType<?> type) {
            metricId = new MetricId<>(tenantId, type, makeSafe(type.getText()));
        }

        public DataRetentionKey(MetricId<?> metricId) {
            this.metricId = metricId;
        }

        public DataRetentionKey(Metric<?> metric) {
            this.metricId = metric.getMetricId();
        }

        @Override
        public boolean equals(Object o) {
            if (this == o) return true;
            if (o == null || getClass() != o.getClass()) return false;

            DataRetentionKey that = (DataRetentionKey) o;

            return metricId.equals(that.metricId);
        }

        @Override
        public int hashCode() {
            return metricId.hashCode();
        }
    }

    /**
     * Note that while user specifies the durations in hours, we store them in seconds.
     */
    private final Map<DataRetentionKey, Integer> dataRetentions = new ConcurrentHashMap<>();

    private ListeningExecutorService metricsTasks;

    private DataAccess dataAccess;

    private ConfigurationService configurationService;

    private MetricRegistry metricRegistry;

    /**
     * Functions used to insert metric data points.
     */
    private Map<MetricType<?>, Func1<Observable<? extends Metric<?>>, Observable<Integer>>> pointsInserter;

    /**
     * Functions used to find metric data points rows.
     */
    private Map<MetricType<?>, Func6<? extends MetricId<?>, Long, Long,
                Integer, Order, Integer, Observable<Row>>> dataPointFinders;

    /**
     * Functions used to transform a row into a data point object.
     */
    private Map<MetricType<?>, Func1<Row, ? extends DataPoint<?>>> dataPointMappers;

    private Map<MetricType<?>, Func1<Row, ? extends DataPoint<?>>> tempDataPointMappers;

    /**
     * Tools that do tag query parsing and execution
     */
    private boolean disableACostOptimization;
    private SimpleTagQueryParser tagQueryParser;
    private ExpressionTagQueryParser expresssionTagQueryParser;

    private int defaultTTL = Duration.standardDays(7).toStandardSeconds().getSeconds();
    private int DEFAULT_RETENTION = (int) Duration.standardSeconds(defaultTTL).getStandardDays();

    private int maxStringSize;

    private long insertRetryMaxDelay;

    private int insertMaxRetries;

    private int defaultPageSize;

    public void startUp(Session session, String keyspace, boolean resetDb, MetricRegistry metricRegistry) {
        startUp(session, keyspace, resetDb, true, metricRegistry);
    }

    public void startUp(Session session, String keyspace, boolean resetDb, boolean createSchema,
            MetricRegistry metricRegistry) {
        session.execute("USE " + keyspace);
        log.infoKeyspaceUsed(keyspace);
        metricsTasks = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(4, new MetricsThreadFactory()));
        loadDataRetentions();

        this.metricRegistry = metricRegistry;

        pointsInserter = ImmutableMap
                .<MetricType<?>, Func1<Observable<? extends Metric<?>>, Observable<Integer>>>builder()
                .put(GAUGE, metric -> {
                    @SuppressWarnings("unchecked")
                    Observable<Metric<Double>> gauge = (Observable<Metric<Double>>) metric;
                    return dataAccess.insertData(gauge);
                })
                .put(COUNTER, metric -> {
                    @SuppressWarnings("unchecked")
                    Observable<Metric<Long>> counter = (Observable<Metric<Long>>) metric;
                    return dataAccess.insertData(counter);
                })
                .put(AVAILABILITY, metric -> {
                    @SuppressWarnings("unchecked")
                    Observable<Metric<AvailabilityType>> avail = (Observable<Metric<AvailabilityType>>) metric;
                    return dataAccess.insertData(avail);
                })
                .put(STRING, metric -> {
                    @SuppressWarnings("unchecked")
                    Observable<Metric<String>> string = (Observable<Metric<String>>) metric;
                    return dataAccess.insertStringDatas(string, this::getTTL, maxStringSize);
                })
                .build();

        dataPointFinders = ImmutableMap
                .<MetricType<?>, Func6<? extends MetricId<?>, Long, Long, Integer, Order, Integer,
                        Observable<Row>>>builder()
                .put(STRING, (metricId, start, end, limit, order, pageSize) -> {
                    @SuppressWarnings("unchecked")
                    MetricId<String> stringId = (MetricId<String>) metricId;
                    return dataAccess.findStringData(stringId, start, end, limit, order, pageSize);
                })
                .build();

        dataPointMappers = ImmutableMap.<MetricType<?>, Func1<Row, ? extends DataPoint<?>>> builder()
                .put(GAUGE, Functions::getGaugeDataPoint)
                .put(AVAILABILITY, Functions::getAvailabilityDataPoint)
                .put(COUNTER, Functions::getCounterDataPoint)
                .put(STRING, Functions::getStringDataPoint)
                .build();

        tempDataPointMappers = ImmutableMap.<MetricType<?>, Func1<Row, ? extends DataPoint<?>>> builder()
                .put(GAUGE, Functions::getTempGaugeDataPoint)
                .put(COUNTER, Functions::getTempCounterDataPoint)
                .put(AVAILABILITY, Functions::getTempAvailabilityDataPoint)
                .build();

        initConfiguration(session);
        setDefaultTTL(session, keyspace);

        verifyAndCreateTempTables();

        int defaultPageSize = session.getCluster().getConfiguration().getQueryOptions().getFetchSize();
        int pageThreshold = Integer.getInteger("hawkular.metrics.page-threshold", 10);
        tagQueryParser = new SimpleTagQueryParser(this.dataAccess, this, disableACostOptimization, defaultPageSize,
                pageThreshold);
        expresssionTagQueryParser = new ExpressionTagQueryParser(this.dataAccess, this);
    }

    void loadDataRetentions() {
        List<String> tenantIds = loadTenantIds();
        CountDownLatch latch = new CountDownLatch(tenantIds.size() * 2);
        for (String tenantId : tenantIds) {
            DataRetentionsMapper gaugeMapper = new DataRetentionsMapper(tenantId, GAUGE);
            DataRetentionsMapper availMapper = new DataRetentionsMapper(tenantId, AVAILABILITY);
            ResultSetFuture gaugeFuture = dataAccess.findDataRetentions(tenantId, GAUGE);
            ResultSetFuture availabilityFuture = dataAccess.findDataRetentions(tenantId, AVAILABILITY);
            ListenableFuture<Set<Retention>> gaugeRetentions = Futures.transform(gaugeFuture, gaugeMapper,
                    metricsTasks);
            ListenableFuture<Set<Retention>> availabilityRetentions = Futures.transform(availabilityFuture, availMapper,
                    metricsTasks);
            Futures.addCallback(gaugeRetentions,
                    new DataRetentionsLoadedCallback(tenantId, GAUGE, latch));
            Futures.addCallback(availabilityRetentions, new DataRetentionsLoadedCallback(tenantId, AVAILABILITY,
                    latch));
        }
        try {
            latch.await();
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        }
    }

    void unloadDataRetentions() {
        dataRetentions.clear();
    }

    /**
     * Measurements of the throughput of inserting data points.
     */
    private Meter getDataPointsInserted() {
        return metricRegistry.meter("DataPointsInserted");
    }

    /**
     * Raw data read metrics
     */
    private Timer getRawDataReadLatency() {
        return metricRegistry.timer("RawDataReadLatency");
    }

    /**
     * Metric tag query metrics
     */
    private Timer getMetricsTagsQueryLatency() {
        return metricRegistry.timer("MetricTagsQueryLatency");
    }

    private void initConfiguration(Session session) {
        Configuration configuration = configurationService.load("org.hawkular.metrics").toBlocking()
                .lastOrDefault(null);
        String configMaxStringSize = configuration.get("string-size");
        if (configMaxStringSize == null) {
            maxStringSize = -1;  // no size limit
        } else {
            maxStringSize = Integer.parseInt(configMaxStringSize);
        }
        log.infoMaxSizeStringMetrics(this.maxStringSize);

        insertRetryMaxDelay = Long.parseLong(configuration.get("ingestion.retry.max-delay", "30000"));
        insertMaxRetries = Integer.parseInt(configuration.get("ingestion.retry.max-retries", "5"));
        log.infoInsertRetryConfig(insertMaxRetries, insertRetryMaxDelay);

        defaultPageSize = Integer.parseInt(configuration.get("page-size", "5000"));
        disableACostOptimization = Boolean.parseBoolean(configuration.get("disable.parser.optimization", "false"));
    }

    private void setDefaultTTL(Session session, String keyspace) {
        ResultSet resultSet = session.execute("select default_time_to_live from system_schema.tables where " +
                "keyspace_name = '" + keyspace + "' and table_name = 'data'");
        List<Row> rows = resultSet.all();
        if (rows.isEmpty()) {
            throw new IllegalStateException("Failed to find " + keyspace + ".data in system_schema.tables. Default " +
                    "data retention cannot be configured.");
        }
        int defaultTTL = rows.get(0).getInt(0);
        if (defaultTTL != this.defaultTTL) {
            session.execute("alter table " + keyspace + ".data with default_time_to_live = " + this.defaultTTL);
        }
        log.infoDefaultDataRetention(this.defaultTTL);
    }

    private class DataRetentionsLoadedCallback implements FutureCallback<Set<Retention>> {

        private final String tenantId;

        private final MetricType<?> type;

        private final CountDownLatch latch;

        public DataRetentionsLoadedCallback(String tenantId, MetricType<?> type, CountDownLatch latch) {
            this.tenantId = tenantId;
            this.type = type;
            this.latch = latch;
        }

        @Override
        public void onSuccess(Set<Retention> dataRetentionsSet) {
            for (Retention r : dataRetentionsSet) {
                dataRetentions.put(new DataRetentionKey(r.getId()), r.getValue());
            }
            latch.countDown();
        }

        @Override
        public void onFailure(Throwable t) {
            log.warnDataRetentionLoadingFailure(tenantId, type, t);
            latch.countDown();
            // TODO We probably should not let initialization proceed on this error (then change log level to FATAL)
        }
    }

    /**
     * This is a test hook.
     */
    DataAccess getDataAccess() {
        return dataAccess;
    }

    /**
     * This is a test hook.
     */
    public void setDataAccess(DataAccess dataAccess) {
        this.dataAccess = dataAccess;
    }

    public void setConfigurationService(ConfigurationService configurationService) {
        this.configurationService = configurationService;
    }

    public void setDefaultTTL(int defaultTTL) {
        this.defaultTTL = Duration.standardDays(defaultTTL).toStandardSeconds().getSeconds();
    }

    @Override
    public Observable<Void> createTenant(final Tenant tenant, boolean overwrite) {
        return Observable.create(subscriber -> {
            Observable<Void> updates = dataAccess.insertTenant(tenant, overwrite).flatMap(resultSet -> {
                if (!resultSet.wasApplied()) {
                    throw new TenantAlreadyExistsException(tenant.getId());
                }

                Observable<Void> retentionUpdates = Observable.from(tenant.getRetentionSettings().entrySet())
                        .flatMap(entry -> dataAccess.updateRetentionsIndex(tenant.getId(), entry.getKey(),
                                ImmutableMap.of(makeSafe(entry.getKey().getText()), entry.getValue())))
                        .map(rs -> null);

                return retentionUpdates;
            });
            updates.subscribe(resultSet -> {
            }, subscriber::onError, subscriber::onCompleted);
        });
    }

    @Override
    public Observable<Tenant> getTenants() {
        return dataAccess.findAllTenantIds()
                .map(row -> row.getString(0))
                .distinct()
                .flatMap(id ->
                                dataAccess.findTenant(id)
                                        .map(Functions::getTenant)
                                        .switchIfEmpty(Observable.just(new Tenant(id)))
                );
    }

    private List<String> loadTenantIds() {
        Iterable<String> tenantIds = dataAccess.findAllTenantIds()
                .map(row -> row.getString(0))
                .distinct()
                .toBlocking()
                .toIterable();
        return ImmutableList.copyOf(tenantIds);
    }

    @Override
    public Observable<Void> createMetric(Metric<?> metric, boolean overwrite) {
        MetricType<?> metricType = metric.getMetricId().getType();
        if (!metricType.isUserType()) {
            throw new IllegalArgumentException(metric + " cannot be created. " + metricType + " metrics are " +
                    "internally generated metrics and cannot be created by clients.");
        }

        ResultSetFuture future = dataAccess.insertMetricInMetricsIndex(metric, overwrite);

        Observable<ResultSet> indexUpdated = ListenableFutureObservable.from(future, metricsTasks);
        return Observable.create(subscriber -> indexUpdated.subscribe(resultSet -> {
            if (!overwrite && !resultSet.wasApplied()) {
                subscriber.onError(new MetricAlreadyExistsException(metric));
            } else {
                // TODO Need error handling if either of the following updates fail
                // If adding tags/retention fails, then we want to report the error to the
                // client. Updating the retentions_idx table could also fail. We need to
                // report that failure as well.
                //
                // The error handling is the same as it was with Guava futures. That is, if any
                // future fails, we treat the entire client request as a failure. We probably
                // eventually want to implement more fine-grained error handling where we can
                // notify the subscriber of what exactly fails.
                List<Observable<ResultSet>> updates = new ArrayList<>();
                updates.add(dataAccess.insertIntoMetricsTagsIndex(metric, metric.getTags()));

                if (metric.getDataRetention() != null) {
                    updates.add(updateRetentionsIndex(metric));
                }

                Observable.merge(updates).subscribe(new VoidSubscriber<>(subscriber));
            }
        }));
    }

    private Observable<ResultSet> updateRetentionsIndex(Metric<?> metric) {
        return ListenableFutureObservable.from(dataAccess.updateRetentionsIndex(metric), metricsTasks)
                .doOnCompleted(() ->
                        dataRetentions.put(new DataRetentionKey(metric), metric.getDataRetention()));
    }

    @Override
    public Observable<MetricId<?>> findAllMetricIdentifiers() {
        return dataAccess.findAllMetricIdentifiersInData()
                .compose(new MetricIdentifierFromFullDataRowTransformer(defaultTTL))
                .distinct();
    }

    public <T> Observable.Transformer<MetricId<T>, Metric<T>> enrichToMetric() {
        return t -> t
                .flatMap(id -> dataAccess.findMetricInMetricsIndex(id)
                        .compose(new MetricsIndexRowTransformer<>(id.getTenantId(), id.getType(), defaultTTL))
                        .switchIfEmpty(dataAccess.findMetricInData(id) // This only verifies it exists..
                                .compose(new MetricFromDataRowTransformer<>(id.getTenantId(), id.getType(), defaultTTL))));
    }

    @Override
    public <T> Observable<Metric<T>> findMetric(final MetricId<T> id) {
        return Observable.just(id)
                .compose(enrichToMetric());
    }

    @SuppressWarnings("unchecked")
    @Override
    public <T> Observable<Metric<T>> findMetrics(String tenantId, MetricType<T> metricType) {
        Observable<Metric<T>> setFromMetricsIndex = null;
        Observable<Metric<T>> setFromData = dataAccess.findAllMetricIdentifiersInData()
                .doOnError(Throwable::printStackTrace)
                .filter(row -> tenantId.equals(row.getString(0)))
                .compose(new MetricIdentifierFromFullDataRowTransformer(defaultTTL))
                .distinct()
                .map(m -> new Metric(m, DEFAULT_RETENTION));

        if (metricType == null) {
            setFromMetricsIndex = Observable.from(MetricType.userTypes())
                    .map(type -> (MetricType<T>) type)
                    .flatMap(type -> dataAccess.findMetricsInMetricsIndex(tenantId, type)
                            .compose(new MetricsIndexRowTransformer<>(tenantId, type, defaultTTL)));
        } else {
            setFromMetricsIndex = dataAccess.findMetricsInMetricsIndex(tenantId, metricType)
                    .compose(new MetricsIndexRowTransformer<>(tenantId, metricType, defaultTTL));

            setFromData = setFromData.filter(m -> metricType.equals(m.getType()));
        }

        return setFromMetricsIndex.concatWith(setFromData).distinct(Metric::getMetricId);
    }

    @Override
    public <T> Observable<Metric<T>> scanAllMetricIndexes() {
        return dataAccess.scanMetricsInMetricsIndex()
                .flatMap(r -> r)
                .flatMap(r -> {
                    MetricId<T> metricId =
                            new MetricId(r.getString(0), MetricType.fromCode(r.getByte(1)),
                                    r.getString(2));
                    Metric<T> metric = new Metric<>(metricId, r.getMap(3, String.class, String.class));
                    return Observable.just(metric);
                });
    }

//    @SuppressWarnings("unchecked")
    @Override
    public <T> Observable<MetricId<T>> findMetricIdentifiersWithFilters(String tenantId, MetricType<T> metricType,
                                                                       String tags) {
        Timer.Context context = getMetricsTagsQueryLatency().time();
        Observable<MetricId<T>> results;
        try {
            results = expresssionTagQueryParser
                    .parse(tenantId, metricType, tags)
                    .map(tMetric -> tMetric);
        } catch (Exception e1) {
            try {
                Tags parsedSimpleTagQuery = TagsConverter.fromString(tags);
                results = tagQueryParser.findMetricIdentifiersWithFilters(tenantId, metricType, parsedSimpleTagQuery.getTags())
                        .map(m -> (MetricId<T>) m);
            } catch (Exception e2) {
                results = Observable.error(new RuntimeApiError("Unparseable tag query expression.", e2));
            }
        }
        return results.doOnCompleted(context::stop);
    }

    public <T> Func1<MetricId<T>, Boolean> idFilter(String regexp) {
        if(Strings.isNullOrEmpty(regexp)) {
            return tMetric -> true;
        }
        boolean positive = (!regexp.startsWith("!"));
        Pattern p = PatternUtil.filterPattern(regexp);
        return tMetric -> positive == p.matcher(tMetric.getName()).matches();
    }

    @Override
    public Observable<Map<String, Set<String>>> getTagValues(String tenantId, MetricType<?> metricType,
                                    Map<String, String> tagsQueries) {
        return tagQueryParser.getTagValues(tenantId, metricType, tagsQueries);
    }

    @Override
    public Observable<Map<String, String>> getMetricTags(MetricId<?> id) {
        return dataAccess.getMetricTags(id)
                .take(1)
                .map(row -> row.getMap(0, String.class, String.class))
                .defaultIfEmpty(new HashMap<>());
    }

    @Override
    public Observable<String> getTagNames(String tenantId, MetricType<?> metricType, String filter) {
        return tagQueryParser.getTagNames(tenantId, metricType, filter);
    }

    // Adding/deleting metric tags currently involves writing to three tables - data,
    // metrics_idx, and metrics_tags_idx. It might make sense to refactor tag related
    // functionality into a separate class.
    @Override
    public Observable<Void> addTags(Metric<?> metric, Map<String, String> tags) {
        try {
            checkArgument(tags != null, "Missing tags");
            checkArgument(isValidTagMap(tags), "Invalid tags; tag key is required");
        } catch (Exception e) {
            return Observable.error(e);
        }

        return dataAccess.insertIntoMetricsTagsIndex(metric, tags).concatWith(dataAccess.addTags(metric, tags))
                .toList().map(l -> null);
    }

    @Override
    public Observable<Void> deleteTags(Metric<?> metric, Set<String> tags) {
        return getMetricTags(metric.getMetricId())
                .map(loadedTags -> {
                    loadedTags.keySet().retainAll(tags);
                    return loadedTags;
                })
                .flatMap(tagsToDelete -> {
                    return dataAccess.deleteTags(metric, tagsToDelete.keySet()).mergeWith(
                            dataAccess.deleteFromMetricsTagsIndex(metric.getMetricId(), tagsToDelete)).toList()
                            .map(r -> null);
                });
    }

    @Override
    public <T> Observable<Void> addDataPoints(MetricType<T> metricType, Observable<Metric<T>> metrics) {
        checkArgument(metricType != null, "metricType is null");

        return pointsInserter
                .get(metricType)
                .call(metrics
                        .filter(metric -> !metric.getDataPoints().isEmpty()))
                .doOnNext(getDataPointsInserted()::mark)
                .map(i -> null);
    }

    @Override
    public <T> Observable<DataPoint<T>> findDataPoints(MetricId<T> metricId, long start, long end, int limit,
            Order order) {
        return findDataPoints(metricId, start, end, limit, order, defaultPageSize);
    }

    @Override
    public <T> Observable<DataPoint<T>> findDataPoints(MetricId<T> metricId, long start, long end, int limit,
                                                       Order order, int pageSize) {

        Timer.Context context = getRawDataReadLatency().time();
        checkArgument(isValidTimeRange(start, end), "Invalid time range");
        Order safeOrder = (null == order) ? Order.ASC : order;
        MetricType<T> metricType = metricId.getType();
        Func1<Row, DataPoint<T>> mapper = getDataPointMapper(metricType);

        if (metricType == GAUGE || metricType == AVAILABILITY || metricType == COUNTER) {
            long sliceStart = DateTimeService.getTimeSlice(start, Duration.standardHours(2));

            Func1<Row, DataPoint<T>> tempMapper = (Func1<Row, DataPoint<T>>) tempDataPointMappers.get(metricType);

            // Calls mostly deprecated methods..
//            Observable<DataPoint<T>> uncompressedPoints = dataAccess.findOldData(metricId, start, end, limit, safeOrder,
//                    pageSize).map(mapper).doOnError(Throwable::printStackTrace);

            Observable<DataPoint<T>> compressedPoints =
                    dataAccess.findCompressedData(metricId, sliceStart, end, limit, safeOrder)
                            .compose(new DataPointDecompressTransformer(metricType, safeOrder, limit, start, end));

            Observable<DataPoint<T>> tempStoragePoints = dataAccess.findTempData(metricId, start, end, limit,
                    safeOrder, pageSize)
                    .map(tempMapper);

            Comparator<DataPoint<T>> comparator = getDataPointComparator(safeOrder);
            List<Observable<? extends DataPoint<T>>> sources = new ArrayList<>(3);
//            sources.add(uncompressedPoints);
            sources.add(compressedPoints);
            sources.add(tempStoragePoints);

            Observable<DataPoint<T>> dataPoints = SortedMerge.create(sources, comparator, false)
                    .distinctUntilChanged(
                            (tDataPoint, tDataPoint2) -> comparator.compare(tDataPoint, tDataPoint2) == 0);

            if (limit > 0) {
                dataPoints = dataPoints.take(limit);
            }

            return dataPoints;
        }
        Func6<MetricId<T>, Long, Long, Integer, Order, Integer, Observable<Row>> finder =
                getDataPointFinder(metricType);

        Observable<DataPoint<T>> results =
                finder.call(metricId, start, end, limit, safeOrder, pageSize).map(mapper);
        return results.doOnCompleted(context::stop);
    }

    private <T> Comparator<DataPoint<T>> getDataPointComparator(Order safeOrder) {
        Comparator<DataPoint<T>> comparator;

        switch(safeOrder) {
            case ASC:
                comparator = (tDataPoint, t1) -> (int) (tDataPoint.getTimestamp() - t1.getTimestamp());
                break;
            case DESC:
                comparator = (tDataPoint, t1) -> (int) (t1.getTimestamp() - tDataPoint.getTimestamp());
                break;
            default:
                throw new RuntimeException(safeOrder.toString() + " is not correct sorting order");
        }

        return comparator;
    }

    private <T> Observable.Transformer<T, T> applyRetryPolicy() {
        return tObservable -> tObservable
                .retryWhen(observable -> {
                    Observable<Integer> range = Observable.range(1, Integer.MAX_VALUE);
                    Observable<Observable<?>> zipWith = observable.zipWith(range, (t, i) -> {
                        log.debug("Attempt #" + i + " to retry the operation after Cassandra client" +
                                " exception");
                        if (t instanceof DriverException) {
                            return Observable.timer(i, TimeUnit.SECONDS).onBackpressureDrop();
                        } else {
                            return Observable.error(t);
                        }
                    });

                    return Observable.merge(zipWith);
                })
                .doOnError(t -> log.error("Failure while trying to apply compression, skipping block", t))
                .onErrorResumeNext(Observable.empty());
    }

    /**
     * Intended to be used at the startup of the MetricsServiceImpl to ensure we have enough tables for processing
     */
    public void verifyAndCreateTempTables() {
        ZonedDateTime currentBlock = ZonedDateTime.ofInstant(Instant.ofEpochMilli(DateTimeService.now.get().getMillis()), UTC)
                .with(DateTimeService.startOfPreviousEvenHour());

        ZonedDateTime lastStartupBlock = currentBlock.plus(6, ChronoUnit.HOURS);
        verifyAndCreateTempTables(currentBlock, lastStartupBlock).await();
    }

    @Override
    public Completable verifyAndCreateTempTables(ZonedDateTime startTime, ZonedDateTime endTime) {
        Set<Long> timestamps = new HashSet<>();

        while(startTime.isBefore(endTime)) {
            // Table sizes are not configurable at this point
            timestamps.add(startTime.toInstant().toEpochMilli());
            startTime = startTime.plus(2, ChronoUnit.HOURS);
        }

        return Completable.fromObservable(dataAccess.createTempTablesIfNotExists(timestamps));
    }

    @Override
    @SuppressWarnings("unchecked")
    public Completable compressBlock(long jobStartTimeSlice, int pageSize, int maxConcurrency) {
        return Completable.fromObservable(
                Observable.from(dataAccess.findExpiredTables(jobStartTimeSlice))
                        .concatMap(startTimeSlice ->
                                dataAccess.findAllDataFromBucket(startTimeSlice, pageSize, maxConcurrency)
                                        .switchIfEmpty(Observable.empty())
                                        .flatMap(rows -> rows
                                                // Each time the tokenrange changes inside the query, create new window, publish allows
                                                // reuse of the observable in two distinct processing phases
                                                .publish(p -> p.window(
                                                        p.map(Row::getPartitionKeyToken)
                                                                .distinctUntilChanged()))
                                                // ConcatMap so we don't mess the order as that's important in the compression job
                                                .concatMap(o -> {
                                                    // Cache the first key from the observable so we can use it to create a key later
                                                    Observable<Row> sharedRows = o.share();
                                                    Observable<CompressedPointContainer> compressed =
                                                            sharedRows.compose(new TempTableCompressTransformer(startTimeSlice));
                                                    Observable<Row> keyTake = sharedRows.take(1);

                                                    // Merge the first row with the compressed package to be able to write to Cassandra
                                                    return compressed.zipWith(keyTake, (cpc, r) -> {
                                                        MetricId<?> metricId =
                                                                new MetricId(r.getString(0), MetricType.fromCode(r.getByte(1)),
                                                                        r.getString(2));
                                                        return dataAccess.insertCompressedData(metricId, startTimeSlice, cpc,
                                                                getTTL(metricId));
                                                    });
                                                }), maxConcurrency)
                                        .flatMap(rs -> rs)
                                        .doOnCompleted(() -> dataAccess.dropTempTable(startTimeSlice)
                                                .compose(applyRetryPolicy())
                                                .subscribeOn(Schedulers.io())
                                                .subscribe())
                        )
        );
    }

    @Override
    @Deprecated
    @SuppressWarnings("unchecked")
    public Completable compressBlock(Observable<? extends MetricId<?>> metrics, long startTimeSlice,
            long endTimeSlice, int pageSize) {

        return Completable.fromObservable(metrics
                .compose(applyRetryPolicy())
                .concatMap(metricId -> findDataPoints(metricId, startTimeSlice, endTimeSlice, 0, ASC, pageSize)
                        .compose(applyRetryPolicy())
                        .compose(new DataPointCompressTransformer(metricId.getType(), startTimeSlice))
                        .concatMap(cpc -> dataAccess.deleteAndInsertCompressedGauge(metricId, startTimeSlice,
                                (CompressedPointContainer) cpc, startTimeSlice, endTimeSlice, getTTL(metricId))
                                .compose(applyRetryPolicy()))));
    }

    @Override
    public <T> Observable<NamedDataPoint<T>> findDataPoints(List<MetricId<T>> metricIds, long start,
            long end, int limit, Order order) {
        return Observable.from(metricIds)
                .concatMap(id -> findDataPoints(id, start, end, limit, order)
                        .map(dataPoint -> new NamedDataPoint<>(id.getName(), dataPoint)));
    }

    @Override
    public <T> Observable<NamedDataPoint<T>> findDataPoints(String tenantId, MetricType<T> metricType,
            String tagFilters, long start, long end, int limit, Order order) {
        return findMetricIdentifiersWithFilters(tenantId, metricType, tagFilters)
                .concatMap(id -> findDataPoints(id, start, end, limit, order)
                        .map(dataPoint -> new NamedDataPoint<>(id.getName(), dataPoint)));
    }

    @SuppressWarnings("unchecked")
    private <T> Func6<MetricId<T>, Long, Long, Integer, Order, Integer, Observable<Row>> getDataPointFinder(
            MetricType<T> metricType) {
        Func6<MetricId<T>, Long, Long, Integer, Order, Integer, Observable<Row>> finder;
        finder = (Func6<MetricId<T>, Long, Long, Integer, Order, Integer, Observable<Row>>) dataPointFinders
                .get(metricType);
        if (finder == null) {
            throw new UnsupportedOperationException(metricType.getText());
        }
        return finder;
    }

    @SuppressWarnings("unchecked")
    private <T> Func1<Row, DataPoint<T>> getDataPointMapper(MetricType<T> metricType) {
        Func1<Row, DataPoint<T>> mapper = (Func1<Row, DataPoint<T>>) dataPointMappers.get(metricType);
        if (mapper == null) {
            throw new UnsupportedOperationException(metricType.getText());
        }
        return mapper;
    }

    @Override
    public Observable<DataPoint<Double>> findRateData(MetricId<? extends Number> id, long start, long end, int limit,
                                                      Order order) {
        checkArgument(isValidTimeRange(start, end), "Invalid time range");
        checkArgument(id.getType() == COUNTER || id.getType() == GAUGE, "Unsupported metric type: %s", id.getType());
        // We can't set the limit here, because some pairs can be discarded (counter resets)
        // But since the loading is reactive, we're not going to fetch more pages than needed (see #take at the end)
        Observable<DataPoint<Double>> dataPoints = this.findDataPoints(id, start, end, 0, order)
                .buffer(2, 1) // emit previous/next pairs
                // adapt pair to the order of traversal
                .map(l -> order == ASC ? l : Lists.reverse(l))
                // Drop the last buffer
                .filter(l -> l.size() == 2)
                // Filter out counter resets
                .filter(l -> id.getType() != COUNTER
                        || l.get(1).getValue().longValue() >= l.get(0).getValue().longValue())
                .map(l -> {
                    DataPoint<? extends Number> point1 = l.get(0);
                    DataPoint<? extends Number> point2 = l.get(1);
                    long timestamp = point2.getTimestamp();
                    double value_diff = point2.getValue().doubleValue() - point1.getValue().doubleValue();
                    double time_diff = point2.getTimestamp() - point1.getTimestamp();
                    double rate = 60_000D * value_diff / time_diff;
                    return new DataPoint<>(timestamp, rate);
                });
        return limit <= 0 ? dataPoints : dataPoints.take(limit);
    }

    @Override
    public <T extends Number> Observable<NamedDataPoint<Double>> findRateData(List<MetricId<T>> ids, long start,
                                                                     long end, int limit, Order order) {
        return Observable.from(ids).concatMap(id -> findRateData(id, start, end, limit, order)
                .map(dataPoint -> new NamedDataPoint<>(id.getName(), dataPoint)));
    }

    @Override
    public Observable<List<NumericBucketPoint>> findRateStats(MetricId<? extends Number> id, BucketConfig bucketConfig,
                                                              List<Percentile> percentiles) {
        TimeRange timeRange = bucketConfig.getTimeRange();
        checkArgument(isValidTimeRange(timeRange.getStart(), timeRange.getEnd()), "Invalid time range");
        return findRateData(id, timeRange.getStart(), timeRange.getEnd(), 0, ASC)
                .compose(new NumericBucketPointTransformer(bucketConfig.getBuckets(), percentiles));
    }

    @SuppressWarnings("unchecked")
    @Override
    public <T> Observable<T> findGaugeData(MetricId<Double> id, long start, long end,
                                           Func1<Observable<DataPoint<Double>>, Observable<T>>... funcs) {
        Observable<DataPoint<Double>> dataCache = this.findDataPoints(id, start, end, 0, Order.DESC).cache();
        return Observable.from(funcs).flatMap(fn -> fn.call(dataCache));
    }

    @Override
    public Observable<List<NumericBucketPoint>> findGaugeStats(MetricId<Double> metricId, BucketConfig bucketConfig,
                List<Percentile> percentiles) {
        TimeRange timeRange = bucketConfig.getTimeRange();
        checkArgument(isValidTimeRange(timeRange.getStart(), timeRange.getEnd()), "Invalid time range");
        return findDataPoints(metricId, timeRange.getStart(), timeRange.getEnd(), 0, Order.DESC)
                .compose(new NumericBucketPointTransformer(bucketConfig.getBuckets(), percentiles));
    }

    @Override
    public Observable<Map<String, TaggedBucketPoint>> findGaugeStats(MetricId<Double> metricId,
            Map<String, String> tags, long start, long end, List<Percentile> percentiles) {
        return findDataPoints(metricId, start, end, 0, Order.DESC)
                .compose(new TaggedBucketPointTransformer(tags, percentiles));
    }

    @Override
    public <T extends Number> Observable<List<NumericBucketPoint>> findNumericStats(
            List<MetricId<T>> metrics, long start, long end, Buckets buckets, List<Percentile>
            percentiles, boolean stacked, boolean isRate) {

        // TODO Stats needs fixing to understand compressed values also..
        checkArgument(isValidTimeRange(start, end), "Invalid time range");
        if (!stacked) {
            if (!isRate) {
                return Observable.from(metrics)
                        .flatMap(metricId -> findDataPoints(metricId, start, end, 0, Order.DESC))
                        .compose(new NumericBucketPointTransformer(buckets, percentiles));
            } else {
                return Observable.from(metrics)
                        .flatMap(metricId -> findRateData(metricId, start, end, 0, ASC))
                        .compose(new NumericBucketPointTransformer(buckets, percentiles));
            }
        } else {
            Observable<Observable<NumericBucketPoint>> individualStats;
            if (!isRate) {
                individualStats = Observable.from(metrics).map(metricId -> {
                    return findDataPoints(metricId, start, end, 0, Order.DESC)
                            .compose(new NumericBucketPointTransformer(buckets, percentiles))
                            .flatMap(Observable::from);
                });
            } else {
                individualStats = Observable.from(metrics).map(metricId -> {
                    return findRateData(metricId, start, end, 0, ASC)
                            .compose(new NumericBucketPointTransformer(buckets, percentiles))
                            .flatMap(Observable::from);
                });
            }

            return Observable.merge(individualStats)
                    .groupBy(BucketPoint::getStart)
                    .flatMap(group -> group.collect(SumNumericBucketPointCollector::new,
                            SumNumericBucketPointCollector::increment))
                    .map(SumNumericBucketPointCollector::toBucketPoint)
                    .toMap(NumericBucketPoint::getStart)
                    .map(pointMap -> NumericBucketPoint.toList(pointMap, buckets));
        }
    }

    @Override
    public Observable<DataPoint<AvailabilityType>> findAvailabilityData(MetricId<AvailabilityType> id, long start,
            long end, boolean distinct, int limit, Order order) {
        checkArgument(isValidTimeRange(start, end), "Invalid time range");
        if (distinct) {
            Observable<DataPoint<AvailabilityType>> availabilityData = findDataPoints(id, start, end, 0, order)
                    .distinctUntilChanged(DataPoint::getValue);
            if (limit <= 0) {
                return availabilityData;
            } else {
                return availabilityData.limit(limit);
            }
        } else {
            return findDataPoints(id, start, end, limit, order);
        }
    }

    @Override
    public Observable<List<AvailabilityBucketPoint>> findAvailabilityStats(MetricId<AvailabilityType> metricId,
            long start, long end, Buckets buckets) {
        checkArgument(isValidTimeRange(start, end), "Invalid time range");
        return this.findDataPoints(metricId, start, end, 0, ASC)
                .groupBy(dataPoint -> buckets.getIndex(dataPoint.getTimestamp()))
                .flatMap(group -> group.collect(() -> new AvailabilityDataPointCollector(buckets, group.getKey()),
                        AvailabilityDataPointCollector::increment))
                .map(AvailabilityDataPointCollector::toBucketPoint)
                .toMap(AvailabilityBucketPoint::getStart)
                .map(pointMap -> AvailabilityBucketPoint.toList(pointMap, buckets));
    }

    @Override
    public Observable<DataPoint<String>> findStringData(MetricId<String> id, long start, long end, boolean distinct,
            int limit, Order order) {
        checkArgument(isValidTimeRange(start, end));
        if (distinct) {
            return findDataPoints(id, start, end, limit, order).distinctUntilChanged(DataPoint::getValue);
        } else {
            return findDataPoints(id, start, end, limit, order);
        }
    }

    @Override
    public Observable<List<NumericBucketPoint>> findCounterStats(MetricId<Long> id, BucketConfig bucketConfig, List<Percentile>
            percentiles) {
        TimeRange timeRange = bucketConfig.getTimeRange();
        checkArgument(isValidTimeRange(timeRange.getStart(), timeRange.getEnd()), "Invalid time range");
        return findDataPoints(id, timeRange.getStart(), timeRange.getEnd(), 0, ASC)
                .doOnError(Throwable::printStackTrace)
                .compose(new NumericBucketPointTransformer(bucketConfig.getBuckets(), percentiles));
    }

    @Override
    public Observable<Map<String, TaggedBucketPoint>> findCounterStats(MetricId<Long> metricId,
            Map<String, String> tags, long start, long end, List<Percentile> percentiles) {
        return findDataPoints(metricId, start, end, 0, ASC)
                .compose(new TaggedBucketPointTransformer(tags, percentiles));
    }

    @Override
    public Observable<List<long[]>> getPeriods(MetricId<Double> id, Predicate<Double> predicate, long start,
            long end) {
        checkArgument(isValidTimeRange(start, end), "Invalid time range");
        return findDataPoints(id, start, end, 0, ASC)
//                .map(Functions::getGaugeDataPoint)
                .toList().map(data -> {
                    List<long[]> periods = new ArrayList<>(data.size());
                    long[] period = null;
                    DataPoint<Double> previous = null;
                    for (DataPoint<Double> d : data) {
                        if (predicate.test(d.getValue())) {
                            if (period == null) {
                                period = new long[2];
                                period[0] = d.getTimestamp();
                            }
                            previous = d;
                        } else if (period != null) {
                            period[1] = previous.getTimestamp();
                            periods.add(period);
                            period = null;
                            previous = null;
                        }
                    }
                    if (period != null) {
                        period[1] = previous.getTimestamp();
                        periods.add(period);
                    }
                    return periods;
                });
    }

    private int getTTL(MetricId<?> metricId) {
        Integer ttl = dataRetentions.get(new DataRetentionKey(metricId));
        if (ttl == null) {
            ttl = dataRetentions.getOrDefault(new DataRetentionKey(metricId.getTenantId(), metricId.getType()),
                    defaultTTL);
        } else {
            ttl = Duration.standardDays(ttl).toStandardSeconds().getSeconds();
        }
        return ttl;
    }

    public void shutdown() {
        metricsTasks.shutdown();
        unloadDataRetentions();
//        dataAccess.shutdown();
    }

    private <T> T time(Timer timer, Callable<T> callable) {
        try {
            // TODO Should this method always return an observable?
            // If so, than we should return Observable.error(e) in the catch block
            return timer.time(callable);
        } catch (Exception e) {
            throw new RuntimeException("There was an error during a timed event", e);
        }
    }

    @Override
    public <T> Observable<Void> deleteMetric(MetricId<T> id) {
        //NOTE: compressed data is not deleted due to the using TWCS compaction strategy
        //      for the compressed data table.
        Observable<Void> result = dataAccess.getMetricTags(id)
                .map(row -> row.getMap(0, String.class, String.class))
                .defaultIfEmpty(new HashMap<>())
                .flatMap(map -> dataAccess.deleteFromMetricsTagsIndex(id, map))
                .map(r -> null);
        Observable<Void> indexes = Observable.merge(
                dataAccess.deleteMetricFromMetricsIndex(id),
                dataAccess.deleteMetricData(id),
                dataAccess.deleteMetricFromRetentionIndex(id))
                .map(r -> null);

        return result.concatWith(indexes);
    }

}


/*
 * Copyright 2014-2017 Red Hat, Inc. and/or its affiliates
 * and other contributors as indicated by the @author tags.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.hawkular.openshift.auth.org.hawkular.openshift.namespace;

import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

import javax.net.ssl.HttpsURLConnection;

import org.jboss.logging.Logger;

import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;

/**
 *
 */
public class NamespaceListener {

    private static final Logger log = Logger.getLogger(NamespaceListener.class);

    // The authentication token used by the filter to connect to the OpenShift Master API
    private final String token;

    // Maps the namespace name to the namespace id
    private final Map<String, String> namespaces;

    private boolean run = true;

    /**
     *
     * @param token The token to be used by the
     */
    public NamespaceListener(String token) {
        log.debug("Creating NamespaceListener");
        this.token = token;

        namespaces = new ConcurrentHashMap<>();
        log.debug("Namespace listener starting.");
        new Thread(new Listener()).start();
    }

    public void stop() {
        log.debug("Namespace listener shutting down.");
        run = false;
    }

    /**
     * Returns the namespace id for a particular namespace name
     * @param namespaceName The name of the namespace
     * @return The id of the namespace or null if the namespace does not exist
     */
    public String getNamespaceId(String namespaceName) {
        return namespaces.computeIfAbsent(namespaceName, n -> getProjectId(namespaceName, token));
    }

    private static String getProjectId(String projectName, String token) {
        String projectId = null;

        HttpsURLConnection connection = null;
        InputStream inputStream = null;
        JsonParser jsonParser = null;

        try {
            String path = "api/v1/namespaces/" + projectName;
            URL url = new URL(NamespaceHandler.KUBERNETES_MASTER_URL + "/" + path);

            connection = (HttpsURLConnection) url.openConnection();
            connection.setRequestMethod("GET");

            connection.setRequestProperty("Accept", "application/json");
            connection.setRequestProperty("Authorization", "Bearer " + token);

            connection.connect();
            int responseCode = connection.getResponseCode();
            if (responseCode == 200) {

                JsonFactory jsonFactory = new JsonFactory();
                jsonFactory.setCodec(new ObjectMapper());

                inputStream = connection.getInputStream();
                jsonParser = jsonFactory.createParser(inputStream);
                JsonNode jsonNode = jsonParser.readValueAsTree();
                if (jsonNode != null) {
                    JsonNode metadata = jsonNode.get("metadata");
                    projectId = metadata.get("uid").asText();
                }

            } else {
                log.warnf("Error getting project metadata. Code %s: %s", responseCode, connection.getResponseMessage());
            }

        } catch (IOException io) {
            log.error(io);
        } finally {
            try {
                if (jsonParser != null) {
                    jsonParser.close();
                }
            }
            catch (IOException io) {
                log.error(io);
            }

            try {
                if (inputStream != null) {
                    inputStream.close();
                }
            } catch (IOException io) {
                log.error(io);
            }

            if (connection != null) {
                connection.disconnect();
            }
        }

        return projectId;
    }

    private class Listener implements Runnable {

        @Override
        public void run() {
            while(run && !Thread.currentThread().isInterrupted()) {
                try {
                    String path = "api/v1/namespaces";
                    URL url = new URL(NamespaceHandler.KUBERNETES_MASTER_URL + "/" + path + "?watch=true");

                    HttpsURLConnection connection = (HttpsURLConnection) url.openConnection();
                    connection.setRequestMethod("GET");

                    connection.setRequestProperty("Accept", "application/json");
                    connection.setRequestProperty("Authorization", "Bearer " + token);

                    connection.setConnectTimeout(5000);

                    // Since the connection is left open for new events, set to 0 which indicates an infinite timeout
                    // Note that the server will eventually close this connection, so we will need to reconnect when this happens
                    connection.setReadTimeout(0);

                    connection.connect();
                    int responseCode = connection.getResponseCode();

                    if (responseCode == 200) {

                        JsonFactory jsonFactory = new JsonFactory();
                        jsonFactory.setCodec(new ObjectMapper());

                        InputStream inputStream = connection.getInputStream();
                        JsonParser jsonParser = jsonFactory.createParser(inputStream);

                        // this will fetch all the available tenants and will then wait until new tenant events occur
                        JsonNode jsonNode = jsonParser.readValueAsTree();

                        // When the connection is closed, the response is null
                        while (jsonNode != null) {
                            JsonNode metaData = jsonNode.get("object").get("metadata");
                            String projectName = metaData.get("name").asText();
                            String projectID = metaData.get("uid").asText();

                            String action = jsonNode.get("type").asText();

                            if (action.equals("ADDED")) {
                                log.debugf("Project %s [%s] has been added.", projectName, projectID);
                                namespaces.put(projectName, projectID);
                            } else if (action.equals("DELETED")) {
                                log.debugf("Project %s [%s] has been removed.", projectName, projectID);
                                namespaces.remove(projectName, projectID);
                            }

                            // this will read the next available tenant if available or wait until the next available
                            // tenant event occurs
                            jsonNode = jsonParser.readValueAsTree();
                        }

                        jsonParser.close();
                        inputStream.close();
                        namespaces.clear();

                    } else {
                        log.warnf("Error getting metadata. Response code %s", responseCode);
                        try {
                            // try again after a second
                            Thread.sleep(1000);
                        } catch (InterruptedException e) {
                            log.error(e);
                            stop();
                        }
                    }

                } catch (Exception e) {
                    log.error(e);
                    try {
                        // try again after a second
                        Thread.sleep(1000);
                    } catch (InterruptedException ie) {
                        log.error(ie);
                        stop();
                    }
                }
            }
        }
    }
}


/*
 * Copyright 2014-2018 Red Hat, Inc. and/or its affiliates
 * and other contributors as indicated by the @author tags.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.hawkular.openshift.auth;

import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;

import static org.hawkular.openshift.auth.Utils.endExchange;

import static io.undertow.util.Headers.ACCEPT;
import static io.undertow.util.Headers.AUTHORIZATION;
import static io.undertow.util.Headers.CONTENT_LENGTH;
import static io.undertow.util.Headers.CONTENT_TYPE;
import static io.undertow.util.Headers.HOST;
import static io.undertow.util.Methods.DELETE;
import static io.undertow.util.Methods.GET;
import static io.undertow.util.Methods.POST;
import static io.undertow.util.Methods.PUT;
import static io.undertow.util.StatusCodes.BAD_REQUEST;
import static io.undertow.util.StatusCodes.CREATED;
import static io.undertow.util.StatusCodes.FORBIDDEN;
import static io.undertow.util.StatusCodes.INTERNAL_SERVER_ERROR;

import java.io.Closeable;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.nio.channels.UnresolvedAddressException;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Queue;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
import java.util.function.Consumer;
import java.util.regex.Pattern;

import org.hawkular.metrics.api.jaxrs.util.MetricRegistryProvider;
import org.jboss.logging.Logger;
import org.xnio.BufferAllocator;
import org.xnio.ByteBufferSlicePool;
import org.xnio.IoUtils;
import org.xnio.OptionMap;
import org.xnio.Xnio;
import org.xnio.XnioExecutor;
import org.xnio.XnioIoThread;
import org.xnio.ssl.XnioSsl;

import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.util.concurrent.Uninterruptibles;

import io.undertow.Undertow;
import io.undertow.client.ClientCallback;
import io.undertow.client.ClientConnection;
import io.undertow.client.ClientExchange;
import io.undertow.client.ClientRequest;
import io.undertow.client.UndertowClient;
import io.undertow.connector.ByteBufferPool;
import io.undertow.protocols.ssl.UndertowXnioSsl;
import io.undertow.server.HttpHandler;
import io.undertow.server.HttpServerExchange;
import io.undertow.server.XnioByteBufferPool;
import io.undertow.util.AttachmentKey;
import io.undertow.util.HttpString;
import io.undertow.util.StatusCodes;
import io.undertow.util.StringReadChannelListener;
import io.undertow.util.StringWriteChannelListener;

/**
 * An authentication/authorization strategy which relies on Openshift's OAuth server, using a non-blocking HTTP client.
 *
 * @author Thomas Segismont
 */
class TokenAuthenticator implements Authenticator {
    private static final Logger log = Logger.getLogger(TokenAuthenticator.class);

    private static final AttachmentKey<AuthContext> AUTH_CONTEXT_KEY = AttachmentKey.create(AuthContext.class);

    private static final HttpString HAWKULAR_TENANT = new HttpString("Hawkular-Tenant");
    static final String BEARER_PREFIX = "Bearer ";
    private static final String MISSING_HEADERS_MSG =
            "The '" + AUTHORIZATION + "' and '" + HAWKULAR_TENANT + "' headers are required";
    private static final String UNAUTHORIZED_USER_EDIT_MSG =
            "Users are not authorized to perform edits on metric data";

    //The resource to check against for security purposes. For this version we are allowing Metrics based on a users
    //access to the pods in in a particular project.
    private static final String RESOURCE = "pods";
    private static final String KIND = "SubjectAccessReview";

    private static final Map<HttpString, String> VERBS;
    private static final String VERBS_DEFAULT;

    static {
        Map<HttpString, String> verbs = new HashMap<>();
        verbs.put(GET, "list");
        verbs.put(PUT, "update");
        verbs.put(POST, "update");
        verbs.put(DELETE, "update");
        verbs.put(new HttpString("PATCH"), "update");
        VERBS = Collections.unmodifiableMap(verbs);
        // default to 'list' verb (which is the lowest level permission)
        VERBS_DEFAULT = VERBS.get(GET);
    }

    private static final String KUBERNETES_MASTER_URL_SYSPROP = "KUBERNETES_MASTER_URL";
    private static final String USER_WRITE_ACCESS_SYSPROP = "USER_WRITE_ACCESS";
    private static final String KUBERNETES_MASTER_URL_DEFAULT = "https://kubernetes.default.svc.cluster.local";
    private static final String KUBERNETES_MASTER_URL =
            System.getProperty(KUBERNETES_MASTER_URL_SYSPROP, KUBERNETES_MASTER_URL_DEFAULT);
    private static final String USER_WRITE_ACCESS = System.getProperty(USER_WRITE_ACCESS_SYSPROP, "false");
    private static final String ACCESS_URI = "/oapi/v1/subjectaccessreviews";
    private static final int MAX_CONNECTIONS_PER_THREAD = 20;
    private static final long CONNECTION_WAIT_TIMEOUT = MILLISECONDS.convert(30, SECONDS);
    private static final String TIMEDOUT_WAITING_CONNECTION = "Could not acquire a Kubernetes client connection";
    private static final long CONNECTION_TTL = MILLISECONDS.convert(10, SECONDS);
    private static final int MAX_RETRY = 5;
    private static final int MAX_PENDING = 32 * 1024;
    private static final String TOO_MANY_PENDING_REQUESTS = "Too many pending requests";
    private static final String CLIENT_REQUEST_FAILURE = "Kubernetes client request failure";

    private static final String METRICS_SCOPE = "OpenShift";
    private static final String METRICS_TYPE = "Security";

    private final HttpHandler containerHandler;
    private final ObjectMapper objectMapper;
    private final URI kubernetesMasterUri;
    private final ConcurrentMap<XnioIoThread, ConnectionPool> connectionPools;
    private final ConnectionFactory connectionFactory;
    private final Timer authLatency;
    private final Timer apiLatency;

    private final Pattern postQuery;
    private final String resourceName;
    private final String componentName;

    TokenAuthenticator(HttpHandler containerHandler, String componentName, String resourceName, Pattern postQuery) {
        this.containerHandler = containerHandler;
        this.resourceName = resourceName;
        this.componentName = componentName;

        this.postQuery = postQuery;
        objectMapper = new ObjectMapper();
        try {
            kubernetesMasterUri = new URI(KUBERNETES_MASTER_URL);
        } catch (URISyntaxException e) {
            throw new RuntimeException(e);
        }
        connectionPools = new ConcurrentHashMap<>(Runtime.getRuntime().availableProcessors(), 1);
        connectionFactory = new ConnectionFactory(kubernetesMasterUri);
        MetricRegistry metrics = MetricRegistryProvider.INSTANCE.getMetricRegistry();
        // Note: HawkularMetricRegistry.registerMetaData cannot be used here because the registry is not yet
        // fully initialized. Calling registerMetaData will result in an NPE.
        authLatency = metrics.timer("openshift-oauth-latency");
        apiLatency = metrics.timer("openshift-oauth-kubernetes-response-time");
    }

    @Override
    public void handleRequest(HttpServerExchange serverExchange) throws Exception {
        AuthContext context = AuthContext.initialize(serverExchange);
        serverExchange.putAttachment(AUTH_CONTEXT_KEY, context);
        // Make sure the exchange attachment is removed in the end
        serverExchange.addExchangeCompleteListener((exchange, nextListener) -> {
            exchange.removeAttachment(AUTH_CONTEXT_KEY);
            nextListener.proceed();
        });
        if (context.isMissingTenantHeader()) {
            endExchange(serverExchange, BAD_REQUEST, MISSING_HEADERS_MSG);
            return;
        }

        // Marks the request as dispatched. If we don't do this, the exchange will be terminated by the container when
        // this method returns, but we need to wait for Kubernetes' master response.
        serverExchange.dispatch();
        XnioIoThread ioThread = serverExchange.getIoThread();
        ConnectionPool connectionPool = connectionPools.computeIfAbsent(ioThread, t -> new ConnectionPool(connectionFactory, componentName));
        PooledConnectionWaiter waiter = createWaiter(serverExchange);
        if (!connectionPool.offer(waiter)) {
            endExchange(serverExchange, INTERNAL_SERVER_ERROR, TOO_MANY_PENDING_REQUESTS);
        }
    }

    //Returns if the request is a query request, eg to perform a READ
    private boolean isQuery(HttpServerExchange serverExchange) {
        if (serverExchange.getRequestMethod().toString().equalsIgnoreCase("GET") || serverExchange.getRequestMethod().toString().equalsIgnoreCase("HEAD")) {
            // all GET requests are considered queries
            return true;
        } else if (serverExchange.getRequestMethod().toString().equalsIgnoreCase("POST")) {
            // some POST requests may be queries we need to check.
            if (postQuery != null && postQuery.matcher(serverExchange.getRelativePath()).find()) {
                return true;
            } else {
                return false;
            }
        } else {
            return false;
        }
    }

    private PooledConnectionWaiter createWaiter(HttpServerExchange serverExchange) {
        Consumer<PooledConnection> onGet = connection -> sendAuthenticationRequest(serverExchange, connection);
        Runnable onTimeout = () -> onPooledConnectionWaitTimeout(serverExchange);
        return new PooledConnectionWaiter(onGet, onTimeout);
    }

    /**
     * Executed when a pooled connection is acquired.
     */
    private void sendAuthenticationRequest(HttpServerExchange serverExchange, PooledConnection connection) {
        AuthContext context = serverExchange.getAttachment(AUTH_CONTEXT_KEY);
        String verb = getVerb(serverExchange);

        String resource;
        // if we are not dealing with a query
        if (!isQuery(serverExchange)) {
            // is USER_WRITE_ACCESS is disabled, then use the legacy check.
            // Otherwise check using the actual resource (eg 'hawkular-metrics', 'hawkular-alerts', etc)
            if (USER_WRITE_ACCESS.equalsIgnoreCase("true")) {
                resource = RESOURCE;
            } else {
                resource= resourceName;
            }
        } else {
            resource = RESOURCE;
        }

        context.subjectAccessReview = generateSubjectAccessReview(context.tenant, verb, resource);
        ClientRequest request = buildClientRequest(context);
        context.clientRequestStarting();
        connection.sendRequest(request, new RequestReadyCallback(serverExchange, connection));
    }

    /**
     * Executed if no poooled connection was made available in a timely manner.
     */
    private void onPooledConnectionWaitTimeout(HttpServerExchange serverExchange) {
        endExchange(serverExchange, INTERNAL_SERVER_ERROR, TIMEDOUT_WAITING_CONNECTION);
    }

    /**
     * Determine the verb we should apply based on the HTTP method being requested.
     *
     * @return the verb to use
     */
    private String getVerb(HttpServerExchange serverExchange) {
        // if its a query type verb, then treat as a GET type call.
        if (isQuery(serverExchange)) {
            return VERBS.get(GET);
        } else {
            String verb = VERBS.get(serverExchange.getRequestMethod());
            if (verb == null) {
                log.debugf("Unhandled http method '%s'. Checking for read access.", serverExchange.getRequestMethod());
                verb = VERBS_DEFAULT;
            }
            return verb;
        }
    }

    /**
     * Generates a SubjectAccessReview object used to request if a user has a certain permission or not.
     *
     * @param namespace the namespace
     * @param verb      the requested permission
     *
     * @return JSON text representation of the SubjectAccessReview object
     */
    private String generateSubjectAccessReview(String namespace, String verb, String resource) {
        ObjectNode objectNode = objectMapper.createObjectNode();
        objectNode.put("apiVersion", "v1");
        objectNode.put("kind", KIND);
        objectNode.put("resource", resource);
        objectNode.put("verb", verb);
        objectNode.put("namespace", namespace);
        return objectNode.toString();
    }

    private ClientRequest buildClientRequest(AuthContext context) {
        ClientRequest request = new ClientRequest().setMethod(POST).setPath(ACCESS_URI);
        String host = kubernetesMasterUri.getHost();
        int port = kubernetesMasterUri.getPort();
        String hostHeader = (port == -1) ? host : (host + ":" + port);
        request.getRequestHeaders()
                .add(HOST, hostHeader)
                .add(ACCEPT, "application/json")
                .add(CONTENT_TYPE, "application/json")
                .add(AUTHORIZATION, context.authorizationHeader)
                .add(CONTENT_LENGTH, context.subjectAccessReview.length());
        return request;
    }

    /**
     * Called when the Kubernetes master server reponse has been inspected.
     */
    private void onRequestResult(HttpServerExchange serverExchange, PooledConnection connection, boolean allowed) {
        connectionPools.get(serverExchange.getIoThread()).release(connection);
        // Remove attachment early to make it eligible for GC
        AuthContext context = serverExchange.removeAttachment(AUTH_CONTEXT_KEY);
        apiLatency.update(context.getClientResponseTime(), NANOSECONDS);
        authLatency.update(context.getLatency(), NANOSECONDS);
        if (allowed) {
            serverExchange.dispatch(containerHandler);
        } else {
            endExchange(serverExchange, FORBIDDEN);
        }
    }

    /**
     * Called if an exception occurs at any stage in the process.
     */
    private void onRequestFailure(HttpServerExchange serverExchange, PooledConnection connection, IOException e,
                                  boolean retry) {
        log.debug("Client request failure", e);
        IoUtils.safeClose(connection);
        ConnectionPool connectionPool = connectionPools.get(serverExchange.getIoThread());
        connectionPool.release(connection);
        AuthContext context = serverExchange.getAttachment(AUTH_CONTEXT_KEY);
        if (context.retries < MAX_RETRY && retry) {
            context.retries++;
            PooledConnectionWaiter waiter = createWaiter(serverExchange);
            if (!connectionPool.offer(waiter)) {
                endExchange(serverExchange, INTERNAL_SERVER_ERROR, TOO_MANY_PENDING_REQUESTS);
            }
        } else {
            endExchange(serverExchange, INTERNAL_SERVER_ERROR, CLIENT_REQUEST_FAILURE);
        }
    }

    @Override
    public void stop() {
        Set<Entry<XnioIoThread, ConnectionPool>> entries = connectionPools.entrySet();
        CountDownLatch latch = new CountDownLatch(entries.size());
        entries.forEach(entry -> {
            // Connection pool is not thread safe and #stop must be called on the corresponding io thread
            entry.getKey().execute(() -> entry.getValue().stop(latch::countDown));
        });
        Uninterruptibles.awaitUninterruptibly(latch, 5, SECONDS);
        connectionFactory.close();
    }

    /**
     * Contextual data needed to perform the authentication process. An instance of this class is attached to the
     * {@link HttpServerExchange}. That makes it easy to retrieve information without passing a lot of arguments
     * around.
     */
    private static final class AuthContext {
        private long creation;
        private String authorizationHeader;
        private String tenant;
        private String subjectAccessReview;
        private int retries;
        private long requestStart;
        private long requestStop;

        private static AuthContext initialize(HttpServerExchange serverExchange) {
            AuthContext context = new AuthContext();
            context.creation = System.nanoTime();
            context.authorizationHeader = serverExchange.getRequestHeaders().getFirst(AUTHORIZATION);
            context.tenant = serverExchange.getRequestHeaders().getFirst(HAWKULAR_TENANT);
            return context;
        }

        private boolean isMissingTenantHeader() {
            return tenant == null;
        }

        private void clientRequestStarting() {
            requestStart = System.nanoTime();
        }

        private void clientResponseReceived() {
            requestStop = System.nanoTime();
        }

        private long getClientResponseTime() {
            return requestStop - requestStart;
        }

        private long getLatency() {
            return requestStop - creation;
        }
    }

    /**
     * Callback invoked when the client exchange is ready for sending data.
     */
    private class RequestReadyCallback implements ClientCallback<ClientExchange> {
        private final HttpServerExchange serverExchange;
        private final PooledConnection connection;

        private RequestReadyCallback(HttpServerExchange serverExchange, PooledConnection connection) {
            this.serverExchange = serverExchange;
            this.connection = connection;
        }

        @Override
        public void completed(ClientExchange clientExchange) {
            clientExchange.setResponseListener(new ResponseListener(serverExchange, connection));
            writeBody(clientExchange);
        }

        private void writeBody(ClientExchange clientExchange) {
            AuthContext context = serverExchange.getAttachment(AUTH_CONTEXT_KEY);
            StringWriteChannelListener writeChannelListener;
            writeChannelListener = new StringWriteChannelListener(context.subjectAccessReview);
            writeChannelListener.setup(clientExchange.getRequestChannel());
        }

        @Override
        public void failed(IOException e) {
            onRequestFailure(serverExchange, connection, e, true);
        }
    }

    /**
     * Callback invoked when the server replied.
     */
    private class ResponseListener implements ClientCallback<ClientExchange> {
        private final HttpServerExchange serverExchange;
        private final PooledConnection connection;

        private ResponseListener(HttpServerExchange serverExchange, PooledConnection connection) {
            this.serverExchange = serverExchange;
            this.connection = connection;
        }

        @Override
        public void completed(ClientExchange clientExchange) {
            StringReadChannelListener readChannelListener;
            readChannelListener = new ResponseBodyListener(serverExchange, connection, clientExchange);
            readChannelListener.setup(clientExchange.getResponseChannel());
        }

        @Override
        public void failed(IOException e) {
            onRequestFailure(serverExchange, connection, e, true);
        }
    }

    /**
     * Callback invoked when the server reponse body has been fully read.
     */
    private class ResponseBodyListener extends StringReadChannelListener {
        private final HttpServerExchange serverExchange;
        private final PooledConnection connection;
        private final ClientExchange clientExchange;

        private ResponseBodyListener(HttpServerExchange serverExchange, PooledConnection connection,
                ClientExchange clientExchange) {
            super(clientExchange.getConnection().getBufferPool());
            this.serverExchange = serverExchange;
            this.connection = connection;
            this.clientExchange = clientExchange;
        }

        @Override
        protected void stringDone(String body) {
            AuthContext context = serverExchange.getAttachment(AUTH_CONTEXT_KEY);
            context.clientResponseReceived();
            int responseCode = clientExchange.getResponse().getResponseCode();
            if (responseCode == CREATED) {
                try {
                    JsonNode jsonNode = objectMapper.readTree(body);
                    JsonNode allowedNode = jsonNode == null ? null : jsonNode.get("allowed");
                    boolean allowed = allowedNode != null && allowedNode.asBoolean();
                    onRequestResult(serverExchange, connection, allowed);
                } catch (IOException e) {
                    onRequestFailure(serverExchange, connection, e, true);
                }
            } else {
                IOException e = new IOException(StatusCodes.getReason(responseCode));
                if(responseCode >= 500) {
                    onRequestFailure(serverExchange, connection, e, true);
                } else {
                    // Retries will not help to solve the issue - client issue
                    onRequestFailure(serverExchange, connection, e, false);
                }
            }
        }

        @Override
        protected void error(IOException e) {
            onRequestFailure(serverExchange, connection, e, true);
        }
    }

    /**
     * A {@link ClientConnection} pool. Each {@link XnioIoThread} has its own pool. While it may not be perfect if
     * the container does not evenly assign requests to IO threads, implementation is easier as no synchronization is
     * needed. But remember that only the corresponding io thread must manipulate its own pool instance.
     *
     * @author snegrea
     */
    private static class ConnectionPool {
        private final ConnectionFactory connectionFactory;
        private final List<PooledConnection> connections;
        private final Queue<PooledConnectionWaiter> waiters;
        private final XnioExecutor.Key periodicTaskKey;
        private int ongoingCreations;
        private boolean stop;
        private volatile int connectionCount;
        private volatile int waiterCount;

        private ConnectionPool(ConnectionFactory connectionFactory, String componentName) {
            this.connectionFactory = connectionFactory;
            connections = new ArrayList<>(MAX_CONNECTIONS_PER_THREAD);
            waiters = new ArrayDeque<>();
            XnioIoThread ioThread = (XnioIoThread) Thread.currentThread();
            periodicTaskKey = ioThread.executeAtInterval(this::periodicTask, 1, SECONDS);
            ongoingCreations = 0;
            stop = false;
        }

        /**
         * This task is executed periodically to make sure no waiter stay in the queue longer than needed and no stale
         * connection occupies the pool.
         */
        private void periodicTask() {
            if (stop) {
                return;
            }
            // Close stale connections and remove them from the pool
            long now = System.currentTimeMillis();
            for (Iterator<PooledConnection> iterator = connections.iterator(); iterator.hasNext(); ) {
                PooledConnection connection = iterator.next();
                if (connection.idle && !connection.canReuse(now)) {
                    iterator.remove();
                    IoUtils.safeClose(connection);
                }
            }
            removeTimedOutWaiters();
            // Create a connection if pool is not full and some clients are waiting
            if (!waiters.isEmpty() && !isFull()) {
                createConnection();
            }
            // Side job, update metrics
            connectionCount = connections.size();
            waiterCount = waiters.size();
        }

        /**
         * @return false if no connection is available immediately and too many clients are waiting, true otherwise
         */
        private boolean offer(PooledConnectionWaiter waiter) {
            if (stop) {
                waiter.onTimeout.run();
                return true;
            }
            removeTimedOutWaiters();
            // We must queue the waiter and not take an available connection immediatly, in order to preserve to the
            // FIFO requirement for clients
            if (waiters.size() < MAX_PENDING) {
                waiters.offer(waiter);
            } else {
                return false;
            }
            // Now try to acquire a connection
            PooledConnection selected = selectIdleConnection();
            if (selected != null) {
                // Got one, poll a waiter (may be the caller or a previous one)
                waiters.poll().onGet.accept(selected);
            }
            return true;
        }

        private PooledConnection selectIdleConnection() {
            // For maximum speed we iterate through the list until we find a reusable connection
            // It's not necessary to close all stale connections here, the periodic task will take care of this
            long now = System.currentTimeMillis();
            for (Iterator<PooledConnection> iterator = connections.iterator(); iterator.hasNext(); ) {
                PooledConnection connection = iterator.next();
                if (connection.idle) {
                    if (connection.canReuse(now)) {
                        connection.idle = false;
                        return connection;
                    } else {
                        iterator.remove();
                        IoUtils.safeClose(connection);
                    }
                }
            }
            return null;
        }

        private void release(PooledConnection connection) {
            connection.idle = true;
            if (stop) {
                return;
            }
            // Don't keep the connection in the pool if we can't reuse it
            if (!connection.canReuse(System.currentTimeMillis())) {
                connections.remove(connection);
                IoUtils.safeClose(connection);
            }
            removeTimedOutWaiters();
            // Stop here if no client is waiting for a connection
            if (!waiters.isEmpty()) {
                PooledConnection selected = selectIdleConnection();
                if (selected != null) {
                    // Got a connection, can poll a waiter now
                    PooledConnectionWaiter waiter = waiters.poll();
                    waiter.onGet.accept(selected);
                } else if (!isFull()) {
                    createConnection();
                }
            }
        }

        private void removeTimedOutWaiters() {
            long now = System.currentTimeMillis();
            for (Iterator<PooledConnectionWaiter> iterator = waiters.iterator(); iterator.hasNext(); ) {
                PooledConnectionWaiter waiter = iterator.next();
                if (waiter.timestamp + CONNECTION_WAIT_TIMEOUT < now) {
                    iterator.remove();
                    waiter.onTimeout.run();
                } else {
                    // waiters is a FIFO queue, so we can stop as soon as we find one which is not timed out
                    break;
                }
            }
        }

        private boolean isFull() {
            return connections.size() + ongoingCreations == MAX_CONNECTIONS_PER_THREAD;
        }

        private void createConnection() {
            ongoingCreations++;
            try {
                connectionFactory.createConnection(new ClientCallback<ClientConnection>() {
                    @Override
                    public void completed(ClientConnection result) {
                        ongoingCreations--;
                        onConnectionCreated(result);
                    }

                    @Override
                    public void failed(IOException e) {
                        ongoingCreations--;
                        onConnectionCreationFailure(e);
                    }
                });
            } catch (UnresolvedAddressException e) {
                ongoingCreations--;
                onConnectionCreationFailure(e);
            }
        }

        private void onConnectionCreated(ClientConnection clientConnection) {
            if (stop) {
                IoUtils.safeClose(clientConnection);
                return;
            }
            PooledConnection connection = new PooledConnection();
            connection.clientConnection = clientConnection;
            connections.add(connection);
            removeTimedOutWaiters();
            if (!waiters.isEmpty()) {
                // If there are waiters, pick up one.
                PooledConnectionWaiter waiter = waiters.poll();
                connection.idle = false;
                waiter.onGet.accept(connection);
            } else {
                // Otherwise make the connection available for future clients
                connection.idle = true;
            }
        }

        private void onConnectionCreationFailure(Exception e) {
            log.debug("Failed to create client connection", e);
            if (stop) {
                return;
            }
            // Wait a bit before trying to create a connection again
            XnioIoThread ioThread = (XnioIoThread) Thread.currentThread();
            ioThread.executeAfter(() -> {
                removeTimedOutWaiters();
                if (!stop && !waiters.isEmpty() && !isFull()) {
                    // It's still necessary to create a connection only if the pool is not stopped, there still is
                    // a client waiting for a connection and the pool is not full
                    createConnection();
                }
            }, 1, SECONDS);
        }

        /**
         * @param onStop callback invoked when waiters have all been notified and all client connections are closed
         */
        private void stop(Runnable onStop) {
            stop = true;
            // Cancel the periodic task
            periodicTaskKey.remove();
            // Simple strategy, tell the waiters they failed to acquire a connection
            while (!waiters.isEmpty()) {
                PooledConnectionWaiter waiter = waiters.poll();
                waiter.onTimeout.run();
            }
            closeAllConnections(onStop);
        }

        private void closeAllConnections(Runnable onAllClosed) {
            for (Iterator<PooledConnection> iterator = connections.iterator(); iterator.hasNext(); ) {
                PooledConnection connection = iterator.next();
                // Don't close a connection if it's still used
                if (connection.idle) {
                    iterator.remove();
                    IoUtils.safeClose(connection);
                }
            }
            // Is there any connection still used?
            if (connections.isEmpty()) {
                // No, invoked the stop callback
                onAllClosed.run();
            } else {
                // Yes, wait a bit and try close idle conections again
                XnioIoThread ioThread = (XnioIoThread) Thread.currentThread();
                ioThread.executeAfter(() -> closeAllConnections(onAllClosed), 500, MILLISECONDS);
            }
        }
    }

    /**
     * Wraps a {@link ClientConnection} with connection pool specific information.
     */
    private static class PooledConnection implements Closeable {
        private ClientConnection clientConnection;
        private boolean idle;
        private long createdOn = System.currentTimeMillis();

        private void sendRequest(ClientRequest request, ClientCallback<ClientExchange> clientCallback) {
            clientConnection.sendRequest(request, clientCallback);
        }

        private boolean isOpen() {
            return clientConnection.isOpen();
        }

        @Override
        public void close() throws IOException {
            clientConnection.close();
        }

        private boolean hasExpired(long now) {
            return createdOn + CONNECTION_TTL < now;
        }

        private boolean canReuse(long now) {
            return isOpen() && !hasExpired(now);
        }
    }

    /**
     * Wraps the callbacks which should be invoked when a connection is acquired of after too much time waiting.
     */
    private static class PooledConnectionWaiter {
        private final Consumer<PooledConnection> onGet;
        private final Runnable onTimeout;
        private final long timestamp;

        private PooledConnectionWaiter(Consumer<PooledConnection> onGet, Runnable onTimeout) {
            this.onGet = onGet;
            this.onTimeout = onTimeout;
            timestamp = System.currentTimeMillis();
        }
    }

    /**
     * Used by the {@link ConnectionPool} in order to setup connections.
     */
    private static class ConnectionFactory {
        private final URI kubernetesMasterUri;
        private final UndertowClient undertowClient;
        private final XnioSsl ssl;
        private final ByteBufferPool byteBufferPool;

        private ConnectionFactory(URI kubernetesMasterUri) {
            this.kubernetesMasterUri = kubernetesMasterUri;
            undertowClient = UndertowClient.getInstance();
            Xnio xnio = Xnio.getInstance(Undertow.class.getClassLoader());
            try {
                ssl = new UndertowXnioSsl(xnio, OptionMap.EMPTY);
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
            byteBufferPool = createByteBufferPool();
        }

        // The code here comes from the Wildfly source
        // It seems the team has spent quite some time searching for good defaults so let's reuse them.
        private ByteBufferPool createByteBufferPool() {
            long maxMemory = Runtime.getRuntime().maxMemory();
            boolean useDirectBuffers;
            int bufferSize, buffersPerRegion;
            if (maxMemory < 64 * 1024 * 1024) {
                //smaller than 64mb of ram we use 512b buffers
                useDirectBuffers = false;
                bufferSize = 512;
                buffersPerRegion = 10;
            } else if (maxMemory < 128 * 1024 * 1024) {
                //use 1k buffers
                useDirectBuffers = true;
                bufferSize = 1024;
                buffersPerRegion = 10;
            } else {
                //use 16k buffers for best performance
                //as 16k is generally the max amount of data that can be sent in a single write() call
                useDirectBuffers = true;
                bufferSize = 1024 * 16;
                buffersPerRegion = 20;
            }
            BufferAllocator<ByteBuffer> allocator;
            if (useDirectBuffers) {
                allocator = BufferAllocator.DIRECT_BYTE_BUFFER_ALLOCATOR;
            } else {
                allocator = BufferAllocator.BYTE_BUFFER_ALLOCATOR;
            }
            int maxRegionSize = buffersPerRegion * bufferSize;
            ByteBufferSlicePool pool = new ByteBufferSlicePool(allocator, bufferSize, maxRegionSize);
            return new XnioByteBufferPool(pool);
        }

        private void createConnection(ClientCallback<ClientConnection> callback) {
            XnioIoThread ioThread = (XnioIoThread) Thread.currentThread();
            undertowClient.connect(callback, kubernetesMasterUri, ioThread, ssl, byteBufferPool, OptionMap.EMPTY);
        }

        private void close() {
            byteBufferPool.close();
        }
    }
}