/*******************************************************************************
 * Copyright (c) 2014, 2015 IBH SYSTEMS GmbH.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *     IBH SYSTEMS GmbH - initial API and implementation
 *******************************************************************************/
package org.eclipse.packagedrone.web.common.tags;

import java.util.Map;
import java.util.SortedSet;
import java.util.TreeSet;

import org.eclipse.packagedrone.repo.MetaKey;
import org.eclipse.packagedrone.web.common.Modifier;

public class Functions
{
    /**
     * Convert a modifier value to a bootstrap type modifier
     *
     * @param prefix
     *            an optional prefix to add
     * @param modifier
     *            the modifier to convert
     * @return the bootstrap type string, optionally with a prefix attached,
     *         never null
     */
    public static String modifier ( final String prefix, final Modifier modifier )
    {
        if ( modifier == null )
        {
            return "";
        }

        String value = null;
        switch ( modifier )
        {
            case DEFAULT:
                value = "default";
                break;
            case PRIMARY:
                value = "primary";
                break;
            case SUCCESS:
                value = "success";
                break;
            case INFO:
                value = "info";
                break;
            case WARNING:
                value = "warning";
                break;
            case DANGER:
                value = "danger";
                break;
            case LINK:
                value = "link";
                break;
        }

        if ( value != null && prefix != null )
        {
            return prefix + value;
        }
        else
        {
            return value != null ? value : "";
        }
    }

    /**
     * Get all meta data values which match namespace and key
     *
     * @param metadata
     *            the map of meta data to filter
     * @param namespace
     *            an optional namespace filter
     * @param key
     *            an optional key filter
     * @return the result set
     */
    public static SortedSet<String> metadata ( final Map<MetaKey, String> metadata, String namespace, String key )
    {
        final SortedSet<String> result = new TreeSet<> ();

        if ( namespace.isEmpty () )
        {
            namespace = null;
        }

        if ( key.isEmpty () )
        {
            key = null;
        }

        for ( final Map.Entry<MetaKey, String> entry : metadata.entrySet () )
        {
            if ( namespace != null && !namespace.equals ( entry.getKey ().getNamespace () ) )
            {
                continue;
            }

            if ( key != null && !key.equals ( entry.getKey ().getKey () ) )
            {
                continue;
            }

            result.add ( entry.getValue () );
        }

        return result;
    }
}


/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright (c) 1997-2011 Oracle and/or its affiliates. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common Development
 * and Distribution License("CDDL") (collectively, the "License").  You
 * may not use this file except in compliance with the License.  You can
 * obtain a copy of the License at
 * https://glassfish.dev.java.net/public/CDDL+GPL_1_1.html
 * or packager/legal/LICENSE.txt.  See the License for the specific
 * language governing permissions and limitations under the License.
 *
 * When distributing the software, include this License Header Notice in each
 * file and include the License file at packager/legal/LICENSE.txt.
 *
 * GPL Classpath Exception:
 * Oracle designates this particular file as subject to the "Classpath"
 * exception as provided by Oracle in the GPL Version 2 section of the License
 * file that accompanied this code.
 *
 * Modifications:
 * If applicable, add the following below the License Header, with the fields
 * enclosed by brackets [] replaced by your own identifying information:
 * "Portions Copyright [year] [name of copyright owner]"
 *
 * Contributor(s):
 * If you wish your version of this file to be governed by only the CDDL or
 * only the GPL Version 2, indicate your decision by adding "[Contributor]
 * elects to include this software in this distribution under the [CDDL or GPL
 * Version 2] license."  If you don't indicate a single choice of license, a
 * recipient has the option to distribute your version of this file under
 * either the CDDL, the GPL Version 2 or to extend the choice of license to
 * its licensees as provided above.  However, if you add GPL Version 2 code
 * and therefore, elected the GPL Version 2 license, then the option applies
 * only if the new code is made subject to such option by the copyright
 * holder.
 *
 *
 * This file incorporates work covered by the following copyright and
 * permission notice:
 *
 * Copyright 2004 The Apache Software Foundation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.jasper.servlet;

// START PWC 6300204
import java.io.FileNotFoundException;
// END PWC 6300204
import java.io.IOException;
import java.io.InputStream;
import java.util.Enumeration;
// START GlassFish 750
import java.net.URL;
import java.net.URLEncoder;
import java.util.concurrent.ConcurrentHashMap;
// END GlassFish 750
import java.util.concurrent.atomic.*;
// START SJSWS 6232180
import java.util.HashSet;
import java.util.StringTokenizer;
// END SJSWS 6232180
// START GlassFish 747
import java.util.HashMap;
// END GlassFish 747
import javax.servlet.ServletConfig;
import javax.servlet.ServletContext;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
// START GlassFish 750
import javax.servlet.jsp.tagext.TagLibraryInfo;
// END GlassFish 750

import java.util.logging.Logger;
import java.util.logging.Level;

import org.apache.jasper.Constants;
import org.apache.jasper.EmbeddedServletOptions;
import org.apache.jasper.Options;
import org.apache.jasper.compiler.JspRuntimeContext;
import org.apache.jasper.compiler.JspUtil;
import org.apache.jasper.compiler.Localizer;
import org.apache.jasper.runtime.JspApplicationContextImpl;
import org.glassfish.jsp.api.JspProbeEmitter;
import org.glassfish.jsp.api.ResourceInjector;

/**
 * The JSP engine (a.k.a Jasper).
 *
 * The servlet container is responsible for providing a
 * URLClassLoader for the web application context Jasper
 * is being used in. Jasper will try get the Tomcat
 * ServletContext attribute for its ServletContext class
 * loader, if that fails, it uses the parent class loader.
 * In either case, it must be a URLClassLoader.
 *
 * @author Anil K. Vijendran
 * @author Harish Prabandham
 * @author Remy Maucherat
 * @author Kin-man Chung
 * @author Glenn Nielsen
 */
public class JspServlet extends HttpServlet {

    // Logger
    private static Logger log = Logger.getLogger(JspServlet.class.getName());

    private static final int CHAR_LIMIT = 256;

    private ServletContext context;
    private ServletConfig config;
    private Options options;
    private JspRuntimeContext rctxt;

    // START S1AS
    // jsp error count
    private AtomicInteger countErrors = new AtomicInteger(0);
    // END S1AS

    // START SJSWS 6232180
    private String httpMethodsString = null;
    private HashSet<String> httpMethodsSet = null;
    // END SJSWS 6232180

    // START GlassFish 750
    private ConcurrentHashMap<String, TagLibraryInfo> taglibs;
    private ConcurrentHashMap<String, URL> tagFileJarUrls;
    // END GlassFish 750

    private JspProbeEmitter jspProbeEmitter;

    /*
     * Initializes this JspServlet.
     */
    public void init(ServletConfig config) throws ServletException {

        super.init(config);
        this.config = config;
        this.context = config.getServletContext();

        // Initialize the JSP Runtime Context
        options = new EmbeddedServletOptions(config, context);
        rctxt = new JspRuntimeContext(context,options);

        // START SJSWS 6232180
        // Determine which HTTP methods to service ("*" means all)
        httpMethodsString = config.getInitParameter("httpMethods");
        if (httpMethodsString != null
                && !httpMethodsString.equals("*")) {
            httpMethodsSet = new HashSet<String>();
            StringTokenizer tokenizer = new StringTokenizer(
                    httpMethodsString, ", \t\n\r\f");
            while (tokenizer.hasMoreTokens()) {
                httpMethodsSet.add(tokenizer.nextToken());
            }
        }
        // END SJSWS 6232180

        // START GlassFish 750
        taglibs = new ConcurrentHashMap<String, TagLibraryInfo>();
        context.setAttribute(Constants.JSP_TAGLIBRARY_CACHE, taglibs);

        tagFileJarUrls = new ConcurrentHashMap<String, URL>();
        context.setAttribute(Constants.JSP_TAGFILE_JAR_URLS_CACHE,
                             tagFileJarUrls);
        // END GlassFish 750

        if (log.isLoggable(Level.FINEST)) {
            log.finest(Localizer.getMessage("jsp.message.scratch.dir.is",
                                           options.getScratchDir().toString()));
            log.finest(Localizer.getMessage("jsp.message.dont.modify.servlets"));
        }

        this.jspProbeEmitter = (JspProbeEmitter)
            config.getServletContext().getAttribute(
                "org.glassfish.jsp.monitor.probeEmitter");
    }

    /**
     * Returns the number of JSPs for which JspServletWrappers exist, i.e.,
     * the number of JSPs that have been loaded into the webapp with which
     * this JspServlet is associated.
     *
     *
This info may be used for monitoring purposes.
     *
     * @return The number of JSPs that have been loaded into the webapp with
     * which this JspServlet is associated
     */
    public int getJspCount() {
        return this.rctxt.getJspCount();
    }

    /**
     * Resets the JSP reload counter.
     *
     * @param count Value to which to reset the JSP reload counter
     */
    public void setJspReloadCount(int count) {
        this.rctxt.setJspReloadCount(count);
    }

    /**
     * Gets the number of JSPs that have been reloaded.
     *
     *
This info may be used for monitoring purposes.
     *
     * @return The number of JSPs (in the webapp with which this JspServlet is
     * associated) that have been reloaded
     */
    public int getJspReloadCount() {
        return this.rctxt.getJspReloadCount();
    }

    // START S1AS
    /**
     * Gets the number of errors triggered by JSP invocations.
     *
     * @return The number of errors triggered by JSP invocations
     */
    public int getJspErrorCount() {
        return countErrors.get();
    }
    // END S1AS

    /**
     *
Look for a precompilation request as described in
     * Section 8.4.2 of the JSP 1.2 Specification.  WARNING -
     * we cannot use request.getParameter() for this, because
     * that will trigger parsing all of the request parameters, and not give
     * a servlet the opportunity to call
     * request.setCharacterEncoding() first.

     *
     * @param request The servlet requset we are processing
     *
     * @exception ServletException if an invalid parameter value for the
     *  jsp_precompile parameter name is specified
     */
    boolean preCompile(HttpServletRequest request) throws ServletException {

        String queryString = request.getQueryString();
        if (queryString == null) {
            return (false);
        }
        int start = queryString.indexOf(Constants.PRECOMPILE);
        if (start < 0) {
            return (false);
        }
        queryString =
            queryString.substring(start + Constants.PRECOMPILE.length());
        if (queryString.length() == 0) {
            return (true);             // ?jsp_precompile
        }
        if (queryString.startsWith("&")) {
            return (true);             // ?jsp_precompile&foo=bar...
        }
        if (!queryString.startsWith("=")) {
            return (false);            // part of some other name or value
        }
        int limit = queryString.length();
        int ampersand = queryString.indexOf("&");
        if (ampersand > 0) {
            limit = ampersand;
        }
        String value = queryString.substring(1, limit);
        if (value.equals("true")) {
            return (true);             // ?jsp_precompile=true
        } else if (value.equals("false")) {
	    // Spec says if jsp_precompile=false, the request should not
	    // be delivered to the JSP page; the easiest way to implement
	    // this is to set the flag to true, and precompile the page anyway.
	    // This still conforms to the spec, since it says the
	    // precompilation request can be ignored.
            return (true);             // ?jsp_precompile=false
        } else {
            throw new ServletException("Cannot have request parameter " +
                                       Constants.PRECOMPILE + " set to " +
                                       value);
        }

    }

    public void service(HttpServletRequest request,
    			HttpServletResponse response)
                throws ServletException, IOException {

        // START SJSWS 6232180
        if (httpMethodsSet != null) {
            String method = request.getMethod();
            if (method == null) {
                return;
            }
            if (!httpMethodsSet.contains(method)) {
                super.service(request, response);
                return;
            }
            if (method.equals("OPTIONS")) {
                response.addHeader("Allow", httpMethodsString);
                return;
            }
        }
        // END SJSWS 6232180

        String jspUri = null;

        String jspFile = (String) request.getAttribute(Constants.JSP_FILE);
        if (jspFile != null) {
            // JSP is specified via <jsp-file> in <servlet> declaration
            jspUri = jspFile;
            request.removeAttribute(Constants.JSP_FILE);
        } else {
            /*
             * Check to see if the requested JSP has been the target of a
             * RequestDispatcher.include()
             */
            jspUri = (String) request.getAttribute(Constants.INC_SERVLET_PATH);
            if (jspUri != null) {
                /*
		 * Requested JSP has been target of
                 * RequestDispatcher.include(). Its path is assembled from the
                 * relevant javax.servlet.include.* request attributes
                 */
                String pathInfo = (String) request.getAttribute(
                                    "javax.servlet.include.path_info");
                if (pathInfo != null) {
                    jspUri += pathInfo;
                }
            } else {
                /*
                 * Requested JSP has not been the target of a
                 * RequestDispatcher.include(). Reconstruct its path from the
                 * request's getServletPath() and getPathInfo()
                 */
                jspUri = request.getServletPath();
                String pathInfo = request.getPathInfo();
                if (pathInfo != null) {
                    jspUri += pathInfo;
                }
            }
        }

        if (log.isLoggable(Level.FINE)) {
            StringBuilder msg = new StringBuilder();
            msg.append("JspEngine --> [" + jspUri);
            msg.append("] ServletPath: [" + request.getServletPath());
            msg.append("] PathInfo: [" + request.getPathInfo());
            msg.append("] RealPath: [" + context.getRealPath(jspUri));
            msg.append("] RequestURI: [" + request.getRequestURI());
            msg.append("] QueryString: [" + request.getQueryString());
            msg.append("]");
            log.fine(msg.toString());
        }

        try {
            boolean precompile = preCompile(request);
            serviceJspFile(request, response, jspUri, null, precompile);
        } catch (RuntimeException e) {
            // STARTS S1AS
            incrementErrorCount(jspUri);
            // END S1AS
            throw e;
        } catch (Error e) {
            incrementErrorCount(jspUri);
            throw e;
        } catch (ServletException e) {
            // STARTS S1AS
            incrementErrorCount(jspUri);
            // END S1AS
            throw e;
        } catch (IOException e) {
            // STARTS S1AS
            incrementErrorCount(jspUri);
            // END S1AS
            throw e;
        } catch (Throwable e) {
            // STARTS S1AS
            incrementErrorCount(jspUri);
            // END S1AS
            throw new ServletException(e);
        }

    }

    public void destroy() {
        if (log.isLoggable(Level.FINE)) {
            log.fine("JspServlet.destroy()");
        }

        rctxt.destroy();
        JspApplicationContextImpl.removeJspApplicationContext(context);

        // START GlassFish 750
        taglibs.clear();
        tagFileJarUrls.clear();
        // END GlassFish 750

        // START GlassFish 747
        HashMap tldUriToLocationMap = (HashMap) context.getAttribute(
            Constants.JSP_TLD_URI_TO_LOCATION_MAP);
        if (tldUriToLocationMap != null) {
            tldUriToLocationMap.clear();
        }
        // END GlassFish 747
    }

    // -------------------------------------------------------- Private Methods

    private void serviceJspFile(HttpServletRequest request,
                                HttpServletResponse response, String jspUri,
                                Throwable exception, boolean precompile)
        throws ServletException, IOException {

        JspServletWrapper wrapper =
            (JspServletWrapper) rctxt.getWrapper(jspUri);
        if (wrapper == null) {
            synchronized(this) {
                wrapper = (JspServletWrapper) rctxt.getWrapper(jspUri);
                if (wrapper == null) {
                    // Check if the requested JSP page exists, to avoid
                    // creating unnecessary directories and files.
                    if (null == context.getResource(jspUri)
                            && !options.getUsePrecompiled()) {
                        String includeRequestUri = (String)
                            request.getAttribute("javax.servlet.include.request_uri");
                        if (includeRequestUri != null) {
                            // Missing JSP resource has been the target of a
                            // RequestDispatcher.include().
                            // Throw an exception (rather than returning a
                            // 404 response error code), because any call to
                            // response.sendError() must be ignored by the
                            // servlet engine when issued from within an
                            // included resource (as per the Servlet spec).
                            throw new FileNotFoundException(
                                JspUtil.escapeXml(jspUri));
                        }

                        response.sendError(HttpServletResponse.SC_NOT_FOUND);
                        log.severe(Localizer.getMessage(
                            "jsp.error.file.not.found",
                            context.getRealPath(jspUri)));
                        return;
                    }
                    boolean isErrorPage = exception != null;
                    wrapper = new JspServletWrapper(config, options, jspUri,
                                                    isErrorPage, rctxt);
                    rctxt.addWrapper(jspUri,wrapper);
                }
            }
        }

        wrapper.service(request, response, precompile);

    }

    // STARTS S1AS
    private void incrementErrorCount(String jspUri) {
        countErrors.incrementAndGet();
        // Fire the jspErrorEvent probe event
        if (jspProbeEmitter != null) {
            jspProbeEmitter.jspErrorEvent(jspUri);
        }
    }
    // END S1AS
}


/*******************************************************************************
 * Copyright (c) 2015, 2016 IBH SYSTEMS GmbH.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *     IBH SYSTEMS GmbH - initial API and implementation
 *******************************************************************************/
package org.eclipse.packagedrone.sec.service.apm;

import static org.eclipse.packagedrone.repo.utils.Tokens.createToken;
import static org.eclipse.packagedrone.sec.service.common.Users.hashIt;

import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeUnit;

import org.eclipse.packagedrone.repo.MetaKey;
import org.eclipse.packagedrone.sec.CreateUser;
import org.eclipse.packagedrone.sec.DatabaseDetails;
import org.eclipse.packagedrone.sec.DatabaseUserInformation;
import org.eclipse.packagedrone.sec.UserDetails;
import org.eclipse.packagedrone.sec.UserInformation;
import org.eclipse.packagedrone.sec.UserStorage;
import org.eclipse.packagedrone.sec.service.LoginException;
import org.eclipse.packagedrone.sec.service.UserService;
import org.eclipse.packagedrone.sec.service.apm.model.UserEntity;
import org.eclipse.packagedrone.sec.service.apm.model.UserModel;
import org.eclipse.packagedrone.sec.service.apm.model.UserStorageModelProvider;
import org.eclipse.packagedrone.sec.service.apm.model.UserWriteModel;
import org.eclipse.packagedrone.sec.service.common.SecurityMailService;
import org.eclipse.packagedrone.sec.service.password.BadPasswordException;
import org.eclipse.packagedrone.sec.service.password.PasswordChecker;
import org.eclipse.packagedrone.storage.apm.StorageManager;
import org.eclipse.packagedrone.storage.apm.StorageRegistration;
import org.eclipse.packagedrone.utils.scheduler.ScheduledTask;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class DatabaseUserService implements UserService, UserStorage, ScheduledTask
{
    private static final MetaKey MODEL_KEY = new MetaKey ( "sec", "users" );

    private final static Logger logger = LoggerFactory.getLogger ( DatabaseUserService.class );

    private static final long MIN_EMAIL_DELAY = TimeUnit.MINUTES.toMillis ( 5 );

    private static final Comparator<DatabaseUserInformation> USER_COMPARATOR;

    static
    {
        Comparator<DatabaseUserInformation> result = Comparator.comparing ( DatabaseUserInformation::getDetails, Comparator.nullsFirst ( Comparator.comparing ( DatabaseDetails::getEmail ) ) );
        result = result.thenComparing ( Comparator.comparing ( DatabaseUserInformation::getId ) );
        USER_COMPARATOR = result;
    }

    private PasswordChecker passwordChecker;

    private SecurityMailService mailService;

    private StorageManager storageManager;

    private StorageRegistration handle;

    public void setSecurityMailService ( final SecurityMailService mailService )
    {
        this.mailService = mailService;
    }

    public void setPasswordChecker ( final PasswordChecker passwordChecker )
    {
        this.passwordChecker = passwordChecker;
    }

    public void setStorageManager ( final StorageManager storageManager )
    {
        this.storageManager = storageManager;
    }

    public void start ()
    {
        this.handle = this.storageManager.registerModel ( 1_000, MODEL_KEY, new UserStorageModelProvider () );
    }

    public void stop ()
    {
        if ( this.handle != null )
        {
            this.handle.unregister ();
            this.handle = null;
        }
    }

    @Override
    public List<DatabaseUserInformation> list ( final int position, final int size )
    {
        return this.storageManager.accessCall ( MODEL_KEY, UserModel.class, users -> {
            final List<DatabaseUserInformation> list = new ArrayList<> ( users.listAll () );
            Collections.sort ( list, USER_COMPARATOR );
            final int end = Math.min ( position + size, list.size () );
            return list.subList ( position, end );
        } );
    }

    @Override
    public DatabaseUserInformation createUser ( final CreateUser data, final boolean emailVerified )
    {
        return this.storageManager.modifyCall ( MODEL_KEY, UserWriteModel.class, users -> {

            if ( !emailVerified )
            {
                // check only for external signup
                checkPassword ( data.getPassword () );
            }

            if ( data.getEmail () != null && users.findByEmail ( data.getEmail () ).isPresent () )
            {
                throw new IllegalArgumentException ( String.format ( "A user with the e-mail '%s' already exists.", data.getEmail () ) );
            }

            final Date now = new Date ();

            final UserEntity user = new UserEntity ();

            user.setId ( UUID.randomUUID ().toString () );
            user.setEmail ( data.getEmail () );
            user.setName ( data.getName () );
            user.setRegistrationDate ( now );
            user.setLocked ( false );

            applyPassword ( user, data.getPassword () );

            final String token;
            if ( emailVerified )
            {
                token = null;
                user.setEmailVerified ( true );
            }
            else
            {
                user.setEmailVerified ( false );
                token = applyNewEmailToken ( now, user );
            }

            // update

            if ( token != null )
            {
                this.mailService.sendVerifyEmail ( data.getEmail (), user.getId (), token );
            }

            users.putUser ( user );

            return new DatabaseUserInformation ( user.getId (), null, user.getRoles (), Helper.toDetails ( user ) );
        } );
    }

    @Override
    public DatabaseUserInformation getUserDetails ( final String userId )
    {
        return this.storageManager.accessCall ( MODEL_KEY, UserModel.class, users -> users.getUser ( userId ).orElse ( null ) );
    }

    @Override
    public DatabaseUserInformation updateUser ( final String userId, final UserDetails data )
    {
        return this.storageManager.modifyCall ( MODEL_KEY, UserWriteModel.class, users -> {

            final UserEntity user = users.getCheckedUser ( userId );

            user.setEmail ( data.getEmail () );
            user.setName ( data.getName () );
            user.setRoles ( new HashSet<> ( data.getRoles () ) );

            users.putUser ( user );

            return new DatabaseUserInformation ( userId, null, data.getRoles (), Helper.toDetails ( user ) );
        } );
    }

    @Override
    public String verifyEmail ( final String userId, final String token )
    {
        return this.storageManager.modifyCall ( MODEL_KEY, UserWriteModel.class, users -> {

            final UserEntity user = users.getUser ( userId ).orElse ( null );

            if ( user == null )
            {
                return "User not found";
            }

            if ( user.isLocked () )
            {
                return "User is locked";
            }

            if ( user.getEmailToken () == null || user.isEmailVerified () )
            {
                // we are already verified
                return null;
            }

            final String salt = user.getEmailTokenSalt ();
            final String hashedToken = hashIt ( salt, token );

            if ( hashedToken.equals ( user.getEmailToken () ) )
            {
                user.setEmailVerified ( true );
                user.setEmailToken ( null );
                user.setEmailTokenSalt ( null );
                user.setEmailTokenDate ( null );

                users.putUser ( user );
                return null;
            }

            return "It may be that you clicked on a verification link which was either expired or superseeded by another e-mail request.";

        } );
    }

    @Override
    public DatabaseUserInformation getUserDetailsByEmail ( final String email )
    {
        return this.storageManager.accessCall ( MODEL_KEY, UserModel.class, users -> users.findUserByEmail ( email ).orElse ( null ) );
    }

    @Override
    public String reRequestEmail ( final String email )
    {
        return this.storageManager.modifyCall ( MODEL_KEY, UserWriteModel.class, users -> {

            final UserEntity user = users.findByEmail ( email ).orElse ( null );

            if ( user == null )
            {
                return "User not found";
            }

            if ( user.isLocked () )
            {
                return "User is locked";
            }

            if ( user.getEmailToken () == null || user.isEmailVerified () )
            {
                // we are already verified
                return "E-Mail is already verified";
            }

            if ( isTooSoon ( user.getEmailTokenDate () ) )
            {
                return MessageFormat.format ( "An e-mail verification was requested at {0,time}. Please wait until {1,time} before requesting the next one!", user.getEmailTokenDate (), nextMailSlot ( user.getEmailTokenDate () ) );
            }

            final String token = applyNewEmailToken ( new Date (), user );

            users.putUser ( user );

            this.mailService.sendVerifyEmail ( user.getEmail (), user.getId (), token );
            return null;
        } );
    }

    @Override
    public String resetPassword ( final String email )
    {
        return this.storageManager.modifyCall ( MODEL_KEY, UserWriteModel.class, users -> {

            final UserEntity user = users.findByEmail ( email ).orElse ( null );

            if ( user == null )
            {
                return "No account for this e-mail address.";
            }

            if ( !user.isEmailVerified () )
            {
                return "The e-mail address for this account is not verified.";
            }

            if ( isTooSoon ( user.getEmailTokenDate () ) )
            {
                return MessageFormat.format ( "A password reset e-mail was requested at {0,time}. Please wait until {1,time} before requesting the next one!", user.getEmailTokenDate (), nextMailSlot ( user.getEmailTokenDate () ) );
            }

            if ( user.isLocked () )
            {
                // we silently fail, since this would give out information about the user's state
                this.mailService.sendEmail ( user.getEmail (), "Password reset request", "lockedUser", null );
                return null;
            }

            final String resetToken = createToken ( 64 );
            final String resetTokenSalt = createToken ( 32 );

            final String resetTokenHash = hashIt ( resetTokenSalt, resetToken );

            user.setEmailTokenSalt ( resetTokenSalt );
            user.setEmailTokenDate ( new Date () );
            user.setEmailToken ( resetTokenHash );

            users.putUser ( user );

            // we don't touch the password for now, could be anybody

            this.mailService.sendResetEmail ( email, resetToken );

            return null;
        } );
    }

    @Override
    public void changePassword ( final String email, final String token, final String password )
    {
        this.storageManager.modifyRun ( MODEL_KEY, UserWriteModel.class, users -> {

            logger.debug ( "Process password change - email: {}, token: {}, password: {}", email, token, password != null ? "***" : null );

            final UserEntity user = users.findByEmail ( email ).orElse ( null );

            if ( user == null )
            {
                throw new RuntimeException ( "User not found" );
            }

            // validate token

            final String salt = user.getEmailTokenSalt ();

            if ( salt == null )
            {
                throw new RuntimeException ( "No token" );
            }

            final String hashedToken = hashIt ( salt, token );

            if ( !hashedToken.equals ( user.getEmailToken () ) )
            {
                throw new RuntimeException ( "Invalid token" );
            }

            // check for "locked" after the token was validated

            if ( user.isLocked () )
            {
                throw new RuntimeException ( "User is locked" );
            }

            checkPassword ( password );

            applyPassword ( user, password );

            user.setEmailToken ( null );
            user.setEmailTokenDate ( null );
            user.setEmailTokenSalt ( null );

            users.putUser ( user );
        } );
    }

    @Override
    public void updatePassword ( final String userId, final String currentPassword, final String newPassword )
    {
        logger.debug ( "Process password update - userId: {},  currentPassword: {},  newPassword: {}", userId, currentPassword != null ? "***" : null, newPassword != null ? "***" : null );

        this.storageManager.modifyRun ( MODEL_KEY, UserWriteModel.class, users -> {

            final UserEntity user = users.getCheckedUser ( userId );

            if ( user.isLocked () )
            {
                throw new RuntimeException ( "User is locked" );
            }

            final String currentSalt = user.getPasswordSalt ();
            final String currentHash = user.getPasswordHash ();

            checkPassword ( newPassword );

            if ( currentPassword != null && currentSalt != null && currentHash != null )
            {
                final String checkHash = hashIt ( currentSalt, currentPassword );
                if ( !currentHash.equals ( checkHash ) )
                {
                    throw new RuntimeException ( "Current password is incorrect" );
                }
            }

            applyPassword ( user, newPassword );

            users.putUser ( user );
        } );
    }

    protected void setUserLocked ( final UserWriteModel users, final String userId, final boolean value )
    {
        final UserEntity user = users.getCheckedUser ( userId );
        user.setLocked ( value );
        users.putUser ( user );
    }

    @Override
    public void lockUser ( final String userId )
    {
        this.storageManager.modifyRun ( MODEL_KEY, UserWriteModel.class, users -> setUserLocked ( users, userId, true ) );
    }

    @Override
    public void unlockUser ( final String userId )
    {
        this.storageManager.modifyRun ( MODEL_KEY, UserWriteModel.class, users -> setUserLocked ( users, userId, false ) );
    }

    @Override
    public void deleteUser ( final String userId )
    {
        this.storageManager.modifyRun ( MODEL_KEY, UserWriteModel.class, users -> users.removeUser ( userId ) );
    }

    private static class LoginResult
    {
        UserInformation userInformation;

        LoginException exception;
    }

    @Override
    public UserInformation checkCredentials ( final String username, final String credentials, final boolean rememberMe ) throws LoginException
    {
        final LoginResult result = loginAndRememberMe ( username, credentials, rememberMe );

        if ( result == null )
        {
            return null;
        }

        if ( result.exception != null )
        {
            throw result.exception;
        }

        return result.userInformation;
    }

    private LoginResult loginAndRememberMe ( final String username, final String credentials, final boolean rememberMe )
    {
        final LoginResult result = this.storageManager.modifyCall ( MODEL_KEY, UserWriteModel.class, users -> {

            final UserEntity user = users.findByEmail ( username ).orElse ( null );

            if ( user == null )
            {
                // let the next one try
                return null;
            }

            boolean valid = false;

            // check for remember me token

            if ( user.getRememberMeTokenHash () != null && user.getRememberMeTokenSalt () != null )
            {
                final String tokenHash = hashIt ( user.getRememberMeTokenSalt (), credentials );
                if ( tokenHash.equals ( user.getRememberMeTokenHash () ) )
                {
                    valid = true;
                }
            }

            // check for password

            if ( user.getPasswordSalt () != null && user.getPasswordHash () != null )
            {
                final String credHash = hashIt ( user.getPasswordSalt (), credentials );
                if ( credHash.equals ( user.getPasswordHash () ) )
                {
                    valid = true;
                }
            }

            if ( !valid )
            {
                // no valid credentials, let other services try
                return null;
            }

            final LoginResult loginResult = new LoginResult ();

            // only fail _after_ the password has been checked, so we should be sure it is the user

            loginResult.exception = validateUserAfterLogin ( user );

            // handle remember me

            String rememberMeToken;

            if ( rememberMe )
            {
                rememberMeToken = createToken ( 128 );
                final String tokenSalt = createToken ( 32 );

                final String tokenHash = hashIt ( tokenSalt, rememberMeToken );

                user.setRememberMeTokenHash ( tokenHash );
                user.setRememberMeTokenSalt ( tokenSalt );

                users.putUser ( user );
            }
            else
            {
                rememberMeToken = null;
            }

            loginResult.userInformation = new DatabaseUserInformation ( user.getId (), rememberMeToken, user.getRoles (), Helper.toDetails ( user ) );

            // we made it

            return loginResult;
        } );
        return result;
    }

    @Override
    public UserInformation refresh ( final UserInformation user )
    {
        return getUserDetails ( user.getId () );
    }

    @Override
    public boolean hasUserBase ()
    {
        return this.storageManager.accessCall ( MODEL_KEY, UserModel.class, users -> !users.isEmpty () );
    }

    private void checkPassword ( final String password )
    {
        try
        {
            this.passwordChecker.checkPassword ( password );
        }
        catch ( final BadPasswordException e )
        {
            throw new RuntimeException ( e );
        }
    }

    private static Date nextMailSlot ( final Date date )
    {
        return new Date ( date.getTime () + MIN_EMAIL_DELAY );
    }

    private static boolean isTooSoon ( final Date date )
    {
        if ( date == null )
        {
            return false;
        }

        return System.currentTimeMillis () - date.getTime () < MIN_EMAIL_DELAY;
    }

    private void applyPassword ( final UserEntity user, final String password )
    {
        if ( password == null || password.isEmpty () )
        {
            return;
        }

        final String salt = createToken ( 32 );
        final String passwordHash = hashIt ( salt, password );

        user.setPasswordSalt ( salt );
        user.setPasswordHash ( passwordHash );
    }

    protected String applyNewEmailToken ( final Date now, final UserEntity user )
    {
        final String token = createToken ( 32 );

        final String tokenSalt = createToken ( 32 );
        final String tokenHash = hashIt ( tokenSalt, token );

        user.setEmailToken ( tokenHash );
        user.setEmailTokenSalt ( tokenSalt );
        user.setEmailTokenDate ( now );
        return token;
    }

    protected LoginException validateUserAfterLogin ( final UserEntity user )
    {
        if ( user.isLocked () )
        {
            return new LoginException ( "User is locked" );
        }

        if ( !user.isEmailVerified () )
        {
            return new LoginException ( "E-mail not verified" );
        }

        return null;
    }

    /**
     * Clean up timed out user registrations
     */
    @Override
    public void run () throws Exception
    {
        this.storageManager.modifyRun ( MODEL_KEY, UserWriteModel.class, users -> {

            final Date timeout = new Date ( System.currentTimeMillis () - getTimeout () );

            final Collection<UserEntity> updates = new LinkedList<> ();
            final Collection<String> removals = new LinkedList<> ();

            for ( final UserEntity user : users.asCollection () )
            {
                if ( user.getEmailTokenDate () == null || user.getEmailTokenDate ().after ( timeout ) )
                {
                    continue;
                }

                // process timeout

                if ( user.isEmailVerified () )
                {
                    user.setEmailToken ( null );
                    user.setEmailTokenDate ( null );
                    user.setEmailTokenSalt ( null );
                    updates.add ( user );
                }
                else
                {
                    // delete
                    removals.add ( user.getId () );
                }
            }

            updates.forEach ( users::putUser );
            removals.forEach ( users::removeUser );
        } );
    }

    private long getTimeout ()
    {
        return TimeUnit.HOURS.toMillis ( 1 );
    }
}


/*******************************************************************************
 * Copyright (c) 2016 IBH SYSTEMS GmbH.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *     IBH SYSTEMS GmbH - initial API and implementation
 *******************************************************************************/
package org.eclipse.packagedrone.repo.adapter.p2.internal.servlet;

import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import org.eclipse.packagedrone.repo.adapter.p2.aspect.P2RepoConstants;
import org.eclipse.packagedrone.repo.channel.ArtifactInformation;
import org.eclipse.packagedrone.repo.channel.ReadableChannel;

public final class BinaryLocator
{
    private BinaryLocator ()
    {
    }

    private static List<ArtifactInformation> filterByMaven ( final ReadableChannel channel, final String id, final String version )
    {
        // all artifacts
        final Collection<ArtifactInformation> arts = channel.getArtifacts ();

        // get a stream to the artifacts which are p2 artifacts fragments and provide this key
        Stream<ArtifactInformation> matchingArts = findByKeys ( arts, "binary", id, version );

        // from the list of p2 fragments, find the matching maven artifact
        matchingArts = matchingArts.flatMap ( art -> {

            Collection<ArtifactInformation> candidates;
            final String parentId = art.getParentId ();

            if ( parentId != null )
            {
                candidates = channel.getChildrenOf ( parentId ).orElse ( Collections.emptyList () );
            }
            else
            {
                candidates = arts;
            }

            if ( candidates == null )
            {
                return Stream.empty ();
            }

            final String groupId = MavenLocator.getGroupId ( art );
            final String artifactId = MavenLocator.getArtifactId ( art );
            final String mavenVersion = MavenLocator.getVersion ( art );
            final String snapshotVersion = MavenLocator.getSnapshotVersion ( art );
            final String extension = MavenLocator.getExtension ( art );
            final String classifier = MavenLocator.getClassifier ( art );

            return MavenLocator.filterByCoordinates ( candidates.stream (), groupId, artifactId, mavenVersion, snapshotVersion, extension, classifier );
        } );

        return matchingArts.collect ( Collectors.toList () );
    }

    public static Optional<ArtifactInformation> findByMaven ( final ReadableChannel channel, final String id, final String version )
    {
        final List<ArtifactInformation> result = filterByMaven ( channel, id, version );
        if ( result.isEmpty () )
        {
            return Optional.empty ();
        }

        Collections.sort ( result, Comparator.comparing ( ArtifactInformation::getCreationInstant ) );

        // get the first one

        return Optional.ofNullable ( result.get ( 0 ) );
    }

    private static Stream<ArtifactInformation> findByKeys ( final Collection<ArtifactInformation> artifacts, final String classifier, final String id, final String version )
    {
        final String key = String.format ( "%s::%s::%s", classifier, id, version );
        return artifacts.stream ().filter ( art -> hasKey ( art, key ) );
    }

    /**
     * Check if the {@code p2.repo:fragment-keys} field contains a matching key
     *
     * @param art
     *            the artifact to check
     * @param key
     *            the required key value
     * @return {@code true} if the artifact has a matching key entry,
     *         {@code false} otherwise
     */
    private static boolean hasKey ( final ArtifactInformation art, final String key )
    {
        if ( key == null )
        {
            return false;
        }

        final String keysString = art.getMetaData ().get ( P2RepoConstants.KEY_FRAGMENT_KEYS );
        if ( keysString == null )
        {
            return false;
        }

        final String[] keys = keysString.split ( P2RepoConstants.ENTRY_DELIMITER );
        for ( final String actualKey : keys )
        {
            if ( key.equals ( actualKey ) )
            {
                return true;
            }
        }
        return false;
    }
}


/*******************************************************************************
 * Copyright (c) 2016 IBH SYSTEMS GmbH and others.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *     IBH SYSTEMS GmbH - initial API and implementation
 *******************************************************************************/
package org.eclipse.packagedrone.utils.io;

public final class FileNames
{
    private FileNames ()
    {
    }

    /**
     * Get the last segment of a path, the filename
     *
     * @param name
     *            the name to process
     * @return the last segment, or {@code null} if the input was {@code null}
     */
    public static String getBasename ( final String name )
    {
        if ( name == null )
        {
            return null;
        }

        final String[] toks = name.split ( "/" );
        if ( toks.length < 1 )
        {
            return name;
        }

        return toks[toks.length - 1];
    }
}


/*
 * XZInputStream
 *
 * Author: Lasse Collin <lasse.collin@tukaani.org>
 *
 * This file has been put into the public domain.
 * You can do whatever you want with this file.
 */

package org.tukaani.xz;

import java.io.InputStream;
import java.io.DataInputStream;
import java.io.IOException;
import java.io.EOFException;
import org.tukaani.xz.common.DecoderUtil;

/**
 * Decompresses a .xz file in streamed mode (no seeking).
 *

 * Use this to decompress regular standalone .xz files. This reads from
 * its input stream until the end of the input or until an error occurs.
 * This supports decompressing concatenated .xz files.
 *
 *
Typical use cases

 *

 * Getting an input stream to decompress a .xz file:
 *

 * InputStream infile = new FileInputStream("foo.xz");
 * XZInputStream inxz = new XZInputStream(infile);
 *

 *

 * It's important to keep in mind that decompressor memory usage depends
 * on the settings used to compress the file. The worst-case memory usage
 * of XZInputStream is currently 1.5 GiB. Still, very few files will
 * require more than about 65 MiB because that's how much decompressing
 * a file created with the highest preset level will need, and only a few
 * people use settings other than the predefined presets.
 *

 * It is possible to specify a memory usage limit for
 * XZInputStream. If decompression requires more memory than
 * the specified limit, MemoryLimitException will be thrown when reading
 * from the stream. For example, the following sets the memory usage limit
 * to 100 MiB:
 *

 * InputStream infile = new FileInputStream("foo.xz");
 * XZInputStream inxz = new XZInputStream(infile, 100 * 1024);
 *

 *
 *
When uncompressed size is known beforehand

 *

 * If you are decompressing complete files and your application knows
 * exactly how much uncompressed data there should be, it is good to try
 * reading one more byte by calling read() and checking
 * that it returns -1. This way the decompressor will parse the
 * file footers and verify the integrity checks, giving the caller more
 * confidence that the uncompressed data is valid. (This advice seems to
 * apply to
 * {@link java.util.zip.GZIPInputStream java.util.zip.GZIPInputStream} too.)
 *
 * @see SingleXZInputStream
 */
public class XZInputStream extends InputStream {
    private final ArrayCache arrayCache;

    private final int memoryLimit;
    private InputStream in;
    private SingleXZInputStream xzIn;
    private final boolean verifyCheck;
    private boolean endReached = false;
    private IOException exception = null;

    private final byte[] tempBuf = new byte[1];

    /**
     * Creates a new XZ decompressor without a memory usage limit.
     *

     * This constructor reads and parses the XZ Stream Header (12 bytes)
     * from in. The header of the first Block is not read
     * until read is called.
     *
     * @param       in          input stream from which XZ-compressed
     *                          data is read
     *
     * @throws      XZFormatException
     *                          input is not in the XZ format
     *
     * @throws      CorruptedInputException
     *                          XZ header CRC32 doesn't match
     *
     * @throws      UnsupportedOptionsException
     *                          XZ header is valid but specifies options
     *                          not supported by this implementation
     *
     * @throws      EOFException
     *                          less than 12 bytes of input was available
     *                          from in
     *
     * @throws      IOException may be thrown by in
     */
    public XZInputStream(InputStream in) throws IOException {
        this(in, -1);
    }

    /**
     * Creates a new XZ decompressor without a memory usage limit.
     *

     * This is identical to XZInputStream(InputStream)
     * except that this takes also the arrayCache argument.
     *
     * @param       in          input stream from which XZ-compressed
     *                          data is read
     *
     * @param       arrayCache  cache to be used for allocating large arrays
     *
     * @throws      XZFormatException
     *                          input is not in the XZ format
     *
     * @throws      CorruptedInputException
     *                          XZ header CRC32 doesn't match
     *
     * @throws      UnsupportedOptionsException
     *                          XZ header is valid but specifies options
     *                          not supported by this implementation
     *
     * @throws      EOFException
     *                          less than 12 bytes of input was available
     *                          from in
     *
     * @throws      IOException may be thrown by in
     *
     * @since 1.7
     */
    public XZInputStream(InputStream in, ArrayCache arrayCache)
            throws IOException {
        this(in, -1, arrayCache);
    }

    /**
     * Creates a new XZ decompressor with an optional memory usage limit.
     *

     * This is identical to XZInputStream(InputStream) except
     * that this takes also the memoryLimit argument.
     *
     * @param       in          input stream from which XZ-compressed
     *                          data is read
     *
     * @param       memoryLimit memory usage limit in kibibytes (KiB)
     *                          or -1 to impose no
     *                          memory usage limit
     *
     * @throws      XZFormatException
     *                          input is not in the XZ format
     *
     * @throws      CorruptedInputException
     *                          XZ header CRC32 doesn't match
     *
     * @throws      UnsupportedOptionsException
     *                          XZ header is valid but specifies options
     *                          not supported by this implementation
     *
     * @throws      EOFException
     *                          less than 12 bytes of input was available
     *                          from in
     *
     * @throws      IOException may be thrown by in
     */
    public XZInputStream(InputStream in, int memoryLimit) throws IOException {
        this(in, memoryLimit, true);
    }

    /**
     * Creates a new XZ decompressor with an optional memory usage limit.
     *

     * This is identical to XZInputStream(InputStream) except
     * that this takes also the memoryLimit and
     * arrayCache arguments.
     *
     * @param       in          input stream from which XZ-compressed
     *                          data is read
     *
     * @param       memoryLimit memory usage limit in kibibytes (KiB)
     *                          or -1 to impose no
     *                          memory usage limit
     *
     * @param       arrayCache  cache to be used for allocating large arrays
     *
     * @throws      XZFormatException
     *                          input is not in the XZ format
     *
     * @throws      CorruptedInputException
     *                          XZ header CRC32 doesn't match
     *
     * @throws      UnsupportedOptionsException
     *                          XZ header is valid but specifies options
     *                          not supported by this implementation
     *
     * @throws      EOFException
     *                          less than 12 bytes of input was available
     *                          from in
     *
     * @throws      IOException may be thrown by in
     *
     * @since 1.7
     */
    public XZInputStream(InputStream in, int memoryLimit,
                         ArrayCache arrayCache) throws IOException {
        this(in, memoryLimit, true, arrayCache);
    }

    /**
     * Creates a new XZ decompressor with an optional memory usage limit
     * and ability to disable verification of integrity checks.
     *

     * This is identical to XZInputStream(InputStream,int) except
     * that this takes also the verifyCheck argument.
     *

     * Note that integrity check verification should almost never be disabled.
     * Possible reasons to disable integrity check verification:
     *

     *
Trying to recover data from a corrupt .xz file.

     *
Speeding up decompression. This matters mostly with SHA-256
     *   or with files that have compressed extremely well. It's recommended
     *   that integrity checking isn't disabled for performance reasons
     *   unless the file integrity is verified externally in some other
     *   way.

     *

     *

     * verifyCheck only affects the integrity check of
     * the actual compressed data. The CRC32 fields in the headers
     * are always verified.
     *
     * @param       in          input stream from which XZ-compressed
     *                          data is read
     *
     * @param       memoryLimit memory usage limit in kibibytes (KiB)
     *                          or -1 to impose no
     *                          memory usage limit
     *
     * @param       verifyCheck if true, the integrity checks
     *                          will be verified; this should almost never
     *                          be set to false
     *
     * @throws      XZFormatException
     *                          input is not in the XZ format
     *
     * @throws      CorruptedInputException
     *                          XZ header CRC32 doesn't match
     *
     * @throws      UnsupportedOptionsException
     *                          XZ header is valid but specifies options
     *                          not supported by this implementation
     *
     * @throws      EOFException
     *                          less than 12 bytes of input was available
     *                          from in
     *
     * @throws      IOException may be thrown by in
     *
     * @since 1.6
     */
    public XZInputStream(InputStream in, int memoryLimit, boolean verifyCheck)
            throws IOException {
        this(in, memoryLimit, verifyCheck, ArrayCache.getDefaultCache());
    }

    /**
     * Creates a new XZ decompressor with an optional memory usage limit
     * and ability to disable verification of integrity checks.
     *

     * This is identical to XZInputStream(InputStream,int,boolean)
     * except that this takes also the arrayCache argument.
     *
     * @param       in          input stream from which XZ-compressed
     *                          data is read
     *
     * @param       memoryLimit memory usage limit in kibibytes (KiB)
     *                          or -1 to impose no
     *                          memory usage limit
     *
     * @param       verifyCheck if true, the integrity checks
     *                          will be verified; this should almost never
     *                          be set to false
     *
     * @param       arrayCache  cache to be used for allocating large arrays
     *
     * @throws      XZFormatException
     *                          input is not in the XZ format
     *
     * @throws      CorruptedInputException
     *                          XZ header CRC32 doesn't match
     *
     * @throws      UnsupportedOptionsException
     *                          XZ header is valid but specifies options
     *                          not supported by this implementation
     *
     * @throws      EOFException
     *                          less than 12 bytes of input was available
     *                          from in
     *
     * @throws      IOException may be thrown by in
     *
     * @since 1.7
     */
    public XZInputStream(InputStream in, int memoryLimit, boolean verifyCheck,
                         ArrayCache arrayCache) throws IOException {
        this.arrayCache = arrayCache;
        this.in = in;
        this.memoryLimit = memoryLimit;
        this.verifyCheck = verifyCheck;
        this.xzIn = new SingleXZInputStream(in, memoryLimit, verifyCheck,
                                            arrayCache);
    }

    /**
     * Decompresses the next byte from this input stream.
     *

     * Reading lots of data with read() from this input stream
     * may be inefficient. Wrap it in {@link java.io.BufferedInputStream}
     * if you need to read lots of data one byte at a time.
     *
     * @return      the next decompressed byte, or -1
     *              to indicate the end of the compressed stream
     *
     * @throws      CorruptedInputException
     * @throws      UnsupportedOptionsException
     * @throws      MemoryLimitException
     *
     * @throws      XZIOException if the stream has been closed
     *
     * @throws      EOFException
     *                          compressed input is truncated or corrupt
     *
     * @throws      IOException may be thrown by in
     */
    public int read() throws IOException {
        return read(tempBuf, 0, 1) == -1 ? -1 : (tempBuf[0] & 0xFF);
    }

    /**
     * Decompresses into an array of bytes.
     *

     * If len is zero, no bytes are read and 0
     * is returned. Otherwise this will try to decompress len
     * bytes of uncompressed data. Less than len bytes may
     * be read only in the following situations:
     *

     *
The end of the compressed data was reached successfully.

     *
An error is detected after at least one but less len
     *       bytes have already been successfully decompressed.
     *       The next call with non-zero len will immediately
     *       throw the pending exception.

     *
An exception is thrown.

     *

     *
     * @param       buf         target buffer for uncompressed data
     * @param       off         start offset in buf
     * @param       len         maximum number of uncompressed bytes to read
     *
     * @return      number of bytes read, or -1 to indicate
     *              the end of the compressed stream
     *
     * @throws      CorruptedInputException
     * @throws      UnsupportedOptionsException
     * @throws      MemoryLimitException
     *
     * @throws      XZIOException if the stream has been closed
     *
     * @throws      EOFException
     *                          compressed input is truncated or corrupt
     *
     * @throws      IOException may be thrown by in
     */
    public int read(byte[] buf, int off, int len) throws IOException {
        if (off < 0 || len < 0 || off + len < 0 || off + len > buf.length)
            throw new IndexOutOfBoundsException();

        if (len == 0)
            return 0;

        if (in == null)
            throw new XZIOException("Stream closed");

        if (exception != null)
            throw exception;

        if (endReached)
            return -1;

        int size = 0;

        try {
            while (len > 0) {
                if (xzIn == null) {
                    prepareNextStream();
                    if (endReached)
                        return size == 0 ? -1 : size;
                }

                int ret = xzIn.read(buf, off, len);

                if (ret > 0) {
                    size += ret;
                    off += ret;
                    len -= ret;
                } else if (ret == -1) {
                    xzIn = null;
                }
            }
        } catch (IOException e) {
            exception = e;
            if (size == 0)
                throw e;
        }

        return size;
    }

    private void prepareNextStream() throws IOException {
        DataInputStream inData = new DataInputStream(in);
        byte[] buf = new byte[DecoderUtil.STREAM_HEADER_SIZE];

        // The size of Stream Padding must be a multiple of four bytes,
        // all bytes zero.
        do {
            // First try to read one byte to see if we have reached the end
            // of the file.
            int ret = inData.read(buf, 0, 1);
            if (ret == -1) {
                endReached = true;
                return;
            }

            // Since we got one byte of input, there must be at least
            // three more available in a valid file.
            inData.readFully(buf, 1, 3);

        } while (buf[0] == 0 && buf[1] == 0 && buf[2] == 0 && buf[3] == 0);

        // Not all bytes are zero. In a valid Stream it indicates the
        // beginning of the next Stream. Read the rest of the Stream Header
        // and initialize the XZ decoder.
        inData.readFully(buf, 4, DecoderUtil.STREAM_HEADER_SIZE - 4);

        try {
            xzIn = new SingleXZInputStream(in, memoryLimit, verifyCheck, buf,
                                           arrayCache);
        } catch (XZFormatException e) {
            // Since this isn't the first .xz Stream, it is more
            // logical to tell that the data is corrupt.
            throw new CorruptedInputException(
                    "Garbage after a valid XZ Stream");
        }
    }

    /**
     * Returns the number of uncompressed bytes that can be read
     * without blocking. The value is returned with an assumption
     * that the compressed input data will be valid. If the compressed
     * data is corrupt, CorruptedInputException may get
     * thrown before the number of bytes claimed to be available have
     * been read from this input stream.
     *
     * @return      the number of uncompressed bytes that can be read
     *              without blocking
     */
    public int available() throws IOException {
        if (in == null)
            throw new XZIOException("Stream closed");

        if (exception != null)
            throw exception;

        return xzIn == null ? 0 : xzIn.available();
    }

    /**
     * Closes the stream and calls in.close().
     * If the stream was already closed, this does nothing.
     *

     * This is equivalent to close(true).
     *
     * @throws  IOException if thrown by in.close()
     */
    public void close() throws IOException {
        close(true);
    }

    /**
     * Closes the stream and optionally calls in.close().
     * If the stream was already closed, this does nothing.
     * If close(false) has been called, a further
     * call of close(true) does nothing (it doesn't call
     * in.close()).
     *

     * If you don't want to close the underlying InputStream,
     * there is usually no need to worry about closing this stream either;
     * it's fine to do nothing and let the garbage collector handle it.
     * However, if you are using {@link ArrayCache}, close(false)
     * can be useful to put the allocated arrays back to the cache without
     * closing the underlying InputStream.
     *

     * Note that if you successfully reach the end of the stream
     * (read returns -1), the arrays are
     * automatically put back to the cache by that read call. In
     * this situation close(false) is redundant (but harmless).
     *
     * @throws  IOException if thrown by in.close()
     *
     * @since 1.7
     */
    public void close(boolean closeInput) throws IOException {
        if (in != null) {
            if (xzIn != null) {
                xzIn.close(false);
                xzIn = null;
            }

            try {
                if (closeInput)
                    in.close();
            } finally {
                in = null;
            }
        }
    }
}


/*
 * Copyright (c) 2016 Rackspace.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * 	http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.rackspacecloud.blueflood.io.datastax;

import com.datastax.driver.core.BoundStatement;
import com.datastax.driver.core.PreparedStatement;
import com.rackspacecloud.blueflood.exceptions.InvalidDataException;
import com.rackspacecloud.blueflood.io.serializers.metrics.GaugeSerDes;
import com.rackspacecloud.blueflood.rollup.Granularity;
import com.rackspacecloud.blueflood.types.BluefloodCounterRollup;
import com.rackspacecloud.blueflood.types.BluefloodGaugeRollup;
import com.rackspacecloud.blueflood.types.IMetric;
import com.rackspacecloud.blueflood.types.Rollup;

import java.nio.ByteBuffer;

/**
 * This class holds the utility methods to read/write Gauge metrics
 * using Datastax driver.
 */
public class DGagueIO extends DAbstractMetricIO {

    private GaugeSerDes serDes = new GaugeSerDes();

    /**
     * Provides a way for the sub class to get a {@link java.nio.ByteBuffer}
     * representation of a certain Rollup object.
     *
     * @param value
     * @return
     */
    @Override
    protected ByteBuffer toByteBuffer(Object value) {
        if ( ! (value instanceof BluefloodGaugeRollup) ) {
            throw new IllegalArgumentException("toByteBuffer(): expecting BluefloodGaugeRollup class but got " + value.getClass().getSimpleName());
        }
        BluefloodGaugeRollup gaugeRollup = (BluefloodGaugeRollup) value;
        return serDes.serialize(gaugeRollup);
    }

    /**
     * Provides a way for the sub class to construct the right Rollup
     * object from a {@link java.nio.ByteBuffer}
     *
     * @param byteBuffer
     * @return
     */
    @Override
    protected BluefloodGaugeRollup fromByteBuffer(ByteBuffer byteBuffer) {
        return serDes.deserialize(byteBuffer);
    }

    /**
     * Retrieves the {@link BoundStatement} for a particular gauge metric
     * @param metric
     * @param granularity
     * @return
     */
    @Override
    protected BoundStatement getBoundStatementForMetric(IMetric metric, Granularity granularity) {
        Object metricValue = metric.getMetricValue();
        if (!(metricValue instanceof BluefloodGaugeRollup)) {
            throw new InvalidDataException(
                    String.format("getBoundStatementForMetric(locator=%s, granularity=%s): metric value %s is not type BluefloodGaugeRollup",
                            metric.getLocator(), granularity, metric.getMetricValue().getClass().getSimpleName())
            );
        }
        PreparedStatement statement = metricsCFPreparedStatements.preaggrGranToInsertStatement.get(granularity);
        return statement.bind(
                metric.getLocator().toString(),
                metric.getCollectionTime(),
                serDes.serialize( (BluefloodGaugeRollup) metricValue ),
                metric.getTtlInSeconds() );
    }

}


/*
 * Copyright 2014 Rackspace
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */
package com.rackspacecloud.blueflood.CloudFilesBackfiller.rollup.handlers;

import com.codahale.metrics.Counter;
import com.codahale.metrics.Meter;
import com.google.gson.Gson;
import com.rackspacecloud.blueflood.CloudFilesBackfiller.download.CloudFilesManager;
import com.rackspacecloud.blueflood.CloudFilesBackfiller.exceptions.OutOFBandException;
import com.rackspacecloud.blueflood.CloudFilesBackfiller.gson.CheckFromJson;
import com.rackspacecloud.blueflood.CloudFilesBackfiller.gson.MetricPoint;
import com.rackspacecloud.blueflood.CloudFilesBackfiller.service.BackFillerConfig;
import com.rackspacecloud.blueflood.CloudFilesBackfiller.service.OutOFBandRollup;
import com.rackspacecloud.blueflood.rollup.Granularity;
import com.rackspacecloud.blueflood.service.Configuration;
import com.rackspacecloud.blueflood.types.*;
import com.rackspacecloud.blueflood.utils.Metrics;
import com.rackspacecloud.blueflood.utils.Util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListMap;

public class BuildStore {
    private static final BuildStore buildStore = new BuildStore();
    private static final Logger log = LoggerFactory.getLogger(BuildStore.class);
    // number of metrics coming in for ranges which we have already rolled up that can be tolerated
    private static final int OUT_OF_RANGE_TOLERATION_THRESHOLD = 10;
    // please getEligibleData for better understanding of what is this parameter
    private static final int RANGE_BUFFER = Configuration.getInstance().getIntegerProperty(BackFillerConfig.NUMBER_OF_BUFFERED_SLOTS);
    private static int outOfRangeToleration = 0;
    // Specifies the sorting order in TreeMap
    private static Comparator<Range> rangeComparator = new Comparator<Range>() {
        @Override public int compare(Range r1, Range r2) {
            return (int) (r1.getStart() - r2.getStart());
        }
    };
    /*
     * This is a concurrent version of TreeMap.
     * Reason behind using a TreeMap : We want to keep the mapping from ranges to metrics sorted
     * Reason for using concurrent version : We do not want to block on simultaneous read(happens in RollupGenerator), write(happens here). Both these operations should not overlap
     * due to the range buffer
     */
    private static ConcurrentSkipListMap<Range, ConcurrentHashMap<Locator, Points>> locatorToTimestampToPoint = new ConcurrentSkipListMap<Range, ConcurrentHashMap<Locator, Points>>(rangeComparator);
    // Fixed list of ranges within the replay period to rollup
    private static List<Range> rangesToRollup = new ArrayList<Range>();
    //Only one thread will be calling this. No need to make this thread safe. Shrinking subset of ranges within replay period
    public static List<Range> rangesStillApplicable = new LinkedList<Range>();
    private static Meter completedRangesReturned = Metrics.meter(BuildStore.class, "Number of Ranges filled up meter");
    private static Meter metricsParsedAndMergedMeter = Metrics.meter(BuildStore.class, "Number of metrics parsed per unit time");
    private static Counter invalidMetricsCounter = Metrics.counter(BuildStore.class, "Invalid metrics found while parsing");
    private static Meter metricCannotBeParsed = Metrics.meter(BuildStore.class, "Unable to parse metrics");
    private static final Collection<Integer> shardsToBackfill = Collections.unmodifiableCollection(
            Util.parseShards(Configuration.getInstance().getStringProperty(BackFillerConfig.SHARDS)));

    static {
        for(Range range : CloudFilesManager.ranges) {
            rangesToRollup.add(range);
            rangesStillApplicable.add(range);
        }
        log.info("Added the first range as "+rangesToRollup.get(0)+" last range as "+rangesToRollup.get(rangesToRollup.size()-1));
    }

    public static BuildStore getBuilder() {
        return buildStore;
    }

    public static void merge (InputStream jsonInput) throws IOException {
        BufferedReader reader = new BufferedReader(new InputStreamReader(jsonInput));
        Gson gson = new Gson();
        String line = reader.readLine();
        try {
            while (line != null) {
                line = line.trim();
                if (line.length() == 0) {
                    line = reader.readLine();
                    continue;
                }

                CheckFromJson checkFromJson = gson.fromJson(line, CheckFromJson.class);

                long timestamp = checkFromJson.getTimestamp();
                long snappedMillis = Granularity.MIN_5.snapMillis(timestamp);

                Range rangeOfThisTimestamp = new Range(snappedMillis, snappedMillis + Granularity.MIN_5.milliseconds() - 1);

                // Do not add timestamps that lie out of range
                if (!rangesToRollup.contains(rangeOfThisTimestamp)) {
                    log.warn("Timestamp of metrics of check found lying out the range: "+rangeOfThisTimestamp+" TS: "+timestamp);
                    line = reader.readLine();
                    continue;
                }

                //These are out of band timestamps lying in the ranges which we have already rolled
                if (rangesToRollup.contains(rangeOfThisTimestamp) && !rangesStillApplicable.contains(rangeOfThisTimestamp)) {
                    log.warn("Range of timestamp of metrics of check " + checkFromJson.getCheckId() + "is out of applicable ranges");
                    outOfRangeToleration++;

                    // If we are seeing a lot of out of band metrics, something is wrong. May be metrics are back logged a lot. stop immediately. try to increase the range buffer?
                    if (outOfRangeToleration > OUT_OF_RANGE_TOLERATION_THRESHOLD) {
                        throw new OutOFBandException("Starting to see a lot of metrics in non-applicable ranges");
                    }
                    line = reader.readLine();
                    continue;
                }

                if (!CheckFromJson.isValid(checkFromJson)) {
                    invalidMetricsCounter.inc();
                } else {
                    for (String metricName : checkFromJson.getMetricNames()) {
                        MetricPoint metricPoint;

                        try {
                            metricPoint = checkFromJson.getMetric(metricName);
                        } catch (RuntimeException e) {
                            //pass. Let the user determine if he wants to continue by just emitting a metric
                            metricCannotBeParsed.mark();
                            continue;
                        }

                        Locator metricLocator;
                        if (checkFromJson.getCheckType().contains("remote")) {
                            String longMetricName = String.format("%s.rackspace.monitoring.entities.%s.checks.%s.%s.%s.%s",
                                    checkFromJson.getTenantId(),
                                    checkFromJson.getEntityId(),
                                    checkFromJson.getCheckType(),
                                    checkFromJson.getCheckId(),
                                    checkFromJson.getMonitoringZoneId(),
                                    metricName).trim();
                            metricLocator = Locator.createLocatorFromDbKey(longMetricName);
                        } else {
                            String longMetricName = String.format("%s.rackspace.monitoring.entities.%s.checks.%s.%s.%s",
                                    checkFromJson.getTenantId(),
                                    checkFromJson.getEntityId(),
                                    checkFromJson.getCheckType(),
                                    checkFromJson.getCheckId(),
                                    metricName).trim();
                            metricLocator = Locator.createLocatorFromDbKey(longMetricName);
                        }

                        if (!shardsToBackfill.contains(Util.computeShard(metricLocator.toString()))) continue;

                        // The following it required because concurrent data structure provides weak consistency. For eg. Two threads both calling get will see different results. putIfAbsent provides atomic operation
                        ConcurrentHashMap<Locator, Points> tsToPoint = locatorToTimestampToPoint.get(rangeOfThisTimestamp);
                        if(tsToPoint == null) {
                            // Need to synchronize this HashMap, because multiple threads might be calling containsKey and then putting values at the same time
                            final ConcurrentHashMap<Locator, Points> tsToPointVal = new ConcurrentHashMap<Locator, Points>();
                            tsToPoint = locatorToTimestampToPoint.putIfAbsent(rangeOfThisTimestamp, tsToPointVal);
                            if (tsToPoint == null) {
                               tsToPoint = tsToPointVal;
                            }
                        }

                        Points points = tsToPoint.get(metricLocator);
                        if (points == null) {
                            Points pointsToPut = new Points();
                            points = tsToPoint.putIfAbsent(metricLocator, pointsToPut);
                            if (points == null) {
                                points = pointsToPut;
                            }
                        }

                        points.add(new Points.Point(timestamp, new SimpleNumber(metricPoint.getValue())));
                        metricsParsedAndMergedMeter.mark();
                    }
                }

                line = reader.readLine();
            }
        } catch (OutOFBandException e) {
            RollupGenerator.rollupExecutors.shutdownNow();
            OutOFBandRollup.getRollupGeneratorThread().interrupt();
            // Stop the monitoring thread
            OutOFBandRollup.getMonitoringThread().interrupt();
            // Stop the file handler thread pool from sending data to buildstore
            FileHandler.handlerThreadPool.shutdownNow();
            throw new RuntimeException(e);
        } catch (Exception e) {
            log.error("Exception encountered while merging file into build store", e);
            throw new RuntimeException(e);
        }
    }

    public void close() {
        locatorToTimestampToPoint.clear();
    }

    /*
     * Consider 5 files getting merged in the buildstore. Assuming the flush rate to be 1 file/min, we need all the files together in order to construct 5 min rollup
     * How are we going to decide that a particular range has been totally filled up and ready to be rolled?
     * One behaviour which we will start seeing in the buildstore is "higher" ranges starting to build up. This means the current range has almost filled up.
     * But there is still a possibility for backed up data, getting merged. So we provide RANGE_BUFFER.
     * Basically, for every call to getEligibleData, it is going to return (n-RANGE_BUFFER) ranges to get rolled up, and keep (RANGE_BUFFER) in buildstore
     * Also, note that returning the range, will eventually remove them from buildstore, after all rollups are completed in RollupGenerator for that range.
     */
    public static Map<Range, ConcurrentHashMap<Locator, Points>> getEligibleData() {
        if (locatorToTimestampToPoint.size() <= RANGE_BUFFER) {
            log.debug("Range buffer still not exceeded. Returning null data to rollup generator");
            return null;
        } else {
            Object[] sortedKeySet = locatorToTimestampToPoint.keySet().toArray();
            Range cuttingPoint = (Range) sortedKeySet[sortedKeySet.length - RANGE_BUFFER - 1];
            log.info("Found completed ranges up to the threshold range of {}", cuttingPoint);
            completedRangesReturned.mark();
            return locatorToTimestampToPoint.headMap(cuttingPoint, true);
        }
    }
}