/**
 * Licensed to DigitalPebble Ltd under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * DigitalPebble licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.digitalpebble.stormcrawler.protocol;

import java.net.URL;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;

import org.apache.commons.lang.StringUtils;
import org.apache.storm.Config;

import com.digitalpebble.stormcrawler.Metadata;
import com.digitalpebble.stormcrawler.util.ConfUtils;
import com.google.common.cache.Cache;
import com.google.common.primitives.Ints;

import crawlercommons.robots.BaseRobotRules;

/**
 * This class is used for parsing robots for urls belonging to HTTP protocol. It
 * extends the generic {@link RobotRulesParser} class and contains Http protocol
 * specific implementation for obtaining the robots file.
 */
public class HttpRobotRulesParser extends RobotRulesParser {

    protected boolean allowForbidden = false;

    HttpRobotRulesParser() {
    }

    public HttpRobotRulesParser(Config conf) {
        setConf(conf);
    }

    @Override
    public void setConf(Config conf) {
        super.setConf(conf);
        allowForbidden = ConfUtils.getBoolean(conf, "http.robots.403.allow",
                true);
    }

    /**
     * Compose unique key to store and access robot rules in cache for given URL
     */
    protected static String getCacheKey(URL url) {
        String protocol = url.getProtocol().toLowerCase(Locale.ROOT);
        String host = url.getHost().toLowerCase(Locale.ROOT);

        int port = url.getPort();
        if (port == -1) {
            port = url.getDefaultPort();
        }
        /*
         * Robot rules apply only to host, protocol, and port where robots.txt
         * is hosted (cf. NUTCH-1752). Consequently
         */
        return protocol + ":" + host + ":" + port;
    }

    /**
     * Returns the robots rules from the cache or empty rules if not found
     *
     * @see com.digitalpebble.stormcrawler.filtering.robots.RobotsFilter
     **/
    public BaseRobotRules getRobotRulesSetFromCache(URL url) {
        String cacheKey = getCacheKey(url);
        BaseRobotRules robotRules = CACHE.getIfPresent(cacheKey);
        if (robotRules != null) {
            return robotRules;
        }
        return EMPTY_RULES;
    }

    /**
     * Get the rules from robots.txt which applies for the given {@code url}.
     * Robot rules are cached for a unique combination of host, protocol, and
     * port. If no rules are found in the cache, a HTTP request is send to fetch
     * {{protocol://host:port/robots.txt}}. The robots.txt is then parsed and
     * the rules are cached to avoid re-fetching and re-parsing it again.
     *
     * @param http
     *            The {@link Protocol} object
     * @param url
     *            URL robots.txt applies to
     *
     * @return {@link BaseRobotRules} holding the rules from robots.txt
     */
    @Override
    public BaseRobotRules getRobotRulesSet(Protocol http, URL url) {

        String cacheKey = getCacheKey(url);

        // check in the error cache first
        BaseRobotRules robotRules = ERRORCACHE.getIfPresent(cacheKey);
        if (robotRules != null) {
            return robotRules;
        }

        // now try the proper cache
        robotRules = CACHE.getIfPresent(cacheKey);
        if (robotRules != null) {
            return robotRules;
        }

        boolean cacheRule = true;
        URL redir = null;

        String keyredir = null;

        LOG.debug("Cache miss {} for {}", cacheKey, url);
        List<Integer> bytesFetched = new LinkedList<>();
        try {
            ProtocolResponse response = http.getProtocolOutput(new URL(url,
                    "/robots.txt").toString(), Metadata.empty);
            int code = response.getStatusCode();
            bytesFetched.add(response.getContent() != null ? response
                    .getContent().length : 0);
            // try one level of redirection ?
            if (code == 301 || code == 302 || code == 307 || code == 308) {
                String redirection = response.getMetadata().getFirstValue(
                        HttpHeaders.LOCATION);
                if (StringUtils.isNotBlank(redirection)) {
                    if (!redirection.startsWith("http")) {
                        // RFC says it should be absolute, but apparently it
                        // isn't
                        redir = new URL(url, redirection);
                    } else {
                        redir = new URL(redirection);
                    }
                    // try from the cache
                    keyredir = getCacheKey(redir);
                    if (cacheKey.equalsIgnoreCase(keyredir)) {
                        keyredir = null;
                    } else {
                        RobotRules cachedRediRobotRules = CACHE
                                .getIfPresent(keyredir);
                        if (cachedRediRobotRules != null) {
                            // cache also for the redirected host
                            // but only if the robots.txt file is at the root
                            if (redir.getPath().equals("/robots.txt")) {
                                LOG.debug(
                                        "Caching robots for {} under key {} in cache",
                                        redir, keyredir);
                                CACHE.put(keyredir, cachedRediRobotRules);
                            }
                            return cachedRediRobotRules;
                        }
                    }

                    response = http.getProtocolOutput(redir.toString(),
                            Metadata.empty);
                    code = response.getStatusCode();
                    bytesFetched.add(response.getContent() != null ? response
                            .getContent().length : 0);
                }
            }
            if (code == 200) // found rules: parse them
            {
                String ct = response.getMetadata().getFirstValue(
                        HttpHeaders.CONTENT_TYPE);
                robotRules = parseRules(url.toString(), response.getContent(),
                        ct, agentNames);
            } else if ((code == 403) && (!allowForbidden)) {
                robotRules = FORBID_ALL_RULES; // use forbid all
            } else if (code >= 500) {
                cacheRule = false;
                robotRules = EMPTY_RULES;
            } else
                robotRules = EMPTY_RULES; // use default rules
        } catch (Throwable t) {
            LOG.info("Couldn't get robots.txt for {} : {}", url, t.toString());
            cacheRule = false;
            robotRules = EMPTY_RULES;
        }

        Cache<String, RobotRules> cacheToUse = CACHE;
        String cacheName = "success";
        if (!cacheRule) {
            cacheToUse = ERRORCACHE;
            cacheName = "error";
        }

        RobotRules cached = new RobotRules(robotRules);

        LOG.debug("Caching robots for {} under key {} in cache {}", url,
                cacheKey, cacheName);
        cacheToUse.put(cacheKey, cached);

        // cache robot for redirections
        // get here only if the target has not been found in the cache
        if (keyredir != null) {
            // cache also for the redirected host
            // but only if the robots.txt file is at the root
            if (redir.getPath().equals("/robots.txt")) {
                LOG.debug("Caching robots for {} under key {} in cache {}",
                        redir, keyredir, cacheName);
                cacheToUse.put(keyredir, cached);
            }
        }

        RobotRules live = new RobotRules(robotRules);
        live.setContentLengthFetched(Ints.toArray(bytesFetched));
        return live;
    }
}


/**
 * Licensed to DigitalPebble Ltd under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * DigitalPebble licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.digitalpebble.stormcrawler.util;

import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;

import org.w3c.dom.DocumentFragment;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import com.digitalpebble.stormcrawler.Metadata;

/**
 * Normalises the robots instructions provided by the HTML meta tags or the HTTP
 * X-Robots-Tag headers.
 **/
public class RobotsTags {

    public final static String ROBOTS_NO_INDEX = "robots.noIndex";

    public final static String ROBOTS_NO_FOLLOW = "robots.noFollow";

    /**
     * Whether to interpret the noFollow directive strictly (remove links) or
     * not (remove anchor and do not track original URL). True by default.
     **/
    public final static String ROBOTS_NO_FOLLOW_STRICT = "robots.noFollow.strict";

    public final static String ROBOTS_NO_CACHE = "robots.noCache";

    private boolean noIndex = false;

    private boolean noFollow = false;

    private boolean noCache = false;

    private final static XPathExpression expression;
    static {
        XPath xpath = XPathFactory.newInstance().newXPath();
        try {
            expression = xpath.compile("/HTML/*/META");
        } catch (XPathExpressionException e) {
            throw new RuntimeException(e);
        }
    }

    /** Get the values from the fetch metadata **/
    public RobotsTags(Metadata metadata) {
        // HTTP headers
        // X-Robots-Tag: noindex
        String[] values = metadata.getValues("X-Robots-Tag");
        if (values == null)
            return;
        if (values.length == 1) {
            // just in case they put all the values on a single line
            values = values[0].split(" *, *");
        }
        parseValues(values);
    }

    public RobotsTags() {
    }

    // set the values based on the meta tags
    // HTML tags
    // <meta name="robots" content="noarchive, nofollow"/>
    // called by the parser bolts
    public void extractMetaTags(DocumentFragment doc)
            throws XPathExpressionException {
        NodeList nodes = (NodeList) expression.evaluate(doc,
                XPathConstants.NODESET);
        if (nodes == null)
            return;
        int numNodes = nodes.getLength();
        for (int i = 0; i < numNodes; i++) {
            Node n = (Node) nodes.item(i);
            // iterate on the attributes
            // and check that it has name=robots and content
            // whatever the case is
            boolean isRobots = false;
            String content = null;
            NamedNodeMap attrs = n.getAttributes();
            for (int att = 0; att < attrs.getLength(); att++) {
                Node keyval = attrs.item(att);
                if ("name".equalsIgnoreCase(keyval.getNodeName())
                        && "robots".equalsIgnoreCase(keyval.getNodeValue())) {
                    isRobots = true;
                    continue;
                }
                if ("content".equalsIgnoreCase(keyval.getNodeName())) {
                    content = keyval.getNodeValue();
                    continue;
                }
            }

            if (isRobots && content != null) {
                // got a value - split it
                String[] vals = content.split(" *, *");
                parseValues(vals);
                return;
            }
        }
    }

    /** Extracts meta tags based on the value of the content attribute **/
    public void extractMetaTags(String content) {
        if (content == null)
            return;
        String[] vals = content.split(" *, *");
        parseValues(vals);
    }

    private void parseValues(String[] values) {
        for (String v : values) {
            v = v.trim();
            if ("noindex".equalsIgnoreCase(v)) {
                noIndex = true;
            } else if ("nofollow".equalsIgnoreCase(v)) {
                noFollow = true;
            } else if ("noarchive".equalsIgnoreCase(v)) {
                noCache = true;
            } else if ("none".equalsIgnoreCase(v)) {
                noIndex = true;
                noFollow = true;
                noCache = true;
            }
        }
    }

    /** Adds a normalised representation of the directives in the metadata **/
    public void normaliseToMetadata(Metadata metadata) {
        metadata.setValue(ROBOTS_NO_INDEX, Boolean.toString(noIndex));
        metadata.setValue(ROBOTS_NO_CACHE, Boolean.toString(noCache));
        metadata.setValue(ROBOTS_NO_FOLLOW, Boolean.toString(noFollow));
    }

    public boolean isNoIndex() {
        return noIndex;
    }

    public boolean isNoFollow() {
        return noFollow;
    }

    public boolean isNoCache() {
        return noCache;
    }

}


/**
 * Licensed to DigitalPebble Ltd under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * DigitalPebble licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.digitalpebble.stormcrawler.persistence;

import java.time.Instant;
import java.time.format.DateTimeFormatter;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;

import org.apache.commons.lang.time.DateUtils;
import org.apache.storm.metric.api.IMetric;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.digitalpebble.stormcrawler.Constants;
import com.digitalpebble.stormcrawler.Metadata;
import com.digitalpebble.stormcrawler.util.ConfUtils;
import com.digitalpebble.stormcrawler.util.MetadataTransfer;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;

/**
 * Abstract bolt used to store the status of URLs. Uses the DefaultScheduler and
 * MetadataTransfer.
 **/
@SuppressWarnings("serial")
public abstract class AbstractStatusUpdaterBolt extends BaseRichBolt {

    private static final Logger LOG = LoggerFactory
            .getLogger(AbstractStatusUpdaterBolt.class);

    /**
     * Parameter name to indicate whether the internal cache should be used for
     * discovered URLs. The value of the parameter is a boolean - true by
     * default.
     **/
    public static String useCacheParamName = "status.updater.use.cache";

    /** Number of successive FETCH_ERROR before status changes to ERROR **/
    public static String maxFetchErrorsParamName = "max.fetch.errors";

    /**
     * Parameter name to configure the cache @see
     * http://docs.guava-libraries.googlecode
     * .com/git/javadoc/com/google/common/cache/CacheBuilderSpec.html Default
     * value is "maximumSize=10000,expireAfterAccess=1h"
     **/
    public static String cacheConfigParamName = "status.updater.cache.spec";

    /**
     * Used for rounding nextFetchDates. Values are hour, minute or second, the
     * latter is the default value.
     **/
    public static String roundDateParamName = "status.updater.unit.round.date";

    /**
     * Key used to pass a preset Date to use as nextFetchDate. The value must
     * represent a valid instant in UTC and be parsable using
     * {@link DateTimeFormatter#ISO_INSTANT}. This also indicates that the
     * storage can be done directly on the metadata as-is.
     **/
    public static final String AS_IS_NEXTFETCHDATE_METADATA = "status.store.as.is.with.nextfetchdate";

    protected OutputCollector _collector;

    private Scheduler scheduler;
    private MetadataTransfer mdTransfer;

    private Cache<Object, Object> cache;
    private boolean useCache = true;

    private int maxFetchErrors = 3;

    private long cacheHits = 0;
    private long cacheMisses = 0;

    private int roundDateUnit = Calendar.SECOND;

    @SuppressWarnings({ "unchecked", "rawtypes" })
    @Override
    public void prepare(Map stormConf, TopologyContext context,
            OutputCollector collector) {
        _collector = collector;

        scheduler = Scheduler.getInstance(stormConf);

        mdTransfer = MetadataTransfer.getInstance(stormConf);

        useCache = ConfUtils.getBoolean(stormConf, useCacheParamName, true);

        if (useCache) {
            String spec = ConfUtils.getString(stormConf, cacheConfigParamName);
            cache = CacheBuilder.from(spec).build();

            context.registerMetric("cache", new IMetric() {
                @Override
                public Object getValueAndReset() {
                    Map<String, Long> statsMap = new HashMap<>();
                    statsMap.put("hits", cacheHits);
                    statsMap.put("misses", cacheMisses);
                    statsMap.put("size", cache.size());
                    cacheHits = 0;
                    cacheMisses = 0;
                    return statsMap;
                }
            }, 30);
        }

        maxFetchErrors = ConfUtils
                .getInt(stormConf, maxFetchErrorsParamName, 3);

        String tmpdateround = ConfUtils.getString(stormConf,
                roundDateParamName, "SECOND");
        if (tmpdateround.equalsIgnoreCase("MINUTE")) {
            roundDateUnit = Calendar.MINUTE;
        } else if (tmpdateround.equalsIgnoreCase("HOUR")) {
            roundDateUnit = Calendar.HOUR;
        }
    }

    @Override
    public void execute(Tuple tuple) {

        String url = tuple.getStringByField("url");
        Status status = (Status) tuple.getValueByField("status");

        boolean potentiallyNew = status.equals(Status.DISCOVERED);

        // if the URL is a freshly discovered one
        // check whether it is already known in the cache
        // if so we've already seen it and don't need to
        // store it again
        if (potentiallyNew && useCache) {
            if (cache.getIfPresent(url) != null) {
                // no need to add it to the queue
                LOG.debug("URL {} already in cache", url);
                cacheHits++;
                _collector.ack(tuple);
                return;
            } else {
                LOG.debug("URL {} not in cache", url);
                cacheMisses++;
            }
        }

        Metadata metadata = (Metadata) tuple.getValueByField("metadata");

        // store directly with the date specified in the metadata without
        // changing the status or scheduling.
        String dateInMetadata = metadata
                .getFirstValue(AS_IS_NEXTFETCHDATE_METADATA);
        if (dateInMetadata != null) {
            Date nextFetch = Date.from(Instant.parse(dateInMetadata));
            try {
                store(url, status, mdTransfer.filter(metadata), nextFetch,
                        tuple);
                return;
            } catch (Exception e) {
                LOG.error("Exception caught when storing", e);
                _collector.fail(tuple);
                return;
            }
        }

        // store last processed or discovery date in UTC
        final String nowAsString = Instant.now().toString();
        if (status.equals(Status.DISCOVERED)) {
            metadata.setValue("discoveryDate", nowAsString);
        } else {
            metadata.setValue("lastProcessedDate", nowAsString);
        }

        // too many fetch errors?
        if (status.equals(Status.FETCH_ERROR)) {
            String errorCount = metadata
                    .getFirstValue(Constants.fetchErrorCountParamName);
            int count = 0;
            try {
                count = Integer.parseInt(errorCount);
            } catch (NumberFormatException e) {
            }
            count++;
            if (count >= maxFetchErrors) {
                status = Status.ERROR;
                metadata.setValue(Constants.STATUS_ERROR_CAUSE,
                        "maxFetchErrors");
            } else {
                metadata.setValue(Constants.fetchErrorCountParamName,
                        Integer.toString(count));
            }
        }

        // delete any existing error count metadata
        // e.g. status changed
        if (!status.equals(Status.FETCH_ERROR)) {
            metadata.remove(Constants.fetchErrorCountParamName);
        }
        // https://github.com/DigitalPebble/storm-crawler/issues/415
        // remove error related key values in case of success
        if (status.equals(Status.FETCHED) || status.equals(Status.REDIRECTION)) {
            metadata.remove(Constants.STATUS_ERROR_CAUSE);
            metadata.remove(Constants.STATUS_ERROR_MESSAGE);
            metadata.remove(Constants.STATUS_ERROR_SOURCE);
        }
        // gone? notify any deleters. Doesn't need to be anchored
        else if (status == Status.ERROR) {
            _collector.emit(Constants.DELETION_STREAM_NAME, new Values(url,
                    metadata));
        }

        // determine the value of the next fetch based on the status
        Date nextFetch = scheduler.schedule(status, metadata);

        // filter metadata just before storing it, so that non-persisted
        // metadata is available to fetch schedulers
        metadata = mdTransfer.filter(metadata);

        // round next fetch date
        nextFetch = DateUtils.round(nextFetch, this.roundDateUnit);

        // extensions of this class will handle the storage
        // on a per document basis
        try {
            store(url, status, metadata, nextFetch, tuple);
        } catch (Exception e) {
            LOG.error("Exception caught when storing", e);
            _collector.fail(tuple);
            return;
        }
    }

    /**
     * Must be called by extending classes to store and collect in one go
     **/
    protected final void ack(Tuple t, String url) {
        // keep the URL in the cache
        if (useCache) {
            cache.put(url, "");
        }

        _collector.ack(t);
    }

    protected abstract void store(String url, Status status, Metadata metadata,
            Date nextFetch, Tuple t) throws Exception;

    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declareStream(Constants.DELETION_STREAM_NAME, new Fields(
                "url", "metadata"));
    }
}


/**
 * Licensed to DigitalPebble Ltd under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * DigitalPebble licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.digitalpebble.stormcrawler.filtering.regex;

import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.digitalpebble.stormcrawler.JSONResource;
import com.digitalpebble.stormcrawler.Metadata;
import com.digitalpebble.stormcrawler.filtering.URLFilter;
import com.digitalpebble.stormcrawler.util.ConfUtils;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;

/**
 * URL filter based on regex patterns and organised by [host | domain | metadata
 * | global]. For a given URL, the scopes are tried in the order given above and
 * the URL is kept or removed based on the first matching rule. The default
 * policy is to accept a URL if no matches are found.
 *
 * The resource file is in JSON and at the following format.
 *
 * <pre>
 * [{
 *         "scope": "GLOBAL",
 *         "patterns": [
 *             "DenyPathQuery \\.jpg"
 *         ]
 *     },
 *     {
 *         "scope": "domain:stormcrawler.net",
 *         "patterns": [
 *             "AllowPath /digitalpebble/",
 *             "DenyPath .+"
 *         ]
 *     },
 *     {
 *         "scope": "metadata:key=value",
 *         "patterns": [
 *             "DenyPath .+"
 *         ]
 *     }
 * ]
 * </pre>
 *
 * Partly inspired by
 * https://github.com/commoncrawl/nutch/blob/cc-fast-url-filter
 * /src/plugin/urlfilter
 * -fast/src/java/org/apache/nutch/urlfilter/fast/FastURLFilter.java
 **/

public class FastURLFilter implements URLFilter, JSONResource {

    public static final Logger LOG = LoggerFactory
            .getLogger(FastURLFilter.class);

    private String resourceFile;

    private Rules rules = new Rules();

    private static final ObjectMapper objectMapper = new ObjectMapper();

    public void configure(@SuppressWarnings("rawtypes") Map stormConf,
            JsonNode filterParams) {

        if (filterParams != null) {
            JsonNode node = filterParams.get("file");
            if (node != null && node.isTextual()) {
                this.resourceFile = node.asText("fast.urlfilter.json");
            }
        }

        // config via json failed - trying from global config
        if (this.resourceFile == null) {
            this.resourceFile = ConfUtils.getString(stormConf,
                    "fast.urlfilter.file", "fast.urlfilter.json");
        }

        try {
            loadJSONResources();
        } catch (Exception e) {
            LOG.error("Exception while loading JSON resources from jar", e);
            throw new RuntimeException(e);
        }
    }

    @Override
    public String getResourceFile() {
        return resourceFile;
    }

    @Override
    public void loadJSONResources(InputStream inputStream)
            throws JsonParseException, JsonMappingException, IOException {

        JsonNode rootNode = objectMapper.readTree(inputStream);
        Rules rules = new Rules();
        Iterator<JsonNode> iter = rootNode.elements();
        while (iter.hasNext()) {
            JsonNode current = iter.next();
            Scope scope = new Scope();
            String scopeval = current.get("scope").asText();
            scopeval = scopeval.trim();
            int offset = 0;
            Scope.Type type;
            String value = null;
            // separate the type from the pattern
            if (scopeval.equals("GLOBAL")) {
                type = Scope.Type.GLOBAL;
            } else if (scopeval.startsWith("domain:")) {
                type = Scope.Type.DOMAIN;
                offset = "domain:".length();
                value = scopeval.substring(offset);
            } else if (scopeval.startsWith("host:")) {
                type = Scope.Type.HOSTNAME;
                offset = "host:".length();
                value = scopeval.substring(offset);
            } else if (scopeval.startsWith("metadata:")) {
                type = Scope.Type.METADATA;
                offset = "metadata:".length();
                value = scopeval.substring(offset);
            } else
                throw new RuntimeException("Invalid scope: " + scopeval);

            JsonNode patternsNode = current.get("patterns");
            if (patternsNode == null)
                throw new RuntimeException("Missing patterns for scope"
                        + scopeval);

            List<Rule> rlist = new LinkedList<>();

            Iterator<JsonNode> iterPatterns = patternsNode.elements();
            while (iterPatterns.hasNext()) {
                JsonNode patternNode = iterPatterns.next();
                rlist.add(new Rule(patternNode.asText()));
            }

            scope.setRules(rlist);

            rules.addScope(scope, type, value);
        }

        this.rules = rules;
    }

    @Override
    public String filter(URL sourceUrl, Metadata sourceMetadata,
            String urlToFilter) {
        try {
            if (rules.filter(urlToFilter, sourceMetadata))
                return null;
        } catch (MalformedURLException e) {
            return null;
        }
        return urlToFilter;
    }
}

class Rules {

    private Scope globalRules;
    private Map<String, Scope> domainRules = new HashMap<>();
    private Map<String, Scope> hostNameRules = new HashMap<>();
    private List<MDScope> metadataRules = new ArrayList<>();

    public void addScope(Scope s, Scope.Type t, String value) {
        if (t.equals(Scope.Type.GLOBAL)) {
            globalRules = s;
        } else if (t.equals(Scope.Type.DOMAIN)) {
            domainRules.put(value, s);
        } else if (t.equals(Scope.Type.HOSTNAME)) {
            hostNameRules.put(value, s);
        } else if (t.equals(Scope.Type.METADATA)) {
            metadataRules.add(new MDScope(value, s.getRules()));
        }
    }

    /**
     * Try the rules from the hostname, domain name, metadata and global scopes
     * in this order. Returns true if the URL should be removed, false
     * otherwise. The value returns the value of the first matching rule, be it
     * positive or negative.
     *
     * @throws MalformedURLException
     **/
    public boolean filter(String url, Metadata metadata)
            throws MalformedURLException {
        URL u = new URL(url);

        // first try the full hostname
        String hostname = u.getHost();
        if (checkScope(hostNameRules.get(hostname), u)) {
            return true;
        }

        // then on the various components of the domain
        String[] domainParts = hostname.split("\\.");
        String domain = null;
        for (int i = domainParts.length - 1; i >= 0; i--) {
            domain = domainParts[i] + (domain == null ? "" : "." + domain);
            if (checkScope(domainRules.get(domain), u)) {
                return true;
            }
        }

        // check on parent's URL metadata
        for (MDScope scope : metadataRules) {
            String[] vals = metadata.getValues(scope.getKey());
            if (vals == null) {
                continue;
            }
            for (String v : vals) {
                if (v.equalsIgnoreCase(scope.getValue())) {
                    FastURLFilter.LOG.debug(
                            "Filtering {} matching metadata {}:{}", url,
                            scope.getKey(), scope.getValue());
                    if (checkScope(scope, u)) {
                        return true;
                    }
                }
            }
        }

        if (checkScope(globalRules, u)) {
            return true;
        }

        return false;
    }

    private boolean checkScope(Scope s, URL u) {
        if (s == null)
            return false;
        for (Rule r : s.getRules()) {
            String haystack = u.getPath();
            // whether to include the query as well?
            if (r.getType().toString().endsWith("QUERY")) {
                if (u.getQuery() != null) {
                    haystack += "?" + u.getQuery();
                }
            }
            if (r.getPattern().matcher(haystack).find()) {
                // matches! returns true for DENY, false for ALLOW
                return r.getType().toString().startsWith("DENY");
            }
        }
        return false;
    }
}

class Scope {

    public enum Type {
        DOMAIN, GLOBAL, HOSTNAME, METADATA
    };

    protected Rule[] rules;

    public void setRules(List<Rule> rlist) {
        this.rules = rlist.toArray(new Rule[rlist.size()]);
    }

    public Rule[] getRules() {
        return rules;
    }

}

class MDScope extends Scope {

    private String key;
    private String value;

    MDScope(String constraint, Rule[] rules) {
        this.rules = rules;
        int eq = constraint.indexOf("=");
        if (eq != -1) {
            key = constraint.substring(0, eq);
            value = constraint.substring(eq + 1);
        } else {
            key = constraint;
        }
    }

    public String getKey() {
        return key;
    }

    public String getValue() {
        return value;
    }

}

class Rule {

    public enum Type {
        DENYPATH, DENYPATHQUERY, ALLOWPATH, ALLOWPATHQUERY
    };

    private Type type;
    private Pattern pattern;

    public Rule(String line) {
        int offset = 0;
        String lcline = line.toLowerCase();
        // separate the type from the pattern
        for (Type t : Type.values()) {
            String start = t.toString().toLowerCase() + " ";
            if (lcline.startsWith(start)) {
                type = t;
                offset = start.length();
                break;
            }
        }
        // no match?
        if (type == null)
            return;

        String patternString = line.substring(offset).trim();
        pattern = Pattern.compile(patternString);
    }

    public Type getType() {
        return type;
    }

    public Pattern getPattern() {
        return pattern;
    }

}


/**
 * Licensed to DigitalPebble Ltd under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * DigitalPebble licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.digitalpebble.stormcrawler.filtering.regex;

import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;

import javax.xml.parsers.DocumentBuilderFactory;

import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.w3c.dom.Text;
import org.xml.sax.InputSource;

import com.digitalpebble.stormcrawler.Metadata;
import com.digitalpebble.stormcrawler.filtering.URLFilter;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;

/**
 * The RegexURLNormalizer is a URL filter that normalizes URLs by matching a
 * regular expression and inserting a replacement string.
 *
 * Adapted from Apache Nutch 1.9.
 */
public class RegexURLNormalizer implements URLFilter {

    private static final Logger LOG = LoggerFactory
            .getLogger(RegexURLNormalizer.class);

    /**
     * Class which holds a compiled pattern and its corresponding substitution
     * string.
     */
    private static class Rule {
        public Pattern pattern;

        public String substitution;
    }

    private List<Rule> rules;

    private static final List<Rule> EMPTY_RULES = Collections.emptyList();

    @Override
    public void configure(Map stormConf, JsonNode paramNode) {
        JsonNode node = paramNode.get("urlNormalizers");
        if (node != null && node.isArray()) {
            rules = readRules((ArrayNode) node);
        } else {
            JsonNode filenameNode = paramNode.get("regexNormalizerFile");
            String rulesFileName;
            if (filenameNode != null) {
                rulesFileName = filenameNode.textValue();
            } else {
                rulesFileName = "default-regex-normalizers.xml";
            }
            rules = readRules(rulesFileName);
        }

    }

    /**
     * This function does the replacements by iterating through all the regex
     * patterns. It accepts a string url as input and returns the altered
     * string. If the normalized url is an empty string, the function will
     * return null.
     */
    @Override
    public String filter(URL sourceUrl, Metadata sourceMetadata,
            String urlString) {

        Iterator<Rule> i = rules.iterator();
        while (i.hasNext()) {
            Rule r = i.next();

            Matcher matcher = r.pattern.matcher(urlString);

            urlString = matcher.replaceAll(r.substitution);
        }

        if (urlString.equals("")) {
            urlString = null;
        }

        return urlString;
    }

    /** Populates a List of Rules off of JsonNode. */
    private List<Rule> readRules(ArrayNode rulesList) {
        List<Rule> rules = new ArrayList<>();
        for (JsonNode regexNode : rulesList) {
            if (regexNode == null || regexNode.isNull()) {
                LOG.warn("bad config: 'regex' element is null");
                continue;
            }
            JsonNode patternNode = regexNode.get("pattern");
            JsonNode substitutionNode = regexNode.get("substitution");

            String substitutionValue = "";
            if (substitutionNode != null) {
                substitutionValue = substitutionNode.asText();
            }
            if (patternNode != null
                    && StringUtils.isNotBlank(patternNode.asText())) {
                Rule rule = createRule(patternNode.asText(), substitutionValue);
                if (rule != null) {
                    rules.add(rule);
                }
            }
        }
        if (rules.size() == 0) {
            rules = EMPTY_RULES;
        }
        return rules;
    }

    /** Reads the configuration file and populates a List of Rules. */
    private List<Rule> readRules(String rulesFile) {
        try {
            InputStream regexStream = getClass().getClassLoader()
                    .getResourceAsStream(rulesFile);
            Reader reader = new InputStreamReader(regexStream,
                    StandardCharsets.UTF_8);
            return readConfiguration(reader);
        } catch (Exception e) {
            LOG.error("Error loading rules from file: {}", e);
            return EMPTY_RULES;
        }
    }

    private List<Rule> readConfiguration(Reader reader) {
        List<Rule> rules = new ArrayList<>();
        try {

            // borrowed heavily from code in Configuration.java
            Document doc = DocumentBuilderFactory.newInstance()
                    .newDocumentBuilder().parse(new InputSource(reader));
            Element root = doc.getDocumentElement();
            if ((!"regex-normalize".equals(root.getTagName()))
                    && (LOG.isErrorEnabled())) {
                LOG.error("bad conf file: top-level element not <regex-normalize>");
            }
            NodeList regexes = root.getChildNodes();
            for (int i = 0; i < regexes.getLength(); i++) {
                Node regexNode = regexes.item(i);
                if (!(regexNode instanceof Element)) {
                    continue;
                }
                Element regex = (Element) regexNode;
                if ((!"regex".equals(regex.getTagName()))
                        && (LOG.isWarnEnabled())) {
                    LOG.warn("bad conf file: element not <regex>");
                }
                NodeList fields = regex.getChildNodes();
                String patternValue = null;
                String subValue = null;
                for (int j = 0; j < fields.getLength(); j++) {
                    Node fieldNode = fields.item(j);
                    if (!(fieldNode instanceof Element)) {
                        continue;
                    }
                    Element field = (Element) fieldNode;
                    if ("pattern".equals(field.getTagName())
                            && field.hasChildNodes()) {
                        patternValue = ((Text) field.getFirstChild()).getData();
                    }
                    if ("substitution".equals(field.getTagName())
                            && field.hasChildNodes()) {
                        subValue = ((Text) field.getFirstChild()).getData();
                    }
                    if (!field.hasChildNodes()) {
                        subValue = "";
                    }
                }
                if (patternValue != null && subValue != null) {
                    Rule rule = createRule(patternValue, subValue);
                    rules.add(rule);
                }
            }
        } catch (Exception e) {
            LOG.error("error parsing conf file", e);
            return EMPTY_RULES;
        }
        if (rules.size() == 0) {
            return EMPTY_RULES;
        }
        return rules;
    }

    private Rule createRule(String patternValue, String subValue) {
        Rule rule = new Rule();
        try {
            rule.pattern = Pattern.compile(patternValue);
        } catch (PatternSyntaxException e) {
            LOG.error(
                    "skipped rule: {} -> {} : invalid regular expression pattern"
                            + patternValue, subValue, e);
            return null;
        }
        rule.substitution = subValue;
        return rule;
    }

}


/**
 * Licensed to DigitalPebble Ltd under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * DigitalPebble licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.digitalpebble.stormcrawler.filtering.basic;

import java.net.IDN;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang.StringUtils;
import org.apache.http.NameValuePair;
import org.apache.http.client.utils.URLEncodedUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.digitalpebble.stormcrawler.Metadata;
import com.digitalpebble.stormcrawler.filtering.URLFilter;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;

public class BasicURLNormalizer implements URLFilter {

    private static final Logger LOG = LoggerFactory
            .getLogger(BasicURLNormalizer.class);
    /**
     * Nutch 1098 - finds URL encoded parts of the URL
     */
    private static final Pattern unescapeRulePattern = Pattern
            .compile("%([0-9A-Fa-f]{2})");

    /** https://github.com/DigitalPebble/storm-crawler/issues/401 **/
    private static final Pattern illegalEscapePattern = Pattern
            .compile("%u([0-9A-Fa-f]{4})");

    // charset used for encoding URLs before escaping
    private static final Charset utf8 = Charset.forName("UTF-8");

    /** look-up table for characters which should not be escaped in URL paths */
    private static final boolean[] unescapedCharacters = new boolean[128];

    private static final Pattern thirtytwobithash = Pattern
            .compile("[a-fA-F\\d]{32}");

    static {
        for (int c = 0; c < 128; c++) {
            /*
             * https://tools.ietf.org/html/rfc3986#section-2.2 For consistency,
             * percent-encoded octets in the ranges of ALPHA (%41-%5A and
             * %61-%7A), DIGIT (%30-%39), hyphen (%2D), period (%2E), underscore
             * (%5F), or tilde (%7E) should not be created by URI producers and,
             * when found in a URI, should be decoded to their corresponding
             * unreserved characters by URI normalizers.
             */
            if ((0x41 <= c && c <= 0x5A) || (0x61 <= c && c <= 0x7A)
                    || (0x30 <= c && c <= 0x39) || c == 0x2D || c == 0x2E
                    || c == 0x5F || c == 0x7E) {
                unescapedCharacters[c] = true;
            } else {
                unescapedCharacters[c] = false;
            }
        }
    }

    boolean removeAnchorPart = true;
    boolean unmangleQueryString = true;
    boolean checkValidURI = true;
    boolean removeHashes = false;
    private boolean hostIDNtoASCII = false;
    final Set<String> queryElementsToRemove = new TreeSet<>();

    @Override
    public String filter(URL sourceUrl, Metadata sourceMetadata,
            String urlToFilter) {

        urlToFilter = urlToFilter.trim();

        final String originalURL = urlToFilter;

        if (removeAnchorPart) {
            try {
                URL theURL = new URL(urlToFilter);
                String anchor = theURL.getRef();
                if (anchor != null)
                    urlToFilter = urlToFilter.replace("#" + anchor, "");
            } catch (MalformedURLException e) {
                return null;
            }
        }

        if (unmangleQueryString) {
            urlToFilter = unmangleQueryString(urlToFilter);
        }

        if (!queryElementsToRemove.isEmpty() || removeHashes) {
            urlToFilter = processQueryElements(urlToFilter);
        }

        try {
            URL theURL = new URL(urlToFilter);
            String file = theURL.getFile();
            String protocol = theURL.getProtocol();
            String host = theURL.getHost();
            boolean hasChanged = false;

            // lowercased protocol
            if (!urlToFilter.startsWith(protocol)) {
                hasChanged = true;
            }

            if (host != null) {
                String newHost = host.toLowerCase(Locale.ROOT);
                if (hostIDNtoASCII && !isAscii(newHost)) {
                    try {
                        newHost = IDN.toASCII(newHost);
                    } catch (IllegalArgumentException ex) {
                        // eg. if the input string contains non-convertible
                        // Unicode codepoints
                        LOG.error("Failed to convert IDN host {} in {}",
                                newHost, urlToFilter);
                        return null;
                    }
                }
                if (!host.equals(newHost)) {
                    host = newHost;
                    hasChanged = true;
                }
            }

            int port = theURL.getPort();
            // properly encode characters in path/file using percent-encoding
            String file2 = unescapePath(file);
            file2 = escapePath(file2);
            if (!file.equals(file2)) {
                hasChanged = true;
            }
            if (hasChanged) {
                urlToFilter = new URL(protocol, host, port, file2).toString();
            }
        } catch (MalformedURLException e) {
            return null;
        }

        if (checkValidURI) {
            try {
                URI uri = URI.create(urlToFilter);
                urlToFilter = uri.normalize().toString();
            } catch (java.lang.IllegalArgumentException e) {
                LOG.info("Invalid URI {} from {} ", urlToFilter, originalURL);
                return null;
            }
        }

        return urlToFilter;
    }

    @Override
    public void configure(Map stormConf, JsonNode paramNode) {
        JsonNode node = paramNode.get("removeAnchorPart");
        if (node != null) {
            removeAnchorPart = node.booleanValue();
        }

        node = paramNode.get("unmangleQueryString");
        if (node != null) {
            unmangleQueryString = node.booleanValue();
        }

        node = paramNode.get("queryElementsToRemove");
        if (node != null) {
            if (!node.isArray()) {
                LOG.warn(
                        "Failed to configure queryElementsToRemove.  Not an array: {}",
                        node.toString());
            } else {
                ArrayNode array = (ArrayNode) node;
                for (JsonNode element : array) {
                    queryElementsToRemove.add(element.asText());
                }
            }
        }

        node = paramNode.get("checkValidURI");
        if (node != null) {
            checkValidURI = node.booleanValue();
        }

        node = paramNode.get("removeHashes");
        if (node != null) {
            removeHashes = node.booleanValue();
        }

        node = paramNode.get("hostIDNtoASCII");
        if (node != null) {
            hostIDNtoASCII = node.booleanValue();
        }
    }

    /**
     * Basic filter to remove query parameters from urls so parameters that
     * don't change the content of the page can be removed. An example would be
     * a google analytics query parameter like "utm_campaign" which might have
     * several different values for a url that points to the same content. This
     * is also called when removing attributes where the value is a hash.
     */
    private String processQueryElements(String urlToFilter) {
        try {
            // Handle illegal characters by making a url first
            // this will clean illegal characters like |
            URL url = new URL(urlToFilter);

            String query = url.getQuery();
            String path = url.getPath();

            // check if the last element of the path contains parameters
            // if so convert them to query elements
            if (path.contains(";")) {
                String[] pathElements = path.split("/");
                String last = pathElements[pathElements.length - 1];
                // replace last value by part without params
                int semicolon = last.indexOf(";");
                if (semicolon != -1) {
                    pathElements[pathElements.length - 1] = last.substring(0,
                            semicolon);
                    String params = last.substring(semicolon + 1).replaceAll(
                            ";", "&");
                    if (query == null) {
                        query = params;
                    } else {
                        query += "&" + params;
                    }
                    // rebuild the path
                    StringBuilder newPath = new StringBuilder();
                    for (String p : pathElements) {
                        if (StringUtils.isNotBlank(p)) {
                            newPath.append("/").append(p);
                        }
                    }
                    path = newPath.toString();
                }
            }

            if (StringUtils.isEmpty(query)) {
                return urlToFilter;
            }

            List<NameValuePair> pairs = URLEncodedUtils.parse(query,
                    StandardCharsets.UTF_8);
            Iterator<NameValuePair> pairsIterator = pairs.iterator();
            while (pairsIterator.hasNext()) {
                NameValuePair param = pairsIterator.next();
                if (queryElementsToRemove.contains(param.getName())) {
                    pairsIterator.remove();
                } else if (removeHashes && param.getValue() != null) {
                    Matcher m = thirtytwobithash.matcher(param.getValue());
                    if (m.matches()) {
                        pairsIterator.remove();
                    }
                }
            }

            StringBuilder newFile = new StringBuilder();
            if (StringUtils.isNotBlank(path)) {
                newFile.append(path);
            }
            if (!pairs.isEmpty()) {
                Collections.sort(pairs, comp);
                String newQueryString = URLEncodedUtils.format(pairs,
                        StandardCharsets.UTF_8);
                newFile.append('?').append(newQueryString);
            }
            if (url.getRef() != null) {
                newFile.append('#').append(url.getRef());
            }

            return new URL(url.getProtocol(), url.getHost(), url.getPort(),
                    newFile.toString()).toString();
        } catch (MalformedURLException e) {
            LOG.warn("Invalid urlToFilter {}. {}", urlToFilter, e);
            return null;
        }
    }

    Comparator<NameValuePair> comp = new Comparator<NameValuePair>() {
        @Override
        public int compare(NameValuePair p1, NameValuePair p2) {
            return p1.getName().compareTo(p2.getName());
        }
    };

    /**
     * A common error to find is a query string that starts with an & instead of
     * a ? This will fix that error. So http://foo.com&a=b will be changed to
     * http://foo.com?a=b.
     *
     * @param urlToFilter
     * @return corrected url
     */
    private String unmangleQueryString(String urlToFilter) {
        int firstAmp = urlToFilter.indexOf('&');
        if (firstAmp > 0) {
            int firstQuestionMark = urlToFilter.indexOf('?');
            if (firstQuestionMark == -1) {
                return urlToFilter.replaceFirst("&", "?");
            }
        }
        return urlToFilter;
    }

    /**
     * Remove % encoding from path segment in URL for characters which should be
     * unescaped according to RFC3986 as
     * well as non-standard implementations of percent encoding, see
     * <https://en.
     * wikipedia.org/wiki/Percent-encoding#Non-standard_implementations>.
     */
    private String unescapePath(String path) {
        Matcher matcher = illegalEscapePattern.matcher(path);

        StringBuilder sb = null;
        int end = 0;

        while (matcher.find()) {
            if (sb == null) {
                sb = new StringBuilder();
            }
            // Append everything up to this group
            sb.append(path.substring(end, matcher.start()));
            String group = matcher.group(1);
            int letter = Integer.valueOf(group, 16);
            sb.append((char) letter);
            end = matcher.end();
        }

        // we got a replacement
        if (sb != null) {
            // append whatever is left
            sb.append(path.substring(end));
            path = sb.toString();
            end = 0;
        }

        matcher = unescapeRulePattern.matcher(path);

        if (!matcher.find()) {
            return path;
        }

        sb = new StringBuilder();

        // Traverse over all encoded groups
        do {
            // Append everything up to this group
            sb.append(path.substring(end, matcher.start()));

            // Get the integer representation of this hexadecimal encoded
            // character
            int letter = Integer.valueOf(matcher.group(1), 16);
            if (letter < 128 && unescapedCharacters[letter]) {
                // character should be unescaped in URLs
                sb.append((char) letter);
            } else {
                // Append the whole sequence as uppercase
                sb.append(matcher.group().toUpperCase(Locale.ROOT));
            }

            end = matcher.end();
        } while (matcher.find());

        // Append the rest if there's anything left
        sb.append(path.substring(end));

        return sb.toString();
    }

    /**
     * Convert path segment of URL from Unicode to UTF-8 and escape all
     * characters which should be escaped according to RFC3986..
     */
    private String escapePath(String path) {
        StringBuilder sb = new StringBuilder(path.length());

        // Traverse over all bytes in this URL
        for (byte b : path.getBytes(utf8)) {
            // Is this a control character?
            if (b < 33 || b == 91 || b == 92 || b == 93 || b == 124) {
                // Start escape sequence
                sb.append('%');

                // Get this byte's hexadecimal representation
                String hex = Integer.toHexString(b & 0xFF).toUpperCase(
                        Locale.ROOT);

                // Do we need to prepend a zero?
                if (hex.length() % 2 != 0) {
                    sb.append('0');
                    sb.append(hex);
                } else {
                    // No, append this hexadecimal representation
                    sb.append(hex);
                }
            } else {
                // No, just append this character as-is
                sb.append((char) b);
            }
        }

        return sb.toString();
    }

    private boolean isAscii(String str) {
        char[] chars = str.toCharArray();
        for (char c : chars) {
            if (c > 127) {
                return false;
            }
        }
        return true;
    }

}


/**
 * Licensed to DigitalPebble Ltd under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * DigitalPebble licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.digitalpebble.stormcrawler.aws.s3;

import java.util.Map;

import org.apache.commons.lang.StringUtils;
import org.apache.storm.metric.api.MultiCountMetric;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.tuple.Fields;

import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.regions.RegionUtils;
import com.amazonaws.services.s3.AmazonS3Client;
import com.digitalpebble.stormcrawler.util.ConfUtils;

@SuppressWarnings("serial")
public abstract class AbstractS3CacheBolt extends BaseRichBolt {

    public static final String S3_PREFIX = "s3.";
    public static final String ENDPOINT = S3_PREFIX + "endpoint";
    public static final String BUCKET = S3_PREFIX + "bucket";

    // is the region needed?
    public static final String REGION = S3_PREFIX + "region";

    public static final String INCACHE = S3_PREFIX + "inCache";

    protected OutputCollector _collector;
    protected MultiCountMetric eventCounter;

    protected AmazonS3Client client;

    protected String bucketName;

    /** Returns an S3 client given the configuration **/
    public static AmazonS3Client getS3Client(Map conf) {
        AWSCredentialsProvider provider = new DefaultAWSCredentialsProviderChain();
        AWSCredentials credentials = provider.getCredentials();
        ClientConfiguration config = new ClientConfiguration();

        AmazonS3Client client = new AmazonS3Client(credentials, config);

        String regionName = ConfUtils.getString(conf, REGION);
        if (StringUtils.isNotBlank(regionName)) {
            client.setRegion(RegionUtils.getRegion(regionName));
        }

        String endpoint = ConfUtils.getString(conf, ENDPOINT);
        if (StringUtils.isNotBlank(endpoint)) {
            client.setEndpoint(endpoint);
        }
        return client;
    }

    @Override
    public void prepare(Map conf, TopologyContext context,
            OutputCollector collector) {
        _collector = collector;
        client = getS3Client(conf);
    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields("url", "content", "metadata"));
    }
}


/**
 * Licensed to DigitalPebble Ltd under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * DigitalPebble licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.digitalpebble.stormcrawler.protocol.selenium;

import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang.StringUtils;
import org.openqa.selenium.remote.RemoteWebDriver;
import org.slf4j.LoggerFactory;

import com.digitalpebble.stormcrawler.Metadata;
import com.digitalpebble.stormcrawler.protocol.ProtocolResponse;
import com.digitalpebble.stormcrawler.util.ConfUtils;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.NullNode;

/**
 * Wrapper for the NavigationFilter defined in a JSON configuration
 */
public class NavigationFilters extends NavigationFilter {

    public static final NavigationFilters emptyNavigationFilters = new NavigationFilters();

    private static final org.slf4j.Logger LOG = LoggerFactory
            .getLogger(NavigationFilters.class);

    private NavigationFilter[] filters;

    private NavigationFilters() {
        filters = new NavigationFilter[0];
    }

    public ProtocolResponse filter(RemoteWebDriver driver, Metadata metadata) {
        for (NavigationFilter filter : filters) {
            ProtocolResponse response = filter.filter(driver, metadata);
            if (response != null)
                return response;
        }
        return null;
    }

    /**
     * Loads and configure the NavigationFilters based on the storm config if
     * there is one otherwise returns an emptyNavigationFilters.
     **/
    @SuppressWarnings("rawtypes")
    public static NavigationFilters fromConf(Map stormConf) {
        String configfile = ConfUtils.getString(stormConf,
                "navigationfilters.config.file");
        if (StringUtils.isNotBlank(configfile)) {
            try {
                return new NavigationFilters(stormConf, configfile);
            } catch (IOException e) {
                String message = "Exception caught while loading the NavigationFilters from "
                        + configfile;
                LOG.error(message);
                throw new RuntimeException(message, e);
            }
        }

        return NavigationFilters.emptyNavigationFilters;
    }

    /**
     * loads the filters from a JSON configuration file
     *
     * @throws IOException
     */
    @SuppressWarnings("rawtypes")
    public NavigationFilters(Map stormConf, String configFile)
            throws IOException {
        // load the JSON configFile
        // build a JSON object out of it
        JsonNode confNode = null;
        InputStream confStream = null;
        try {
            confStream = getClass().getClassLoader().getResourceAsStream(
                    configFile);

            ObjectMapper mapper = new ObjectMapper();
            confNode = mapper.readValue(confStream, JsonNode.class);
        } catch (Exception e) {
            throw new IOException("Unable to build JSON object from file", e);
        } finally {
            if (confStream != null) {
                confStream.close();
            }
        }

        configure(stormConf, confNode);
    }

    @SuppressWarnings("rawtypes")
    @Override
    public void configure(Map stormConf, JsonNode filtersConf) {
        // initialises the filters
        List<NavigationFilter> filterLists = new ArrayList<>();

        // get the filters part
        String name = getClass().getCanonicalName();
        filtersConf = filtersConf.get(name);

        if (filtersConf == null) {
            LOG.info("No field {} in JSON config. Skipping", name);
            filters = new NavigationFilter[0];
            return;
        }

        // conf node contains a list of objects
        Iterator<JsonNode> filterIter = filtersConf.elements();
        while (filterIter.hasNext()) {
            JsonNode afilterConf = filterIter.next();
            String filterName = "<unnamed>";
            JsonNode nameNode = afilterConf.get("name");
            if (nameNode != null) {
                filterName = nameNode.textValue();
            }
            JsonNode classNode = afilterConf.get("class");
            if (classNode == null) {
                LOG.error("Filter {} doesn't specified a 'class' attribute",
                        filterName);
                continue;
            }
            String className = classNode.textValue().trim();
            filterName += '[' + className + ']';
            // check that it is available and implements the interface
            // NavigationFilter
            try {
                Class<?> filterClass = Class.forName(className);
                boolean subClassOK = NavigationFilter.class
                        .isAssignableFrom(filterClass);
                if (!subClassOK) {
                    LOG.error("Filter {} does not extend NavigationFilter",
                            filterName);
                    continue;
                }
                NavigationFilter filterInstance = (NavigationFilter) filterClass
                        .newInstance();

                JsonNode paramNode = afilterConf.get("params");
                if (paramNode != null) {
                    filterInstance.configure(stormConf, paramNode);
                } else {
                    // Pass in a nullNode if missing
                    filterInstance.configure(stormConf, NullNode.getInstance());
                }

                filterLists.add(filterInstance);
                LOG.info("Setup {}", filterName);
            } catch (Exception e) {
                LOG.error("Can't setup {}: {}", filterName, e);
                throw new RuntimeException("Can't setup " + filterName, e);
            }
        }

        filters = filterLists.toArray(new NavigationFilter[filterLists.size()]);
    }

}


/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.digitalpebble.stormcrawler.warc;

import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.storm.hdfs.bolt.AbstractHdfsBolt;
import org.apache.storm.hdfs.bolt.format.FileNameFormat;
import org.apache.storm.hdfs.bolt.format.RecordFormat;
import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy;
import org.apache.storm.hdfs.bolt.sync.SyncPolicy;
import org.apache.storm.hdfs.common.AbstractHDFSWriter;
import org.apache.storm.hdfs.common.HDFSWriter;
import org.apache.storm.hdfs.common.rotation.RotationAction;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.utils.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Unlike the standard HdfsBolt this one writes to a gzipped stream with
 * per-record compression.
 **/
@SuppressWarnings("serial")
public class GzipHdfsBolt extends AbstractHdfsBolt {

    private static final Logger LOG = LoggerFactory
            .getLogger(GzipHdfsBolt.class);

    protected transient FSDataOutputStream out = null;

    protected RecordFormat format;

    public static class GzippedRecordFormat implements RecordFormat {

        private RecordFormat baseFormat;

        /**
         * whether to skip empty records (byte[] of length 0), i.e. do not
         * create an gzip container containing nothing
         */
        protected boolean compressEmpty = false;

        public GzippedRecordFormat(RecordFormat format) {
            baseFormat = format;
        }

        @Override
        public byte[] format(Tuple tuple) {
            byte[] bytes = baseFormat.format(tuple);
            if (bytes.length == 0 && !compressEmpty) {
                return new byte[] {};
            }
            return Utils.gzip(bytes);
        }
    }

    public static class MultipleRecordFormat implements RecordFormat {

        private List<RecordFormat> formats = new ArrayList<>();

        public MultipleRecordFormat(RecordFormat format) {
            addFormat(format, 0);
        }

        public void addFormat(RecordFormat format, int position) {
            if (position < 0 || position > formats.size()) {
                formats.add(format);
            } else {
                formats.add(position, format);
            }
        }

        @Override
        public byte[] format(Tuple tuple) {
            byte[][] tmp = new byte[formats.size()][];
            int i = -1;
            int size = 0;
            for (RecordFormat format : formats) {
                tmp[++i] = format.format(tuple);
                size += tmp[i].length;
            }
            byte[] res = new byte[size];
            int pos = 0;
            for (i = 0; i < tmp.length; i++) {
                System.arraycopy(tmp[i], 0, res, pos, tmp[i].length);
                pos += tmp[i].length;
            }
            return res;
        }
    }

    public GzipHdfsBolt withFsUrl(String fsUrl) {
        this.fsUrl = fsUrl;
        return this;
    }

    public GzipHdfsBolt withConfigKey(String configKey) {
        this.configKey = configKey;
        return this;
    }

    public GzipHdfsBolt withFileNameFormat(FileNameFormat fileNameFormat) {
        this.fileNameFormat = fileNameFormat;
        return this;
    }

    /** Sets the record format, overwriting the existing one(s) */
    public GzipHdfsBolt withRecordFormat(RecordFormat format) {
        this.format = new GzippedRecordFormat(format);
        return this;
    }

    /** Add an additional record format at end of existing ones */
    public GzipHdfsBolt addRecordFormat(RecordFormat format) {
        return addRecordFormat(format, -1);
    }

    /** Add an additional record format at given position */
    public GzipHdfsBolt addRecordFormat(RecordFormat format, int position) {
        MultipleRecordFormat formats;
        if (this.format == null) {
            formats = new MultipleRecordFormat(format);
            this.format = formats;
        } else {
            if (this.format instanceof MultipleRecordFormat) {
                formats = (MultipleRecordFormat) this.format;
            } else {
                formats = new MultipleRecordFormat(this.format);
                this.format = formats;
            }
            formats.addFormat(new GzippedRecordFormat(format), position);
        }
        return this;
    }

    public GzipHdfsBolt withSyncPolicy(SyncPolicy syncPolicy) {
        this.syncPolicy = syncPolicy;
        return this;
    }

    public GzipHdfsBolt withRotationPolicy(FileRotationPolicy rotationPolicy) {
        this.rotationPolicy = rotationPolicy;
        return this;
    }

    public GzipHdfsBolt addRotationAction(RotationAction action) {
        this.rotationActions.add(action);
        return this;
    }

    @Override
    public void doPrepare(Map conf, TopologyContext topologyContext,
            OutputCollector collector) throws IOException {
        LOG.info("Preparing HDFS Bolt...");
        this.fs = FileSystem.get(URI.create(this.fsUrl), hdfsConfig);
    }

    @Override
    public void cleanup() {
        LOG.info("Cleanup called on bolt");
        try {
            this.out.flush();
            this.out.close();
        } catch (IOException e) {
            LOG.error("Exception while calling cleanup");
        }
    }

    @Override
    protected String getWriterKey(Tuple tuple) {
        return "CONSTANT";
    }

    @Override
    protected AbstractHDFSWriter makeNewWriter(Path path, Tuple tuple)
            throws IOException {
        out = this.fs.create(path);
        return new HDFSWriter(rotationPolicy, path, out, format);
    }
}


/**
 * Licensed to DigitalPebble Ltd under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * DigitalPebble licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.digitalpebble.stormcrawler.persistence;

import java.time.Instant;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;

import org.apache.storm.metric.api.MultiCountMetric;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichSpout;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;
import org.apache.storm.utils.Utils;

import com.digitalpebble.stormcrawler.util.CollectionMetric;
import com.digitalpebble.stormcrawler.util.ConfUtils;
import com.google.common.base.Optional;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;

/**
 * Common features of spouts which query a backend to generate tuples. Tracks
 * the URLs being processes, with an optional delay before they are removed from
 * the cache. Throttles the rate a which queries are emitted and provides a
 * buffer to store the URLs waiting to be sent.
 *
 * @since 1.11
 **/

public abstract class AbstractQueryingSpout extends BaseRichSpout {

    /**
     * Time in seconds for which acked or failed URLs will be considered for
     * fetching again, default 30 secs.
     **/
    protected static final String StatusTTLPurgatory = "spout.ttl.purgatory";
    /**
     * Min time to allow between 2 successive queries to the backend. Value in
     * msecs, default 2000.
     **/
    protected static final String StatusMinDelayParamName = "spout.min.delay.queries";
    protected long minDelayBetweenQueries = 2000;

    /**
     * Max time to allow between 2 successive queries to the backend. Value in
     * msecs, default 20000.
     **/
    protected static final String StatusMaxDelayParamName = "spout.max.delay.queries";
    protected long maxDelayBetweenQueries = 20000;

    /**
     * Delay in seconds after which the nextFetchDate filter is set to the
     * current time, default 120. Is used to prevent the search to be limited to
     * a handful of sources.
     **/
    protected static final String resetFetchDateParamName = "spout.reset.fetchdate.after";
    protected int resetFetchDateAfterNSecs = 120;

    protected Instant lastTimeResetToNOW;

    private long timeLastQuerySent = 0;
    private long timeLastQueryReceived = 0;

    private long timestampEmptyBuffer = -1;

    protected MultiCountMetric eventCounter;

    protected Queue<Values> buffer = new LinkedList<>();
    protected HashSet<String> in_buffer = new HashSet<>();

    protected SpoutOutputCollector _collector;

    /** Required for implementations doing asynchronous calls **/
    protected AtomicBoolean isInQuery = new AtomicBoolean(false);

    protected CollectionMetric queryTimes;

    @Override
    public void open(Map stormConf, TopologyContext context,
            SpoutOutputCollector collector) {

        int ttlPurgatory = ConfUtils.getInt(stormConf, StatusTTLPurgatory, 30);

        minDelayBetweenQueries = ConfUtils.getLong(stormConf,
                StatusMinDelayParamName, 2000);

        maxDelayBetweenQueries = ConfUtils.getLong(stormConf,
                StatusMaxDelayParamName, maxDelayBetweenQueries);

        beingProcessed = new InProcessMap<>(ttlPurgatory, TimeUnit.SECONDS);

        eventCounter = context.registerMetric("counters",
                new MultiCountMetric(), 10);

        context.registerMetric("buffer_size", () -> buffer.size(), 10);
        context.registerMetric("beingProcessed", () -> beingProcessed.size(), 10);
        context.registerMetric("inPurgatory", () -> beingProcessed.inCache(), 10);

        queryTimes = new CollectionMetric();
        context.registerMetric("spout_query_time_msec", queryTimes, 10);

        resetFetchDateAfterNSecs = ConfUtils.getInt(stormConf,
                resetFetchDateParamName, resetFetchDateAfterNSecs);

        _collector = collector;
    }

    /**
     * Method where specific implementations query the storage. Implementations
     * should call markQueryReceivedNow when the documents have been received.
     **/
    protected abstract void populateBuffer();

    /**
     * Map to keep in-process URLs, with the URL as key and optional value
     * depending on the spout implementation. The entries are kept in a cache
     * for a configurable amount of time to avoid that some items are fetched a
     * second time if new items are queried shortly after they have been acked.
     */
    protected InProcessMap<String, Object> beingProcessed;
    private boolean active;

    /** Map which holds elements some additional time after the removal. */
    public class InProcessMap<K, V> extends HashMap<K, V> {

        private Cache<K, Optional<V>> deletionCache;

        public InProcessMap(long maxDuration, TimeUnit timeUnit) {
            deletionCache = CacheBuilder.newBuilder()
                    .expireAfterWrite(maxDuration, timeUnit).build();
        }

        @Override
        public boolean containsKey(Object key) {
            boolean incache = super.containsKey(key);
            if (!incache) {
                incache = (deletionCache.getIfPresent(key) != null);
            }
            return incache;
        }

        @Override
        public V remove(Object key) {
            deletionCache.put((K) key, Optional.absent());
            return super.remove(key);
        }

        public long inCache() {
            return deletionCache.size();
        }
    }

    @Override
    public void nextTuple() {
        if (!active)
            return;

        // synchronize access to buffer needed in case of asynchronous
        // queries to the backend
        synchronized (buffer) {

            // force the refresh of the buffer even if the buffer is not empty
            if (!isInQuery.get() && triggerQueries()) {
                populateBuffer();
                timeLastQuerySent = System.currentTimeMillis();
            }

            if (!buffer.isEmpty()) {
                // track how long the buffer had been empty for
                if (timestampEmptyBuffer != -1) {
                    eventCounter.scope("empty.buffer").incrBy(
                            System.currentTimeMillis() - timestampEmptyBuffer);
                    timestampEmptyBuffer = -1;
                }
                List<Object> fields = buffer.remove();
                String url = fields.get(0).toString();
                this._collector.emit(fields, url);
                beingProcessed.put(url, null);
                in_buffer.remove(url);
                eventCounter.scope("emitted").incrBy(1);
                return;
            } else if (timestampEmptyBuffer == -1) {
                timestampEmptyBuffer = System.currentTimeMillis();
            }
        }

        if (isInQuery.get() || throttleQueries() > 0) {
            // sleep for a bit but not too much in order to give ack/fail a
            // chance
            Utils.sleep(10);
            return;
        }

        // re-populate the buffer
        populateBuffer();

        timeLastQuerySent = System.currentTimeMillis();
    }

    /**
     * Returns the amount of time to wait if the backend was queried too
     * recently and needs throttling or -1 if the backend can be queried
     * straight away.
     **/
    private long throttleQueries() {
        if (timeLastQuerySent != 0) {
            // check that we allowed some time between queries
            long difference = System.currentTimeMillis() - timeLastQuerySent;
            if (difference < minDelayBetweenQueries) {
                return minDelayBetweenQueries - difference;
            }
        }
        return -1;
    }

    /**
     * Indicates whether enough time has elapsed since receiving the results of
     * the previous query so that a new one can be sent even if the buffer is
     * not empty. Applies to asynchronous clients only.
     **/
    private boolean triggerQueries() {
        if (timeLastQueryReceived != 0 && maxDelayBetweenQueries > 0) {
            // check that we allowed some time between queries
            long difference = System.currentTimeMillis()
                    - timeLastQueryReceived;
            if (difference > maxDelayBetweenQueries) {
                return true;
            }
        }
        return false;
    }

    protected long getTimeLastQuerySent() {
        return timeLastQuerySent;
    }

    /**
     * sets the marker that we are in a query to false and timeLastQueryReceived
     * to now
     **/
    protected void markQueryReceivedNow() {
        isInQuery.set(false);
        timeLastQueryReceived = System.currentTimeMillis();
    }

    @Override
    public void activate() {
        active = true;
    }

    @Override
    public void deactivate() {
        active = false;
    }

    @Override
    public void ack(Object msgId) {
        beingProcessed.remove(msgId);
        eventCounter.scope("acked").incrBy(1);
    }

    @Override
    public void fail(Object msgId) {
        beingProcessed.remove(msgId);
        eventCounter.scope("failed").incrBy(1);
    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields("url", "metadata"));
    }

}


/**
 * Licensed to DigitalPebble Ltd under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * DigitalPebble licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.digitalpebble.stormcrawler.parse;

import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang.StringUtils;
import org.slf4j.LoggerFactory;
import org.w3c.dom.DocumentFragment;

import com.digitalpebble.stormcrawler.JSONResource;
import com.digitalpebble.stormcrawler.util.ConfUtils;
import com.digitalpebble.stormcrawler.util.Configurable;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;

/**
 * Wrapper for the ParseFilters defined in a JSON configuration
 */
public class ParseFilters extends ParseFilter implements JSONResource {

    public static final ParseFilters emptyParseFilter = new ParseFilters();

    private static final org.slf4j.Logger LOG = LoggerFactory
            .getLogger(ParseFilters.class);

    private ParseFilter[] filters;

    private ParseFilters() {
        filters = new ParseFilter[0];
    }

    private String configFile = "parsefilters.config.file";

    private Map stormConf;

    /**
     * Loads and configure the ParseFilters based on the storm config if there
     * is one otherwise returns an emptyParseFilter.
     **/
    @SuppressWarnings("rawtypes")
    public static ParseFilters fromConf(Map stormConf) {
        String parseconfigfile = ConfUtils.getString(stormConf,
                "parsefilters.config.file");
        if (StringUtils.isNotBlank(parseconfigfile)) {
            try {
                return new ParseFilters(stormConf, parseconfigfile);
            } catch (IOException e) {
                String message = "Exception caught while loading the ParseFilters from "
                        + parseconfigfile;
                LOG.error(message);
                throw new RuntimeException(message, e);
            }
        }

        return ParseFilters.emptyParseFilter;
    }

    /**
     * loads the filters from a JSON configuration file
     *
     * @throws IOException
     */
    @SuppressWarnings("rawtypes")
    public ParseFilters(Map stormConf, String configFile) throws IOException {
        this.configFile = configFile;
        this.stormConf = stormConf;
        try {
            loadJSONResources();
        } catch (Exception e) {
            throw new IOException("Unable to build JSON object from file", e);
        }
    }

    @Override
    public void loadJSONResources(InputStream inputStream)
            throws JsonParseException, JsonMappingException, IOException {
        ObjectMapper mapper = new ObjectMapper();
        JsonNode confNode = mapper.readValue(inputStream, JsonNode.class);
        configure(stormConf, confNode);
    }

    @Override
    public String getResourceFile() {
        return this.configFile;
    }

    @SuppressWarnings("rawtypes")
    @Override
    public void configure(Map stormConf, JsonNode filtersConf) {
        List<ParseFilter> list = Configurable.configure(stormConf, filtersConf,
                ParseFilter.class, this.getClass().getName());
        filters = list.toArray(new ParseFilter[list.size()]);
    }

    @Override
    public boolean needsDOM() {
        for (ParseFilter filter : filters) {
            boolean needsDOM = filter.needsDOM();
            if (needsDOM) {
                return true;
            }
        }
        return false;
    }

    @Override
    public void filter(String URL, byte[] content, DocumentFragment doc,
            ParseResult parse) {

        for (ParseFilter filter : filters) {
            long start = System.currentTimeMillis();
            if (doc == null && filter.needsDOM()) {
                LOG.info(
                        "ParseFilter {} needs DOM but has none to work on - skip : {}",
                        filter.getClass().getName(), URL);
                continue;
            }
            filter.filter(URL, content, doc, parse);
            long end = System.currentTimeMillis();
            LOG.debug("ParseFilter {} took {} msec", filter.getClass()
                    .getName(), end - start);
        }
    }
}