/**
 * Licensed to DigitalPebble Ltd under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * DigitalPebble licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * XXX NUTCH COMMENT
 * XXX ab@apache.org: This class is copied verbatim from Xalan-J 2.6.0
 * XXX distribution, org.apache.xml.utils.DOMBuilder, in order to
 * avoid dependency on Xalan.
 */

package com.digitalpebble.stormcrawler.tika;

import java.util.Locale;
import java.util.Stack;

import org.w3c.dom.CDATASection;
import org.w3c.dom.Comment;
import org.w3c.dom.Document;
import org.w3c.dom.DocumentFragment;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.Text;
import org.xml.sax.Attributes;
import org.xml.sax.ContentHandler;
import org.xml.sax.Locator;
import org.xml.sax.ext.LexicalHandler;

/**
 * This class takes SAX events (in addition to some extra events that SAX
 * doesn't handle yet) and adds the result to a document or document fragment.
 */
public class DOMBuilder implements ContentHandler, LexicalHandler {

    private boolean upperCaseElementNames = true;

    /** Root document */
    private Document m_doc;

    /** Current node */
    private Node m_currentNode = null;

    /** First node of document fragment or null if not a DocumentFragment */
    private DocumentFragment m_docFrag = null;

    /** Vector of element nodes */
    private Stack<Element> m_elemStack = new Stack<>();

    /**
     * Element recorded with this namespace will be converted to Node without a
     * namespace
     */
    private String defaultNamespaceURI = null;

    /**
     * DOMBuilder instance constructor... it will add the DOM nodes to the
     * document fragment.
     *
     * @param doc
     *            Root document
     * @param node
     *            Current node
     */
    public DOMBuilder(Document doc, Node node) {
        m_doc = doc;
        m_currentNode = node;
    }

    /**
     * DOMBuilder instance constructor... it will add the DOM nodes to the
     * document fragment.
     *
     * @param doc
     *            Root document
     * @param docFrag
     *            Document fragment
     */
    DOMBuilder(Document doc, DocumentFragment docFrag) {
        m_doc = doc;
        m_docFrag = docFrag;
    }

    /**
     * DOMBuilder instance constructor... it will add the DOM nodes to the
     * document.
     *
     * @param doc
     *            Root document
     */
    DOMBuilder(Document doc) {
        m_doc = doc;
    }

    /**
     * Get the root node of the DOM being created. This is either a Document or
     * a DocumentFragment.
     *
     * @return The root document or document fragment if not null
     */
    Node getRootNode() {
        return (null != m_docFrag) ? (Node) m_docFrag : (Node) m_doc;
    }

    /**
     * Get the node currently being processed.
     *
     * @return the current node being processed
     */
    Node getCurrentNode() {
        return m_currentNode;
    }

    /**
     * Return null since there is no Writer for this class.
     *
     * @return null
     */
    java.io.Writer getWriter() {
        return null;
    }

    /**
     * Append a node to the current container.
     *
     * @param newNode
     *            New node to append
     */
    protected void append(Node newNode) throws org.xml.sax.SAXException {

        Node currentNode = m_currentNode;

        if (null != currentNode) {
            currentNode.appendChild(newNode);

            // System.out.println(newNode.getNodeName());
        } else if (null != m_docFrag) {
            m_docFrag.appendChild(newNode);
        } else {
            boolean ok = true;
            short type = newNode.getNodeType();

            if (type == Node.TEXT_NODE) {
                String data = newNode.getNodeValue();

                if ((null != data) && (data.trim().length() > 0)) {
                    throw new org.xml.sax.SAXException(
                            "Warning: can't output text before document element!  Ignoring...");
                }

                ok = false;
            } else if (type == Node.ELEMENT_NODE) {
                if (m_doc.getDocumentElement() != null) {
                    throw new org.xml.sax.SAXException(
                            "Can't have more than one root on a DOM!");
                }
            }

            if (ok) {
                m_doc.appendChild(newNode);
            }
        }
    }

    /**
     * Receive an object for locating the origin of SAX document events.
     *
     *

     * SAX parsers are strongly encouraged (though not absolutely required) to
     * supply a locator: if it does so, it must supply the locator to the
     * application by invoking this method before invoking any of the other
     * methods in the ContentHandler interface.
     *

     *
     *

     * The locator allows the application to determine the end position of any
     * document-related event, even if the parser is not reporting an error.
     * Typically, the application will use this information for reporting its
     * own errors (such as character content that does not match an
     * application's business rules). The information returned by the locator is
     * probably not sufficient for use with a search engine.
     *

     *
     *

     * Note that the locator will return correct information only during the
     * invocation of the events in this interface. The application should not
     * attempt to use it at any other time.
     *

     *
     * @param locator
     *            An object that can return the location of any SAX document
     *            event.
     * @see org.xml.sax.Locator
     */
    @Override
    public void setDocumentLocator(Locator locator) {

        // No action for the moment.
    }

    /**
     * Receive notification of the beginning of a document.
     *
     *

     * The SAX parser will invoke this method only once, before any other
     * methods in this interface or in DTDHandler (except for
     * setDocumentLocator).
     *

     */
    @Override
    public void startDocument() throws org.xml.sax.SAXException {

        // No action for the moment.
    }

    /**
     * Receive notification of the end of a document.
     *
     *

     * The SAX parser will invoke this method only once, and it will be the last
     * method invoked during the parse. The parser shall not invoke this method
     * until it has either abandoned parsing (because of an unrecoverable error)
     * or reached the end of input.
     *

     */
    @Override
    public void endDocument() throws org.xml.sax.SAXException {

        // No action for the moment.
    }

    /**
     * Receive notification of the beginning of an element.
     *
     *

     * The Parser will invoke this method at the beginning of every element in
     * the XML document; there will be a corresponding endElement() event for
     * every startElement() event (even when the element is empty). All of the
     * element's content will be reported, in order, before the corresponding
     * endElement() event.
     *

     *
     *

     * If the element name has a namespace prefix, the prefix will still be
     * attached. Note that the attribute list provided will contain only
     * attributes with explicit values (specified or defaulted): #IMPLIED
     * attributes will be omitted.
     *

     *
     *
     * @param ns
     *            The namespace of the node
     * @param localName
     *            The local part of the qualified name
     * @param name
     *            The element name.
     * @param atts
     *            The attributes attached to the element, if any.
     * @see #endElement
     * @see org.xml.sax.Attributes
     */
    @Override
    public void startElement(String ns, String localName, String name,
            Attributes atts) throws org.xml.sax.SAXException {

        Element elem;

        if (upperCaseElementNames) {
            name = name.toUpperCase(Locale.ROOT);
        }

        // Note that the namespace-aware call must be used to correctly
        // construct a Level 2 DOM, even for non-namespaced nodes.
        if ((null == ns) || (ns.length() == 0)
                || ns.equals(defaultNamespaceURI)) {
            elem = m_doc.createElementNS(null, name);
        } else {
            elem = m_doc.createElementNS(ns, name);
        }

        append(elem);

        try {
            int nAtts = atts.getLength();

            if (0 != nAtts) {
                for (int i = 0; i < nAtts; i++) {
                    // First handle a possible ID attribute
                    if (atts.getType(i).equalsIgnoreCase("ID")) {
                        setIDAttribute(atts.getValue(i), elem);
                    }

                    String attrNS = atts.getURI(i);

                    if ("".equals(attrNS)) {
                        attrNS = null; // DOM represents no-namespace as null
                    }

                    // System.out.println("attrNS: "+attrNS+", localName: "+atts.getQName(i)
                    // +", qname: "+atts.getQName(i)+", value: "+atts.getValue(i));
                    // Crimson won't let us set an xmlns: attribute on the DOM.
                    String attrQName = atts.getQName(i);

                    // In SAX, xmlns: attributes have an empty namespace, while
                    // in DOM they should have the xmlns namespace
                    if (attrQName.startsWith("xmlns:")) {
                        attrNS = "http://www.w3.org/2000/xmlns/";
                    }

                    // ALWAYS use the DOM Level 2 call!
                    elem.setAttributeNS(attrNS, attrQName, atts.getValue(i));
                }
            }

            // append(elem);

            m_elemStack.push(elem);

            m_currentNode = elem;

            // append(elem);
        } catch (java.lang.Exception de) {
            // de.printStackTrace();
            throw new org.xml.sax.SAXException(de);
        }

    }

    /**
     *
     *
     *
     * Receive notification of the end of an element.
     *
     *

     * The SAX parser will invoke this method at the end of every element in the
     * XML document; there will be a corresponding startElement() event for
     * every endElement() event (even when the element is empty).
     *

     *
     *

     * If the element name has a namespace prefix, the prefix will still be
     * attached to the name.
     *

     *
     *
     * @param ns
     *            the namespace of the element
     * @param localName
     *            The local part of the qualified name of the element
     * @param name
     *            The element name
     */
    @Override
    public void endElement(String ns, String localName, String name)
            throws org.xml.sax.SAXException {
        m_elemStack.pop();
        m_currentNode = m_elemStack.isEmpty() ? null : (Node) m_elemStack
                .peek();
    }

    /**
     * Set an ID string to node association in the ID table.
     *
     * @param id
     *            The ID string.
     * @param elem
     *            The associated ID.
     */
    public void setIDAttribute(String id, Element elem) {

        // Do nothing. This method is meant to be overiden.
    }

    /**
     * Receive notification of character data.
     *
     *

     * The Parser will call this method to report each chunk of character data.
     * SAX parsers may return all contiguous character data in a single chunk,
     * or they may split it into several chunks; however, all of the characters
     * in any single event must come from the same external entity, so that the
     * Locator provides useful information.
     *

     *
     *

     * The application must not attempt to read from the array outside of the
     * specified range.
     *

     *
     *

     * Note that some parsers will report whitespace using the
     * ignorableWhitespace() method rather than this one (validating parsers
     * must do so).
     *

     *
     * @param ch
     *            The characters from the XML document.
     * @param start
     *            The start position in the array.
     * @param length
     *            The number of characters to read from the array.
     * @see #ignorableWhitespace
     * @see org.xml.sax.Locator
     */
    @Override
    public void characters(char ch[], int start, int length)
            throws org.xml.sax.SAXException {
        if (isOutsideDocElem()
                && XMLCharacterRecognizer.isWhiteSpace(ch, start, length)) {
            return; // avoid DOM006 Hierarchy request error
        }

        if (m_inCData) {
            cdata(ch, start, length);

            return;
        }

        String s = new String(ch, start, length);
        Node childNode;
        childNode = m_currentNode != null ? m_currentNode.getLastChild() : null;
        if (childNode != null && childNode.getNodeType() == Node.TEXT_NODE) {
            ((Text) childNode).appendData(s);
        } else {
            Text text = m_doc.createTextNode(s);
            append(text);
        }
    }

    /**
     * If available, when the disable-output-escaping attribute is used, output
     * raw text without escaping. A PI will be inserted in front of the node
     * with the name "lotusxsl-next-is-raw" and a value of "formatter-to-dom".
     *
     * @param ch
     *            Array containing the characters
     * @param start
     *            Index to start of characters in the array
     * @param length
     *            Number of characters in the array
     */
    public void charactersRaw(char ch[], int start, int length)
            throws org.xml.sax.SAXException {
        if (isOutsideDocElem()
                && XMLCharacterRecognizer.isWhiteSpace(ch, start, length)) {
            return; // avoid DOM006 Hierarchy request error
        }

        String s = new String(ch, start, length);

        append(m_doc.createProcessingInstruction("xslt-next-is-raw",
                "formatter-to-dom"));
        append(m_doc.createTextNode(s));
    }

    /**
     * Report the beginning of an entity.
     *
     * The start and end of the document entity are not reported. The start and
     * end of the external DTD subset are reported using the pseudo-name
     * "[dtd]". All other events must be properly nested within start/end entity
     * events.
     *
     * @param name
     *            The name of the entity. If it is a parameter entity, the name
     *            will begin with '%'.
     * @see #endEntity
     * @see org.xml.sax.ext.DeclHandler#internalEntityDecl
     * @see org.xml.sax.ext.DeclHandler#externalEntityDecl
     */
    @Override
    public void startEntity(String name) throws org.xml.sax.SAXException {

        // Almost certainly the wrong behavior...
        // entityReference(name);
    }

    /**
     * Report the end of an entity.
     *
     * @param name
     *            The name of the entity that is ending.
     * @see #startEntity
     */
    @Override
    public void endEntity(String name) throws org.xml.sax.SAXException {
    }

    /**
     * Receive notivication of a entityReference.
     *
     * @param name
     *            name of the entity reference
     */
    public void entityReference(String name) throws org.xml.sax.SAXException {
        append(m_doc.createEntityReference(name));
    }

    /**
     * Receive notification of ignorable whitespace in element content.
     *
     *

     * Validating Parsers must use this method to report each chunk of ignorable
     * whitespace (see the W3C XML 1.0 recommendation, section 2.10):
     * non-validating parsers may also use this method if they are capable of
     * parsing and using content models.
     *

     *
     *

     * SAX parsers may return all contiguous whitespace in a single chunk, or
     * they may split it into several chunks; however, all of the characters in
     * any single event must come from the same external entity, so that the
     * Locator provides useful information.
     *

     *
     *

     * The application must not attempt to read from the array outside of the
     * specified range.
     *

     *
     * @param ch
     *            The characters from the XML document.
     * @param start
     *            The start position in the array.
     * @param length
     *            The number of characters to read from the array.
     * @see #characters
     */
    @Override
    public void ignorableWhitespace(char ch[], int start, int length)
            throws org.xml.sax.SAXException {
        if (isOutsideDocElem()) {
            return; // avoid DOM006 Hierarchy request error
        }

        String s = new String(ch, start, length);

        append(m_doc.createTextNode(s));
    }

    /**
     * Tell if the current node is outside the document element.
     *
     * @return true if the current node is outside the document element.
     */
    private boolean isOutsideDocElem() {
        return (null == m_docFrag)
                && m_elemStack.size() == 0
                && (null == m_currentNode || m_currentNode.getNodeType() == Node.DOCUMENT_NODE);
    }

    /**
     * Receive notification of a processing instruction.
     *
     *

     * The Parser will invoke this method once for each processing instruction
     * found: note that processing instructions may occur before or after the
     * main document element.
     *

     *
     *

     * A SAX parser should never report an XML declaration (XML 1.0, section
     * 2.8) or a text declaration (XML 1.0, section 4.3.1) using this method.
     *

     *
     * @param target
     *            The processing instruction target.
     * @param data
     *            The processing instruction data, or null if none was supplied.
     */
    @Override
    public void processingInstruction(String target, String data)
            throws org.xml.sax.SAXException {
        append(m_doc.createProcessingInstruction(target, data));
    }

    /**
     * Report an XML comment anywhere in the document.
     *
     * This callback will be used for comments inside or outside the document
     * element, including comments in the external DTD subset (if read).
     *
     * @param ch
     *            An array holding the characters in the comment.
     * @param start
     *            The starting position in the array.
     * @param length
     *            The number of characters to use from the array.
     */
    @Override
    public void comment(char ch[], int start, int length)
            throws org.xml.sax.SAXException {
        // tagsoup sometimes submits invalid values here
        if (ch == null || start < 0 || length >= (ch.length - start)
                || length < 0) {
            return;
        }
        append(m_doc.createComment(new String(ch, start, length)));
    }

    /** Flag indicating that we are processing a CData section */
    protected boolean m_inCData = false;

    /**
     * Report the start of a CDATA section.
     *
     * @see #endCDATA
     */
    @Override
    public void startCDATA() throws org.xml.sax.SAXException {
        m_inCData = true;
        append(m_doc.createCDATASection(""));
    }

    /**
     * Report the end of a CDATA section.
     *
     * @see #startCDATA
     */
    @Override
    public void endCDATA() throws org.xml.sax.SAXException {
        m_inCData = false;
    }

    /**
     * Receive notification of cdata.
     *
     *

     * The Parser will call this method to report each chunk of character data.
     * SAX parsers may return all contiguous character data in a single chunk,
     * or they may split it into several chunks; however, all of the characters
     * in any single event must come from the same external entity, so that the
     * Locator provides useful information.
     *

     *
     *

     * The application must not attempt to read from the array outside of the
     * specified range.
     *

     *
     *

     * Note that some parsers will report whitespace using the
     * ignorableWhitespace() method rather than this one (validating parsers
     * must do so).
     *

     *
     * @param ch
     *            The characters from the XML document.
     * @param start
     *            The start position in the array.
     * @param length
     *            The number of characters to read from the array.
     * @see #ignorableWhitespace
     * @see org.xml.sax.Locator
     */
    public void cdata(char ch[], int start, int length) {
        if (isOutsideDocElem()
                && XMLCharacterRecognizer.isWhiteSpace(ch, start, length)) {
            return; // avoid DOM006 Hierarchy request error
        }

        String s = new String(ch, start, length);

        // XXX ab@apache.org: modified from the original, to accomodate TagSoup.
        Node n = m_currentNode.getLastChild();
        if (n instanceof CDATASection) {
            ((CDATASection) n).appendData(s);
        } else if (n instanceof Comment) {
            ((Comment) n).appendData(s);
        }
    }

    /**
     * Report the start of DTD declarations, if any.
     *
     * Any declarations are assumed to be in the internal subset unless
     * otherwise indicated.
     *
     * @param name
     *            The document type name.
     * @param publicId
     *            The declared public identifier for the external DTD subset, or
     *            null if none was declared.
     * @param systemId
     *            The declared system identifier for the external DTD subset, or
     *            null if none was declared.
     * @see #endDTD
     * @see #startEntity
     */
    @Override
    public void startDTD(String name, String publicId, String systemId)
            throws org.xml.sax.SAXException {

        // Do nothing for now.
    }

    /**
     * Report the end of DTD declarations.
     *
     * @see #startDTD
     */
    @Override
    public void endDTD() throws org.xml.sax.SAXException {

        // Do nothing for now.
    }

    /**
     * Begin the scope of a prefix-URI Namespace mapping.
     *
     *

     * The information from this event is not necessary for normal Namespace
     * processing: the SAX XML reader will automatically replace prefixes for
     * element and attribute names when the
     * http://xml.org/sax/features/namespaces feature is true (the default).
     *

     *
     *

     * There are cases, however, when applications need to use prefixes in
     * character data or in attribute values, where they cannot safely be
     * expanded automatically; the start/endPrefixMapping event supplies the
     * information to the application to expand prefixes in those contexts
     * itself, if necessary.
     *

     *
     *

     * Note that start/endPrefixMapping events are not guaranteed to be properly
     * nested relative to each-other: all startPrefixMapping events will occur
     * before the corresponding startElement event, and all endPrefixMapping
     * events will occur after the corresponding endElement event, but their
     * order is not guaranteed.
     *

     *
     * @param prefix
     *            The Namespace prefix being declared.
     * @param uri
     *            The Namespace URI the prefix is mapped to.
     * @see #endPrefixMapping
     * @see #startElement
     */
    @Override
    public void startPrefixMapping(String prefix, String uri)
            throws org.xml.sax.SAXException {

        /*
         * // Not sure if this is needed or wanted // Also, it fails in the
         * stree. if((null != m_currentNode) && (m_currentNode.getNodeType() ==
         * Node.ELEMENT_NODE)) { String qname; if(((null != prefix) &&
         * (prefix.length() == 0)) || (null == prefix)) qname = "xmlns"; else
         * qname = "xmlns:"+prefix;
         *
         * Element elem = (Element)m_currentNode; String val =
         * elem.getAttribute(qname); // Obsolete, should be DOM2...? if(val ==
         * null) { elem.setAttributeNS("http://www.w3.org/XML/1998/namespace",
         * qname, uri); } }
         */
    }

    /**
     * End the scope of a prefix-URI mapping.
     *
     *

     * See startPrefixMapping for details. This event will always occur after
     * the corresponding endElement event, but the order of endPrefixMapping
     * events is not otherwise guaranteed.
     *

     *
     * @param prefix
     *            The prefix that was being mapping.
     * @see #startPrefixMapping
     * @see #endElement
     */
    @Override
    public void endPrefixMapping(String prefix) throws org.xml.sax.SAXException {
    }

    /**
     * Receive notification of a skipped entity.
     *
     *

     * The Parser will invoke this method once for each entity skipped.
     * Non-validating processors may skip entities if they have not seen the
     * declarations (because, for example, the entity was declared in an
     * external DTD subset). All processors may skip external entities,
     * depending on the values of the
     * http://xml.org/sax/features/external-general-entities and the
     * http://xml.org/sax/features/external-parameter-entities properties.
     *

     *
     * @param name
     *            The name of the skipped entity. If it is a parameter entity,
     *            the name will begin with '%'.
     */
    @Override
    public void skippedEntity(String name) throws org.xml.sax.SAXException {
    }

    public boolean isUpperCaseElementNames() {
        return upperCaseElementNames;
    }

    public void setUpperCaseElementNames(boolean upperCaseElementNames) {
        this.upperCaseElementNames = upperCaseElementNames;
    }

    public String getDefaultNamespaceURI() {
        return defaultNamespaceURI;
    }

    public void setDefaultNamespaceURI(String defaultNamespaceURI) {
        this.defaultNamespaceURI = defaultNamespaceURI;
    }
}


/**
 * Licensed to DigitalPebble Ltd under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * DigitalPebble licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.digitalpebble.stormcrawler.protocol;

import java.net.URL;
import java.util.ArrayList;
import java.util.StringTokenizer;

import javax.security.auth.login.Configuration;

import org.apache.storm.Config;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.digitalpebble.stormcrawler.util.ConfUtils;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;

import crawlercommons.robots.BaseRobotRules;
import crawlercommons.robots.SimpleRobotRules;
import crawlercommons.robots.SimpleRobotRules.RobotRulesMode;
import crawlercommons.robots.SimpleRobotRulesParser;

/**
 * This class uses crawler-commons for handling the parsing of
 * {@code robots.txt} files. It emits SimpleRobotRules objects, which describe
 * the download permissions as described in SimpleRobotRulesParser.
 */
public abstract class RobotRulesParser {

    public static final Logger LOG = LoggerFactory
            .getLogger(RobotRulesParser.class);

    protected static Cache<String, RobotRules> CACHE;

    // if a server or client error happened while fetching the robots
    // cache the result for a shorter period before trying again
    protected static Cache<String, RobotRules> ERRORCACHE;

    /**
     * Parameter name to configure the cache for robots @see
     * http://docs.guava-libraries.googlecode
     * .com/git/javadoc/com/google/common/cache/CacheBuilderSpec.html Default
     * value is "maximumSize=10000,expireAfterWrite=6h"
     **/
    public static final String cacheConfigParamName = "robots.cache.spec";

    /**
     * Parameter name to configure the cache for robots errors @see
     * http://docs.guava-libraries.googlecode
     * .com/git/javadoc/com/google/common/cache/CacheBuilderSpec.html Default
     * value is "maximumSize=10000,expireAfterWrite=1h"
     **/
    public static final String errorcacheConfigParamName = "robots.error.cache.spec";

    /**
     * A {@link BaseRobotRules} object appropriate for use when the
     * {@code robots.txt} file is empty or missing; all requests are allowed.
     */
    public static final BaseRobotRules EMPTY_RULES = new SimpleRobotRules(
            RobotRulesMode.ALLOW_ALL);

    /**
     * A {@link BaseRobotRules} object appropriate for use when the
     * {@code robots.txt} file is not fetched due to a {@code 403/Forbidden}
     * response; all requests are disallowed.
     */
    public static final BaseRobotRules FORBID_ALL_RULES = new SimpleRobotRules(
            RobotRulesMode.ALLOW_NONE);

    private static SimpleRobotRulesParser robotParser = new SimpleRobotRulesParser();
    protected String agentNames;

    public RobotRulesParser() {
    }

    /**
     * Set the {@link Configuration} object
     */
    public void setConf(Config conf) {

        // Grab the agent names we advertise to robots files.
        String agentName = ConfUtils.getString(conf, "http.agent.name");
        if (null == agentName) {
            throw new RuntimeException("Agent name not configured!");
        }

        String configuredAgentNames = ConfUtils.getString(conf,
                "http.robots.agents", "");
        StringTokenizer tok = new StringTokenizer(configuredAgentNames, ",");
        ArrayList<String> agents = new ArrayList<>();
        while (tok.hasMoreTokens()) {
            agents.add(tok.nextToken().trim());
        }

        /**
         * If there are no agents for robots-parsing, use the default
         * agent-string. If both are present, our agent-string should be the
         * first one we advertise to robots-parsing.
         */
        if (agents.isEmpty()) {
            LOG.info(
                    "No agents listed in 'http.robots.agents' property! Using http.agent.name [{}]",
                    agentName);
            this.agentNames = agentName;
        } else {
            int index = 0;
            if ((agents.get(0)).equalsIgnoreCase(agentName)) {
                index++;
            } else {
                LOG.info(
                        "Agent we advertise ({}) not listed first in 'http.robots.agents' property!",
                        agentName);
            }

            StringBuilder combinedAgentsString = new StringBuilder(agentName);
            // append all the agents from the http.robots.agents property
            for (; index < agents.size(); index++) {
                combinedAgentsString.append(", ").append(agents.get(index));
            }

            this.agentNames = combinedAgentsString.toString();
        }

        String spec = ConfUtils.getString(conf, cacheConfigParamName,
                "maximumSize=10000,expireAfterWrite=6h");
        CACHE = CacheBuilder.from(spec).build();

        spec = ConfUtils.getString(conf, errorcacheConfigParamName,
                "maximumSize=10000,expireAfterWrite=1h");
        ERRORCACHE = CacheBuilder.from(spec).build();
    }

    /**
     * Parses the robots content using the {@link SimpleRobotRulesParser} from
     * crawler commons
     *
     * @param url
     *            A string containing url
     * @param content
     *            Contents of the robots file in a byte array
     * @param contentType
     *            The
     * @param robotName
     *            A string containing value of
     * @return BaseRobotRules object
     */
    public BaseRobotRules parseRules(String url, byte[] content,
            String contentType, String robotName) {
        return robotParser.parseContent(url, content, contentType, robotName);
    }

    public BaseRobotRules getRobotRulesSet(Protocol protocol, String url) {
        URL u;
        try {
            u = new URL(url);
        } catch (Exception e) {
            return EMPTY_RULES;
        }
        return getRobotRulesSet(protocol, u);
    }

    public abstract BaseRobotRules getRobotRulesSet(Protocol protocol, URL url);

}


/**
 * Licensed to DigitalPebble Ltd under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * DigitalPebble licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.digitalpebble.stormcrawler.indexing;

import java.net.MalformedURLException;
import java.net.URL;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.digitalpebble.stormcrawler.Metadata;
import com.digitalpebble.stormcrawler.util.ConfUtils;
import com.digitalpebble.stormcrawler.util.RobotsTags;
import com.digitalpebble.stormcrawler.util.URLUtil;

import crawlercommons.domains.PaidLevelDomain;

/** Abstract class to simplify writing IndexerBolts **/
@SuppressWarnings("serial")
public abstract class AbstractIndexerBolt extends BaseRichBolt {

    private final Logger LOG = LoggerFactory.getLogger(getClass());

    /**
     * Mapping between metadata keys and field names for indexing Can be a list
     * of values separated by a = or a single string
     **/
    public static final String metadata2fieldParamName = "indexer.md.mapping";

    /**
     * list of metadata key + values to be used as a filter. A document will be
     * indexed only if it has such a md. Can be null in which case we don't
     * filter at all.
     **/
    public static final String metadataFilterParamName = "indexer.md.filter";

    /** Field name to use for storing the text of a document **/
    public static final String textFieldParamName = "indexer.text.fieldname";

    /** Trim length of text to index. Defaults to -1 to keep it intact **/
    public static final String textLengthParamName = "indexer.text.maxlength";

    /** Field name to use for storing the url of a document **/
    public static final String urlFieldParamName = "indexer.url.fieldname";

    /** Field name to use for reading the canonical property of the metadata */
    public static final String canonicalMetadataParamName = "indexer.canonical.name";

    private String[] filterKeyValue = null;

    private Map<String, String> metadata2field = new HashMap<>();

    private String fieldNameForText = null;

    private int maxLengthText = -1;

    private String fieldNameForURL = null;

    private String canonicalMetadataName = null;

    @SuppressWarnings({ "rawtypes", "unchecked" })
    @Override
    public void prepare(Map conf, TopologyContext context,
            OutputCollector collector) {

        String mdF = ConfUtils.getString(conf, metadataFilterParamName);
        if (StringUtils.isNotBlank(mdF)) {
            // split it in key value
            int equals = mdF.indexOf('=');
            if (equals != -1) {
                String key = mdF.substring(0, equals);
                String value = mdF.substring(equals + 1);
                filterKeyValue = new String[] { key.trim(), value.trim() };
            } else {
                LOG.error("Can't split into key value : {}", mdF);
            }
        }

        fieldNameForText = ConfUtils.getString(conf, textFieldParamName);

        maxLengthText = ConfUtils.getInt(conf, textLengthParamName, -1);

        fieldNameForURL = ConfUtils.getString(conf, urlFieldParamName);

        canonicalMetadataName = ConfUtils.getString(conf,
                canonicalMetadataParamName);

        for (String mapping : ConfUtils.loadListFromConf(
                metadata2fieldParamName, conf)) {
            int equals = mapping.indexOf('=');
            if (equals != -1) {
                String key = mapping.substring(0, equals);
                String value = mapping.substring(equals + 1);
                metadata2field.put(key.trim(), value.trim());
            } else {
                LOG.error("Can't split into key value : {}", mapping);
            }
        }
    }

    /**
     * Determine whether a document should be indexed based on the presence of a
     * given key/value or the RobotsTags.ROBOTS_NO_INDEX directive.
     *
     * @return true if the document should be kept.
     **/
    protected boolean filterDocument(Metadata meta) {
        String noindexVal = meta.getFirstValue(RobotsTags.ROBOTS_NO_INDEX);
        if ("true".equalsIgnoreCase(noindexVal))
            return false;

        if (filterKeyValue == null)
            return true;
        String[] values = meta.getValues(filterKeyValue[0]);
        // key not found
        if (values == null)
            return false;
        return ArrayUtils.contains(values, filterKeyValue[1]);
    }

    /** Returns a mapping field name / values for the metadata to index **/
    protected Map<String, String[]> filterMetadata(Metadata meta) {

        Pattern indexValuePattern = Pattern.compile("\\[(\\d+)\\]");

        Map<String, String[]> fieldVals = new HashMap<>();
        Iterator<Entry<String, String>> iter = metadata2field.entrySet()
                .iterator();
        while (iter.hasNext()) {
            Entry<String, String> entry = iter.next();
            // check whether we want a specific value or all of them?
            int index = -1;
            String key = entry.getKey();
            Matcher match = indexValuePattern.matcher(key);
            if (match.find()) {
                index = Integer.parseInt(match.group(1));
                key = key.substring(0, match.start());
            }
            String[] values = meta.getValues(key);
            // not found
            if (values == null || values.length == 0)
                continue;
            // want a value index that it outside the range given
            if (index >= values.length)
                continue;
            // store all values available
            if (index == -1)
                fieldVals.put(entry.getValue(), values);
            // or only the one we want
            else
                fieldVals.put(entry.getValue(), new String[] { values[index] });
        }

        return fieldVals;
    }

    /**
     * Returns the value to be used as the URL for indexing purposes, if present
     * the canonical value is used instead
     */
    protected String valueForURL(Tuple tuple) {

        String url = tuple.getStringByField("url");
        Metadata metadata = (Metadata) tuple.getValueByField("metadata");

        // functionality deactivated
        if (StringUtils.isBlank(canonicalMetadataParamName)) {
            return url;
        }

        String canonicalValue = metadata.getFirstValue(canonicalMetadataName);

        // no value found?
        if (StringUtils.isBlank(canonicalValue)) {
            return url;
        }

        try {
            URL sURL = new URL(url);
            URL canonical = URLUtil.resolveURL(sURL, canonicalValue);

            String sDomain = PaidLevelDomain.getPLD(sURL.getHost());
            String canonicalDomain = PaidLevelDomain
                    .getPLD(canonical.getHost());

            // check that the domain is the same
            if (sDomain.equalsIgnoreCase(canonicalDomain)) {
                return canonical.toExternalForm();
            } else {
                LOG.info(
                        "Canonical URL references a different domain, ignoring in {} ",
                        url);
            }
        } catch (MalformedURLException e) {
            LOG.error("Malformed canonical URL {} was found in {} ",
                    canonicalValue, url);
        }

        return url;
    }

    /**
     * Returns the field name to use for the text or null if the text must not
     * be indexed
     **/
    protected String fieldNameForText() {
        return fieldNameForText;
    }

    /**
     * Returns a trimmed string or the original one if it is below the threshold
     * set in the configuration.
     **/
    protected String trimText(String text) {
        if (maxLengthText == -1)
            return text;
        if (text == null)
            return text;
        if (text.length() <= maxLengthText)
            return text;
        return text.substring(0, maxLengthText);
    }

    /**
     * Returns the field name to use for the URL or null if the URL must not be
     * indexed
     **/
    protected String fieldNameForURL() {
        return fieldNameForURL;
    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declareStream(
                com.digitalpebble.stormcrawler.Constants.StatusStreamName,
                new Fields("url", "metadata", "status"));
    }
}


/**
 * Licensed to DigitalPebble Ltd under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * DigitalPebble licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.digitalpebble.stormcrawler.persistence;

import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.digitalpebble.stormcrawler.Constants;
import com.digitalpebble.stormcrawler.Metadata;
import com.digitalpebble.stormcrawler.util.ConfUtils;

/**
 * Schedules a nextFetchDate based on the configuration
 **/
public class DefaultScheduler extends Scheduler {

    /** Date far in the future used for never-refetch items. */
    public static final Date NEVER = new Calendar.Builder()
            .setCalendarType("iso8601").setDate(2099, Calendar.DECEMBER, 31)
            .build().getTime();

    /**
     * Key used to pass a custom delay via metadata. Used by the sitemaps to
     * stagger the scheduling of URLs.
     */
    public static final String DELAY_METADATA = "scheduler.delay.mins";

    // fetch intervals in minutes
    private int defaultfetchInterval;
    private int fetchErrorFetchInterval;
    private int errorFetchInterval;

    private CustomInterval[] customIntervals;

    /*
     * (non-Javadoc)
     *
     * @see
     * com.digitalpebble.stormcrawler.persistence.Scheduler#init(java.util.Map)
     */
    @SuppressWarnings("rawtypes")
    @Override
    public void init(Map stormConf) {
        defaultfetchInterval = ConfUtils.getInt(stormConf,
                Constants.defaultFetchIntervalParamName, 1440);
        fetchErrorFetchInterval = ConfUtils.getInt(stormConf,
                Constants.fetchErrorFetchIntervalParamName, 120);
        errorFetchInterval = ConfUtils.getInt(stormConf,
                Constants.errorFetchIntervalParamName, 44640);

        // loads any custom key values
        // must be of form fetchInterval(.STATUS)?.keyname=value
        // e.g. fetchInterval.isFeed=true
        // e.g. fetchInterval.FETCH_ERROR.isFeed=true
        Map<String, CustomInterval> intervals = new HashMap<>();
        Pattern pattern = Pattern
                .compile("^fetchInterval(\\..+?)?\\.(.+)=(.+)");
        Iterator<String> keyIter = stormConf.keySet().iterator();
        while (keyIter.hasNext()) {
            String key = keyIter.next();
            Matcher m = pattern.matcher(key);
            if (!m.matches()) {
                continue;
            }
            Status status = null;
            // was a status specified?
            if (m.group(1) != null) {
                status = Status.valueOf(m.group(1).substring(1));
            }
            String mdname = m.group(2);
            String mdvalue = m.group(3);
            int customInterval = ConfUtils.getInt(stormConf, key, -1);
            if (customInterval != -1) {
                CustomInterval interval = intervals.get(mdname + mdvalue);
                if (interval == null) {
                    interval = new CustomInterval(mdname, mdvalue, status,
                            customInterval);
                } else {
                    interval.setDurationForStatus(status, customInterval);
                }
                // specify particular interval for this status
                intervals.put(mdname + mdvalue, interval);
            }
        }
        customIntervals = intervals.values().toArray(
                new CustomInterval[intervals.size()]);
    }

    /*
     * (non-Javadoc)
     *
     * @see com.digitalpebble.stormcrawler.persistence.Scheduler#schedule(com.
     * digitalpebble. stormcrawler.persistence .Status,
     * com.digitalpebble.stormcrawler.Metadata)
     */
    @Override
    public Date schedule(Status status, Metadata metadata) {

        int minutesIncrement = 0;

        Optional<Integer> customInterval = Optional.empty();

        // try with a value set in the metadata
        String customInMetadata = metadata.getFirstValue(DELAY_METADATA);
        if (customInMetadata != null) {
            customInterval = Optional.of(Integer.parseInt(customInMetadata));
        }
        // try with the rules from the configuration
        if (!customInterval.isPresent()) {
            customInterval = checkCustomInterval(metadata, status);
        }

        if (customInterval.isPresent()) {
            minutesIncrement = customInterval.get();
        } else {
            switch (status) {
            case FETCHED:
                minutesIncrement = defaultfetchInterval;
                break;
            case FETCH_ERROR:
                minutesIncrement = fetchErrorFetchInterval;
                break;
            case ERROR:
                minutesIncrement = errorFetchInterval;
                break;
            case REDIRECTION:
                minutesIncrement = defaultfetchInterval;
                break;
            default:
                // leave it to now e.g. DISCOVERED
            }
        }

        // a value of -1 means never fetch
        // we use a conventional value
        if (minutesIncrement == -1) {
            return NEVER;
        }

        Calendar cal = Calendar.getInstance();
        cal.add(Calendar.MINUTE, minutesIncrement);

        return cal.getTime();
    }

    /**
     * Returns the first matching custom interval
     **/
    protected final Optional<Integer> checkCustomInterval(Metadata metadata,
            Status s) {
        if (customIntervals == null)
            return Optional.empty();

        for (CustomInterval customInterval : customIntervals) {
            String[] values = metadata.getValues(customInterval.key);
            if (values == null) {
                continue;
            }
            for (String v : values) {
                if (v.equals(customInterval.value)) {
                    return customInterval.getDurationForStatus(s);
                }
            }
        }

        return Optional.empty();
    }

    private class CustomInterval {
        private String key;
        private String value;
        private Map<Status, Integer> durationPerStatus;
        private Integer defaultDuration = null;

        private CustomInterval(String key, String value, Status status,
                int minutes) {
            this.key = key;
            this.value = value;
            this.durationPerStatus = new HashMap<>();
            setDurationForStatus(status, minutes);
        }

        private void setDurationForStatus(Status s, int minutes) {
            if (s == null) {
                defaultDuration = minutes;
            } else {
                this.durationPerStatus.put(s, minutes);
            }
        }

        private Optional<Integer> getDurationForStatus(Status s) {
            // do we have a specific value for this status?
            Integer customD = durationPerStatus.get(s);
            if (customD != null) {
                return Optional.of(customD);
            }
            // is there a default one set?
            if (defaultDuration != null) {
                return Optional.of(defaultDuration);
            }
            // no default value or custom one for that status
            return Optional.empty();
        }
    }
}


/**
 * Licensed to DigitalPebble Ltd under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * DigitalPebble licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.digitalpebble.stormcrawler.filtering;

import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang.StringUtils;
import org.slf4j.LoggerFactory;

import com.digitalpebble.stormcrawler.JSONResource;
import com.digitalpebble.stormcrawler.Metadata;
import com.digitalpebble.stormcrawler.util.ConfUtils;
import com.digitalpebble.stormcrawler.util.Configurable;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;

/**
 * Wrapper for the URLFilters defined in a JSON configuration
 */
public class URLFilters implements URLFilter, JSONResource {

    public static final URLFilters emptyURLFilters = new URLFilters();

    private static final org.slf4j.Logger LOG = LoggerFactory
            .getLogger(URLFilters.class);

    private URLFilter[] filters;

    private URLFilters() {
        filters = new URLFilters[0];
    }

    private String configFile = "urlfilters.config.file";

    private Map stormConf;

    /**
     * Loads and configure the URLFilters based on the storm config if there is
     * one otherwise returns an empty URLFilter.
     **/
    public static URLFilters fromConf(Map stormConf) {

        String configFile = ConfUtils.getString(stormConf,
                "urlfilters.config.file");
        if (StringUtils.isNotBlank(configFile)) {
            try {
                return new URLFilters(stormConf, configFile);
            } catch (IOException e) {
                String message = "Exception caught while loading the URLFilters from "
                        + configFile;
                LOG.error(message);
                throw new RuntimeException(message, e);
            }
        }

        return URLFilters.emptyURLFilters;
    }

    /**
     * Loads the filters from a JSON configuration file
     *
     * @throws IOException
     */
    public URLFilters(Map stormConf, String configFile) throws IOException {
        this.configFile = configFile;
        this.stormConf = stormConf;
        try {
            loadJSONResources();
        } catch (Exception e) {
            throw new IOException("Unable to build JSON object from file", e);
        }
    }

    @Override
    public void loadJSONResources(InputStream inputStream)
            throws JsonParseException, JsonMappingException, IOException {
        ObjectMapper mapper = new ObjectMapper();
        JsonNode confNode = mapper.readValue(inputStream, JsonNode.class);
        configure(stormConf, confNode);
    }

    @Override
    public String filter(URL sourceUrl, Metadata sourceMetadata,
            String urlToFilter) {
        String normalizedURL = urlToFilter;
        try {
            for (URLFilter filter : filters) {
                long start = System.currentTimeMillis();
                normalizedURL = filter.filter(sourceUrl, sourceMetadata,
                        normalizedURL);
                long end = System.currentTimeMillis();
                LOG.debug("URLFilter {} took {} msec", filter.getClass()
                        .getName(), end - start);
                if (normalizedURL == null)
                    break;
            }
        } catch (Exception e) {
            LOG.error("URL filtering threw exception", e);
        }
        return normalizedURL;
    }

    @Override
    public String getResourceFile() {
        return this.configFile;
    }

    @SuppressWarnings("rawtypes")
    @Override
    public void configure(Map stormConf, JsonNode filtersConf) {
        List<URLFilter> list = Configurable.configure(stormConf, filtersConf,
                URLFilter.class, this.getClass().getName());
        filters = list.toArray(new URLFilter[list.size()]);
    }

}


/**
 * Licensed to DigitalPebble Ltd under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * DigitalPebble licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.digitalpebble.stormcrawler.persistence;

import java.util.Date;
import java.util.Map;

import org.apache.commons.lang.StringUtils;

import com.digitalpebble.stormcrawler.Metadata;
import com.digitalpebble.stormcrawler.util.ConfUtils;

public abstract class Scheduler {

    /**
     * Class to use for Scheduler. Must extend the class Scheduler.
     */
    public static final String schedulerClassParamName = "scheduler.class";

    @SuppressWarnings("rawtypes")
    /** Configuration of the scheduler based on the config. Should be called by Scheduler.getInstance() **/
    protected abstract void init(Map stormConf);

    /**
     * Returns a Date indicating when the document should be refetched next,
     * based on its status.
     **/
    public abstract Date schedule(Status status, Metadata metadata);

    /** Returns a Scheduler instance based on the configuration **/
    @SuppressWarnings({ "rawtypes", "unchecked" })
    public static Scheduler getInstance(Map stormConf) {
        Scheduler scheduler;

        String className = ConfUtils.getString(stormConf,
                schedulerClassParamName);

        if (StringUtils.isBlank(className)) {
            throw new RuntimeException("Missing value for config  "
                    + schedulerClassParamName);
        }

        try {
            Class<?> schedulerc = Class.forName(className);
            boolean interfaceOK = Scheduler.class.isAssignableFrom(schedulerc);
            if (!interfaceOK) {
                throw new RuntimeException("Class " + className
                        + " must extend Scheduler");
            }
            scheduler = (Scheduler) schedulerc.newInstance();
        } catch (Exception e) {
            throw new RuntimeException("Can't instanciate " + className);
        }

        scheduler.init(stormConf);
        return scheduler;
    }
}


/**
 * Licensed to DigitalPebble Ltd under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * DigitalPebble licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.digitalpebble.stormcrawler;

import java.io.FileNotFoundException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang.StringUtils;

import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.StormSubmitter;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.utils.Utils;

import com.digitalpebble.stormcrawler.util.ConfUtils;

public abstract class ConfigurableTopology {

    protected Config conf = new Config();
    protected boolean isLocal = false;
    protected int ttl = -1;

    public static void start(ConfigurableTopology topology, String args[]) {
        // loads the default configuration file
        Map defaultSCConfig = Utils.findAndReadConfigFile(
                "crawler-default.yaml", false);
        topology.conf.putAll(ConfUtils.extractConfigElement(defaultSCConfig));

        String[] remainingArgs = topology.parse(args);
        topology.run(remainingArgs);
    }

    protected Config getConf() {
        return conf;
    }

    protected abstract int run(String args[]);

    /** Submits the topology with the name taken from the configuration **/
    protected int submit(Config conf, TopologyBuilder builder) {
        String name = ConfUtils.getString(conf, Config.TOPOLOGY_NAME);
        if (StringUtils.isBlank(name))
            throw new RuntimeException("No value found for "
                    + Config.TOPOLOGY_NAME);
        return submit(name, conf, builder);
    }

    /** Submits the topology under a specific name **/
    protected int submit(String name, Config conf, TopologyBuilder builder) {

        // register Metadata for serialization with FieldsSerializer
        Config.registerSerialization(conf, Metadata.class);

        if (isLocal) {
            LocalCluster cluster = new LocalCluster();
            cluster.submitTopology(name, conf, builder.createTopology());
            if (ttl != -1) {
                Utils.sleep(ttl * 1000);
                cluster.shutdown();
            }
        }

        else {
            try {
                StormSubmitter.submitTopology(name, conf,
                        builder.createTopology());
            } catch (Exception e) {
                e.printStackTrace();
                return -1;
            }
        }
        return 0;
    }

    private String[] parse(String args[]) {

        List<String> newArgs = new ArrayList<>();
        Collections.addAll(newArgs, args);

        Iterator<String> iter = newArgs.iterator();
        while (iter.hasNext()) {
            String param = iter.next();
            if (param.equals("-conf")) {
                if (!iter.hasNext()) {
                    throw new RuntimeException("Conf file not specified");
                }
                iter.remove();
                String resource = iter.next();
                try {
                    ConfUtils.loadConf(resource, conf);
                } catch (FileNotFoundException e) {
                    throw new RuntimeException("File not found : " + resource);
                }
                iter.remove();
            } else if (param.equals("-local")) {
                isLocal = true;
                iter.remove();
            } else if (param.equals("-ttl")) {
                if (!iter.hasNext()) {
                    throw new RuntimeException("ttl value not specified");
                }
                iter.remove();
                String ttlValue = iter.next();
                try {
                    ttl = Integer.parseInt(ttlValue);
                } catch (NumberFormatException nfe) {
                    throw new RuntimeException("ttl value incorrect");
                }
                iter.remove();
            }
        }

        return newArgs.toArray(new String[newArgs.size()]);
    }

}


package com.poiji.bind;

import com.poiji.bind.mapping.UnmarshallerHelper;
import com.poiji.exception.IllegalCastException;
import com.poiji.exception.InvalidExcelFileExtension;
import com.poiji.exception.PoijiExcelType;
import com.poiji.exception.PoijiException;
import com.poiji.option.PoijiOptions;
import com.poiji.option.PoijiOptions.PoijiOptionsBuilder;
import com.poiji.util.Files;

import java.io.File;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.function.Consumer;

import static com.poiji.util.PoijiConstants.XLSX_EXTENSION;
import static com.poiji.util.PoijiConstants.XLS_EXTENSION;

/**
 * The entry point of the mapping process.
 *

 * Example:
 *

 * List<Employee> employees = Poiji.fromExcel(new File("employees.xls"), Employee.class);
 * employees.size();
 * // 3
 * Employee firstEmployee = employees.get(0);
 * // Employee{employeeId=123923, name='Joe', surname='Doe', age=30, single=true, birthday='4/9/1987'}
 *

 *

 * Created by hakan on 16/01/2017.
 */
public final class Poiji {

    private static final Files files = Files.getInstance();

    private Poiji() {
    }

    /**
     * converts excel rows into a list of objects
     *
     * @param file
     *         excel file ending with .xls or .xlsx.
     * @param type
     *         type of the root object.
     * @param <T>
     *         type of the root object.
     * @return
     *         the newly created a list of objects
     *
     * @throws PoijiException
     *          if an internal exception occurs during the mapping process.
     * @throws InvalidExcelFileExtension
     *          if the specified excel file extension is invalid.
     * @throws IllegalCastException
     *          if this Field object is enforcing Java language access control and the underlying field is either inaccessible or final.
     *
     * @see Poiji#fromExcel(File, Class, PoijiOptions)
     */
    public static synchronized <T> List<T> fromExcel(final File file, final Class<T> type) {
        final ArrayList<T> list = new ArrayList<>();
        fromExcel(file, type, list::add);
        return list;
    }

    /**
     * converts excel rows into a list of objects
     *
     * @param file excel file ending with .xls or .xlsx.
     * @param type type of the root object.
     * @param <T> type of the root object.
     * @param consumer output retrieves records
     * @throws PoijiException if an internal exception occurs during the mapping
     * process.
     * @throws InvalidExcelFileExtension if the specified excel file extension
     * is invalid.
     * @throws IllegalCastException if this Field object is enforcing Java language access control and the underlying field is either inaccessible or final.
     *
     * @see Poiji#fromExcel(File, Class, PoijiOptions)
     */
    public static synchronized <T> void fromExcel(final File file, final Class<T> type, final Consumer<? super T> consumer) {
        final Unmarshaller unmarshaller = deserializer(file, PoijiOptionsBuilder.settings().build());
        unmarshaller.unmarshal(type, consumer);
    }

    /**
     * converts excel rows into a list of objects
     *
     * @param inputStream excel file stream
     * @param excelType   type of the excel file, xls or xlsx
     * @param type        type of the root object.
     * @param <T>         type of the root object.
     * @return the newly created a list of objects
     * @throws PoijiException            if an internal exception occurs during the mapping process.
     * @throws InvalidExcelFileExtension if the specified excel file extension is invalid.
     * @throws IllegalCastException      if this Field object is enforcing Java language access control and the underlying field is either inaccessible or final.
     * @see Poiji#fromExcel(File, Class, PoijiOptions)
     */
    public static synchronized <T> List<T> fromExcel(final InputStream inputStream,
                                                     PoijiExcelType excelType,
                                                     final Class<T> type) {
        final ArrayList<T> list = new ArrayList<>();
        fromExcel(inputStream, excelType, type, list::add);
        return list;
    }

    /**
     * converts excel rows into a list of objects
     *
     * @param inputStream excel file stream
     * @param excelType type of the excel file, xls or xlsx
     * @param type type of the root object.
     * @param <T> type of the root object.
     * @param consumer represents an operation that accepts the type argument
     * @throws PoijiException if an internal exception occurs during the mapping process.
     * @throws InvalidExcelFileExtension if the specified excel file extension is invalid.
     * @throws IllegalCastException if this Field object is enforcing Java language access control and the underlying field is either inaccessible or final.
     * @see Poiji#fromExcel(File, Class, PoijiOptions)
     */
    public static synchronized <T> void fromExcel(final InputStream inputStream,
                                                  PoijiExcelType excelType,
                                                  final Class<T> type,
                                                  final Consumer<? super T> consumer) {
        Objects.requireNonNull(excelType);

        final Unmarshaller unmarshaller = deserializer(inputStream, excelType, PoijiOptionsBuilder.settings().build());
        unmarshaller.unmarshal(type, consumer);
    }

    /**
     * converts excel rows into a list of objects
     *
     * @param file    excel file ending with .xls or .xlsx.
     * @param type    type of the root object.
     * @param <T>     type of the root object.
     * @param options specifies to change the default behaviour of the poiji.
     * @return the newly created a list of objects
     * @throws PoijiException            if an internal exception occurs during the mapping process.
     * @throws InvalidExcelFileExtension if the specified excel file extension is invalid.
     * @throws IllegalCastException      if this Field object is enforcing Java language access control and the underlying field is either inaccessible or final.
     * @see Poiji#fromExcel(File, Class)
     */
    public static synchronized <T> List<T> fromExcel(final File file, final Class<T> type, final PoijiOptions options) {
        final ArrayList<T> list = new ArrayList<>();
        fromExcel(file, type, options,list::add);
        return list;
    }

    /**
     * converts excel rows into a list of objects
     *
     * @param file excel file ending with .xls or .xlsx.
     * @param type type of the root object.
     * @param <T> type of the root object.
     * @param options specifies to change the default behaviour of the poiji.
     * @param consumer represents an operation that accepts the type argument
     * @throws PoijiException if an internal exception occurs during the mapping process.
     * @throws InvalidExcelFileExtension if the specified excel file extension is invalid.
     * @throws IllegalCastException if this Field object is enforcing Java language access control and the underlying field is either inaccessible or final.
     * @see Poiji#fromExcel(File, Class)
     */
    public static synchronized <T> void fromExcel(final File file, final Class<T> type, final PoijiOptions options, final Consumer<? super T> consumer) {
        final Unmarshaller unmarshaller = deserializer(file, options);
        unmarshaller.unmarshal(type, consumer);
    }

    /**
     * converts excel rows into a list of objects
     *
     * @param inputStream excel file stream
     * @param excelType   type of the excel file, xls or xlsx
     * @param type        type of the root object.
     * @param <T>         type of the root object.
     * @param options     specifies to change the default behaviour of the poiji.
     * @return the newly created a list of objects
     * @throws PoijiException            if an internal exception occurs during the mapping process.
     * @throws InvalidExcelFileExtension if the specified excel file extension is invalid.
     * @throws IllegalCastException      if this Field object is enforcing Java language access control and the underlying field is either inaccessible or final.
     * @see Poiji#fromExcel(File, Class)
     */
    public static synchronized <T> List<T> fromExcel(final InputStream inputStream,
                                                     final PoijiExcelType excelType,
                                                     final Class<T> type,
                                                     final PoijiOptions options) {
        Objects.requireNonNull(excelType);
        final ArrayList<T> list = new ArrayList<>();
        fromExcel(inputStream, excelType,type, options, list::add);
        return list;
    }

    /**
     * converts excel rows into a list of objects
     *
     * @param inputStream excel file stream
     * @param excelType type of the excel file, xls or xlsx
     * @param type type of the root object.
     * @param <T> type of the root object.
     * @param options specifies to change the default behaviour of the poiji.
     * @param consumer represents an operation that accepts the type argument
     * @throws PoijiException if an internal exception occurs during the mapping process.
     * @throws InvalidExcelFileExtension if the specified excel file extension is invalid.
     * @throws IllegalCastException if this Field object is enforcing Java
     * language access control and the underlying field is either inaccessible or final.
     * @see Poiji#fromExcel(File, Class)
     */
    public static synchronized <T> void fromExcel(final InputStream inputStream,
                                                  final PoijiExcelType excelType,
                                                  final Class<T> type,
                                                  final PoijiOptions options,
                                                  final Consumer<? super T> consumer) {
        Objects.requireNonNull(excelType);

        final Unmarshaller unmarshaller = deserializer(inputStream, excelType, options);
        unmarshaller.unmarshal(type, consumer);
    }

    @SuppressWarnings("unchecked")
    private static Unmarshaller deserializer(final File file, final PoijiOptions options) {
        final PoijiFile poijiFile = new PoijiFile(file);

        String extension = files.getExtension(file.getName());

        if (XLS_EXTENSION.equals(extension)) {
            return UnmarshallerHelper.HSSFInstance(poijiFile, options);
        } else if (XLSX_EXTENSION.equals(extension)) {
            return UnmarshallerHelper.XSSFInstance(poijiFile, options);
        } else {
            throw new InvalidExcelFileExtension("Invalid file extension (" + extension + "), excepted .xls or .xlsx");
        }
    }

    @SuppressWarnings("unchecked")
    private static Unmarshaller deserializer(final InputStream inputStream, PoijiExcelType excelType, final PoijiOptions options) {
        final PoijiInputStream poijiInputStream = new PoijiInputStream<>(inputStream);

        if (excelType == PoijiExcelType.XLS) {
            return UnmarshallerHelper.HSSFInstance(poijiInputStream, options);
        } else if (excelType == PoijiExcelType.XLSX) {
            return UnmarshallerHelper.XSSFInstance(poijiInputStream, options);
        } else {
            throw new InvalidExcelFileExtension("Invalid file extension (" + excelType + "), excepted .xls or .xlsx");
        }
    }
}


package com.poiji.bind.mapping;

import com.poiji.annotation.ExcelCell;
import com.poiji.annotation.ExcelCellName;
import com.poiji.annotation.ExcelCellRange;
import com.poiji.annotation.ExcelRow;
import com.poiji.config.Casting;
import com.poiji.exception.IllegalCastException;
import com.poiji.exception.PoijiInstantiationException;
import com.poiji.option.PoijiOptions;
import org.apache.poi.ss.util.CellAddress;
import org.apache.poi.xssf.eventusermodel.XSSFSheetXMLHandler.SheetContentsHandler;
import org.apache.poi.xssf.usermodel.XSSFComment;

import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.util.HashMap;
import java.util.Map;
import java.util.function.Consumer;

import static java.lang.String.valueOf;

/**
 * This class handles the processing of a .xlsx file,
 * and generates a list of instances of a given type
 * <p>
 * Created by hakan on 22/10/2017
 */
final class PoijiHandler<T> implements SheetContentsHandler {

    private T instance;
    private Consumer<? super T> consumer;
    private int internalCount;

    private Class<T> type;
    private PoijiOptions options;

    private final Casting casting;
    private Map<String, Integer> titles;
    // New maps used to speed up computing and handle inner objects
    private Map<String, Object> fieldInstances;
    private Map<Integer, Field> columnToField;
    private Map<Integer, Field> columnToSuperClassField;

    PoijiHandler(Class<T> type, PoijiOptions options, Consumer<T> consumer) {
        this.type = type;
        this.options = options;
        this.consumer = consumer;

        casting = options.getCasting();
        titles = new HashMap<String, Integer>();
        columnToField = new HashMap<>();
        columnToSuperClassField = new HashMap<>();
    }

    private <T> T newInstanceOf(Class<T> type) {
        T newInstance;
        try {
            newInstance = type.getDeclaredConstructor().newInstance();
        } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException | InstantiationException e) {
            throw new IllegalCastException("Cannot create a new instance of " + type.getName());
        }

        return newInstance;
    }

    private void setFieldValue(String content, Class<? super T> subclass, int column) {
        if (subclass != Object.class) {
            if(setValue(content, subclass, column)) {
                return;
            }

            setFieldValue(content, subclass.getSuperclass(), column);
        }
    }
    /**
     *  Using this to hold inner objects that will be mapped to the main object
     * **/
    private Object getInstance(Field field) {
        Object ins = null;
        try {
            if (fieldInstances.containsKey(field.getName())) {
                ins = fieldInstances.get(field.getName());
            } else {
                ins = field.getType().getDeclaredConstructor().newInstance();
                fieldInstances.put(field.getName(), ins);
            }
        } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException
                | InstantiationException e) {
            throw new PoijiInstantiationException("Cannot create a new instance of " + type.getName());
        }
        return ins;
    }

    /***
     * 	Modified this method so that for each time it reads a cell
     * 	it doesn't need to check all fields even after it as found
     * 	the matching field
     *
     * */
    private boolean setValue(String content, Class<? super T> type, int column) {
        // For ExcelRow annotation
        if(columnToField.containsKey(-1)) {
            Field field = columnToField.get(-1);
            Object o = casting.castValue(field.getType(), valueOf(internalCount), options);
            setFieldData(field, o, instance);
        }
        if(columnToField.containsKey(column)) {
            Field field = columnToField.get(column);
            if (columnToSuperClassField.containsKey(column)) {
                Object ins = null;
                ins = getInstance(columnToSuperClassField.get(column));
                if (setValue(field, column, content, ins)) {
                    setFieldData(columnToSuperClassField.get(column), ins, instance);
                    return true;
                } else {
                    return false;
                }
            }
            return setValue(field, column, content, instance);
        }
        for (Field field : type.getDeclaredFields()) {

            ExcelRow excelRow = field.getAnnotation(ExcelRow.class);
            if (excelRow != null) {
                Object o = casting.castValue(field.getType(), valueOf(internalCount), options);
                setFieldData(field, o, instance);
                columnToField.put(-1, field);
            }
            ExcelCellRange range = field.getAnnotation(ExcelCellRange.class);
            if (range != null) {
                Object ins = null;
                ins = getInstance(field);
                for (Field f : field.getType().getDeclaredFields()) {
                    if (setValue(f, column, content, ins)) {
                        setFieldData(field, ins, instance);
                        columnToField.put(column, f);
                        columnToSuperClassField.put(column, field);
                        return true;
                    }
                }
            } else {
                if(setValue(field, column, content, instance)) {
                    columnToField.put(column, field);
                    return true;
                }
            }
        }
        return false;
    }

    private boolean setValue(Field field, int column, String content, Object ins) {
           ExcelCell index = field.getAnnotation(ExcelCell.class);
           if (index != null) {
               Class<?> fieldType = field.getType();
               if (column == index.value()) {
                   Object o = casting.castValue(fieldType, content, options);
                   setFieldData(field, o, ins);
               }
           } else {
               ExcelCellName excelCellName = field.getAnnotation(ExcelCellName.class);
               if (excelCellName != null) {
                   Class<?> fieldType = field.getType();
                   Integer titleColumn = titles.get(excelCellName.value() );
                   //Fix both columns mapped to name passing this condition below
                   if (titleColumn != null && titleColumn == column) {
                       Object o = casting.castValue(fieldType, content, options);
                       setFieldData(field, o, ins);
                       return true;
                   }
               }
           }
           return false;
    }

    private void setFieldData(Field field, Object o, Object instance) {
        try {
            field.setAccessible(true);
            field.set(instance, o);
        } catch (IllegalAccessException e) {
            throw new IllegalCastException("Unexpected cast type {" + o + "} of field" + field.getName());
        }
    }

    @Override
    public void startRow(int rowNum) {
        if (rowNum + 1 > options.skip()) {
            instance = newInstanceOf(type);
            fieldInstances = new HashMap<>();
        }
    }

    @Override
    public void endRow(int rowNum) {

        if (internalCount != rowNum)
			return;

        if (rowNum + 1 > options.skip()) {
            consumer.accept(instance);
        }
    }

    @Override
    public void cell(String cellReference, String formattedValue, XSSFComment comment) {

        CellAddress cellAddress = new CellAddress(cellReference);
        int row = cellAddress.getRow();

        internalCount = row;
        int column = cellAddress.getColumn();
        int headers = options.getHeaderStart();

        if (row <= headers) {
            titles.put(formattedValue, column);
        }

        if (row + 1 <= options.skip()) {
            return;
        }

        setFieldValue(formattedValue, type, column);
    }

    @Override
    public void headerFooter(String text, boolean isHeader, String tagName) {
        //no-op
    }
}