/**
 * Bundle loader
 *
 * Flag "bundle"
 *
 * This plugin provides require.bundle() function
 */

/**
 * @name sandbox
 */
(function (sb) {
    var callbackName = sb.options.bundle,
        pendingBundlesLength = 0;

    /**
     * Cases:
     *     ({main}, {modules}, {options})
     *     ({modules}, {options})
     *     ({modules})
     *
     * @param {Function} _main
     * @param {Object}   _modules
     * @param {Object}   _modules_options
     */
    var processBundleJSONP = function (_main, _modules, _modules_options) {
        if (typeof _main === "object") {
            _modules_options = _modules;
            _modules = _main;
        }

        for (var moduleName in _modules) {
            // if already initialized - skip
            if (moduleName in sb.modules) {
                continue;
            }

            // declare new modules
            sb.modules[moduleName] = _modules[moduleName];
            sb.initialized[moduleName] = 0;

            // declare module options
            if (_modules_options && moduleName in _modules_options) {
                sb.modules_options[moduleName] = _modules_options[moduleName];
            }
        }

        if (typeof _main === "function") {
            var output = {'exports': {}};
            _main(sb.trigger('lmd-register:decorate-require', "<bundle:main>", sb.require)[1], output.exports, output);
        }
    };

    var trap = function () {
        pendingBundlesLength++;
        // make trap
        sb.global[callbackName] = processBundleJSONP;
    };

    var cleanup = function (callback, scriptTag) {
        // Be sure that callback will be called after script eval
        setTimeout(function () {
            pendingBundlesLength--;
            // cleanup if no pending bundles
            if (!pendingBundlesLength) {
                sb.global[callbackName] = sb.undefined;
            }
            callback(scriptTag);
        }, 10);
    };

    /**
     * Loads LMD bundle
     *
     * @param {String|Array} bundleSrc path to file
     * @param {Function}     [callback]   callback(result) undefined on error HTMLScriptElement on success
     */
    sb.require.bundle = function (bundleSrc, callback) {
        var replacement = sb.trigger('*:request-off-package', bundleSrc, callback, 'image'), // [[returnResult, bundleSrc, bundle, true], callback, type]
            returnResult = replacement[0][0];

        if (replacement[0][3]) { // isReturnASAP
            return returnResult;
        }

        callback = replacement[1];
        bundleSrc = replacement[0][1];

        trap();

        sb.trigger('*:load-script', bundleSrc, function (scriptTag) {
            cleanup(callback, scriptTag);
        });

        return returnResult;
    };

}(sandbox));


/**
 * LMD require.js() and shortcuts example
 */

function drawImageOnCanvas(img) {
    var ctx = $('canvas')[0].getContext('2d');

    ctx.drawImage(img, 0, 0);
    ctx.rotate(-Math.PI / 12);
    ctx.translate(0, 150);
}

$(function () {
    var $button = $('.b-button');

    $button.click(function () {
        require.image("images/html5.png", function (img) {
            !img && console.log('Fail to load');
            drawImageOnCanvas(img);
        });
    });
});


/**
 * LMD
 *
 * @author  Mikhail Davydov
 * @licence MIT
 */

var lmdPackage = require(__dirname + '/../lmd_builder.js'),
    fs = require('fs');

/**
 *
 *
 * @type {String[]}
 */
var availableModes = ['main', 'watch'];

/**
 * Simple argv parser
 *
 * @see https://gist.github.com/1497865
 *
 * @param {String} a an argv string
 *
 * @returns {Object}
 */
var parseArgv = function (a,b,c,d) {
    c={};for(a=a.split(/\s*\B[-]+([\w-]+)[\s=]*/),d=1;b=a[d++];c[b]=a[d++]||!0);return c
};

var createWritableFile = function (fileName) {
    return fs.createWriteStream(fileName, {
        flags: "w",
        encoding: "utf8",
        mode: 0666
    });
};

/**
 * Formats lmd config
 *
 * @param  {String|Object} data
 *
 * @return {Object}
 */
var parseData = function (data) {
    var config;

    // case data is argv string
    if (typeof data === "string") {
        // try to parse new version
        config = parseArgv(data);

        // its new config argv string
        if (Object.keys(config).length) {
            // translate short params to long one
            config.mode = config.mode || config.m;
            config.output = config.output || config.o;
            config.log = config.log || config.l;
            config.config = config.config || config.c;
            config['no-warn'] = config['no-warn'] || config['no-w'];
            config['source-map'] = config['source-map'] || config['sm'];
            config['source-map-root'] = config['source-map-root'] || config['sm-root'];
            config['source-map-www'] = config['source-map-www'] || config['sm-www'];
            config['source-map-inline'] = config['source-map-inline'] || config['sm-inline'];
        } else {
            // an old argv format, split argv and parse manually
            data = data.split(' ');

            // without mode
            if (availableModes.indexOf(data[2]) === -1) {
                config = {
                    mode: 'main',
                    config: data[2],
                    output: data[3]
                };
            } else { // with mode
                config = {
                    mode: data[2],
                    config: data[3],
                    output: data[4]
                };
            }
        }

    // case data is config object
    } else if (typeof config === "object") {
        // use as is
        config = data;

    // case else
    } else {
        // wut?
        throw new Error('Bad config data');
    }

    config.mode = config.mode || 'main';
    config.warn = !config['no-warn'];
    config.sourcemap = config['source-map'] || false;
    config.www_root = config['source-map-root'] || "";
    config.sourcemap_www = config['source-map-www'] || "";
    config.sourcemap_inline = config['source-map-inline'] || false;
    config.log = config.log || false;

    return config;
};

module.exports = function (argv) {

    var config = parseData(argv.join(' ')),
        builderOptions = {
            warn: config.warn,
            sourcemap: config.sourcemap,
            www_root: config.www_root,
            sourcemap_www: config.sourcemap_www,
            sourcemap_inline: config.sourcemap_inline,
            output: config.output
        };

    switch (config.mode) {
        case 'main':
            var buildResult = new lmdPackage(config.config, builderOptions);
            // usage log
            if (!config.config) {
                buildResult.log.pipe(process.stdout);
                break;
            }

            if (builderOptions.sourcemap) {
                buildResult.sourceMap.pipe(createWritableFile(builderOptions.sourcemap));
            }

            if (builderOptions.output) {
                buildResult.pipe(createWritableFile(builderOptions.output));
                if (config.log) {
                    buildResult.log.pipe(process.stdout);
                }
            } else {
                buildResult.pipe(process.stdout);
            }
            break;
        case 'watch':
            var watcher = new lmdPackage.watch(config.config, builderOptions);
            // usage log
            if (!config.config || !builderOptions.output) {
                watcher.log.pipe(process.stdout);
                break;
            }

            if (config.log) {
                watcher.log.pipe(process.stdout);
            }
            break;
    }

};


/**
 * sitemap.js
 *
 * An input generator for html-snapshots that uses a sitemap.xml file.
 * Creates the snapshot arguments driven from sitemap.xml urls.
 *
 * Copyright (c) 2013 - 2019 Alex Grant, LocalNerve, contributors
 * Licensed under the MIT license.
 */
"use strict";

var fs = require("fs");
var path = require("path");
var urlm = require("url");
var zlib = require("zlib");
var _ = require("lodash");
var xml2js = require("xml2js");
var request = require("request");
var common = require("./index");
var base = require("../input-generators/_base");

// the out of date fallback
var unixStart = "1970-01-01T00:00:00.000Z";

// milliseconds for changefreq
var changeFreq = {
  always: -Number.MAX_VALUE,
  hourly: 3600000,
  daily: 86400000,
  weekly: 604800000,
  monthly: 2629740000,
  yearly: 31560000000,
  never: Number.MAX_VALUE
};

/**
 * Return true if the urlNode is still current (not stale) by sitemap policy.
 * If both lastmod and changefreq tags are supplied both are used.
 * If only one of lastmod or changefreq are supplied,
 *   then they are compared against previous output if it exists.
 * If a test fails for any reason, it is characterized as stale (returns false).
 *
 * @param {Object} urlNode - Sitemap XML url node as an object.
 * @param {Object} options - Sitemap options object, passed through.
 */
function stillCurrent (urlNode, options) {
  var lesser, greater, oPath;
  var now = Date.now();
  var lMod = _.first(urlNode.lastmod);
  var cFreq = _.first(urlNode.changefreq) ? _.first(urlNode.changefreq).toLowerCase() : null;

  // only lastmod specified
  if (lMod && !cFreq) {
    // if sitemap is malformed, just blow up
    oPath = base.outputFile(options, urlNode.loc[0]);

    lesser = now - ( (fs.existsSync(oPath) && fs.statSync(oPath).mtime.getTime()) || unixStart );
    greater = now - Date.parse(lMod);
  }
  // only changefreq specified
  else if (!lMod && cFreq) {
    // if sitemap is malformed, just blow up
    oPath = base.outputFile(options, urlNode.loc[0]);

    lesser = now - ( (fs.existsSync(oPath) && fs.statSync(oPath).mtime.getTime()) || unixStart );
    greater = changeFreq[cFreq] || changeFreq.always;
  }
  // both or neither were specified
  else {
    lesser = now - Date.parse(lMod || unixStart);
    greater = changeFreq[cFreq] || changeFreq.always;
  }

  return lesser <= greater;
}

/**
 * Parse a sitemap document
 * For each qualifying url element in urlset, call base.input
 * Stops processing if an error occurs.
 *
 * @param {Object} options - Sitemap options.
 * @param {String} options.source - The sitemap url or file.
 * @param {Boolean} options.sitemapPolicy - True if sitemapPolicy should be enforced.
 * @param {String} document - An xml document string.
 * @param {Function} callback - The completion callback.
 */
function parse (options, document, callback) {
  xml2js.parseString(document, {
    trim: true,
    normalizeTags: true,
    normalize: true
  }, function (err, result) {
    var source = options.source;

    if (!err) {

      // Process the url input, but break if base.input returns false.
      //   In other words, _.find is looking for a non-falsy err.
      // For now, this can only happen if no outputDir is defined,
      //   which is a fatal bad option problem and will happen immediately.
      _.find(
        // if the sitemap is malformed, just blow up
        result.urlset.url,
        function (urlNode) {
          // optionally ignore current urls by sitemap policy
          var url,
              process = !options.sitemapPolicy ||
                !stillCurrent(urlNode, options);

          if (process) {
            // if sitemap is malformed, just blow up
            url = urlm.parse(urlNode.loc[0]);
            if (!base.input(_.extend({}, options, {
                protocol: url.protocol,
                auth: url.auth,
                hostname: url.hostname,
                port: url.port
              }),
              urlNode.loc[0])
            ) {
              source = urlNode.loc[0];
              err = base.generatorError();
            }
          }

          return err;
        }
      );
    }

    callback(common.prependMsgToErr(err, source, true));
  });
}

/**
 * Convert Buffer input, call next or callback with error
 *
 * @param {Object} options - Sitemap options.
 * @param {String} options.source - Sitemap URL.
 * @param {Buffer} buffer - A node buffer.
 * @param {Function} next - A function to run after optional conversion applied.
 * @param {Function} callback - A completion callback.
 */
function convert(options, buffer, next, callback) {
  var gunzip = path.extname(options.source) === ".gz";

  if (gunzip) {
    zlib.gunzip(buffer, function (err, result) {
      if (err) {
        callback(common.prependMsgToErr(err, options.source, true));
      } else {
        next(options, result && result.toString(), callback);
      }
    });
  }
  else {
    next(options, buffer.toString(), callback);
  }
}

/**
 * Retrieve the sitemap from a url and call to parse it.
 *
 * @param {Object} options - Sitemap options.
 * @param {String} options.source - Sitemap URL.
 * @param {Function} options.timeout - User supplied run timeout.
 * @param {Function} parseFn - A data parser function.
 * @param {Function} callback - A completion callback function.
 */
function getUrl (options, parseFn, callback) {
  request({
    url: options.source,
    encoding: null,
    timeout: options.timeout() // get the default timeout
  }, function (err, res, body) {
    var error = err || common.checkResponse(res, ["text/xml", "application/xml"]);

    if (error) {
      callback(common.prependMsgToErr(error, options.source, true));
    } else {
      convert(options, body, parseFn, callback);
    }
  });
}

/**
 * Retrieve the sitemap from a file and call to parse it.
 *
 * @param {Object} options - Sitemap options.
 * @param {String} options.source - Sitemap file path.
 * @param {Function} parseFn - A data parser function.
 * @param {Function} callback - A completion callback.
 */
function getFile (options, parseFn, callback) {
  fs.readFile(options.source, function (err, data) {
    if (err) {
      callback(common.prependMsgToErr(err, options.source, true));
    } else {
      convert(options, data, parseFn, callback);
    }
  });
}

module.exports = {
  getFile: getFile,
  getUrl: getUrl,
  stillCurrent: stillCurrent,
  parse: parse
};


/*
 * cli.js
 *
 * Handles phantomjs command line interface.
 * Reads the command line arguments and returns options.
 *
 * This is a module for a phantomJS script that runs in phantomjs.
 *
 * Copyright (c) 2013 - 2019 Alex Grant, LocalNerve, contributors
 * Licensed under the MIT license.
 */

var system = require("system");
var globals = require("./globals");

module.exports = {

  /**
   * command line argument handling for selector based snapshot scripts
   *  args: PHANTOM_SCRIPT OUTPUTFILE URL SELECTOR [TIMEOUT INTERVAL USEJQUERY FILTER]
   */
  selector: function () {

    var options = {};
    var defaults = {

      // outputFile (required)
      // The file to write the snapshot to
      // encoding is the same as served from url
      outputFile: "snapshot.html",

      // url (required)
      // The url to produce the snapshot from
      url: "http://localhost/",

      // selector (required)
      // The selector to wait for before taking the snapshot
      // When the selector is visible, this defines document readiness
      selector: "body",

      // timeout (optional)
      // The maximum amount of time (ms) to wait before giving up on the snapshot
      timeout: 10000,

      // checkInterval (optional)
      // The frequency (ms) to check for document readiness
      checkInterval: 250,

      // useJQuery (optional)
      // flag to indicate use jQuery selector or not. jQuery must already exist in page.
      useJQuery: false,

      // verbose (optional)
      // flag to indicate verbose output is desired.
      verbose: false

      // module (optional)
      // An external module to load
      // (default undefined)
    };

    if (system.args.length < 4) {
      globals.exit(3, "phantomjs script '"+system.args[0]+"' expected these arguments: OUTPUTFILE URL SELECTOR [TIMEOUT INTERVAL JQUERY MODULE]");
    } else {

      options.outputFile = system.args[1];
      options.url = system.args[2];
      options.selector = system.args[3];
      if (system.args[4]) {
        options.timeout = parseInt(system.args[4], 10);
      }
      if (system.args[5]) {
        options.checkInterval = parseInt(system.args[5], 10);
      }
      if (system.args[6]) {
        var useJQuery = system.args[6].toLowerCase();
        options.useJQuery =
          (useJQuery === "true" || useJQuery === "yes" || useJQuery === "1");
      }
      if (system.args[7]) {
        var verbose = system.args[7].toLowerCase();
        options.verbose =
          (verbose === "true" || verbose === "yes" || verbose === "1");
      }
      if (system.args[8] && system.args[8] !== "undefined") {
        options.module = system.args[8];
      }

      // ensure defaults, replaces false with false in two cases (useJQuery, verbose)
      Object.keys(defaults).forEach(function(prop) {
        if (!options[prop]) {
          options[prop] = defaults[prop];
        }
      });
    }

    return options;

  }
};


/**
 * common code
 *
 * Copyright (c) 2013 - 2019, Alex Grant, LocalNerve, contributors
 * Licensed under the MIT license.
 */
"use strict";

module.exports = {
  /**
   * Ensures that options at least contains propties and values from must
   * if they're not already defined and not null.
   * Differs from underscore by replacing
   *   undefined *or null* falsies, and only one defaults source allowed.
   */
  ensure: function (options, must) {
    if (must) {
      for (var prop in must) {
        if (options[prop] === void 0 || options[prop] === null) {
          options[prop] = must[prop];
        }
      }
    }
    return options;
  },

  /**
   * simple test for url
   * If you can think of a more approriate test for this use case,
   *   please let me know in the issues...
   */
  isUrl: function (obj) {
    var result = false;
    if (typeof obj === "string") {
      return (/^https?:\/\//).test(obj);
    }
    return result;
  },

  /**
   * Prepend a message to an Error message.
   *
   * @param {Object} [error] - An instance of Error. If not, converted to Error.
   * If undefined, then pass through undefined.
   * @param {String} [message] - The message to prepend.
   * @param {Boolean} [quoteInput] - True if the message should be quoted.
   * @returns {Error} The updated Error instance, undefined if no error.
   */
  prependMsgToErr: function (error, message, quoteInput) {
    var result, prepend, empty = "", quote = "'";

    if (error) {
      if (message) {
        prepend = quoteInput ? empty.concat(quote, message, quote) : message;
      }

      // Force Error instance, coerce given error to a string
      error = error instanceof Error ? error : new Error(empty + error);

      // If message supplied, prepend it
      error.message =
        prepend ? empty.concat(prepend, ": ", error.message) : error.message;

      result = error;
    }

    return result;
  },

  /**
   * Simple response checker for remote files.
   * Expected use in robots.txt or sitemap.xml only.
   *
   * @param {IncomingMessage} res - The IncomingMessage response to check.
   * @param {Array} mediaTypes - array of acceptable content-type media type strings.
   * @returns {String} Error message, empty string (falsy) if OK.
   */
  checkResponse: function (res, mediaTypes) {
    var contentTypeOk,
        result = "status: '" + res.statusCode + "', GET failed.";

    mediaTypes = !Array.isArray(mediaTypes) ? [mediaTypes] : mediaTypes;

    if (res.statusCode === 200) {
      // if content-type exists, and media type found then contentTypeOk
      contentTypeOk =
        res.headers["content-type"] &&
        // empty array and none found return true
        !mediaTypes.every(function(mediaType) {
          // flip -1 to 0 and NOT, so that true == NOT found, found stops loop w/false
          return !~res.headers["content-type"].indexOf(mediaType);
        });

      result = contentTypeOk ? "" : "content-type not one of '"+mediaTypes.join(",")+"'";
    }

    return result;
  }

};


/***
 * nodeCall
 *
 * Copyright (c) 2013 - 2019, Alex Grant, LocalNerve, contributors
 * Licensed under the MIT license.
 */
/* global Promise */
"use strict";

/**
 * Utility to promisify a Node function
 *
 * @param {Function} nodeFunc - The node-style function to Promisify.
 */
function nodeCall (nodeFunc /* args... */) {
  var nodeArgs = Array.prototype.slice.call(arguments, 1);

  return new Promise(function (resolve, reject) {
    /**
     * Resolve a node callback
     */
    function nodeResolver (err, value) {
      if (err) {
        reject(err);
      } else {
        resolve(value);
      }
    }

    nodeArgs.push(nodeResolver);
    nodeFunc.apply(nodeFunc, nodeArgs);
  });
}

module.exports = nodeCall;


/**
 * sitemap-index.js
 *
 * An input generator for html-snapshots that uses a sitemap index xml file.
 * Creates the snapshot arguments driven from multiple sitemap.xml urls.
 *
 * Copyright (c) 2013 - 2019 Alex Grant, LocalNerve, contributors
 * Licensed under the MIT license.
 */
/* global Promise */
"use strict";

var fs = require("fs");
var pathLib = require("path");
var _ = require("lodash");
var mkdirp = require("mkdirp");
var xml2js = require("xml2js");
var common = require("../common");
var nodeCall = require("../common/node");
var smLib = require("../common/sitemap");
var base = require("./_base");

// The default options
var defaults = Object.freeze({
  source: "./sitemap-index.xml",
  sitemapPolicy: false,
  // Truthy, directs sitemapIndex sitemap file storage for sitemapPolicy purpose.
  sitemapOutputDir: "_sitemaps_"
});

var sitemapIndex;

/**
 * Make sure path exists so a sitemap write can succeed.
 *
 * @param {String} outputPath - File path and name.
 * @param {Funcion} callback - Completion callback.
 */
function prepareWrite (outputPath, callback) {
  var path = pathLib.parse(outputPath);
  var dir = pathLib.join(path.root, path.dir);
  mkdirp(dir, callback);
}

/**
 * Parse a sitemap, generate input, and optionally update the local file
 * for sitemapPolicy purposes.
 * This writes the sitemap files out EVEN IF sitemapPolicy is false.
 * The thinking is that it's better/cheaper to do this b/c if sitemapPolicy is
 * enabled later, then there is something to compare against, for more savings.
 *
 * @param {Object} options - Sitemap options.
 * @param {Object} options.__sitemapIndex - Sitemap Index options.
 * @param {String} document - xml document string.
 * @param {Function} callback - A completion callback.
 */
function processSitemap (options, document, callback) {
  smLib.parse(options, document, function (err) {
    var sitemapIndexOpts = options.__sitemapIndex;
    var outputPath = base.outputFile(sitemapIndexOpts, options.source);

    if (!err && sitemapIndexOpts.sitemapOutputDir) {
      prepareWrite(outputPath, function (err) {
        if (!err) {
          fs.writeFile(outputPath, document, callback);
        } else {
          callback(common.prependMsgToErr(err, outputPath, true));
        }
      });
    } else {
      callback(err);
    }
  });
}

/**
 * Parse a sitemap index document
 * For each qualifying sitemap element, download, parse the sitemap.
 * Stops processing if an error occurs.
 *
 * @param {Object} options - Sitemap options.
 * @param {Boolean} options.sitemapPolicy - True if sitemap policy should be used.
 * @param {String} document - xml document string.
 * @param {Function} callback - A completion callback.
 */
function parse (options, document, callback) {
  xml2js.parseString(document, {
    trim: true,
    normalizeTags: true,
    normalize: true
  }, function (err, result) {
    var sitemapUrls = [];
    var sitemapIndexOptions = Object.assign({}, options, {
      outputPath: undefined
    });

    if (!err) {
      // Check if we should process each sitemap in the index.
      _.forEach(
        // if the sitemap index is malformed, just blow up
        result.sitemapindex.sitemap,
        function (sitemapNode) {
          // optionally ignore current sitemaps by sitemap policy
          var shouldProcess = !options.sitemapPolicy ||
            !smLib.stillCurrent(sitemapNode, sitemapIndexOptions);

          if (shouldProcess) {
            // if sitemap index is malformed, just blow up
            sitemapUrls.push(sitemapNode.loc[0]);
          }
        }
      );

      // Edge case message for clarity
      if (sitemapUrls.length === 0) {
        console.log("[*] No sitemaps qualified for processing");
      }

      // Get all the sitemaps, parse them, and generate input for each
      Promise.all(sitemapUrls.map(function (sitemapUrl) {
        var sitemapOptions = Object.assign({}, options, {
          source: sitemapUrl,
          input: "sitemap",
          sitemapOutputDir: false,
          __sitemapIndex: sitemapIndexOptions
        });

        console.log("[+] Loading the sitemap: '"+sitemapUrl+"'");

        return nodeCall(smLib.getUrl, sitemapOptions, processSitemap);
      }))
        .then(function () {
          // Ignore the array of results to this function
          callback();
        })
        .catch(callback);
    } else {
      callback(err);
    }
  });
}

/**
 * generateInput
 *
 * Called back by base.run to generate the input for this input type.
 * This can return true on error for true async. An async error is supplied to listener
 * in this case via _abort.
 *
 * @param {Object} options - Sitemap options.
 * @param {String} options.source - Sitemap URL or file path.
 */
function generateInput (options) {
  return nodeCall(
    common.isUrl(options.source) ? smLib.getUrl : smLib.getFile,
    options,
    parse
  )
    .catch(function (err) {
      options._abort(err);
    })
    .then(function () {
      base.EOI(sitemapIndex);
    });
}

sitemapIndex = module.exports = {
  /**
   * Generate the input arguments for snapshots from a robots.txt file.
   * Each input argument generated calls the listener passing the input object.
   *
   * @param {Object} options - Sitemap index options.
   * @param {Function} listener - Callback to receive each input.
   * @returns {Promise} Resolves to undefined on completion.
   */
  run: function (options, listener) {
    var opts = Object.assign({}, base.defaults(defaults), options);

    return base.run(opts, generateInput, listener);
  }
};


/*
 * textfile.js
 *
 * An input generator for html-snapshots that uses a simple text file
 *   with the host relative page urls, one per line.
 *
 * Copyright (c) 2013 - 2019 Alex Grant, LocalNerve, contributors
 * Licensed under the MIT license.
 */
"use strict";

var fs = require("fs");
var base = require("./_base");
var common = require("../common");
var nodeCall = require("../common/node");

// The default options
var defaults = Object.freeze({
  source: "./line.txt",
  hostname: "localhost"
});

var textfile;

/**
 * Generate the snapshot arguments from a line oriented text file.
 * Each line contains a single url we need a snapshot for.
 *
 * @param {Object} options - input generator options object.
 * @param {String} options.source - path to the local input file.
 * @returns {Promise} resolves when complete.
 */
function generateInput (options) {
  return nodeCall(
    fs.readFile,
    options.source
  )
    .catch(function (err) {
      options._abort(err);
    })
    .then(function (data) {
      var error;

      if (data) {
        data.toString().split('\n').every(function (line) {
          var page = line.replace(/^\s+|\s+$/g, "");

          if (!base.input(options, page)) {
            error = common.prependMsgToErr(base.generatorError(), page, true);
            return false;
          }
          return true;
        });

        if (error) {
          console.error(error);
          options._abort(error);
        }
      }

      base.EOI(textfile);
    });
}

textfile = module.exports = {
  /**
   * Generate the input arguments for snapshots from a simple, line oriented
   * text file.
   *
   * @param {Object} options - Text file options.
   * @param {Function} listener - Callback to receive each input.
   * @returns {Promise} Resolves to undefined on completion.
   */
  run: function (options, listener) {
    var opts = Object.assign({}, base.defaults(defaults), options);

    return base.run(opts, generateInput, listener);
  }
};


/*
 * array.js
 *
 * An input generator for html-snapshots that uses a javascript array
 *   to generate snapshot input from the host relative page urls.
 *
 * Copyright (c) 2013 - 2019 Alex Grant, LocalNerve, contributors
 * Licensed under the MIT license.
 */
/* global Promise */
"use strict";

var urlm = require("url");
var common = require("../common");
var base = require("./_base");

var array;

/**
 * Generate the snapshot arguments from an array of pages.
 * Stops processing if one fails.
 *
 * @param {Object} options - input generator options object.
 * @param {String} options.source - Array of urls.
 * @param {Function} options._abort - Called to abort the output watcher.
 * @returns {Promise} Resolves to undefined on success.
 */
function generateInput (options) {
  var result = new Promise(function (resolve, reject) {
    var all;

    if (Array.isArray(options.source)) {
      all = options.source.every(function (sourceUrl) {
        var url = urlm.parse(sourceUrl);
        var opts = Object.assign({}, options, {
          protocol: url.protocol,
          auth: url.auth,
          hostname: url.hostname,
          port: url.port
        });

        if (!base.input(opts, sourceUrl)) {
          reject(
            common.prependMsgToErr(base.generatorError(), sourceUrl, true)
          );
          return false;
        }

        return true;
      });

      if (all) {
        resolve();
      }
    } else {
      reject(new Error("options.source must be an array"));
    }
  });

  return result
    .catch(function (error) {
      options._abort(error);
    })
    .then(function () {
      base.EOI(array);
    });
}

array = module.exports = {
  /**
   * Generate the input arguments for snapshots from an array.
   *
   * @param {Object} options - Array generator options.
   * @param {Function} listener - Callback to receive each input.
   * @returns {Promise} Resolves to undefined on completion.
   */
  run: function (options, listener) {
    return base.run(options, generateInput, listener);
  }
};


/*
 * verbose.js
 *
 * Adds handlers to WebPage module to produce additional output for debugging.
 *
 * This is a module for a phantomJS script that runs in phantomjs.
 *
 * Copyright (c) 2013 - 2019 Alex Grant, LocalNerve, contributors
 * Licensed under the MIT license.
 */
/* global window */
var system = require("system");

/**
 * Decorate the given WebPage with handlers to produce verbose output.
 * Source:
 * https://newspaint.wordpress.com/2013/04/25/getting-to-the-bottom-of-why-a-phantomjs-page-load-fails/
 */
function verbose (page) {

  page.onResourceError = function(resourceError) {
      system.stderr.writeLine('= onResourceError()');
      system.stderr.writeLine('  - unable to load url: "' + resourceError.url + '"');
      system.stderr.writeLine('  - error code: ' + resourceError.errorCode + ', description: ' + resourceError.errorString );
  };

  page.onError = function(msg, trace) {
      system.stderr.writeLine('= onError()');
      var msgStack = ['  ERROR: ' + msg];
      if (trace) {
          msgStack.push('  TRACE:');
          trace.forEach(function(t) {
              msgStack.push('    -> ' + t.file + ': ' + t.line + (t.function ? ' (in function "' + t.function + '")' : ''));
          });
      }
      system.stderr.writeLine(msgStack.join('\n'));
  };

  page.onResourceRequested = function (request) {
      system.stderr.writeLine('= onResourceRequested()');
      system.stderr.writeLine('  request: ' + JSON.stringify(request, undefined, 4));
  };

  page.onResourceReceived = function(response) {
      system.stderr.writeLine('= onResourceReceived()' );
      system.stderr.writeLine('  id: ' + response.id + ', stage: "' + response.stage + '", response: ' + JSON.stringify(response));
  };

  page.onLoadStarted = function() {
      system.stderr.writeLine('= onLoadStarted()');
      var currentUrl = page.evaluate(function() {
          return window.location.href;
      });
      system.stderr.writeLine('  leaving url: ' + currentUrl);
  };

  page.onNavigationRequested = function(url, type, willNavigate, main) {
      system.stderr.writeLine('= onNavigationRequested');
      system.stderr.writeLine('  destination_url: ' + url);
      system.stderr.writeLine('  type (cause): ' + type);
      system.stderr.writeLine('  will navigate: ' + willNavigate);
      system.stderr.writeLine('  from page\'s main frame: ' + main);
  };

  page.onLoadFinished = function(status) {
      system.stderr.writeLine('= onLoadFinished()');
      system.stderr.writeLine('  status: ' + status);
  };
}

module.exports = verbose;


/**
 * index.js
 *
 * Input generator factory
 * Create an input generator that produces snapshots arguments
 *
 * Copyright (c) 2013 - 2019 Alex Grant, LocalNerve, contributors
 * Licensed under the MIT license.
 */
"use strict";

module.exports = {
  /**
   * create
   * Creates the input generator based on the input string, which must match a
   * js file in this directory.
   * So, if input === "robots", then executes require("./robots");
   * Unsupported input generator types return a "null" generator.
   *
   * @param {String} input - The name of the input generator to create.
   * @returns {Object} The input generator, a null generator on error.
   */
  create: function (input) {
    var result = {
      run: function () {
        return [];
      },
      __null: true
    },
    hasInput;

    if (input) {
      input = (""+input).replace(" ", "").toLowerCase();
      hasInput = input && input.charAt(0) !== '_' && input !== "index";
    }

    try {
      if (hasInput) {
        result = require( "./" + input);
      }
    } catch (e) {
      console.error("Input generator load failed '" + input + "'", e)
      /* return the "null" generator on error */
    }

    return result;
  },

  /**
   * Determine if an input generator is null (bogus).
   * If the generator has __null defined, then its bogus.
   *
   * @param {Object} generator - An input generator.
   * @returns {Boolean} True if the given generator is bogus, false otherwise.
   */
  isNull: function (generator) {
    return generator.__null !== void 0;
  }
};


/**
 * _base.js
 *
 * The common, base functionality of an input generator.
 *
 * Copyright (c) 2013 - 2019 Alex Grant, LocalNerve, contributors
 * Licensed under the MIT license.
 */
"use strict";

var EventEmitter = require("events").EventEmitter;
var path = require("path");
var urlm = require("url");
var _ = require("lodash");
var common = require("../common");

// Defaults for this module
var defaults = Object.freeze({
  protocol: "http",
  hostname: "localhost",
  port: undefined,
  auth: undefined,
  outputPath: undefined,
  outputDir: "./snapshots",
  selector: "body",
  timeout: 10000,
  checkInterval: 250,
  useJQuery: false,
  verbose: false,
  phantomjsOptions: ""
});

/**
 * Normalize the given object to a function.
 *
 * If the input is a function, return it as is.
 * If the input is undefined, then a passthru function is generated.
 * If the input is NOT an Object, then a function returns that value.
 * If the input is an Object, then a function receives a key and
 *   returns the property value if it exists, otherwise return the __default value.
 *   If a passthru is requested, the key is returned when no __default is found.
 *
 * @param {Any} obj - Some input to be represented by a function.
 * @returns {Function} A function to represent the given input.
 */
function normalize (obj) {
  var result = obj;
  if (typeof obj !== "function") {
    if (typeof obj !== "undefined") {
      if (Object.prototype.toString.call(obj) !== "[object Object]") {
        result = (function (value) { return function () { return value; }; }(obj));
      } else {
        result = (function (o) { return function (key, passthru) {
          if (o[key] === void 0) {
            return o.__default || (passthru ? key : undefined);
          } else {
            return o[key];
          }
        }; }(obj));
      }
    } else {
      result = function (passthru) { return passthru; };
    }
  }
  return result;
}

/**
 * If a normalized option was overriden, but a default still undefined, supply one.
 * The result could still be undefined, but only if the default is undefined.
 *
 * If calling the normalized option with no args is undefined, supply missing default.
 *
 * @param {Object} options - Input generator options.
 * @param {String} name - The name of the property to supply missing default for.
 */
function supplyMissingDefault (options, name) {
  if (options[name]() === void 0) {
    options[name] = _.wrap(options[name], function(func, key) {
      var res = func(key);
      return res === void 0 ? defaults[name] : res;
    });
  }
}

/**
 * Prepare options for use by an input generator.
 *
 * @param {Object} options - Input generator options with or without defaults.
 * @param {Function} listener - The input generator listener.
 */
function prepOptions (options, listener) {
  // Ensure defaults are represented
  common.ensure(options, defaults);

  // Normalize certain arguments if they are not functions.
  // The certain arguments are per-page options.
  // However, outputPath is a special case
  [
    "selector", "timeout", "useJQuery", "verbose", "phantomjsOptions"
  ]
  .forEach(function (perPageOption) {
    options[perPageOption] = normalize(options[perPageOption]);
    supplyMissingDefault(options, perPageOption);
  });

  options._inputEmitter = new EventEmitter();
  options._inputEmitter.on("input", listener);
}

/**
 * Get an output path for a page.
 *
 * Parse a given page as a url and return its corresponding outputPath.
 * If parse is an object, parse.url will be a reference to the url parse.
 * Returns false if no output path found.
 *
 * @param {Object} options - output path options.
 * @param {Function} options.outputPath - Derives the outputPath from page.
 * @param {String} page - The url to the page resource.
 * @param {Object} [parse] - Supply to get page (url) parse output in prop `url`.
 * @returns {String} The 'path' part of the file output path.
 */
function getOutputPath (options, page, parse) {
  var pagePart = urlm.parse(page),
      // if outputPath was normalized with an object, let the key passthru
      outputPath = options.outputPath(page, true);

  // check for bad output path
  if (!outputPath) {
    return false;
  }

  // if the outputPath is really still a url, fix it to path+hash
  if (common.isUrl(outputPath)) {
    outputPath = pagePart.path + (pagePart.hash ? pagePart.hash : "");
  }

  // if the caller wants the url parse output, return it
  if (parse) {
    parse.url = pagePart;
  }

  return outputPath;
}

/**
 * Using the options, map the given page to an output path.
 *
 * @param {Object} options - The input generator options.
 * @param {String} options.outputDir - The output directory root path.
 * @param {String} [options.outputPath] - optional function or object to map pages to outputPaths.
 * @param {String} [options.sitemapOutputDir] - optional root path for sitemaps within outputDir.
 * @returns {String|Boolean} The full path to the output file or false on failure.
 */
function mapOutputFile (options, page, parse) {
  if (!_.isFunction(options.outputPath)) {
    options.outputPath = normalize(options.outputPath);
  }

  var outputPath = getOutputPath(options, page, parse);
  var outputDir = options.outputDir;
  var fileName = "index.html";

  if (options.sitemapOutputDir) {
    outputDir = path.join(options.outputDir, options.sitemapOutputDir);
    fileName = "";
  }

  return ( outputPath && path.join(outputDir, outputPath, fileName) ) || false;
}

/*
 * Interface consumed by input generators
 *
 * Consumption pattern call order:
 * <Setup Phase>
 *   [defaults]
 * <Start Processing>
 *   run
 * <For each page, specialized>
 *   input
 * <End Processing>
 *   EOI
 */
module.exports = {

  /**
   * Add specific items to the defaults.
   * Not required if the generator has no defaults itself.
   *
   * @param {Object} specific - Input generator specific defaults.
   * @return {Object} The fully mixed defaults.
   */
  defaults: function (specific) {
    return Object.assign({}, defaults, specific);
  },

  /**
   * Run the input generator.
   *
   * @param {Object} options - Input generator options.
   * @param {Function} generator - The input generator to run.
   * @param {Function} listener - The callback that receives each input generated.
   * @returns {Promise} The promise returned by the input generator.
   */
  run: function (options, generator, listener) {
    options = options || {};
    prepOptions(options, listener);
    return generator(options);
  },

  /**
   * Generate the input
   * Emit the event that contains the input hash
   *
   * @param {Object} options - input generator options.
   * @param {String} options.protocol - https?
   * @param {String} options.auth - user:pass for old style urls.
   * @param {String} options.hostname - The hostname part of the url.
   * @param {Number} options.port - The port part of the url.
   * @param {Function} options.selector - Produces a selector given a page url.
   * @param {Function} options.timeout - Produces a timeout given a page url.
   * @param {Number} options.checkInterval - The millisecond resolute poll interval.
   * @param {Function} options.userJQuery - Produces a jquery bool given a page url.
   * @param {Function} options.verbose - Produces a verbose bool given a page url.
   * @param {Function} options.phantomjsOptions - Produces phantomjs options given a page url.
   * @param {Function} options._inputEmitter - The EventEmitter to generate input.
   * @param {String} page - The url to the page resource.
   * @returns {String} The full output file path.
   */
  input: function (options, page) {
    var parse = {};
    var outputFile = mapOutputFile(options, page, parse);

    if (outputFile) {

      options._inputEmitter.emit("input", {
        outputFile: outputFile,
        // make the url
        url: urlm.format({
                protocol: options.protocol,
                auth: options.auth,
                hostname: options.hostname,
                port: options.port,
                pathname: parse.url.pathname,
                search: parse.url.search,
                hash: parse.url.hash
              }),
        // map the input page to a selector
        selector: options.selector(page),
        // map the input page to a timeout
        timeout: options.timeout(page),
        checkInterval: options.checkInterval,
        // map the input page to a useJQuery flag
        useJQuery: options.useJQuery(page),
        // map the input page to a verbose flag
        verbose: options.verbose(page),
        // map the input page to phantomJS options
        phantomjsOptions: options.phantomjsOptions(page),
        // useful for testing, debugging
        __page: page
      });

    }

    return outputFile;
  },

  /**
   * Signal the end of input
   *
   * @param {Object} generator - The input generator.
   */
  EOI: function (generator) {
    generator.EOI = function () { return true; };
  },

  /**
   * Expose the output file mapping utility function.
   *
   * @see mapOutputFile.
   */
  outputFile: mapOutputFile,

  /**
   * Unified input generator error.
   *
   * @returns {Error} The generic input generator failure error.
   */
  generatorError: function () {
    return new Error("Failed to generate input");
  }

};


/**
 * exists.js
 *
 * Robust exists check for snapshot files.
 * Empirically, I found that either one of these NodeJS checks can fail,
 * but both don't. This is a workaround.
 *
 * Copyright (c) 2013 - 2019, Alex Grant, LocalNerve, contributors
 * Licensed under the MIT license.
 */

var fs = require("fs");
var nodeCall = require("../common/node");

/**
 * Check if a path exists.
 *
 * @param {String} path - The full path to the file to check.
 * @param {Object} [options] - fileExists options.
 * @param {Boolean} [options.returnFile] - If true, resolve to input filename on
 * success. Otherwise, resolve to boolean. Defaults to false (boolean).
 * @returns {Promise} Resolves to true (or file) if exists, false otherwise.
 */
function pathExists (path, options) {
  options = options || {
    returnFile: false
  };

  // Defaults to F_OK
  return nodeCall(fs.access, path)
    .then(function () {
      return options.returnFile ? path : true;
    })
    .catch(function () {
      if (fs.existsSync(path)) {
        return options.returnFile ? path : true;
      }
      return false;
    });
}

module.exports = pathExists;


/*
 * html-snapshots.js
 *
 * Produce html snapshots using for a website for SEO purposes.
 * This is required for javascript SPAs or ajax page output.
 * By default, uses a selector to search content to determine if
 *   a page is "ready" for its html snapshot.
 *
 * Copyright (c) 2013 - 2019 Alex Grant, LocalNerve, contributors
 * Licensed under the MIT license.
 */
/* global Promise */
"use strict";

var spawn = require("child_process").spawn;
var path = require("path");
var EventEmitter = require("events").EventEmitter;
var rimraf = require("rimraf").sync;
var asyncLib = require("async");
var _ = require("lodash");

var common = require("./common");
var inputFactory = require("./input-generators");
var Notifier = require("./async").Notifier;
var phantomDir = "./phantom";
var snapshotScript = path.join(phantomDir, "default.js");

/**
 * Determine the default phantomJS module path. This is overridden by the
 * phatomjs option.
 *
 * This function, (technique and concerns) originated from karma-phantomjs-launcher:
 *
 * The MIT License
 * Copyright (C) 2011-2013 Google, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of
 * this software and associated documentation files (the "Software"), to deal in
 * the Software without restriction, including without limitation the rights to
 * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
 * of the Software, and to permit persons to whom the Software is furnished to do
 * so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 */
var defaultPhantomJSExePath = function () {
  // If the path we're given by phantomjs is to a .cmd, it is pointing to a global copy.
  // Using the cmd as the process to execute causes problems cleaning up the processes
  // so we walk from the cmd to the phantomjs.exe and use that instead.

  var phantomSource = require("phantomjs-prebuilt").path;

  if (path.extname(phantomSource).toLowerCase() === ".cmd") {
    return path.join(path.dirname( phantomSource ), "//node_modules//phantomjs-prebuilt//lib//phantom//bin//phantomjs.exe");
  }

  return phantomSource;
};

/**
 * This module's defaults
 */
var defaults = Object.freeze({
  input: "robots",
  phantomjs: defaultPhantomJSExePath(),
  snapshotScript: path.join(__dirname, snapshotScript),
  outputDirClean: false,
  pollInterval: 500,
  processLimit: 4
});

/**
 * The worker task that launches phantomjs.
 *
 * @param {Object} input - An input object from an input generator.
 * @param {Array} input.phantomjsOptions - Actual Phantomjs options.
 * @param {String} input.outputFile - The output file that the html is written to.
 * @param {String} input.url - The url to get the html from.
 * @param {String} input.selector - The selector to wait for.
 * @param {String} input.timeout - The phantomjs timeout.
 * @param {String} input.checkInterval - The interval to poll for results.
 * @param {Boolean} input.useJQuery - True if selector is jquery selector.
 * @param {Boolean} input.verbose - True if verbose output is desired.
 * @param {Object} options - Input options.
 * @param {Object} options.snapshotScript - Snapshot script options.
 * @param {String} options.snapshotScript.script - Snapshot script name.
 * @param {String} options.snapshotScript.module - Snapshot script module.
 * @param {String} options.phantomjs - Full path to the Phantomjs exe.
 * @param {Object} notifier - The async Notifier instance for this run.
 * @param {Function} notifier.remove - Remove an outputFile from notifier.
 * @param {Function} notifier.setError - Set an error on the notifier.
 * @param {Function} notifier.add - Add a file, timeout to the notifier.
 * @param {Function} notifier.known - Check if a file is or has been processed.
 * @param {Function} qcb - async.queue callback function.
 */
function worker (input, options, notifier, qcb) {
  var cp,
      customModule,
      snapshotScript = options.snapshotScript,
      phantomjsOptions = Array.isArray(input.phantomjsOptions) ? input.phantomjsOptions : [input.phantomjsOptions];

  // If the outputFile has NOT already been seen by the notifier, process.
  if (!notifier.known(input.outputFile)) {

    // map snapshotScript object script to a real path
    if (_.isObject(options.snapshotScript)) {
      snapshotScript = path.join(__dirname, phantomDir, options.snapshotScript.script) + ".js";
      customModule = options.snapshotScript.module;
    }

    cp = spawn(
      options.phantomjs,
      phantomjsOptions.concat([
        snapshotScript,
        input.outputFile,
        input.url,
        input.selector,
        input.timeout,
        input.checkInterval,
        input.useJQuery,
        input.verbose,
        customModule
      ]), { cwd: process.cwd(), stdio: "inherit", detached: true }
    );

    cp.on("error", function (e) {
      notifier.remove(input.outputFile);
      notifier.setError(e);
      console.error(e);
      qcb(e);
    });

    cp.on("exit", function (code) {
      qcb(code);
    });

    // start counting
    notifier.add(input.outputFile, input.timeout);
  }
  else {
    // The input.outputFile is being or has been processed this run.
    qcb(0);
  }
}

/**
 * Prepare html snapshots options.
 *
 * @param {Object} options - html snapshots options.
 * @param {Array|String} options.source - html snapshots data source.
 */
function prepOptions (options) {
  // ensure this module's defaults are represented in the options.
  common.ensure(options, defaults);

  // if array data source, ensure input type is "array".
  if (Array.isArray(options.source)) {
    options.input = "array";
  }
}

module.exports = {
  /**
   * Run all the snapshots using the requested inputGenerator
   *
   * @param {Object} options - ALL user supplied html snapshots options.
   * @param {String} options.outputDir - Directory to write the html files to.
   * @param {String} [options.input] - Input source type "robots", "sitemap", "array";
   * Defaults to "robots".
   * @param {Boolean} [options.outputDirClean] - True if output dir should be rm -rf;
   * Defaults to false.
   * @param {Number} [options.pollInterval] -
   * @param {Function} [listener] - User supplied optional callback.
   * @returns {Promise} Resolves on completion.
   */
  run: function (options, listener) {
    var inputGenerator, notifier, started, result, q, emitter, completion;

    options = options || {};
    prepOptions(options);

    // create the inputGenerator, default to robots
    inputGenerator = inputFactory.create(options.input);

    // clean the snapshot output directory
    if (options.outputDirClean) {
      rimraf(options.outputDir);
    }

    // start async completion notification.
    notifier = new Notifier();
    emitter = new EventEmitter();
    started = notifier.start(options.pollInterval, inputGenerator,
      function (err, completed) {
        emitter.emit("complete", err, completed);
      });

    if (started) {
      // create the completion Promise.
      completion = new Promise(function (resolve, reject) {
        function completionResolver (err, completed) {
          try {
            _.isFunction(listener) && listener(err, completed);
          } catch (e) {
            console.error("User supplied listener exception", e);
          }
          if (err) {
            err.notCompleted = notifier.filesNotDone;
            err.completed = completed;
            reject(err);
          } else {
            resolve(completed);
          }
        }
        emitter.addListener("complete", completionResolver);
      });

      // create a worker queue with a parallel process limit.
      q = asyncLib.queue(function (task, callback) {
        task(_.once(callback));
      }, options.processLimit);

      // have the queue call notifier.empty when last item
      //  from the queue is given to a worker.
      q.empty = notifier.qEmpty.bind(notifier);

      // expose abort callback to input generators via options.
      options._abort = function (err) {
        notifier.abort(q, err);
      };

      // generate input for the snapshots.
      result = inputGenerator.run(options, function (input) {
        // give the worker the input and place into the queue
        q.push(_.partial(worker, input, options, notifier));
      })
        // after input generation, resolve on browser completion.
        .then(function () {
          return completion;
        });
    } else {
      result = Promise.reject("failed to start async notifier");
    }

    return result;
  }
};


/**
 * index.js
 *
 * Async stuff for html-snapshots
 *   Notifier class
 *
 * Copyright (c) 2013 - 2019, Alex Grant, LocalNerve, contributors
 * Licensed under the MIT license.
 */
/* global Promise */
"use strict";

var crypto = require("crypto");
var AsyncLock = require("async-lock");
var combineErrors = require("combine-errors");
var pathExists = require("./exists");

var L_WAIT = 10000;
var TIMEOUT_PAD_FLOOR = 800;
function NOOP () {}

/**
 * Create the lock factory.
 */
function createLockFactory () {
  // Create the per instance async lock.
  var lock = new AsyncLock();

  // Make a random id
  var rid = crypto.randomBytes(16).toString("hex");

  /**
   * Force a serial execution context.
   *
   * @param {Function} fn - The function to guard.
   * @param {Number} timeout - The max time to wait for the lock.
   */
  return function lockFactory (fn, timeout) {
    return function protectedContext () {
      lock.acquire("cs-guard-" + rid, function (done) {
        fn(function () {
          done(null, 0);
        });
      }, NOOP, {
        timeout: timeout
      });
    };
  };
}

/**
 * Notifier Constructor
 * Polls the outputDir and when all the files exist,
 * calls the listener indicating the snapshots are done.
 */
function Notifier () {

  // Create the serial execution context mechanism.
  this.csFactory = createLockFactory();

  // The private files collection
  // Contains a file and timer: "filename": {timer: timerid}
  // Used for tracking work left to do. When empty, work is done.
  this.files = {};

  // Contains files successfully processed
  this.filesDone = [];

  // Contains files unsuccessfully processed
  this.filesNotDone = [];

  // true if a timeout occurred, or set by abort
  this.errors = [];

  // the holder of the current failure timeout padding
  this.padTimeout = TIMEOUT_PAD_FLOOR;

  // our reference to the listener
  this.callback = null;

  // our reference to the watcher (an interval id)
  // initial value undefined is important
  // this.watcher;

  // the working pollInterval for this run
  // this.interval

  // flag set by qEmpty callback
  // when the last item from the queue is given to a worker
  this.qempty = false;
}

Notifier.prototype = {
  /**
   * Set the qempty flag to true.
   */
  qEmpty: function qEmpty () {
    this.qempty = true;
  },

  /**
   * Start a watch interval, when a file exists, remove it from our files array.
   * If the files array is empty, call the listener and stop the watch interval.
   *
   * @param {Number} pollInterval - The millisecond poll interval to check for
   * file completion.
   * @param {Object} input - The inputGenerator.
   * @param {Function} listener - The callback to notify when complete.
   * @param {Function} input.EOI - The inputGenerator function that signals end
   * of input.
   * @returns {Boolean} true if started successfully, false otherwise.
   */
  start: function start (pollInterval, input, listener) {
    var result = (
      pollInterval > 0 &&
      typeof listener === "function" &&
      (!!input)
      );

    if (result) {
      if (this.isStarted()) {
        throw new Error("Notifier already started");
      }

      this.callback = listener;
      this.interval = parseInt(pollInterval, 10);

      // Poll the filesystem for the files to exist
      // Checks the child process expected output to determine success or failure
      // if the file exists, then it succeeded.
      this.watcher = setInterval(this.csFactory(function (done) {
        var self = this;
        var eoi = typeof input.EOI === "function" && input.EOI();

        if (eoi) {
          Promise.all(Object.keys(self.files).map(function (file) {
            return pathExists(file, {
              returnFile: true
            });
          }))
            .then(function (files) {
              var callback = self.callback;

              try {
                files.forEach(function (file) {
                  file && self._remove(file, true);
                });

                if (self._isDone()) {
                  self._closeWatcher();
                  if (self.callback) {
                    self.callback = null;
                    setImmediate(function () {
                      callback(self.getError(), self.filesDone);
                    });
                  }
                }
              } catch (e) {
                console.error(e);
              }
              done();
            });
        } else {
          done();
        }
      }.bind(this), L_WAIT), this.interval);

    } else {
      console.error("Bad poll interval, async listener, or input generator supplied");
    }

    return result;
  },

  /**
   * Indicates if start has been called.
   *
   * @returns {Boolean} true if start has been called, false otherwise.
   */
  isStarted: function isStarted () {
    return (typeof this.watcher !== "undefined");
  },

  /**
   * Add a file to the files array if it's not there.
   *
   * @param {String} outputFile - The full path to the output file to add.
   * @param {Number} timeout - timeout milliseconds to wait (max).
   */
  add: function add (outputFile, timeout) {
    var failTimeout = timeout;
    var timer;

    if (!this.isStarted()) {
      throw new Error("MUST call `start` before `add`");
    }

    if (!this._exists(outputFile)) {
      // make sure we evaluate after the child process
      failTimeout = parseInt(timeout, 10) + parseInt(this.padTimeout, 10);
      // Stagger and grow the failure timeout padding, add 1s every 10 processes
      this.padTimeout += 100;

      // setup a timeout handler to detect failure
      timer = setTimeout(this.csFactory(function (done) {
        var self = this;

        // if the output file has not already been removed
        if (self._exists(outputFile)) {
          pathExists(outputFile)
            .then(function (fsExists) {
              var callback = self.callback;

              try {
                if (!fsExists) {
                  self._setError(new Error(
                    "'" + outputFile + "' did not get a snapshot before timeout"
                  ));
                }
                self._remove(outputFile, fsExists);

                if (self._isDone()) {
                  self._closeWatcher();
                  if (self.callback) {
                    self.callback = null;
                    setImmediate(function () {
                      callback(self.getError(), self.filesDone);
                    });
                  }
                }
              } catch (e) {
                console.error(e);
              }
              done();
            });
        } else {
          done();
        }
      }.bind(this), L_WAIT), parseInt(failTimeout, 10));

      // add the file tracking object
      this.files[outputFile] = {
        timer: timer
      };
    }
  },

  /**
   * Check if a file exists in the internal files array.
   *
   * @param {String} outputFile - The outputFile to check.
   * @returns {Boolean} true if the file exists, false otherwise.
   */
  _exists: function _exists (outputFile) {
    return (!!this.files[outputFile]);
  },

  /**
   * Check if a file is being processed or has already been processed.
   *
   * @param {String} outputFile - The outputFile to check.
   * @returns {Boolean} true if the fi
   */
  known: function known (outputFile) {
    var result = false;
    this.csFactory(function (done) {
      result =
        this._exists(outputFile) || this.filesDone.indexOf(outputFile) > -1;
      done();
    }.bind(this), L_WAIT)();
    return result;
  },

  /**
   * Remove a file from the files array if it's there.
   * Unprotected version.
   *
   * @param {String} outputFile - The outputFile to remove.
   * @param {Boolean} done - If true, add to filesDone collection.
   */
  _remove: function _remove (outputFile, done) {
    if (this._exists(outputFile)) {
      if (done) {
        this.filesDone.push(outputFile);
      } else {
        this.filesNotDone.push(outputFile);
      }
      clearTimeout(this.files[outputFile].timer);
      delete this.files[outputFile];
    }
  },

  /**
   * Remove a file from the files array if it's there.
   * Protected version.
   *
   * @param {String} outputFile - The outputFile to remove.
   * @param {Boolean} done - If true, add to filesDone collection.
   */
  remove: function remove (outputFile, done) {
    this.csFactory(function (_done) {
      this._remove(outputFile, done);
      _done();
    }.bind(this), L_WAIT)();
  },

  /**
   * Audit the file count.
   *
   * @returns {Number} the number of files being watched at the moment.
   */
  _fileCount: function _fileCount () {
    return Object.keys(this.files).length;
  },

  /**
   * Check if there is anything left to watch.
   *
   * @returns {Boolean} true if processing is done, false otherwise.
   */
  _isDone: function _isDone () {
    return this._fileCount() === 0 && this.qempty;
  },

  /**
   * Set the error, saves to a collection of errors.
   * Unprotected version.
   *
   * @param {Object} value - The error to set.
   */
  _setError: function _setError (value) {
    this.errors.push(value);
  },

  /**
   * Set the error property, if not already set.
   * Protected version.
   *
   * @param {Object} value - The error to set.
   */
  setError: function setError (value) {
    this.csFactory(function (done) {
      this._setError(value);
      done();
    }.bind(this), L_WAIT)();
  },

  /**
   * Get the error property.
   *
   * @returns {Object} The error property.
   */
  getError: function getError () {
    return this.errors.length > 0 ? combineErrors(this.errors) : undefined;
  },

  /**
   * End file watching.
   */
  _closeWatcher: function _closeWatcher () {
    if (this.watcher) {
      clearInterval(this.watcher);
      this.watcher = null;
    }
  },

  /**
   * Provides a way to abort what this is doing.
   * Causes conditions so that, if isStarted, the poll interval will exit,
   *   cleanup, and call the listener back.
   * If not started, the listener does not get called. This relationship
   *   is set in html-snapshots.js: listener == notifier.start
   *
   * @param {Object} q - The worker queue, expected created by async.queue.
   * @param {Function} q.length - Gets the worker queue length.
   * @param {Object} [err] - The error object to set.
   */
  abort: function abort (q, err) {
    this.csFactory(function (done) {
      try {
        // for each file, clearTimeout and delete the object
        Object.keys(this.files).forEach(function(file) {
          clearTimeout(this.files[file].timer);
          delete this.files[file];
        }, this);

        // if nothing is waiting, make sure empty is set
        this.qempty = !q.length();

        // set the error
        this._setError(err);
      } catch (e) {
        console.error(e);
      }
      done();
    }.bind(this), L_WAIT)();
  }
};

module.exports = {
  Notifier: Notifier,
  TIMEOUT_PAD_FLOOR: TIMEOUT_PAD_FLOOR
};


/*
 * selectorDetect.js
 *
 * Produce a single snapshot for a web page when a given selector is detected.
 *
 * This is a module for a phantomJS script that runs in phantomjs.
 *
 * Copyright (c) 2013 - 2019 Alex Grant, LocalNerve, contributors
 * Licensed under the MIT license.
 */

// Get the start time immediately since we're already in a PhantomJS process instance
// The spawner is already counting down (with allowances)...
var start = new Date().getTime();

/**
 * Dependencies, phantomjs
 */
var page = require("webpage").create();
var fs = require("fs");
var globals = require("./globals");
var verbose = require("./verbose");

/**
 * waitFor
 *
 * Heavily borrowed from phantomJS 'waitFor' example
 */
function waitFor (testFx, onReady, onTimeout, timeout, checkInterval) {
  var condition = false,
  interval = setInterval(function () {
    if ( (new Date().getTime() - start < timeout) && !condition ) {
      // If not timeout yet and condition not yet fulfilled
      condition = testFx();
    } else {
      clearInterval(interval); // Stop this interval
      if ( !condition ) {
        // If condition still not fulfilled (timeout but condition is 'false')
        onTimeout();
      } else {
        // Condition fulfilled (timeout and/or condition is 'true')
        onReady((new Date().getTime() - start));
      }
    }
  }, checkInterval);
}

/**
 * snapshot
 *
 * Opens the page and waits for the selector to become visible. Then takes the html snapshot.
 * Applies an optional output filter to the html content.
 *
 */
function snapshot (options, detector, filter) {
  filter = filter || function(content) { return content; };

  console.log("Creating snapshot for "+options.url+"...");

  // https://github.com/ariya/phantomjs/issues/10930
  page.customHeaders = {
    "Accept-Encoding": "identity"
  };

  // add optional verbose output
  if (options.verbose) {
    verbose(page);
  }

  // create the snapshot
  page.open(options.url, {
    resourceTimeout: options.timeout - 300 // a little space to report problems.
  }, function (status) {
    if (status !== "success") {
      // if phantomJS could not load the page, so end right now
      globals.exit(2, "Unable to load page " + options.url);
    } else {
      // phantomJS loaded the page, so wait for it to be ready
      waitFor(

        // The test to determine readiness
        function () {
          return page.evaluate(detector, {
            selector: options.selector,
            url: options.url
          });
        },

        // The onReady callback
        function (time) {
          fs.write(options.outputFile, filter(page.content), "w");
          globals.exit(0, "snapshot for "+options.url+" finished in "+time+" ms\n  written to "+options.outputFile);
        },

        // The onTimeout callback
        function () {
          globals.exit(1, "timed out waiting for "+options.selector+" to become visible for "+options.url);
        },

        options.timeout,
        options.checkInterval

      );
    }
  });
}

module.exports = snapshot;


/*
 * robots.js
 *
 * An input generator for html-snapshots that uses a simple robots.txt file.
 * Creates the snapshot arguments driven from robots.txt "Allow: ".
 * Does not support wildcards.
 * If you need wildcards, use the sitemap input generator.
 *
 * Copyright (c) 2013 - 2019 Alex Grant, LocalNerve, contributors
 * Licensed under the MIT license.
 */
"use strict";

var fs = require("fs");
var request = require("request");
var common = require("../common");
var nodeCall = require("../common/node");
var base = require("./_base");

// The default options
var defaults = Object.freeze({
  source: "./robots.txt",
  hostname: "localhost"
});

var robots;

/**
 * Generate input for one line of a simple robots.txt file
 * Does not support wildcards.
 */
function oneline (line, options) {
  var key = "Allow: ",
      index = line.indexOf(key);

  if (index !== -1) {
    var page = line.substr(index + key.length).replace(/^\s+|\s+$/g, "");
    return page.indexOf("*") === -1 && base.input(options, page);
  }

  return true;
}

/**
 * Retrieves robots.txt from url and parses it.
 *
 * @param {Object} options - Robots.txt options
 * @param {String} options.source - The URL to a robots.txt
 * @param {Function} options.timeout - Returns the robots.txt request timeout.
 * @param {Function} callback - A completion callback.
 */
function getRobotsUrl (options, callback) {
  request({
    url: options.source,
    timeout: options.timeout()
  }, function (err, res, body) {
    var error = err || common.checkResponse(res, "text/plain");

    if (error) {
      callback(common.prependMsgToErr(error, options.source, true));
    } else {
      body.toString().split('\n').every(function(line) {
        // Process the line input, but break if base.input returns false.
        // For now, this can only happen if no outputDir is defined,
        //   which is a fatal bad option problem and will happen immediately.
        if (!oneline(line, options)) {
          error = common.prependMsgToErr(base.generatorError(), line, true);
          return false;
        }
        return true;
      });

      callback(error);
    }
  });
}

/**
 * Reads the robots.txt file and parses it.
 */
function getRobotsFile (options, callback) {
  fs.readFile(options.source, function (err, data) {
    if (!err) {
      data.toString().split('\n').every(function (line) {
        // Process the line input, but break if base.input returns false.
        // For now, this can only happen if no outputDir is defined,
        //   which is a fatal bad option problem and will happen immediately.
        if (!oneline(line, options)) {
          err = common.prependMsgToErr(base.generatorError(), line, true);
          return false;
        }
        return true;
      });
    }

    callback(err);
  });
}

/**
 * Generate the snapshot arguments from a robots.txt file.
 * Each line that has "Allow:" contains a url we need a snapshot for.
 * This can return true on error for true async. An async error is supplied to listener
 * in this case via _abort.
 *
 * @param {Object} options - input generator options object.
 * @param {String} options.source - A url or file path.
 * @param {Function} options._abort - Abort function to stop the async notifier.
 * @returns {Promise} Resolves to undefined on completion.
 */
function generateInput (options) {
  return nodeCall(
    common.isUrl(options.source) ? getRobotsUrl : getRobotsFile,
    options
  )
    .catch(function (err) {
      options._abort(err);
    })
    .then(function () {
      base.EOI(robots);
    });
}

robots = module.exports = {
  /**
   * Generate the input arguments for snapshots from a robots.txt file
   * Each input argument generated calls the listener passing the input object.
   *
   * @param {Object} options - Robots options.
   * @param {Function} listener - Callback to receive each input.
   * @returns {Promise} Resolves to undefined on completion.
   */
  run: function (options, listener) {
    var opts = Object.assign({}, base.defaults(defaults), options);

    return base.run(opts, generateInput, listener);
  }
};


/*
 * detectors.js
 *
 * A collection of detector functions that detect when a page is ready for a snapshot.
 *
 * The detector functions are run in PhantomJS page.evaluate in the context of a webpage,
 * in a sandboxed execution context.
 * Other special rules apply: http://phantomjs.org/api/webpage/method/evaluate.html
 *
 * The detector functions return true when the page is ready, false if not.
 *
 * This is a module for a phantomJS script that runs in phantomjs.
 *
 * Copyright (c) 2013 - 2019 Alex Grant, LocalNerve, contributors
 * Licensed under the MIT license.
 */
/* global document, $ */
module.exports = {

  // Use plain Web API detection, with standard selectors
  // https://developer.mozilla.org/en-US/docs/Web/Guide/CSS/Getting_Started/Selectors
  // Returns true on element visibility
  querySelector: function(options) {
    var result = false;
    var el = document.querySelector(options.selector);

    if (el) {
      // el must be an HTMLElement that is visible
      result = el.offsetWidth && el.offsetHeight;
    }

    return result;
  },

  // Use JQuery selectors
  // Returns true on element visibility
  jquerySelector: function(options) {
    var result = false;

    if (typeof $ !== "undefined") {
      // the definition of document readiness is visibility
      result = $(options.selector).is(":visible");
    }

    return result;
  }

};


import {nest} from "d3-collection";

/**
    @function nest
    @summary Extends the base behavior of d3.nest to allow for multiple depth levels.
    @param {Array} *data* The data array to be nested.
    @param {Array} *keys* An array of key accessors that signify each nest level.
*/
export default function(data, keys) {

  if (!(keys instanceof Array)) keys = [keys];

  const dataNest = nest();
  for (let i = 0; i < keys.length; i++) dataNest.key(keys[i]);
  const nestedData = dataNest.entries(data);

  return bubble(nestedData);

}

/**
    Bubbles up values that do not nest to the furthest key.
    @param {Array} *values* The "values" of a nest object.
    @private
*/
function bubble(values) {

  return values.map(d => {

    if (d.key && d.values) {
      if (d.values[0].key === "undefined") return d.values[0].values[0];
      else d.values = bubble(d.values);
    }

    return d;

  });

}


"use strict";

const cidrTools = module.exports = {};
const IPCIDR = require("ip-cidr");
const isCidr = require("is-cidr");
const ipv6Normalize = require("ipv6-normalize");
const naturalCompare = require("string-natural-compare");
const Address4 = require("ip-address").Address4;
const Address6 = require("ip-address").Address6;
const BigInteger = require("jsbn").BigInteger;

const bits = {
  "v4": 32,
  "v6": 128,
};

const bigint = numberstring => new BigInteger(numberstring);

const zero = bigint("0");
const one = bigint("1");
const two = bigint("2");

function parse(str) {
  if (isCidr(str)) {
    return new IPCIDR(cidrTools.normalize(str));
  } else {
    const parsed = new IPCIDR(cidrTools.normalize(str));
    if (parsed && parsed.address) {
      return new IPCIDR(cidrTools.normalize(`${str}/${bits[parsed.address.v4 ? "v4" : "v6"]}`));
    } else {
      throw new Error(`Network is not a CIDR or IP: ${str}`);
    }
  }
}

function format(number, v) {
  const cls = v === "v6" ? Address6 : Address4;
  if (number.constructor.name !== "BigInteger") number = bigint(number);
  return cidrTools.normalize(cls.fromBigInteger(number).address);
}

function uniq(arr) {
  return [...new Set(arr)];
}

function overlap(a, b) {
  const aStart = a.start({type: "bigInteger"});
  const bStart = b.start({type: "bigInteger"});
  const aEnd = a.end({type: "bigInteger"});
  const bEnd = b.end({type: "bigInteger"});

  //    aaa
  // bbb
  if (aStart.compareTo(bEnd) > 0) return false; // a starts after b

  // aaa
  //    bbb
  if (bStart.compareTo(aEnd) > 0) return false; // b starts after a

  return true;
}

// exclude b from a and return remainder cidrs
function exclude(a, b, v) {
  const aStart = a.start({type: "bigInteger"});
  const bStart = b.start({type: "bigInteger"});
  const aEnd = a.end({type: "bigInteger"});
  const bEnd = b.end({type: "bigInteger"});
  const parts = [];

  // compareTo returns negative if left is less than right

  //       aaa
  //   bbb
  //   aaa
  //       bbb
  if (aStart.compareTo(bEnd) > 0 || aEnd.compareTo(bStart) < 0) {
    return [a.cidr];
  }

  //   aaa
  //   bbb
  if (aStart.compareTo(bStart) === 0 && aEnd.compareTo(bEnd) === 0) {
    return [];
  }

  //   aa
  //  bbbb
  if (aStart.compareTo(bStart) > 0 && aEnd.compareTo(bEnd) < 0) {
    return [];
  }

  // aaaa
  //   bbbb
  // aaaa
  //   bb
  if (aStart.compareTo(bStart) < 0 && aEnd.compareTo(bEnd) <= 0) {
    parts.push({
      start: aStart,
      end: bStart.subtract(one),
    });
  }

  //    aaa
  //   bbb
  //   aaaa
  //   bbb
  if (aStart.compareTo(bStart) >= 0 && aEnd.compareTo(bEnd) > 0) {
    parts.push({
      start: bEnd.add(one),
      end: aEnd,
    });
  }

  //  aaaa
  //   bb
  if (aStart.compareTo(bStart) < 0 && aEnd.compareTo(bEnd) > 0) {
    parts.push({
      start: aStart,
      end: bStart.subtract(one),
    });
    parts.push({
      start: bEnd.add(one),
      end: aEnd,
    });
  }

  const remaining = [];
  for (const part of parts) {
    for (const subpart of subparts(part, v)) {
      remaining.push(formatPart(subpart, v));
    }
  }

  return cidrTools.merge(remaining);
}

function biggestPowerOfTwo(num) {
  if (num.compareTo(zero) === 0) return zero;
  const power = bigint(String(num.toString(2).length - 1));
  return two.pow(power);
}

function subparts(part) {
  // special case for when part is length 1
  if (part.end.subtract(part.start).compareTo(one) === 0) {
    if (part.end.remainder(two).equals(zero)) {
      return [{start: part.start, end: part.start}, {start: part.end, end: part.end}];
    } else {
      return [{start: part.start, end: part.end}];
    }
  }

  const size = diff(part.end, part.start);
  let biggest = biggestPowerOfTwo(size);

  let start, end;
  if (size.equals(biggest) && part.start.add(size).compareTo(part.end) === 0) {
    return [part];
  } else if (part.start.remainder(biggest).equals(zero)) {
    // start is matching on the size-defined boundary - ex: 0-12, use 0-8
    start = part.start;
    end = start.add(biggest).subtract(one);
  } else {
    start = part.end.divide(biggest).multiply(biggest);

    // start is not matching on the size-defined boundary - 4-16, use 8-16
    if (start.add(biggest).subtract(one).compareTo(part.end) > 0) {
      // divide will floor to nearest integer
      start = part.end.divide(biggest).subtract(one).multiply(biggest);

      while (start.compareTo(part.start) < 0) {
        biggest = biggest.divide(two);
        start = part.end.divide(biggest).subtract(one).multiply(biggest);
      }

      end = start.add(biggest).subtract(one);
    } else {
      start = part.end.divide(biggest).multiply(biggest);
      end = start.add(biggest).subtract(one);
    }
  }

  let parts = [{start, end}];

  // additional subnets on left side
  if (!start.equals(part.start)) {
    parts = parts.concat(subparts({start: part.start, end: start.subtract(one)}));
  }

  // additional subnets on right side
  if (!end.equals(part.end)) {
    parts = parts.concat(subparts({start: end.add(one), end: part.end}));
  }

  return parts;
}

function diff(a, b) {
  if (a.constructor.name !== "BigInteger") a = bigint(a);
  if (b.constructor.name !== "BigInteger") b = bigint(b);
  a = a.add(one);
  return a.subtract(b);
}

function formatPart(part, v) {
  const ip = format(part.start, v);
  const zeroes = diff(part.end, part.start).toString(2);
  const prefix = bits[v] - (zeroes.match(/0/g) || []).length;
  return `${ip}/${prefix}`;
}

cidrTools.normalize = (cidr) => {
  const cidrVersion = isCidr(cidr);
  if (cidrVersion === 4) {
    return cidr;
  } else if (cidrVersion === 6) {
    const [ip, prefix] = cidr.split("/");
    return `${ipv6Normalize(ip)}/${prefix}`;
  }

  const parsed = new IPCIDR(cidr);
  if (parsed && parsed.address && parsed.address.v4) {
    return cidr;
  } else if (parsed && parsed.address && parsed.address.v4 === false) {
    return ipv6Normalize(cidr);
  }

  throw new Error(`Invalid network: ${cidr}`);
};

function mapNets(nets) {
  const maps = {v4: {}, v6: {}};
  for (const net of nets) {
    const start = net.start({type: "bigInteger"}).toString();
    const end = net.end({type: "bigInteger"}).toString();
    const v = `v${isCidr(net)}`;

    if (!maps[v][start]) maps[v][start] = {};
    if (!maps[v][end]) maps[v][end] = {};

    if (maps[v][start].start) {
      maps[v][start].start += 1;
    } else {
      maps[v][start].start = 1;
    }

    if (maps[v][end].end) {
      maps[v][end].end += 1;
    } else {
      maps[v][end].end = 1;
    }
  }
  return maps;
}

cidrTools.merge = function(nets) {
  nets = uniq((Array.isArray(nets) ? nets : [nets]).map(parse));
  const maps = mapNets(nets);

  const merged = {v4: [], v6: []};
  const start = {v4: null, v6: null};
  const end = {v4: null, v6: null};

  for (const v of ["v4", "v6"]) {
    const numbers = Object.keys(maps[v]).sort(naturalCompare);
    let depth = 0;

    for (const [index, number] of numbers.entries()) {
      const marker = maps[v][number];

      if (!start[v] && marker.start) {
        start[v] = bigint(number);
      }
      if (marker.end) {
        end[v] = bigint(number);
      }

      if (marker.start) depth += marker.start;
      if (marker.end) depth -= marker.end;

      if (marker.end && depth === 0 && ((numbers[index + 1] - numbers[index]) > 1)) {
        for (const sub of subparts({start: start[v], end: end[v]})) {
          merged[v].push(formatPart(sub, v));
        }
        start[v] = null;
        end[v] = null;
      } else if (index === (numbers.length - 1)) {
        for (const sub of subparts({start: start[v], end: end[v]})) {
          merged[v].push(formatPart(sub, v));
        }
      }
    }
  }

  merged.v4 = merged.v4.sort(naturalCompare);
  merged.v6 = merged.v6.sort(naturalCompare);
  return merged.v4.concat(merged.v6);
};

cidrTools.exclude = function(basenets, exclnets) {
  basenets = uniq(Array.isArray(basenets) ? basenets : [basenets]);
  exclnets = uniq(Array.isArray(exclnets) ? exclnets : [exclnets]);

  basenets = cidrTools.merge(basenets);
  exclnets = cidrTools.merge(exclnets);

  const bases = {v4: [], v6: []};
  const excls = {v4: [], v6: []};

  for (const basenet of basenets) {
    bases[`v${isCidr(basenet)}`].push(basenet);
  }

  for (const exclnet of exclnets) {
    excls[`v${isCidr(exclnet)}`].push(exclnet);
  }

  for (const v of ["v4", "v6"]) {
    for (const exclcidr of excls[v]) {
      for (const [index, basecidr] of bases[v].entries()) {
        const base = parse(basecidr);
        const excl = parse(exclcidr);
        const remainders = exclude(base, excl, v);
        if (base.toString() !== remainders.toString()) {
          bases[v] = bases[v].concat(remainders);
          bases[v].splice(index, 1);
        }
      }
    }
  }

  return bases.v4.concat(bases.v6);
};

cidrTools.expand = function(nets) {
  nets = uniq(Array.isArray(nets) ? nets : [nets]);

  let ips = [];
  for (const net of cidrTools.merge(nets)) {
    ips = ips.concat((new IPCIDR(net)).toArray());
  }
  return ips.map(cidrTools.normalize);
};

cidrTools.overlap = (a, b) => {
  const aNets = uniq(Array.isArray(a) ? a : [a]);
  const bNets = uniq(Array.isArray(b) ? b : [b]);

  for (let a of aNets) {
    for (let b of bNets) {
      a = parse(a);
      b = parse(b);

      if (a.address.v4 !== b.address.v4) {
        continue;
      }

      if (overlap(a, b)) {
        return true;
      }
    }
  }

  return false;
};