import * as t from 'babel-types';
import plur from 'plur';
import * as validators from '../validators';
import arrayOfStrings from '../util/arrayOfStrings';
import {notCallExpression, callExpression} from '../util/callExpressions';
import {ifAllTruthy, ifAnyTruthy, allTruthy} from '../util/conditionals';
import {createConst, createLet} from '../util/createVariable';
import dataValidator from '../util/dataValidator';
import globals from '../util/globals';
import importMethod from '../util/importMethod';
import template from '../util/moduleTemplate';
import {returnFalse} from '../util/returnBooleans';
import generateProgram from './program';
import requireModules from './requireModules';
import validator from './validator';
import validatorMap from './validatorMap';

/*
 * Common Babel nodes.
 */

const evenIdentifier           = t.identifier('even');
const indexIdentifier          = t.identifier('index');
const lengthIdentifier         = t.identifier('length');
const nodeIdentifier           = t.identifier('node');
const validIdentifier          = t.identifier('valid');
const valueParserASTIdentifier = t.identifier('valueParserAST');

const valueParserASTNodes = t.memberExpression(
    valueParserASTIdentifier,
    t.identifier('nodes')
);

const valueParserASTNodesLength = t.memberExpression(
    valueParserASTNodes,
    lengthIdentifier
);

const firstValueParserNode = createConst(
    nodeIdentifier,
    t.memberExpression(
        valueParserASTNodes,
        t.numericLiteral(0),
        true
    )
);

/**
 * Test that the nodes from postcss-value-parser are a certain length. This
 * uses strict equality by default, but you can supply an alternate operator.
 *
 * @private
 * @param  {number} length          The length to check
 * @param  {type} operator = '==='  The operator to use.
 * @return {Babel}                  The binary expression.
 * @example
 * const equality = valueParserNodesLength(5);
 * //=> valueParserAST.nodes.length === 5;
 *
 * const lessThan = valueParserNodesLength(5, '<');
 * //=> valueParserAST.nodes.length < 5;
 */

function valueParserNodesLength (length, operator = '===') {
    return t.binaryExpression(
        operator,
        valueParserASTNodesLength,
        t.numericLiteral(length)
    );
}

function getValidatorResult (identifier, cache) {
    const result = t.identifier(`${identifier}Result`);
    return [
        createConst(result, cache),
        t.ifStatement(
            callExpression(
                'shouldReturnResult',
                result
            ),
            t.blockStatement([
                t.returnStatement(result),
            ])
        ),
    ];
}

const dependencies = {
    valueParser: 'postcss-value-parser',
    shouldReturnResult: './util/shouldReturnResult',
    isKeyword: './validators/isKeyword',
    isVariable: './validators/isVariable',
};

function addDependency (dep) {
    if (!dependencies[dep]) {
        dependencies[dep] = `./validators/${dep}`;
    }
}

function generatePositionValidator ({candidates, identifier}) {
    const func = candidates[0].separator === ',' ? 'isPositionRepeat' : 'isPositionNoRepeat';
    return createConst(t.identifier(identifier), t.identifier(func));
}

function genericValidatorStub (name, {identifier}) {
    addDependency(name);
    return createConst(t.identifier(identifier), t.identifier(name));
}

function keyword (config, {value}) {
    config.keywords.push(value);
    return config;
}

function dataString (config, candidate) {
    const camel = candidate.type === 'string' ? 'isString' : dataValidator(candidate.value);
    if (!validators[camel]) {
        return config;
    }
    addDependency(camel);
    if (camel === 'isPosition') {
        config.preConditions.push(
            ...getValidatorResult(camel, callExpression('isPositionRepeat', valueParserASTIdentifier))
        );
        return config;
    }
    if (camel === 'isTransformList' || camel === 'isShadowT' || camel === 'isFilterFunctionList') {
        config.preConditions.push(
            ...getValidatorResult(camel, callExpression(camel, valueParserASTIdentifier))
        );
        return config;
    }
    if (candidate.min === 1) {
        const method = candidate.separator === ',' ? 'isComma' : 'isSpace';
        addDependency(method);
        addDependency('isEven');
        const result = `${camel}Result`;
        config.repeatingConditions.push(
            t.ifStatement(
                evenIdentifier,
                t.blockStatement([
                    createConst(t.identifier(result), callExpression(
                        camel,
                        nodeIdentifier
                    )),
                    t.ifStatement(
                        callExpression(
                            'shouldReturnResult',
                            t.identifier(result)
                        ),
                        t.blockStatement([
                            t.expressionStatement(
                                t.assignmentExpression(
                                    '=',
                                    validIdentifier,
                                    t.identifier(`${camel}Result`)
                                ),
                            ),
                            returnFalse,
                        ])
                    ),
                    createConst(t.identifier('isVariableResult'), callExpression(
                        'isVariable',
                        nodeIdentifier
                    )),
                    t.expressionStatement(
                        t.assignmentExpression(
                            '=',
                            validIdentifier,
                            t.identifier(`isVariableResult`)
                        ),
                    ),
                    returnFalse,
                ])
            )
        );
        config.repeatingConditions.push(
            ifAnyTruthy([
                allTruthy(
                    t.unaryExpression('!', evenIdentifier),
                    notCallExpression(method, nodeIdentifier)
                ),
            ], [
                t.expressionStatement(
                    t.assignmentExpression(
                        '=',
                        validIdentifier,
                        t.booleanLiteral(false)
                    ),
                ),
            ])
        );
        config.preConditions.push(
            t.ifStatement(
                callExpression('isEven', valueParserASTNodesLength),
                t.blockStatement([
                    returnFalse,
                ])
            )
        );
        config.repeatingReturn = [t.returnStatement(validIdentifier)];
        if (candidate.max !== false) {
            config.preConditions.push(
                t.ifStatement(
                    valueParserNodesLength((candidate.max * 2) - 1, '>'),
                    t.blockStatement([
                        t.returnStatement(callExpression(
                            'invalidMessage',
                            t.stringLiteral(`Expected a maximum of ${candidate.max} ${plur('value', candidate.max)}.`)
                        )),
                    ])
                )
            );
        }
        return config;
    }
    config.validators[camel] = callExpression(camel, nodeIdentifier);
    return config;
}

const candidateTypes = {
    keyword,
    data: dataString,
    string: dataString,
};

function createValidator (opts) {
    const {candidates, identifier} = opts;
    /*
     * These candidates are excluded from the automatic generation
     * of a validator because they work on the whole AST from
     * postcss-value-parser, whereas automatically generated
     * conditions tend to work on a single node instead.
     */
    if (candidates.length === 1) {
        const {value} = candidates[0];
        switch (value) {
        case 'bg-size':
        case 'repeat-style':
        case 'clip-path-property':
            return genericValidatorStub(dataValidator(value), opts);
        case 'position':
            return generatePositionValidator(opts);
        }
    }
    const settings = candidates.reduce((config, candidate) => {
        const {type} = candidate;
        if (candidateTypes[type]) {
            return candidateTypes[type](config, candidate);
        }
        return config;
    }, {
        keywords: [],
        preConditions: [],
        repeatingConditions: [],
        repeatingReturn: false,
        validators: {},
    });

    let keywords = [];

    if (settings.keywords.length) {
        const keywordList = arrayOfStrings(settings.keywords.filter(Boolean));
        /*
         * This handles the simplest case; where the grammar defines a list
         * of keywords and nothing else. In this case we can use the
         * isKeywordFactory function which saves us from having to generate
         * a validator ourselves. It produces output such as:
         *
         * const property = isKeywordFactory(['foo', 'bar', 'baz']);
         */
        if (
            !Object.keys(settings.validators).length &&
            !settings.preConditions.length &&
            !settings.repeatingConditions.length
        ) {
            const keywordFactory = 'isKeywordFactory';
            addDependency(keywordFactory);
            return [
                createConst(t.identifier(identifier), callExpression(
                    keywordFactory,
                    keywordList
                )),
            ];
        }
        /*
         * Otherwise, we need to generate a list of keywords that are used
         * as another validation condition. If we only have one keyword,
         * this can be inlined inside the `isKeyword` function - i.e.
         * `isKeyword(node, 'foo')`. Otherwise, we create a new list
         * and use it as a reference - `isKeyword(node, propertyKeywords)`,
         * where `propertyKeywords` is `const propertyKeywords = ['foo', 'bar']`
         */
        const keywordsList = t.identifier(`${identifier}Keywords`);
        const list = settings.keywords.length === 1 ? t.stringLiteral(settings.keywords[0]) : keywordsList;
        settings.validators.isKeyword = callExpression('isKeyword', nodeIdentifier, list);
        if (settings.keywords.length > 1) {
            keywords.push(createConst(
                keywordsList,
                keywordList
            ));
        }
    }

    const validatorKeys = Object.keys(settings.validators);

    const validatorList = validatorKeys.sort(k => k === 'isKeyword' ? -1 : 1).reduce((list, key) => {
        list.push(
            ...getValidatorResult(key, settings.validators[key])
        );
        return list;
    }, []);

    let body = [
        ...settings.preConditions,
    ];

    if (settings.repeatingConditions.length) {
        /*
         * Handle the case where the validator needs to validate *multiple*
         * values; in this case, we need to iterate over postcss-value-parser's
         * AST and check that each *even* index fulfills any of the validation
         * conditions, and that the correct separator is supplied in each
         * *odd* index.
         */
        body.unshift(createLet(
            validIdentifier,
            t.booleanLiteral(true)
        ));

        if (validatorKeys.length) {
            body.unshift(
                firstValueParserNode,
                ifAllTruthy([
                    valueParserNodesLength(1),
                ], validatorList)
            );
        }

        body.push(
            t.expressionStatement(
                t.callExpression(
                    t.memberExpression(
                        valueParserASTIdentifier,
                        t.identifier('walk')
                    ), [
                        t.arrowFunctionExpression([
                            nodeIdentifier,
                            indexIdentifier,
                        ], t.blockStatement([
                            t.ifStatement(
                                t.binaryExpression(
                                    '!==',
                                    validIdentifier,
                                    t.booleanLiteral(true)
                                ),
                                t.blockStatement([
                                    t.returnStatement(),
                                ])
                            ),
                            createConst(
                                evenIdentifier,
                                callExpression('isEven', indexIdentifier)
                            ),
                            ...settings.repeatingConditions,
                            returnFalse,
                        ])
                    )]
                )
            ),
            ...settings.repeatingReturn
        );
    } else {
        /*
         * Handle the case where the validator should only need to validate
         * a *single* value; i.e, an AST yielded from postcss-value-parser
         * should only have a length of 1. If it is 1, then check that this
         * node fulfills any of the validation conditions, otherwise
         * return false.
         */
        body.push(
            firstValueParserNode,
            t.ifStatement(
                valueParserNodesLength(1, '!=='),
                t.blockStatement([
                    t.returnStatement(callExpression(
                        'invalidMessage',
                        t.stringLiteral('Expected a single value to be passed.')
                    )),
                ])
            ),
            ...validatorList,
            returnFalse
        );
    }

    return validator(identifier, keywords, body);
}

const defaultExport = template(`
/**
 * The main entry point of this module takes a CSS property/value
 * pair, and validates it. It will return either \`true\` if valid,
 * or a message object if either invalid or unknown.
 *
 * @param {string} property The CSS property to validate.
 * @param {string|valueParser} value Either a string or an AST yielded
 * by postcss-value-parser.
 * @return {boolean|object}
 * @example
Valid CSS (string)

 * import cssValues from 'css-values';
 *
 * cssValues('color', 'transparent');
 * //=> true
 * @example
Valid CSS (valueParser)

 * import valueParser from 'postcss-value-parser';
 * import cssValues from 'css-values';
 *
 * cssValues('color', valueParser('transparent'));
 * //=> true
 * @example
Invalid CSS (string, recognised properties)

 * import cssValues from 'css-values';
 *
 * cssValues('color', 'traansparent');
 * //=> {type: 'invalid', message: '"traansparent" is not a valid value for "color".'}
 * @example
Invalid CSS (string, unknown properties)

 * import cssValues from 'css-values';
 *
 * cssValues('colr', 'transparent');
 * //=> {type: 'unknown', message: '"colr" is not a recognised property.'}
 */
export default function cssValues (property, value) {
    if (typeof value === 'string') {
        value = valueParser(value);
    }
    const first = value.nodes[0];
    if (value.nodes.length === 1 && (isKeyword(first, cssGlobals) || isVariable(first))) {
        return true;
    }
    if (validators[property]) {
        const result = validators[property](value);
        if (result.type) {
            return result;
        }
        if (!!result === false) {
            return invalidMessage('"' + value + '" is not a valid value for "' + property + '".');
        }
        return true;
    }
    // Pass through unknown properties
    return unknownMessage('"' + property + '" is not a recognised property.');
}
`, {preserveComments: true});

export default config => {
    const funcs = config.reduce((list, descriptor) => {
        const fn = createValidator(descriptor);
        if (fn) {
            return [...list, fn];
        }
        return list;
    }, []);
    return generateProgram([
        requireModules(...Object.keys(dependencies).map(key => {
            return {
                identifier: key,
                module: dependencies[key],
            };
        })),
        t.importDeclaration([
            importMethod(t.identifier('invalidMessage')),
            importMethod(t.identifier('unknownMessage')),
        ], t.stringLiteral('./util/validationMessages')),
        t.importDeclaration([
            importMethod(t.identifier('isPositionRepeat')),
            importMethod(t.identifier('isPositionNoRepeat')),
        ], t.stringLiteral('./validators/isPosition')),
        ...funcs,
        validatorMap(config),
        createConst(
            t.identifier('cssGlobals'),
            arrayOfStrings(globals)
        ),
        defaultExport(),
    ]);
};


"use strict";

/**
 * Wrap up a selenium server process.
 */
var _ = require("lodash");
var treeKill = require("tree-kill");
var log = require("./log");
var _log = log.bind(log, "[SERVER]".yellow);
var NOT_FOUND = -1;

// Globals
var STARTED = [
  "Started org.openqa.jetty.jetty.Server",
  "Selenium Server is up and running"
];
var ERRORS = {
  ALREADY_RUNNING: "java.net.BindException: Selenium is already running",
  FAILED_TO_START: "Failed to start: SocketListener",
  EXECUTE_EXCEPTION: "org.apache.commons.exec.ExecuteException"
};
var SEL_LOG_COLORS = {
  " DEBUG ": "blue",
  " INFO ": "green",
  " WARN ": "orange",
  " ERROR ": "red"
};

var Server = module.exports = function (config) {
  this.config = config || {};
  this._process = null;
  this._isStarted = false;
  this._err = null;
};

// Static props.
Server.log = _log;

// Detect started state.
Server.prototype._parseOutput = function (data) {
  data = (data || "").toString().trim();

  // Startup errors.
  var startErr = _.find(ERRORS, function (errText) {
    return data.indexOf(errText) > NOT_FOUND;
  });
  if (startErr) {
    this._process.emit("error", new Error(data));
  }

  // Detect started
  _.each(STARTED, function (startToken) {
    /*eslint-disable no-invalid-this*/
    if (!this._isStarted && data.indexOf(startToken) > NOT_FOUND) {
      this._isStarted = true;
      this._process.emit("started");
    }
  }, this);
};

// Add colors to selenium log messages.
// **Note**: Naive, but fun.
var _colorSelLog = function (data) {
  data = (data || "").toString().trim();

  _.any(SEL_LOG_COLORS, function (val, key) {
    if (data.indexOf(key) > NOT_FOUND) {
      data = data.split(key).join(key[val]);
      return true;
    }
  });

  return data;
};

/**
 * Start wrapper.
 *
 * @param {Function} callback Callback `fn(err, proc)` with server process
 * @returns {void}
 */
Server.prototype.start = function (callback) {
  var portscanner = require("portscanner"); // eslint-disable-line global-require

  var self = this;
  var DEFAULT_PORT = 4444;
  var port = self.config.options.server.port || DEFAULT_PORT;
  var useExisting = !!self.config.options.server.useExisting;

  if (!useExisting) {
    return self.startProcess(callback);
  }

  // Check for existing process first.
  portscanner.checkPortStatus(port, "127.0.0.1", function (err, status) {
    if (err) { return callback(err); }

    if (status === "open") {
      // In use, pass null for server.
      return callback(null, null);
    }

    // Otherwise, start up process.
    self.startProcess(callback);
  });
};

/**
 * Start process.
 *
 * @param {Function} callback Callback `fn(err, proc)` with server process
 */
// TODO: Refactor and decompose out the `max-statements` warning.
/*eslint-disable max-statements*/
Server.prototype.startProcess = function (callback) {
  var selenium = require("selenium-standalone"); // eslint-disable-line global-require

  callback = callback || function () {};

  var self = this;
  var port = self.config.options.server.port;
  var doLog = self.config.options.server.logger;
  var debug = self.config.options.server.debug;
  var startTimeout = self.config.options.server.startTimeout;

  var phantomPath = self.config.setting.server.phantomPath;
  var timerId;

  // Verify and clean state.
  if (self._process) { return callback(new Error("Process already exists")); }
  self._err = null;

  // Ready listeners.
  var _done = _.once(function (err) {
    if (timerId) { clearTimeout(timerId); }
    if (self._process) { self._process.removeListener("started", _done); }
    self._err = err || self._err;
    callback(self._err, self._process);
  });

  // Start selenium process.
  selenium.start({
    seleniumArgs: [].concat(
      port ? ["-port", port] : [],
      debug ? ["-debug"] : [],
      phantomPath ? ["-Dphantomjs.binary.path=" + phantomPath] : []
    ),
    spawnOptions: {
      stdio: "pipe"
    },
    spawnCb: function (proc) {
      // Stash running process.
      self._process = proc;

      // Communication listeners.
      proc.on("error", function (err) {
        _log("[error]".red, err.toString().trim());
        _done(err);
      });
      proc.stderr.on("data", function (data) {
        self._parseOutput(data);
        if (doLog) { _log("[stderr]", _colorSelLog(data)); }
      });
      proc.stdout.on("data", function (data) {
        self._parseOutput(data);
        if (doLog) { _log("[stdout]", _colorSelLog(data)); }
      });

      // Max wait.
      timerId = setTimeout(function () {
        _done(new Error("Server start timeout"));
      }, startTimeout);

      proc.on("started", _done);
    }
  }, function (err) {
    // Only call done if error, otherwise allow Spawn CB to finish.
    if (err) { return _done(err); }
  });
};

/**
 * Stop wrapper.
 *
 * @param {Function} callback Callback `fn(err)` when stopped
 * @returns {void}
 */
Server.prototype.stop = function (callback) {
  callback = callback || function () {};
  var self = this;
  var stopTimeout = self.config.options.server.stopTimeout;
  var proc = self._process;
  var _done;

  if (!proc || self._err) {
    return callback(self._err);
  }

  // Max wait.
  var timerId = setTimeout(function () {
    _done(new Error("Server stop timeout"));
  }, stopTimeout);

  // Done listener.
  _done = _.once(function (codeOrErr) {
    clearTimeout(timerId);
    proc.removeListener("exit", _done);
    self._err = codeOrErr instanceof Error ? codeOrErr : self._err;
    callback(self._err);
  });
  proc.on("exit", _done);

  // Kill it the "official" way.
  proc.kill();

  // Kill it the unofficial way.
  treeKill(proc.pid, "SIGKILL", _done);
};
/*eslint-enable max-statements*/


import {resolve} from 'path';
import camelCase from 'camelcase';
import nanoEqual from 'nano-equal';
import rename from 'gulp-rename';
import File from 'vinyl';
import vfs from 'vinyl-fs';
import map from 'map-stream';
import * as generator from './generators/index';
import * as log from './loggers/html';
import dataValidator from './util/dataValidator';
import handleError from './util/handleError';
import writeBundle from './util/writeBundle';
import {properties} from './data';
import Parser from './parser';
import prefixer from './prefixer';
import * as validators from './validators/index';

function mergeProperties (data) {
    return Object.keys(data).reduce((list, key) => {
        let values = data[key];
        if (!list.length) {
            list.push({
                properties: [key],
                values,
            });
        } else {
            const copy = list.filter(prop => nanoEqual(prop.values, values));
            if (copy[0]) {
                copy[0].properties.push(key);
            } else {
                list.push({
                    properties: [key],
                    values,
                });
            }
        }
        return list;
    }, []);
}

/*
 * This is a temporary function to be removed
 * once generation of all functions has been
 * achieved.
 */

function known (parsed) {
    return parsed.every(node => {
        return node.type === 'keyword' ||
        node.type === 'string' ||
        node.type === 'group' && !node.order && !node.min && node.values.every(n => n.type === 'keyword') ||
        (node.type === 'data' && validators[dataValidator(node.value)]); // eslint-disable-line
    });
}

function getExclusives (parsed) {
    return parsed.reduce((list, node) => {
        if (node.type === 'keyword') {
            list.push(node.value);
        }
        if (node.type === 'group') {
            node.values.forEach(({value}) => list.push(value));
        }
        return list;
    }, []);
}

let promises = [];

properties.forEach(property => {
    let parsed = new Parser(property.syntax);
    if (parsed.length === 1 && parsed[0].type === 'property') {
        const {syntax} = properties.filter(prop => prop.name === parsed[0].value)[0];
        parsed = new Parser(syntax);
    }
    let hasUnknown = !known(parsed);
    if (hasUnknown) {
        log.fail(property.name, property.syntax, parsed);
        return;
    }
    log.pass(property.name, property.syntax, parsed);
    let promise = prefixer({
        property: property.name,
        values: getExclusives(parsed),
    })
        .then(results => {
            if (!Object.keys(results).length) {
                results[property.name] = [];
            }
            let merged = mergeProperties(results);
            let config = [];
            merged.forEach(merge => {
                // Assume the specification property is on the bottom of the array
                let identifier = camelCase(merge.properties.slice(0).reverse()[0]) + 'Validator';

                let candidates = merge.values.reduce((list, value) => {
                    if (list.some(p => p.type === 'keyword' && p.value === value)) {
                        return list;
                    }
                    list.push({type: 'keyword', value});
                    return list;
                }, parsed.slice(0));

                // The all property is already covered by the global
                // validator, so it's not necessary to generate a validator
                // or tests for it.
                if (identifier === 'allValidator') {
                    return;
                }

                config.push({
                    identifier,
                    properties: merge.properties,
                    candidates,
                });
            });

            return config;
        })
        .catch(handleError);

    promises.push(promise);
});

function canMergeValidators (a, b) {
    return nanoEqual({
        candidates: a.candidates,
    }, {
        candidates: b.candidates,
    });
}

function renamer (path) {
    path.dirname = path.dirname.replace('src', 'packages/css-values');
}

Promise.all(promises).then((configs) => {
    const fileSystem = [];
    const outputs = configs.reduce((list, configArray) => {
        configArray.forEach(config => {
            const canMerge = Object.keys(list).some(key => {
                const value = list[key];
                if (canMergeValidators(value, config)) {
                    list[key].properties = [
                        ...value.properties,
                        ...config.properties,
                    ];
                    return true;
                }
                return false;
            });
            if (!canMerge) {
                list[config.identifier] = config;
            }
        });
        return list;
    }, {});
    const validatorConfig = Object.keys(outputs).reduce((conf, output) => {
        return [
            ...conf,
            outputs[output],
        ];
    }, []);

    fileSystem.push(new File({
        path: resolve(`packages/css-values/index.js`),
        contents: new Buffer(generator.property(validatorConfig)),
    }), new File({
        path: resolve(`packages/css-values/test.js`),
        contents: new Buffer(generator.test(validatorConfig)),
    }));

    const stream = vfs.src(['./src/util/**/*.js', './src/validators/**/*.js'], {base: process.cwd()})
        .pipe(rename(renamer))
        .pipe(map((file, cb) => {
            fileSystem.push(file);
            cb(null, file);
        }));

    stream.on('close', () => {
        return writeBundle({
            entry: './packages/css-values/index.js',
            dest: './packages/css-values/index.js',
            files: fileSystem,
        }).then(writeBundle({
            entry: './packages/css-values/test.js',
            dest: './packages/css-values/test.js',
            files: fileSystem,
            external: [
                resolve('./packages/css-values/index.js'),
            ],
        })).catch(handleError);
    });
}).catch(handleError);


/**
 *  Very rude refactoring of https://github.com/fengdh/mdict-js to make it work with node.js by Jeka Kiselyov ( https://github.com/jeka-kiselyov ).
 *  Please feel free to post pull requests with optimizations
 *  Also released under terms of the MIT License.
 */

/*
 * A pure Javascript implementation of MDict Parser which supports dictionary file in mdx/mdd format.
 * By Feng Dihai <fengdh@gmail.com>, 2015/07/01
 *
 * For my wife, my kids and family to whom I'm in love with life.
 *
 * Based on:
 *  - An Analysis of MDX/MDD File Format by Xiaoqiang Wang (xwang)
 *    https://bitbucket.org/xwang/mdict-analysis/
 *  - GitHub: zhansliu/writemdict
 *    https://github.com/zhansliu/writemdict/blob/master/fileformat.md
 *  - Source code of mdictparser.cc, part of goldendict
 *    https://github.com/goldendict/goldendict/blob/master/mdictparser.cc
 *
 * This is free software released under terms of the MIT License.
 * You can get a copy on http://opensource.org/licenses/MIT.
 *
 * NOTE - Unsupported features:
 *
 *    i. 64-bit number used in data offset or length.
 *       Only lower 32-bit is recognized that validate value must be lower than 2^32 or 4G,
 *       due to number format supported in current Javascript standard (ECMAScript5).
 *       Huge dictionary file larger than 4G is considered out of scope for a web app IMHO.
 *
 *   ii. Encrypted keyword header which requires external or embedded regkey.
 *       Most of shared MDict dictionary files are not encrypted,
 *       and I have no intention to break protected ones.
 *       However keyword index encryption is common and supported.
 *
 *  iii. Stylesheet substitution.
 *       Have no example yet, I suppose it is not a popular feature.
 *       Contact me if you have one in need of support.
 *
 * MDict software and its file format is developed by Rayman Zhang(张文伟),
 * read more on http://www.mdict.cn/ or http://www.octopus-studio.com/.
 */

/**
 * Usage:
 *   mdict-parser.js is defined as an AMD (or Node::todo) module.
 *   To initialize it, you have to provide/define a module named with "mdict-parseXml",
 *   which will be used to covert a string to XML dom object when parsing dictionary head.
 *
 *   For use inside modern browser, DOMParser is available:
 *
 *     define('mdict-parseXml', function() {
 *         return function (str) { return (new DOMParser()).parseFromString(str, 'text/xml'); };
 *     });
 *
 *   For node server, there are many xml-to-dom module available.
 */
var pako = require('pako');
var lzo = require(__dirname+'/lzo.js');
var Promise = require('bluebird');
var common = require(__dirname+'/mdict-common.js');
var TextDecoder = require('text-encoding').TextDecoder;
var fs = require('fs');
var DataView = require('buffer-dataview');
var DOMParser = require('xmldom').DOMParser;
var ripemd128 = require(__dirname+"/ripemd128.js");

// (function (root, factory) {
//   "use strict";

//   if (typeof define === 'function') {
//     if  (define.amd) {
//       // AMD. Register as an anonymous module.
//       define(['pako', 'lzo', 'ripemd128', 'bluebird', 'mdict-parseXml', 'mdict-common'], factory);
//     } else {
//       // TODO: For Node
//       /*
//       define(function(require, exports, module)) {
//         exports = factory();
//       });
//       */
//     }
//   }

// }(this, function(pako, lzo, ripemd128, Promise, parseXml, common) {
  // Value of undefined.
  var UNDEFINED = void 0;

  // A shared UTF-16LE text decorder used to read dictionary header string.
  var UTF_16LE = new TextDecoder('utf-16le');

  /**
   * Return the first argument as result.
   * This function is used to simulate consequence, i.e. read data and return it, then forward to a new position.
   * @param any data or function call
   * @return the first arugment
   */
  function conseq(/* args... */) { return arguments[0]; }

  function newUint8Array(buf, offset, len) {
    var ret = new Uint8Array(len);
    buf.copy(ret, 0, offset, offset + len);
    return ret;
  };

  /*
   * Decrypt encrypted data block of keyword index (attrs.Encrypted = "2").
   * @see https://github.com/zhansliu/writemdict/blob/master/fileformat.md#keyword-index-encryption
   * @param buf an ArrayBuffer containing source data
   * @param key an ArrayBuffer holding decryption key, which will be supplied to ripemd128() before decryption
   * @return an ArrayBuffer carrying decrypted data, occupying the same memory space of source buffer
   */
  function decrypt(buf, key) {
    key = ripemd128.ripemd128(key);
    var byte, keylen = key.length, prev = 0x36, i = 0, len = buf.length;
    for (; i < len; i++) {
      byte = buf[i];
      byte = ((byte >> 4) | (byte << 4) );                  // & 0xFF;  <-- it's already a byte
      byte = byte  ^ prev ^ (i & 0xFF) ^ key[i % keylen];
      prev = buf[i];
      buf[i] = byte;
    }
    return buf;
  }

  /**
   * For sliceThen(..).exec(proc, ..), mark what proc function returns is multiple values
   * to be passed to further Promise#spread(..) call.
   */
  function spreadus() {
    var args = Array.prototype.slice.apply(arguments);
    args._spreadus_ = true;
    return args;
  }

  /**
   * Slice part of a file/blob object, return a promise object which will resolve to an ArrayBuffer to feed subsequent process.
   * The returned promise object is extened with an exec(proc, args...) method which can be chained with further process.
   * @param file file or blob object
   * @param offset start position to slice
   * @param len length to slice
   * @return a promise object which will resolve to an ArrayBuffer containing data been read
   */

  // var fd = null;
  // var fdFileName = null;
  function sliceThen(file, offset, len) {
    var p = new Promise(function(_resolve) {
      fs.open(file, 'r', function(err, fd){
        if (err) {
          throw err;
        }

        var res = new Buffer(len);
        fs.read(fd, res, 0, len, offset, function(err, bytesRead, buffer){
          if (err) {
            throw err;
          }

          _resolve(buffer);
        });
      });
    });

    /**
     * Call proc with specified arguments prepending with sliced file/blob data (ArrayBuffer) been read.
     * @param the first argument is a function to be executed
     * @param other optional arguments are passed to the function following auto supplied input ArrayBuffer
     * @return a promise object which can be chained with further process through spread() method
     */
    p.exec = function(proc /*, args... */) {
      var args = Array.prototype.slice.call(arguments, 1);
      return p.then(function(data) {
          args.unshift(data);
          var ret = proc.apply(null, args);
          return resolve(ret !== UNDEFINED && ret._spreadus_ ? ret : [ret]);
      });
    };

    return p;
  }

  /**
   * Wrap value as a resolved promise.
   */
  function resolve(value) { return Promise.resolve(value); }

  /**
   * Wrap value as a rejected promise.
   */
  function reject(reason) { return Promise.reject(reason); }

  /**
   * Harvest any resolved promises, if all failed then return reasons.
   */
  function harvest(outcomes) {
    return Promise.settle(outcomes).then(function(results) {
      if (results.length === 0) {
        return reject("** NOT FOUND **");
      }

      var solved = [], failed = [];
      for (var i = 0; i < results.length; i++) {
        if (results[i].isResolved()) {
          solved.push(results[i].value());
        } else {
          failed.push(results[i].reason());
        }
      }
      return solved.length ? solved : failed;
    });
  }

  /*
   * Create a Record Block Table object to load record block info from record section in mdx/mdd file.
   * Retrived data is stored in an Uint32Array which contains N pairs of (offset_comp, offset_decomp) value,
   * where N is number of record blocks.
   *
   * When looking up a given key for its definition:
   *   1. Search KEY_INDEX to locate keyword block containing the given key.
   *   2. Scanning the found keyword block to get its record offset and size.
   *   3. Search RECORD_BLOCK_TABLE to get record block containing the record.
   *   4. Load the found record block, using its offset and size to retrieve record content.
   *
   * @see https://github.com/zhansliu/writemdict/blob/master/fileformat.md#record-section
   */
  function createRecordBlockTable() {
    var pos = 0, // current position
        arr;     // backed Uint32Array
    return {
      // Allocate required ArrayBuffer for storing record block table, where len is number of record blocks.
      alloc:  function(len) {
                arr = new Uint32Array(len * 2);
              },
      // Store offset pair value (compressed & decompressed) for a record block
      // NOTE: offset_comp is absolute offset counted from start of mdx/mdd file.
      put:    function(offset_comp, offset_decomp) {
                arr[pos++] = offset_comp; arr[pos++] = offset_decomp;
              },
      // Given offset of a keyword after decompression, return a record block info containing it, else undefined if not found.
      find:   function(keyAt) {
                var hi = (arr.length >> 1) - 1, lo = 0, i = (lo + hi) >> 1, val = arr[(i << 1) + 1];

                if (keyAt > arr[(hi << 1) + 1] || keyAt < 0) {
                  return;
                }

                while (true) {
                  if (hi - lo <= 1) {
                    if (i < hi) {
                      return {
                        block_no:     i,
                        comp_offset:  arr[i <<= 1],
                        comp_size:    arr[i + 2] - arr[i],
                        decomp_offset:arr[i + 1],
                        decomp_size:  arr[i + 3] - arr[i + 1]
                      };
                    } else {
                      return;
                    }
                  }

                  (keyAt < val)  ? hi = i : lo = i;
                  i = (lo + hi) >> 1;
                  val = arr[(i << 1) + 1];
                }
              },
    };
  }

  /**
   * Test if a value of dictionary attribute is true or not.
   */
  function isTrue(v) {
    v = ((v || false) + '').toLowerCase();
    return v === 'yes' || v === 'true';
  }

  /**
   * Parse a MDict dictionary/resource file (mdx/mdd).
   * @param file a File/Blob object
   * @param ext file extension, mdx/mdd
   * @return a Promise object which will resolve to a lookup function.
   */
  function parse_mdict(file, ext) {

    var KEY_INDEX,                                       // keyword index array
        RECORD_BLOCK_TABLE = createRecordBlockTable();   // record block table

    var attrs = {},     // storing dictionary attributes
        _v2,            // true if enginge version > 2
        _bpu,           // bytes per unit when converting text size to byte length for text data
        _tail,          // need to skip extra tail bytes after decoding text
        _decoder,       // text decorder

        _decryptors  = [false, false],
                        // [keyword_header_decryptor, keyword_index_decryptor], only keyword_index_decryptor is supported

        _searchTextLen, // search NUL to get text length

        _readShort   = function(scanner) { return scanner.readUint8(); },
                        // read a "short" number representing kewword text size, 8-bit for version < 2, 16-bit for version >= 2

        _readNum     = function(scanner) { return scanner.readInt(); },
                        // Read a number representing offset or data block size, 16-bit for version < 2, 32-bit for version >= 2

        _checksum_v2 = function() {},
                        // Version >= 2.0 only checksum

        _adaptKey    = function(key) { return key; },
                        // adapt key by converting to lower case or stripping punctuations according to dictionary attributes (KeyCaseSensitive, StripKey)

        _slice       = sliceThen.bind(null, file);
                        // bind sliceThen() with file argument

    /**
     * Config scanner according to dictionary attributes.
     */
    function config() {
      attrs.Encoding = attrs.Encoding || 'UTF-16';

      _searchTextLen = (attrs.Encoding === 'UTF-16')
                          ?   function(dv, offset) {
                                offset = offset;
                                var mark = offset;
                                while (dv.getUint16(offset++)) { /* scan for NUL */ };
                                return offset - mark;
                          } : function(dv, offset) {
                                offset = offset;
                                var mark = offset;
                                while (dv.getUint8(offset++)) { /* scan for NUL */ }
                                return offset - mark - 1;
                          };

      _decoder = new TextDecoder(attrs.Encoding || 'UTF-16LE');

      _bpu = (attrs.Encoding === 'UTF-16') ? 2 : 1;

      if (parseInt(attrs.GeneratedByEngineVersion, 10) >= 2.0) {
        _v2 = true;
        _tail = _bpu;

          // HUGE dictionary file (>4G) is not supported, take only lower 32-bit
        _readNum     = function(scanner) { return scanner.forward(4), scanner.readInt(); };
        _readShort   = function(scanner) { return scanner.readUint16(); };
        _checksum_v2 = function(scanner) { return scanner.checksum(); };
      } else {
        _tail = 0;
      }

      // keyword index decrypted?
      if (attrs.Encrypted & 0x02) {
        _decryptors[1] = decrypt;
      }

      var regexp = common.REGEXP_STRIPKEY[ext];
      if (isTrue(attrs.KeyCaseSensitive)) {
        _adaptKey = isTrue(attrs.StripKey)
                      ? function(key) { return key.replace(regexp, '$1'); }
                      : function(key) { return key; };
      } else {
        _adaptKey = isTrue(attrs.StripKey || (_v2 ? '' : 'yes'))
                      ? function(key) { return key.toLowerCase().replace(regexp, '$1'); }
                      : function(key) { return key.toLowerCase(); };
      }
    }

    // Read data in current offset from target data ArrayBuffer
    function Scanner(buf, len) {
      var offset = 0, dv = new DataView(buf);
      var methods = {
        // target data size in bytes
        size:     function() { return len || buf.byteLength; },
        // update offset to new position
        forward:  function(len) { return offset += len; },
        // return current offset
        offset:   function() { return offset; },

        // MDict file format uses big endian to store number

        // 32-bit unsigned int
        readInt:    function() { return conseq(dv.getUint32(offset, false), this.forward(4)); },
        readUint16: function() { return conseq(dv.getUint16(offset, false), this.forward(2)); },
        readUint8:  function() { return conseq(dv.getUint8(offset, false),  this.forward(1)); },

        // Read a "short" number representing keyword text size, 8-bit for version < 2, 16-bit for version >= 2
        readShort:  function() { return _readShort(this); },
        // Read a number representing offset or data block size, 16-bit for version < 2, 32-bit for version >= 2
        readNum:    function() { return _readNum(this); },

        readUTF16:  function(len) { return conseq(UTF_16LE.decode(newUint8Array(buf, offset, len)), this.forward(len)); },

        // Read data to an Uint8Array and decode it to text with specified encoding.
        // Text length in bytes is determined by searching terminated NUL.
        // NOTE: After decoding the text, it is need to forward extra "tail" bytes according to specified encoding.
        readText: function() {
          var len = _searchTextLen(dv, offset);
          return conseq(_decoder.decode(newUint8Array(buf, offset, len)), this.forward(len + _bpu));
        },
        // Read data to an Uint8Array and decode it to text with specified encoding.
        // @param len length in basic unit, need to multiply byte per unit to get length in bytes
        // NOTE: After decoding the text, it is need to forward extra "tail" bytes according to specified encoding.
        readTextSized: function(len) {
          len *= _bpu;
          var read = conseq(_decoder.decode(newUint8Array(buf, offset, len)), this.forward(len + _tail));
          return read;
        },

        // Skip checksum, just ignore it anyway.
        checksum:    function() { this.forward(4); },
        // Version >= 2.0 only
        checksum_v2: function() { return _checksum_v2(this); },

        // Read data block of keyword index, key block or record content.
        // These data block are maybe in compressed (gzip or lzo) format, while keyword index maybe be encrypted.
        // @see https://github.com/zhansliu/writemdict/blob/master/fileformat.md#compression (with typo mistake)
        readBlock: function(len, expectedBufSize, decryptor) {
          var comp_type = dv.getUint8(offset, false);  // compression type, 0 = non, 1 = lzo, 2 = gzip
          if (comp_type === 0) {
            if (_v2) {
              this.forward(8);  // for version >= 2, skip comp_type (4 bytes with tailing \x00) and checksum (4 bytes)
            }
            return this;
          } else {
            // skip comp_type (4 bytes with tailing \x00) and checksum (4 bytes)
            offset += 8; len -= 8;
            var tmp = new Uint8Array(len);
            buf.copy(tmp, 0, offset, offset + len);
            if (decryptor) {
              var passkey = new Uint8Array(8);
              var q = new Buffer(4);
              buf.copy(passkey, 0, offset-4, offset);
              // var q = new Buffer(4);
              passkey.set([0x95, 0x36, 0x00, 0x00], 4);         // key part 2: fixed data
              tmp = decryptor(tmp, passkey);
            }
            tmp = comp_type === 2 ? pako.inflate(tmp) : lzo.decompress(tmp, expectedBufSize, 1308672);
            this.forward(len);
            var d = new Buffer(tmp);
            return Scanner(d, tmp.length);
          }
        },

        // Read raw data as Uint8Array from current offset with specified length in bytes
        readRaw: function(len) {
          return conseq(newUint8Array(buf, offset, len), this.forward(len === UNDEFINED ? buf.length - offset : len));
        },
      };

      return Object.create(methods);
    }

    /**
     * Read the first 4 bytes of mdx/mdd file to get length of header_str.
     * @see https://github.com/zhansliu/writemdict/blob/master/fileformat.md#file-structure
     * @param input sliced file (start = 0, length = 4)
     * @return length of header_str
     */
    function read_file_head(input) {
      return Scanner(input).readInt();
    }

    /**
     * Read header section, parse dictionary attributes and config scanner according to engine version attribute.
     * @see https://github.com/zhansliu/writemdict/blob/master/fileformat.md#header-section
     * @param input sliced file (start = 4, length = len + 48), header string + header section (max length 48)
     * @param len lenghth of header_str
     * @return [remained length of header section (header_str and checksum, = len + 4), original input]
     */
    function read_header_sect(input, len) {
      var scanner = Scanner(input),
          header_str = scanner.readUTF16(len).replace(/\0$/, ''); // need to remove tailing NUL

      // parse dictionary attributes
      var doc = new DOMParser().parseFromString(header_str,'text/xml');

      var elem = doc.getElementsByTagName('Dictionary')[0];
      if (!elem) {
        elem = doc.getElementsByTagName('Library_Data')[0];
      }

      // console.log(doc.getElementsByTagName('Dictionary')[0].attributes);

      // var xml = parseXml(header_str).querySelector('Dictionary, Library_Data').attributes;

      for (var i = 0, item; i < elem.attributes.length; i++) {
        item = elem.attributes[i];
        attrs[item.nodeName] = item.nodeValue;
      }

      attrs.Encrypted = parseInt(attrs.Encrypted, 10) || 0;

      config();
      return spreadus(len + 4, input);
    }

    /**
     * Read keyword summary at the begining of keyword section.
     * @see https://github.com/zhansliu/writemdict/blob/master/fileformat.md#keyword-section
     * @param input sliced file, same as input passed to read_header_sect()
     * @param offset start position of keyword section in sliced file, equals to length of header string plus checksum.\
     * @return keyword_sect object
     */
    function read_keyword_summary(input, offset) {
      var scanner = Scanner(input);
      scanner.forward(offset);
      return {
        num_blocks:           scanner.readNum(),
        num_entries:          scanner.readNum(),
        key_index_decomp_len: _v2 && scanner.readNum(),  // Ver >= 2.0 only
        key_index_comp_len:   scanner.readNum(),
        key_blocks_len:       scanner.readNum(),
        chksum:               scanner.checksum_v2(),
        // extra field
        len:                  scanner.offset() - offset,  // actual length of keyword section, varying with engine version attribute
      };
    }

    /**
     * Read keyword index part of keyword section.
     * @see https://github.com/zhansliu/writemdict/blob/master/fileformat.md#keyword-header-encryption
     * @see https://github.com/zhansliu/writemdict/blob/master/fileformat.md#keyword-index
     * @param input sliced file, remained part of keyword section after keyword summary which can also be used to read following key blocks.
     * @param keyword_summary
     * @return [keyword_summary, array of keyword index]
     */
    function read_keyword_index(input, keyword_summary) {
      var scanner = Scanner(input).readBlock(keyword_summary.key_index_comp_len, keyword_summary.key_index_decomp_len, _decryptors[1]),
          keyword_index = Array(keyword_summary.num_blocks),
          offset = 0;
      for (var i = 0, size; i < keyword_summary.num_blocks; i++) {
        keyword_index[i] = {
          num_entries: conseq(scanner.readNum(), size = scanner.readShort()),
// UNUSED, can be ignored
//          first_size:  size = scanner.readShort(),
          first_word:  conseq(scanner.readTextSized(size), size = scanner.readShort()),
// UNUSED, can be ignored
//          last_size:   size = scanner.readShort(),
          last_word:   scanner.readTextSized(size),
          comp_size:   size = scanner.readNum(),
          decomp_size: scanner.readNum(),
          // extra fields
          offset: offset,     // offset of the first byte for the target key block in mdx/mdd file
          index: i            // index of this key index, used to search previous/next block
        };
        offset += size;
      }
      return spreadus(keyword_summary, keyword_index);
    }

    /**
     * Read keyword entries inside a keyword block and fill KEY_TABLE.
     * @param scanner scanner object to read key entries, which starts at begining of target key block
     * @param kdx corresponding keyword index object
     * NOTE: no need to read keyword block anymore, for debug only.
     */
    function read_key_block(scanner, kdx) {
      var scanner = scanner.readBlock(kdx.comp_size, kdx.decomp_size);
      for (var i = 0; i < kdx.num_entries; i++) {
//        scanner.readNum(); scanner.readText();
        var kk = [scanner.readNum(), scanner.readText()];
      }
    }

    /**
     * Delay to scan key table, for debug onyl.
     * @param slicedKeyBlock a promise object which will resolve to an ArrayBuffer containing keyword blocks
     *                       sliced from mdx/mdd file.
     * @param num_entries number of keyword entries
     * @param keyword_index array of keyword index
     * @param delay time to delay for scanning key table
     */
    function willScanKeyTable(slicedKeyBlock, num_entries, keyword_index, delay) {
      slicedKeyBlock.delay(delay).then(function (input) {
        var scanner = Scanner(input);
        for (var i = 0, size = keyword_index.length; i < size; i++) {
        // common.log('z',keyword_index[i]);
          read_key_block(scanner, keyword_index[i]);
        }
      });
    }

    /**
     * Read record summary at the begining of record section.
     * @see https://github.com/zhansliu/writemdict/blob/master/fileformat.md#record-section
     * @param input sliced file, start = begining of record section, length = 32 (max length of record summary)
     * @param pos begining of record section
     * @returj record summary object
     */
    function read_record_summary(input, pos) {
      var scanner = Scanner(input),
          record_summary = {
            num_blocks:   scanner.readNum(),
            num_entries:  scanner.readNum(),
            index_len:    scanner.readNum(),
            blocks_len:   scanner.readNum(),
            // extra field
            len:          scanner.offset(),   // actual length of record section (excluding record block index), varying with engine version attribute
          };

      // start position of record block from head of mdx/mdd file
      record_summary.block_pos = pos + record_summary.index_len + record_summary.len;

      return record_summary;
    }

    /**
     * Read record block index part in record section, and fill RECORD_BLOCK_TABLE
     * @see https://github.com/zhansliu/writemdict/blob/master/fileformat.md#record-section
     * @param input sliced file, start = begining of record block index, length = record_summary.index_len
     * @param record_summary record summary object
     */
    function read_record_block(input, record_summary) {
      var scanner = Scanner(input),
          size = record_summary.num_blocks,
          record_index = Array(size),
          p0 = record_summary.block_pos,
          p1 = 0;

      RECORD_BLOCK_TABLE.alloc(size + 1);
      for (var i = 0, rdx; i < size; i++) {
        record_index[i] = rdx = {
          comp_size:   scanner.readNum(),
          decomp_size: scanner.readNum()
        };
        RECORD_BLOCK_TABLE.put(p0, p1);
        p0 += rdx.comp_size;
        p1 += rdx.decomp_size;
      }
      RECORD_BLOCK_TABLE.put(p0, p1);
    }

    /**
     * Read definition in text for given keyinfo object.
     * @param input record block sliced from the file
     * @param block record block index
     * @param keyinfo a object with property of record's offset and optional size for the given keyword
     * @return definition in text
     */
    function read_definition(input, block, keyinfo) {
      var scanner = Scanner(input).readBlock(block.comp_size, block.decomp_size);
      scanner.forward(keyinfo.offset - block.decomp_offset);
      return scanner.readText();
    }

    /**
     * Following link to find actual definition of keyword.
     * @param definition maybe starts with "@@@LINK=" which links to another keyword
     * @param lookup search function
     * @return resolved actual definition
     */
    function followLink(definition, lookup) {
      return (definition.substring(0, 8) !== '@@@LINK=')
                ? definition
                : lookup(definition.substring(8));
    }

    /**
     * Read content in ArrayBuffer for give keyinfo object
     * @param input record block sliced from the file
     * @param block record block index
     * @param keyinfo a object with property of record's offset and optional size for the given keyword
     * @return an ArrayBuffer containing resource of image/audio/css/font etc.
     */
    function read_object(input, block, keyinfo) {
      if (input.byteLength > 0) {
        var scanner = Scanner(input).readBlock(block.comp_size, block.decomp_size);
        scanner.forward(keyinfo.offset - block.decomp_offset);
        return scanner.readRaw(keyinfo.size);
      } else {
        throw '* OUT OF FILE RANGE * ' + keyinfo + ' @offset=' + block.comp_offset;
      }
    }

    /**
     * Find word definition for given keyinfo object.
     * @param keyinfo a object with property of record's offset and optional size for the given keyword
     * @return a promise object which will resolve to definition in text. Link to other keyword is followed to get actual definition.
     */
    function findWord(keyinfo) {
      var block = RECORD_BLOCK_TABLE.find(keyinfo.offset);
      return _slice(block.comp_offset, block.comp_size)
                .exec(read_definition, block, keyinfo)
                  .spread(function (definition) { return resolve(followLink(definition, LOOKUP.mdx)); });
    }

    /**
     * Find resource (image, sound etc.) for given keyinfo object.
     * @param keyinfo a object with property of record's offset and optional size for the given keyword
     * @return a promise object which will resolve to an ArrayBuffer containing resource of image/audio/css/font etc.
     * TODO: Follow link, maybe it's too expensive and a rarely used feature?
     */
    function findResource(keyinfo) {
      var block = RECORD_BLOCK_TABLE.find(keyinfo.offset);
      return _slice(block.comp_offset, block.comp_size)
                .exec(read_object, block, keyinfo)
                  .spread(function (blob) { return resolve(blob); });
    }

    //------------------------------------------------------------------------------------------------
    // Implementation for look-up
    //------------------------------------------------------------------------------------------------
    var slicedKeyBlock,
        _cached_keys,             // cache latest keys
        _trail,                   // store latest visited record block & position when search for candidate keys
        mutual_ticket = 0;        // a oneway increased ticket used to cancel unfinished pattern match

    /**
      * Reduce the key index array to an element which contains or is the nearest one matching a given phrase.
      */
    function reduce(arr, phrase) {
      var len = arr.length;
      if (len > 1) {
        len = len >> 1;
        return phrase > _adaptKey(arr[len - 1].last_word)
                  ? reduce(arr.slice(len), phrase)
                  : reduce(arr.slice(0, len), phrase);
      } else {
        return arr[0];
      }
    }

    /**
      * Reduce the array to index of an element which contains or is the nearest one matching a given phrase.
      */
    function shrink(arr, phrase) {
      var len = arr.length, sub;
      if (len > 1) {
        len = len >> 1;
        var key = _adaptKey(arr[len]);
        if (phrase < key) {
          sub = arr.slice(0, len);
          sub.pos = arr.pos;
        } else {
          sub = arr.slice(len);
          sub.pos = (arr.pos || 0) + len;
        }
        return shrink(sub, phrase);
      } else {
        return (arr.pos || 0) + (phrase <= _adaptKey(arr[0]) ? 0 : 1);
      }
    }

    /**
     * Load keys for a keyword index object from mdx/mdd file.
     * @param kdx keyword index object
     */
    function loadKeys(kdx) {
      if (_cached_keys && _cached_keys.pilot === kdx.first_word) {
        return resolve(_cached_keys.list);
      } else {
        return slicedKeyBlock.then(function(input) {
          var scanner = Scanner(input), list = Array(kdx.num_entries);
          scanner.forward(kdx.offset);
          scanner = scanner.readBlock(kdx.comp_size, kdx.decomp_size);

          for (var i = 0; i < kdx.num_entries; i++) {
            var offset = scanner.readNum();
            list[i] = new Object(scanner.readText());
            list[i].offset = offset;
            if (i > 0) {
              list[i - 1].size = offset - list[i - 1].offset;
            }
          }
          _cached_keys = {list: list, pilot: kdx.first_word};
          return list;
        });
      }
    }

    /**
     * Search for the first keyword match given phrase.
     */
    function seekVanguard(phrase) {
      phrase = _adaptKey(phrase);
      var kdx = reduce(KEY_INDEX, phrase);

      // look back for the first record block containing keyword for the specified phrase
      if (phrase <= _adaptKey(kdx.last_word)) {
        var index = kdx.index - 1, prev;
        while (prev = KEY_INDEX[index]) {
          if (_adaptKey(prev.last_word) !== _adaptKey(kdx.last_word)) {
            break;
          }
          kdx = prev;
          index--;
        }
      }

      return loadKeys(kdx).then(function (list) {
          var idx = shrink(list, phrase);
          // look back for the first matched keyword position
          while (idx > 0) {
            if (_adaptKey(list[--idx]) !== _adaptKey(phrase)) {
              idx++;
              break;
            }
          }
          return [kdx, Math.min(idx, list.length - 1), list];
        });
    }

    // TODO: have to restrict max count to improve response
    /**
     * Append more to word list according to a filter or expected size.
     */
    function appendMore(word, list, nextKdx, expectedSize, filter, ticket) {
      if (ticket !== mutual_ticket) {
        throw 'force terminated';
      }

      if (filter) {
        if (_trail.count < expectedSize && nextKdx && nextKdx.first_word.substr(0, word.length) === word) {
          return loadKeys(nextKdx).delay(30).then(function(more) {
            _trail.offset = 0;
            _trail.block = nextKdx.index;
            Array.prototype.push.apply(list, more.filter(filter, _trail));
            return appendMore(word, list, KEY_INDEX[nextKdx.index + 1], expectedSize, filter, ticket);
          });
        } else {
          if (list.length === 0) {
            _trail.exhausted = true;
          }
          return resolve(list);
        }
      } else {
        var shortage = expectedSize - list.length;
        if (shortage > 0 && nextKdx) {
          _trail.block = nextKdx.index;
          return loadKeys(nextKdx).then(function(more) {
            _trail.offset = 0;
            _trail.pos = Math.min(shortage, more.length);
            Array.prototype.push.apply(list, more.slice(0, shortage));
            return appendMore(word, list, KEY_INDEX[nextKdx.index + 1], expectedSize, filter, ticket);
          });
        } else {
          if (_trail.pos > expectedSize) {
            _trail.pos = expectedSize;
          }
          list = list.slice(0, expectedSize);
          _trail.count = list.length;
          _trail.total += _trail.count;
          return resolve(list);
        }
      }
    }

    function followUp() {
      var kdx = KEY_INDEX[_trail.block];
      return loadKeys(kdx).then(function (list) {
                return [kdx, Math.min(_trail.offset + _trail.pos, list.length - 1), list];
              });
    }

    function matchKeys(phrase, expectedSize, follow) {
      expectedSize = Math.max(expectedSize || 0, 10);
      var str = phrase.trim().toLowerCase(),
          m = /([^?*]+)[?*]+/.exec(str),
          word;
      if (m) {
        word = m[1];
        var wildcard = new RegExp('^' + str.replace(/([\.\\\+\[\^\]\$\(\)])/g, '\\$1').replace(/\*+/g, '.*').replace(/\?/g, '.') + '$'),
            tester = phrase[phrase.length - 1] === ' '
                        ? function(s) { return wildcard.test(s); }
                        : function(s) { return wildcard.test(s) && !/ /.test(s); },
            filter = function (s, i) {
                       if (_trail.count < expectedSize && tester(s)) {
                         _trail.count++;
                         _trail.total++;
                         _trail.pos = i + 1;
                         return true;
                       }
                       return false;
                     };
      } else {
        word = phrase.trim();
      }

      if (_trail && _trail.phrase !== phrase) {
        follow = false;
      }

      if (follow && _trail && _trail.exhausted) {
        return resolve([]);
      }

      var startFrom = follow && _trail ? followUp() : seekVanguard(word);

      return startFrom.spread(function(kdx, idx, list) {
        list = list.slice(idx);
        _trail = {phrase: phrase,
                  block:  kdx.index,
                  offset: idx,
                  pos:    list.length,
                  count:  0,
                  total:  follow ? _trail && _trail.total || 0 : 0
                };
        if (filter) {
          list = list.filter(filter, _trail);
        }
        return appendMore(word, list, KEY_INDEX[kdx.index + 1], expectedSize, filter, ++mutual_ticket)
                  .then(function(result) {
                    if (_trail.block === KEY_INDEX.length - 1) {
                      if (_trail.offset + _trail.pos >= KEY_INDEX[_trail.block].num_entries) {
                        _trail.exhausted = true;
// console.log('EXHAUSTED!!!!');
                      }
                    }
// console.log('trail: ', _trail);
                    return result;
                  });
      });
    };

    /**
     * Match the first element in list with given offset.
     */
    function matchOffset(list, offset) {
      return list.some(function(el) { return el.offset === offset ? list = [el] : false; }) ? list : [];
    }

    // Lookup functions
    var LOOKUP = {
      /**
       * @param query
       *          String
       *          {phrase: .., max: .., follow: true} object
       */
      mdx: function(query) {
        if (typeof query === 'string' || query instanceof String) {
          _trail = null;
          var word = query.trim().toLowerCase(), offset = query.offset;

          return seekVanguard(word).spread(function(kdx, idx, list) {
            list = list.slice(idx);
            if (offset !== UNDEFINED) {
              list = matchOffset(list, offset);
            } else {
              list = list.filter(function(el) { return el.toLowerCase() === word; });
            }
            return harvest(list.map(findWord));
          });
        } else {
          return matchKeys(query.phrase, query.max, query.follow);
        }
      },

      // TODO: chain multiple mdd file
      mdd: function(phrase) {
        var word = phrase.trim().toLowerCase();
        word = '\\' + word.replace(/(^[/\\])|([/]$)/, '');
        word = word.replace(/\//g, '\\');
        return seekVanguard(word).spread(function(kdx, idx, list) {
          return list.slice(idx).filter(function(one) {
            return one.toLowerCase() === word;
          });
        }).then(function(candidates) {
          if (candidates.length === 0) {
            throw '*RESOURCE NOT FOUND* ' + phrase;
          } else {
            return findResource(candidates[0]);
          }
        });
      }
    };

    // ------------------------------------------
    // start to load mdx/mdd file
    // ------------------------------------------

    var pos = 0;

    // read first 4 bytes to get header length
    return _slice(pos, 4).exec(read_file_head).then(function(len) {
      len = parseInt(len, 10);
      pos += 4;                                   // start of header string in header section
      return _slice(pos, len + 48)
                .exec(read_header_sect, len);

    }).then(function(ret) {
      var header_remain_len = ret[0];
      var input = ret[1];
      pos += header_remain_len;                   // start of keyword section
      return read_keyword_summary(input, header_remain_len);

    }).then(function(keyword_summary) {
      pos += keyword_summary.len;            // start of key index in keyword section
      return _slice(pos, keyword_summary.key_index_comp_len)
                .exec(read_keyword_index, keyword_summary);

    }).then(function (data){
      var keyword_summary = data[0];
      var keyword_index = data[1];      // console.log(data[1]); // console.log(data);
      pos += keyword_summary.key_index_comp_len;  // start of keyword block in keyword section

      slicedKeyBlock = _slice(pos, keyword_summary.key_blocks_len);

      // Now it's fast enough to look up word without key table, which scans keyword from the specified key blocks in an effcient way.
      // No need to scan the whole key table in ahead.
      // willScanKeyTable(slicedKeyBlock, keyword_summary.num_entries, keyword_index, 00); dasd;
      //

      pos += keyword_summary.key_blocks_len;    // start of record section
      KEY_INDEX = keyword_index;

    }).then(function () {
      return _slice(pos, 32)
                .exec(read_record_summary, pos);

    }).spread(function (record_summary) {
      pos += record_summary.len;                // start of record blocks in record section
      return _slice(pos, record_summary.index_len)
                .exec(read_record_block, record_summary);

    }).spread(function() {
      // resolve and return lookup() function according to file extension (mdx/mdd)
      LOOKUP[ext].description = attrs.Description;
      return resolve(LOOKUP[ext]);
    });
  };

  // -------------------------
  // END OF parse_mdict()
  // -------------------------

  /**
   * Load a set of files which will be parsed as MDict dictionary & resource (mdx/mdd).
   */
exports.load = function(files) {
      var resources = [];
      Array.prototype.forEach.call(files, function(f) {
        var ext =  common.getExtension(f.name, 'mdx');

        resources.push(resources[ext] = parse_mdict(f, ext));
      });

      return Promise.all(resources)
                    .then(function() { return resolve(resources); });
    };

// }));