# encoding: binary
# frozen_string_literal: true

module RbNaCl
  # Various utility functions
  module Util
    extend Sodium

    sodium_function :c_verify16, :crypto_verify_16, %i[pointer pointer]
    sodium_function :c_verify32, :crypto_verify_32, %i[pointer pointer]
    sodium_function :c_verify64, :crypto_verify_64, %i[pointer pointer]

    module_function

    # Returns a string of n zeros
    #
    # Lots of the functions require us to create strings to pass into functions of a specified size.
    #
    # @param [Integer] n the size of the string to make
    #
    # @return [String] A nice collection of zeros
    def zeros(n = 32)
      zeros = "\0" * n
      # make sure they're 8-bit zeros, not 7-bit zeros.  Otherwise we might get
      # encoding errors later
      zeros.respond_to?(:force_encoding) ? zeros.force_encoding("ASCII-8BIT") : zeros
    end

    # Prepends a message with zeros
    #
    # Many functions require a string with some zeros prepended.
    #
    # @param [Integer] n The number of zeros to prepend
    # @param [String] message The string to be prepended
    #
    # @return [String] a bunch of zeros
    def prepend_zeros(n, message)
      zeros(n) + message
    end

    # Remove zeros from the start of a message
    #
    # Many functions require a string with some zeros prepended, then need them removing after.
    # Note: this modifies the passed in string
    #
    # @param [Integer] n The number of zeros to remove
    # @param [String] message The string to be slice
    #
    # @return [String] less a bunch of zeros
    def remove_zeros(n, message)
      message.slice!(n, message.bytesize - n)
    end

    # Pad a string out to n characters with zeros
    #
    # @param [Integer] n The length of the resulting string
    # @param [String]  message the message to be padded
    #
    # @raise [RbNaCl::LengthError] If the string is too long
    #
    # @return [String] A string, n bytes long
    def zero_pad(n, message)
      len = message.bytesize
      if len == n
        message
      elsif len > n
        raise LengthError, "String too long for zero-padding to #{n} bytes"
      else
        message + zeros(n - len)
      end
    end

    # Check the length of the passed in string
    #
    # In several places through the codebase we have to be VERY strict with
    # what length of string we accept.  This method supports that.
    #
    # @raise [RbNaCl::LengthError] If the string is not the right length
    #
    # @param string [String] The string to compare
    # @param length [Integer] The desired length
    # @param description [String] Description of the string (used in the error)
    def check_length(string, length, description)
      if string.nil?
        # code below is runs only in test cases
        # nil can't be converted to str with #to_str method
        raise LengthError,
              "#{description} was nil (Expected #{length.to_int})",
              caller
      end

      if string.bytesize != length.to_int
        raise LengthError,
              "#{description} was #{string.bytesize} bytes (Expected #{length.to_int})",
              caller
      end
      true
    end

    # Check a passed in string, converting the argument if necessary
    #
    # In several places through the codebase we have to be VERY strict with
    # the strings we accept.  This method supports that.
    #
    # @raise [ArgumentError] If we cannot convert to a string with #to_str
    # @raise [RbNaCl::LengthError] If the string is not the right length
    #
    # @param string [#to_str] The input string
    # @param length [Integer] The only acceptable length of the string
    # @param description [String] Description of the string (used in the error)
    def check_string(string, length, description)
      check_string_validation(string)
      string = string.to_s
      check_length(string, length, description)

      string
    end

    # Check a passed in string, convertion if necessary
    #
    # This method will check the key, and raise error
    # if argument is not a string, and if it's empty string.
    #
    # RFC 2104 HMAC
    # The key for HMAC can be of any length (keys longer than B bytes are
    # first hashed using H). However, less than L bytes is strongly
    # discouraged as it would decrease the security strength of the
    # function.  Keys longer than L bytes are acceptable but the extra
    # length would not significantly increase the function strength. (A
    # longer key may be advisable if the randomness of the key is
    # considered weak.)
    #
    # see https://tools.ietf.org/html/rfc2104#section-3
    #
    #
    # @raise [ArgumentError] If we cannot convert to a string with #to_str
    # @raise [RbNaCl::LengthError] If the string is empty
    #
    # @param string [#to_str] The input string
    def check_hmac_key(string, _description)
      check_string_validation(string)

      string = string.to_str

      if string.bytesize.zero?
        raise LengthError,
              "#{Description} was #{string.bytesize} bytes (Expected more than 0)",
              caller
      end

      string
    end

    # Check a passed string is it valid
    #
    # Raise an error if passed argument is invalid
    #
    # @raise [TypeError] If string cannot convert to a string with #to_str
    # @raise [EncodingError] If string have wrong encoding
    #
    # @param string [#to_str] The input string
    def check_string_validation(string)
      raise TypeError, "can't convert #{string.class} into String with #to_str" unless string.respond_to? :to_str

      string = string.to_str

      raise EncodingError, "strings must use BINARY encoding (got #{string.encoding})" if string.encoding != Encoding::BINARY
    end

    # Compare two 64 byte strings in constant time
    #
    # This should help to avoid timing attacks for string comparisons in your
    # application.  Note that many of the functions (such as HmacSha512#verify)
    # use this method under the hood already.
    #
    # @param [String] one String #1
    # @param [String] two String #2
    #
    # @return [Boolean] Well, are they equal?
    def verify64(one, two)
      return false unless two.bytesize == 64 && one.bytesize == 64

      c_verify64(one, two)
    end

    # Compare two 64 byte strings in constant time
    #
    # This should help to avoid timing attacks for string comparisons in your
    # application.  Note that many of the functions (such as HmacSha512#verify)
    # use this method under the hood already.
    #
    # @param [String] one String #1
    # @param [String] two String #2
    #
    # @raise [ArgumentError] If the strings are not equal in length
    #
    # @return [Boolean] Well, are they equal?
    def verify64!(one, two)
      check_length(one, 64, "First message")
      check_length(two, 64, "Second message")
      c_verify64(one, two)
    end

    # Compare two 32 byte strings in constant time
    #
    # This should help to avoid timing attacks for string comparisons in your
    # application.  Note that many of the functions (such as HmacSha256#verify)
    # use this method under the hood already.
    #
    # @param [String] one String #1
    # @param [String] two String #2
    #
    # @return [Boolean] Well, are they equal?
    def verify32(one, two)
      return false unless two.bytesize == 32 && one.bytesize == 32

      c_verify32(one, two)
    end

    # Compare two 32 byte strings in constant time
    #
    # This should help to avoid timing attacks for string comparisons in your
    # application.  Note that many of the functions (such as HmacSha256#verify)
    # use this method under the hood already.
    #
    # @param [String] one String #1
    # @param [String] two String #2
    #
    # @raise [ArgumentError] If the strings are not equal in length
    #
    # @return [Boolean] Well, are they equal?
    def verify32!(one, two)
      check_length(one, 32, "First message")
      check_length(two, 32, "Second message")
      c_verify32(one, two)
    end

    # Compare two 16 byte strings in constant time
    #
    # This should help to avoid timing attacks for string comparisons in your
    # application.  Note that many of the functions (such as OneTime#verify)
    # use this method under the hood already.
    #
    # @param [String] one String #1
    # @param [String] two String #2
    #
    # @return [Boolean] Well, are they equal?
    def verify16(one, two)
      return false unless two.bytesize == 16 && one.bytesize == 16

      c_verify16(one, two)
    end

    # Compare two 16 byte strings in constant time
    #
    # This should help to avoid timing attacks for string comparisons in your
    # application.  Note that many of the functions (such as OneTime#verify)
    # use this method under the hood already.
    #
    # @param [String] one String #1
    # @param [String] two String #2
    #
    # @raise [ArgumentError] If the strings are not equal in length
    #
    # @return [Boolean] Well, are they equal?
    def verify16!(one, two)
      check_length(one, 16, "First message")
      check_length(two, 16, "Second message")
      c_verify16(one, two)
    end

    # Hex encodes a message
    #
    # @param [String] bytes The bytes to encode
    #
    # @return [String] Tasty, tasty hexadecimal
    def bin2hex(bytes)
      bytes.to_s.unpack("H*").first
    end

    # Hex decodes a message
    #
    # @param [String] hex hex to decode.
    #
    # @return [String] crisp and clean bytes
    def hex2bin(hex)
      [hex.to_s].pack("H*")
    end
  end
end


# encoding: binary
# frozen_string_literal: true

module RbNaCl
  # Secret Key Authenticators
  #
  # These provide a means of verifying the integrity of a message, but only
  # with the knowledge of a shared key.  This can be a preshared key, or one
  # that is derived through some cryptographic protocol.
  class Auth
    # Number of bytes in a valid key
    KEYBYTES = 0

    # Number of bytes in a valid authenticator
    BYTES = 0

    attr_reader :key
    private :key

    # A new authenticator, ready for auth and verification
    #
    # @param [#to_str] key the key used for authenticators, 32 bytes.
    def initialize(key)
      @key = Util.check_string(key, key_bytes, "#{self.class} key")
    end

    # Compute authenticator for message
    #
    # @param [#to_str] key the key used for the authenticator
    # @param [#to_str] message message to construct an authenticator for
    #
    # @return [String] The authenticator, as raw bytes
    def self.auth(key, message)
      new(key).auth(message)
    end

    # Verifies the given authenticator with the message.
    #
    # @param [#to_str] key the key used for the authenticator
    # @param [#to_str] authenticator to be checked
    # @param [#to_str] message the message to be authenticated
    #
    # @raise [BadAuthenticatorError] if the tag isn't valid
    # @raise [LengthError]  if the tag is of the wrong length
    #
    # @return [Boolean] Was it valid?
    def self.verify(key, authenticator, message)
      new(key).verify(authenticator, message)
    end

    # Compute authenticator for message
    #
    # @param [#to_str] message the message to authenticate
    #
    # @return [String] the authenticator as raw bytes
    def auth(message)
      authenticator = Util.zeros(tag_bytes)
      message = message.to_str
      compute_authenticator(authenticator, message)
      authenticator
    end

    # Verifies the given authenticator with the message.
    #
    # @param [#to_str] authenticator to be checked
    # @param [#to_str] message the message to be authenticated
    #
    # @raise [BadAuthenticatorError] if the tag isn't valid
    # @raise [LengthError]  if the tag is of the wrong length
    #
    # @return [Boolean] Was it valid?
    def verify(authenticator, message)
      auth = authenticator.to_s
      Util.check_length(auth, tag_bytes, "Provided authenticator")
      verify_message(auth, message) || raise(BadAuthenticatorError, "Invalid authenticator provided, message is corrupt")
    end

    # The crypto primitive for this authenticator instance
    #
    # @return [Symbol] The primitive used
    def primitive
      self.class.primitive
    end

    # The number of key bytes for this Auth class
    #
    # @return [Integer] number of key bytes
    def self.key_bytes
      self::KEYBYTES
    end

    # The number of key bytes for this Auth instance
    #
    # @return [Integer] number of key bytes
    def key_bytes
      self.class.key_bytes
    end

    # The number bytes in the tag or authenticator from this Auth class
    #
    # @return [Integer] number of tag bytes
    def self.tag_bytes
      self::BYTES
    end

    # The number of bytes in the tag or authenticator for this Auth instance
    #
    # @return [Integer] number of tag bytes
    def tag_bytes
      self.class.tag_bytes
    end

    private

    def compute_authenticator(_authenticator, _message)
      raise NotImplementedError
    end

    def verify_message(_authenticator, _message)
      raise NotImplementedError
    end
  end
end


# encoding: binary
# frozen_string_literal: true

require "forwardable"

# NaCl/libsodium for Ruby
module RbNaCl
  # The simplest nonce strategy that could possibly work
  #
  # This class implements the simplest possible nonce generation strategy to
  # wrap a RbNaCl::Box or RbNaCl::SecretBox.  A 24-byte random nonce is used
  # for the encryption and is prepended to the message.  When it is time to
  # open the box, the message is split into nonce and ciphertext, and then the
  # box is decrypted.
  #
  # Thanks to the size of the nonce, the chance of a collision is negligible.  For
  # example, after encrypting 2^64 messages, the odds of their having been
  # repeated nonce is approximately 2^-64.  As an additional convenience, the
  # ciphertexts may be encoded or decoded by any of the encoders implemented in
  # the library.
  #
  # The resulting ciphertexts are 40 bytes longer than the plain text (24 byte
  # nonce plus a 16 byte authenticator).  This might be annoying if you're
  # encrypting tweets, but for files represents a fairly small overhead.
  #
  # Some caveats:
  #
  # * If your random source is broken, so is the security of the messages.  You
  #   have bigger problems than just this library at that point, but it's worth
  #   saying.
  # * The confidentiality of your messages is assured with this strategy, but
  #   there is no protection against messages being reordered and replayed by an
  #   active adversary.
  class SimpleBox
    extend Forwardable
    def_delegators :@box, :nonce_bytes, :primitive

    # Create a new SimpleBox
    #
    # @param box [SecretBox, Box] the SecretBox or Box to use.
    #
    # @return [SimpleBox] Ready for use
    def initialize(box)
      @box = box
    end

    # Use a secret key to create a SimpleBox
    #
    # This is a convenience method.  It takes a secret key and instantiates a
    # SecretBox under the hood, then returns the new SimpleBox.
    #
    # @param secret_key [String] The secret key, 32 bytes long.
    #
    # @return [SimpleBox] Ready for use
    def self.from_secret_key(secret_key)
      new(SecretBox.new(secret_key))
    end

    # Use a pair of keys to create a SimpleBox
    #
    # This is a convenience method.  It takes a pair of keys and instantiates a
    # Box under the hood, then returns the new SimpleBox.
    #
    # @param public_key  [PublicKey,  String] The RbNaCl public key, as class or string
    # @param private_key [PrivateKey, String] The RbNaCl private key, as class or string
    #
    # @return [SimpleBox] Ready for use
    def self.from_keypair(public_key, private_key)
      new(Box.new(public_key, private_key))
    end

    # Encrypts the message with a random nonce
    #
    # Encrypts the message with a random nonce, then returns the ciphertext with
    # the nonce prepended.  Optionally encodes the message using an encoder.
    #
    # @param message [String] The message to encrypt
    #
    # @return [String] The enciphered message
    def box(message)
      nonce = generate_nonce
      cipher_text = @box.box(nonce, message)
      nonce + cipher_text
    end
    alias encrypt box

    # Decrypts the ciphertext with a random nonce
    #
    # Takes a ciphertext, optionally decodes it, then splits the nonce off the
    # front and uses this to decrypt.  Returns the message.
    #
    # @param enciphered_message [String] The message to decrypt.
    #
    # @raise [CryptoError] If the message has been tampered with.
    #
    # @return [String] The decoded message
    def open(enciphered_message)
      nonce, ciphertext = extract_nonce(enciphered_message.to_s)
      @box.open(nonce, ciphertext)
    end
    alias decrypt open

    private

    def generate_nonce
      Random.random_bytes(nonce_bytes)
    end

    def extract_nonce(bytes)
      nonce = bytes.slice(0, nonce_bytes)
      [nonce, bytes.slice(nonce_bytes..-1)]
    end
  end

  # Backwards compatibility with the old RandomNonceBox name
  RandomNonceBox = SimpleBox
end


module SemanticLogger
  # Manage a collection of appenders.
  class Appenders < Concurrent::Array
    attr_accessor :logger

    def initialize(logger = Processor.logger.dup)
      @logger      = logger
      @logger.name = self.class.name
    end

    def add(options, deprecated_level = nil, &block)
      options  = options.is_a?(Hash) ? options.dup : convert_old_appender_args(options, deprecated_level)
      appender = SemanticLogger::Appender.factory(options, &block)
      self << appender
      appender
    end

    def log(log)
      each do |appender|
        begin
          appender.log(log) if appender.should_log?(log)
        rescue Exception => exc
          logger.error "Failed to log to appender: #{appender.inspect}", exc
        end
      end
    end

    def flush
      each do |appender|
        begin
          logger.trace "Flushing appender: #{appender.name}"
          appender.flush
        rescue Exception => exc
          logger.error "Failed to flush appender: #{appender.inspect}", exc
        end
      end
      logger.trace 'All appenders flushed'
    end

    def close
      each do |appender|
        begin
          logger.trace "Closing appender: #{appender.name}"
          appender.flush
          appender.close
          delete(appender)
        rescue Exception => exc
          logger.error "Failed to close appender: #{appender.inspect}", exc
        end
      end
      logger.trace 'All appenders closed and removed from appender list'
    end

    # After a fork the appender thread is not running, start it if it is not running.
    def reopen
      each do |appender|
        begin
          next unless appender.respond_to?(:reopen)

          logger.trace "Reopening appender: #{appender.name}"
          appender.reopen
        rescue Exception => exc
          logger.error "Failed to re-open appender: #{appender.inspect}", exc
        end
      end
      logger.trace 'All appenders re-opened'
    end

    private

    # Backward compatibility
    def convert_old_appender_args(appender, level)
      options         = {}
      options[:level] = level if level

      if appender.is_a?(String)
        options[:file_name] = appender
      elsif appender.is_a?(IO)
        options[:io] = appender
      elsif appender.is_a?(Symbol) || appender.is_a?(Subscriber)
        options[:appender] = appender
      else
        options[:logger] = appender
      end
      warn "[DEPRECATED] SemanticLogger.add_appender parameters have changed. Please use: #{options.inspect}"
      options
    end
  end
end


# Abstract Subscriber
#
#   Abstract base class for all appenders.
module SemanticLogger
  class Subscriber < SemanticLogger::Base
    # Every appender has its own formatter
    attr_reader :formatter
    attr_writer :application, :host, :logger, :metrics

    # Returns the current log level if set, otherwise it logs everything it receives.
    def level
      @level || :trace
    end

    # A subscriber should implement flush if it can.
    def flush
      # NOOP
    end

    # A subscriber should implement close if it can.
    def close
      # NOOP
    end

    # Returns [SemanticLogger::Formatters::Default] default formatter for this subscriber.
    def default_formatter
      SemanticLogger::Formatters::Default.new
    end

    # Allow application name to be set globally or on a per subscriber basis.
    def application
      @application || SemanticLogger.application
    end

    # Allow host name to be set globally or on a per subscriber basis.
    def host
      @host || SemanticLogger.host
    end

    # Give each appender its own logger for logging.
    # For example trace messages sent to services or errors when something fails.
    def logger
      @logger ||= begin
        logger      = SemanticLogger::Processor.logger.clone
        logger.name = self.class.name
        logger
      end
    end

    # Set the formatter from Symbol|Hash|Block
    def formatter=(formatter)
      @formatter =
        if formatter.nil?
          respond_to?(:call) ? self : default_formatter
        else
          Formatters.factory(formatter)
        end
    end

    # Whether this log entry meets the criteria to be logged by this appender.
    def should_log?(log)
      super(log) && (log.metric_only? ? metrics? : true)
    end

    private

    # Initializer for Abstract Class SemanticLogger::Subscriber
    #
    # Parameters
    #   level: [:trace | :debug | :info | :warn | :error | :fatal]
    #     Override the log level for this subscriber.
    #
    #   formatter: [Object|Proc]
    #     An instance of a class that implements #call, or a Proc to be used to format
    #     the output from this subscriber
    #     Default: Use the built-in formatter (See: #call)
    #
    #   filter: [Regexp|Proc]
    #     RegExp: Only include log messages where the class name matches the supplied.
    #     regular expression. All other messages will be ignored.
    #     Proc: Only include log messages where the supplied Proc returns true
    #           The Proc must return true or false.
    #
    #   application: [String]
    #     Name of this application to appear in log messages.
    #     Default: SemanticLogger.application
    #
    #   host: [String]
    #     Name of this host to appear in log messages.
    #     Default: SemanticLogger.host
    #
    #   metrics: [Boolean]
    #     Whether to log metric only entries with this subscriber.
    #     Default: false
    def initialize(level: nil, formatter: nil, filter: nil, application: nil, host: nil, metrics: false, &block)
      self.formatter = block || formatter
      @application   = application
      @host          = host
      @metrics       = metrics

      # Subscribers don't take a class name, so use this class name if a subscriber
      # is logged to directly.
      super(self.class, level, filter)
    end

    # Return the level index for fast comparisons.
    # Returns the lowest level index if the level has not been explicitly
    # set for this instance.
    def level_index
      @level_index || 0
    end

    # Whether to log metric only entries with this subscriber
    def metrics?
      @metrics
    end
  end
end


# Base logger
#
#   Abstract base class for loggers
#
#   Implements common behavior such as log level, default text formatter etc
#
module SemanticLogger
  class Base
    # Class name to be logged
    attr_accessor :name, :filter

    # Set the logging level for this logger
    #
    # Note: This level is only for this particular instance. It does not override
    #   the log level in any logging instance or the default log level
    #   SemanticLogger.default_level
    #
    # Must be one of the values in SemanticLogger::LEVELS, or
    # nil if this logger instance should use the global default level
    def level=(level)
      if level.nil?
        # Use the global default level for this logger
        @level_index = nil
        @level       = nil
      else
        @level_index = Levels.index(level)
        @level       = Levels.level(@level_index)
      end
    end

    # Returns the current log level if set, otherwise it returns the global
    # default log level
    def level
      @level || SemanticLogger.default_level
    end

    # Implement the log level calls
    #   logger.debug(message, hash|exception=nil, &block)
    #
    # Implement the log level query
    #   logger.debug?
    #
    # Parameters:
    #   message
    #     [String] text message to be logged
    #     Should always be supplied unless the result of the supplied block returns
    #     a string in which case it will become the logged message
    #     Default: nil
    #
    #   payload
    #     [Hash|Exception] Optional hash payload or an exception to be logged
    #     Default: nil
    #
    #   exception
    #     [Exception] Optional exception to be logged
    #     Allows both an exception and a payload to be logged
    #     Default: nil
    #
    # Examples:
    #    require 'semantic_logger'
    #
    #    # Enable trace level logging
    #    SemanticLogger.default_level = :info
    #
    #    # Log to screen
    #    SemanticLogger.add_appender(io: STDOUT, formatter: :color)
    #
    #    # And log to a file at the same time
    #    SemanticLogger.add_appender(file_name: 'application.log', formatter: :color)
    #
    #    logger = SemanticLogger['MyApplication']
    #    logger.debug("Only display this if log level is set to Debug or lower")
    #
    #    # Log semantic information along with a text message
    #    logger.info("Request received", user: "joe", duration: 100)
    #
    #    # Log an exception in a semantic way
    #    logger.info("Parsing received XML", exc)
    #
    SemanticLogger::LEVELS.each_with_index do |level, index|
      class_eval <<~METHODS, __FILE__, __LINE__ + 1
        def #{level}(message=nil, payload=nil, exception=nil, &block)
          if level_index <= #{index}
            log_internal(:#{level}, #{index}, message, payload, exception, &block)
            true
          else
            false
          end
        end

        def #{level}?
          level_index <= #{index}
        end

        def measure_#{level}(message, params = {}, &block)
          if level_index <= #{index}
            measure_internal(:#{level}, #{index}, message, params, &block)
          else
            block.call(params) if block
          end
        end

        def benchmark_#{level}(message, params = {}, &block)
          if level_index <= #{index}
            measure_internal(:#{level}, #{index}, message, params, &block)
          else
            block.call(params) if block
          end
        end
      METHODS
    end

    # Dynamically supply the log level with every measurement call
    def measure(level, message, params = {}, &block)
      index = Levels.index(level)
      if level_index <= index
        measure_internal(level, index, message, params, &block)
      elsif block
        yield(params)
      end
    end

    # Backward compatibility
    alias benchmark measure

    # Log a thread backtrace
    def backtrace(thread: Thread.current,
                  level: :warn,
                  message: 'Backtrace:',
                  payload: nil,
                  metric: nil,
                  metric_amount: nil)

      log = Log.new(name, level)
      return false unless meets_log_level?(log)

      backtrace =
        if thread == Thread.current
          Utils.extract_backtrace
        else
          log.thread_name = thread.name
          log.tags        = (thread[:semantic_logger_tags] || []).clone
          log.named_tags  = (thread[:semantic_logger_named_tags] || {}).clone
          thread.backtrace
        end
      # TODO: Keep backtrace instead of transforming into a text message at this point
      # Maybe log_backtrace: true
      if backtrace
        message += "\n"
        message << backtrace.join("\n")
      end

      if log.assign(message:       message,
                    backtrace:     backtrace,
                    payload:       payload,
                    metric:        metric,
                    metric_amount: metric_amount) && !filtered?(log)
        self.log(log)
      else
        false
      end
    end

    # Add the tags or named tags to the list of tags to log for this thread whilst the supplied block is active.
    #
    # Returns result of block.
    #
    # Tagged example:
    #   SemanticLogger.tagged(12345, 'jack') do
    #     logger.debug('Hello World')
    #   end
    #
    # Named Tags (Hash) example:
    #   SemanticLogger.tagged(tracking_number: 12345) do
    #     logger.debug('Hello World')
    #   end
    #
    # Notes:
    # - Named tags are the recommended approach since the tag consists of a name value pair this is more useful
    #   than just a string value in the logs, or centralized logging system.
    # - This method is slow when using multiple text tags since it needs to flatten the tags and
    #   remove empty elements to support Rails 4.
    # - It is recommended to keep tags as a list without any empty values, or contain any child arrays.
    #   However, this api will convert:
    #     `logger.tagged([['first', nil], nil, ['more'], 'other'])`
    #   to:
    #     `logger.tagged('first', 'more', 'other')`
    # - For better performance with clean tags, see `SemanticLogger.tagged`.
    def tagged(*tags, &block)
      # Allow named tags to be passed into the logger
      if tags.size == 1
        tag = tags[0]
        return yield if tag.nil? || tag == ''
        return tag.is_a?(Hash) ? SemanticLogger.named_tagged(tag, &block) : SemanticLogger.fast_tag(tag.to_s, &block)
      end

      # Need to flatten and reject empties to support calls from Rails 4
      new_tags = tags.flatten.collect(&:to_s).reject(&:empty?)
      SemanticLogger.tagged(*new_tags, &block)
    end

    # :nodoc:
    alias with_tags tagged

    # :nodoc:
    def tags
      SemanticLogger.tags
    end

    def named_tags
      SemanticLogger.named_tags
    end

    # Returns the list of tags pushed after flattening them out and removing blanks
    #
    # Note:
    # - This method is slow since it needs to flatten the tags and remove empty elements
    #   to support Rails 4.
    # - For better performance with clean tags, use `SemanticLogger.push_tags`
    def push_tags(*tags)
      # Need to flatten and reject empties to support calls from Rails 4
      new_tags = tags.flatten.collect(&:to_s).reject(&:empty?)
      SemanticLogger.push_tags(*new_tags)
    end

    # :nodoc:
    def pop_tags(quantity = 1)
      SemanticLogger.pop_tags(quantity)
    end

    # :nodoc:
    def silence(new_level = :error, &block)
      SemanticLogger.silence(new_level, &block)
    end

    # Deprecated. Use `SemanticLogger.tagged`
    def fast_tag(tag, &block)
      SemanticLogger.fast_tag(tag, &block)
    end

    # :nodoc:
    def with_payload(payload, &block)
      warn '#with_payload is deprecated, use SemanticLogger.named_tagged'
      SemanticLogger.named_tagged(payload, &block)
    end

    # :nodoc:
    def payload
      warn '#payload is deprecated, use SemanticLogger.named_tags'
      SemanticLogger.named_tags
    end

    # Write log data to underlying data storage
    def log(_log_)
      raise NotImplementedError, 'Logging Appender must implement #log(log)'
    end

    # Whether this log entry meets the criteria to be logged by this appender.
    def should_log?(log)
      meets_log_level?(log) && !filtered?(log)
    end

    private

    # Initializer for Abstract Class SemanticLogger::Base
    #
    # Parameters
    #  klass [String]
    #   Name of the class, module, or other identifier for which the log messages
    #   are being logged
    #
    #  level [Symbol]
    #    Only allow log entries of this level or higher to be written to this appender
    #    For example if set to :warn, this appender would only log :warn and :fatal
    #    log messages when other appenders could be logging :info and lower
    #
    #  filter [Regexp|Proc]
    #    RegExp: Only include log messages where the class name matches the supplied
    #    regular expression. All other messages will be ignored
    #    Proc: Only include log messages where the supplied Proc returns true
    #          The Proc must return true or false
    def initialize(klass, level = nil, filter = nil)
      # Support filtering all messages to this logger using a Regular Expression or Proc
      raise ':filter must be a Regexp or Proc' unless filter.nil? || filter.is_a?(Regexp) || filter.is_a?(Proc)

      @filter = filter.is_a?(Regexp) ? filter.freeze : filter
      @name   = klass.is_a?(String) ? klass : klass.name
      if level.nil?
        # Allow the global default level to determine this loggers log level
        @level_index = nil
        @level       = nil
      else
        self.level = level
      end
    end

    # Return the level index for fast comparisons
    # Returns the global default level index if the level has not been explicitly
    # set for this instance
    def level_index
      @level_index || SemanticLogger.default_level_index
    end

    # Whether to log the supplied message based on the current filter if any
    def filtered?(log)
      return false if @filter.nil?

      @filter.is_a?(Regexp) ? (@filter =~ log.name).nil? : @filter.call(log) != true
    end

    # Ensure minimum log level is met
    def meets_log_level?(log)
      (level_index <= (log.level_index || 0))
    end

    # Log message at the specified level
    def log_internal(level, index, message = nil, payload = nil, exception = nil, &block)
      log        = Log.new(name, level, index)
      should_log =
        if payload.nil? && exception.nil? && message.is_a?(Hash)
          # Check if someone just logged a hash payload instead of meaning to call semantic logger
          if message.key?(:message) || message.key?(:payload) || message.key?(:exception) || message.key?(:metric)
            log.assign(message)
          else
            log.assign_positional(nil, message, nil, &block)
          end
        else
          log.assign_positional(message, payload, exception, &block)
        end

      # Log level may change during assign due to :on_exception_level
      self.log(log) if should_log && should_log?(log)
    end

    # Measure the supplied block and log the message
    def measure_internal(level, index, message, params)
      exception = nil
      result    = nil
      # Single parameter is a hash
      if params.empty? && message.is_a?(Hash)
        params  = message
        message = nil
      end
      start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
      begin
        if block_given?
          result =
            if (silence_level = params[:silence])
              # In case someone accidentally sets `silence: true` instead of `silence: :error`
              silence_level = :error if silence_level == true
              silence(silence_level) { yield(params) }
            else
              yield(params)
            end
        end
      rescue Exception => exc
        exception = exc
      ensure
        # Must use ensure block otherwise a `return` in the yield above will skip the log entry
        log       = Log.new(name, level, index)
        exception ||= params[:exception]
        message   = params[:message] if params[:message]
        duration  =
          if block_given?
            1_000.0 * (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start)
          else
            params[:duration] || raise('Mandatory block missing when :duration option is not supplied')
          end

        # Extract options after block completes so that block can modify any of the options
        payload = params[:payload]

        # May return false due to elastic logging
        should_log = log.assign(
          message:            message,
          payload:            payload,
          min_duration:       params[:min_duration] || 0.0,
          exception:          exception,
          metric:             params[:metric],
          metric_amount:      params[:metric_amount],
          duration:           duration,
          log_exception:      params[:log_exception] || :partial,
          on_exception_level: params[:on_exception_level]
        )

        # Log level may change during assign due to :on_exception_level
        self.log(log) if should_log && should_log?(log)
        raise exception if exception
        result
      end
    end

    # For measuring methods and logging their duration.
    def measure_method(index:,
                       level:,
                       message:,
                       min_duration:,
                       metric:,
                       log_exception:,
                       on_exception_level:)

      # Ignores filter, silence, payload
      exception = nil
      start     = Process.clock_gettime(Process::CLOCK_MONOTONIC)
      begin
        yield
      rescue Exception => exc
        exception = exc
      ensure
        log = Log.new(name, level, index)
        # May return false due to elastic logging
        should_log = log.assign(
          message:            message,
          min_duration:       min_duration,
          exception:          exception,
          metric:             metric,
          duration:           1_000.0 * (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start),
          log_exception:      log_exception,
          on_exception_level: on_exception_level
        )

        # Log level may change during assign due to :on_exception_level
        log(log) if should_log && should_log?(log)
        raise exception if exception
      end
    end
  end
end


require 'concurrent'
module SemanticLogger
  # Logger stores the class name to be used for all log messages so that every
  # log message written by this instance will include the class name
  class Logger < Base
    include SemanticLogger::Concerns::Compatibility

    def self.subscribe(object = nil, &block)
      subscriber = block || object

      unless subscriber.is_a?(Proc) || subscriber.respond_to?(:call)
        raise('When supplying an on_log subscriber, it must support the #call method')
      end

      subscribers = (@subscribers ||= Concurrent::Array.new)
      subscribers << subscriber unless subscribers.include?(subscriber)
    end

    class << self
      attr_reader :subscribers
    end

    def self.processor
      @processor
    end

    # Returns a Logger instance
    #
    # Return the logger for a specific class, supports class specific log levels
    #   logger = SemanticLogger::Logger.new(self)
    # OR
    #   logger = SemanticLogger::Logger.new('MyClass')
    #
    # Parameters:
    #  klass
    #    A class, module or a string with the application/class name
    #    to be used in the logger
    #
    #  level
    #    The initial log level to start with for this logger instance
    #    Default: SemanticLogger.default_level
    #
    #  filter [Regexp|Proc]
    #    RegExp: Only include log messages where the class name matches the supplied
    #    regular expression. All other messages will be ignored
    #    Proc: Only include log messages where the supplied Proc returns true
    #          The Proc must return true or false
    def initialize(klass, level = nil, filter = nil)
      super(klass, level, filter)
    end

    # Place log request on the queue for the Appender thread to write to each
    # appender in the order that they were registered
    #
    # Subscribers are called inline before handing off to the queue so that
    # they can capture additional context information as needed.
    def log(log, message = nil, progname = nil, &block)
      # Compatibility with ::Logger
      return add(log, message, progname, &block) unless log.is_a?(SemanticLogger::Log)

      Logger.call_subscribers(log)

      Logger.processor.log(log)
    end

    private

    @processor   = Processor.new
    @subscribers = nil

    def self.call_subscribers(log)
      return unless @subscribers

      @subscribers.each do |subscriber|
        begin
          subscriber.call(log)
        rescue Exception => exc
          self.class.processor.logger.error('Exception calling :on_log subscriber', exc)
        end
      end
    end
  end
end


module SemanticLogger
  # Log
  #
  #   Class to hold all log entry information
  #
  # level
  #   Log level of the supplied log call
  #   :trace, :debug, :info, :warn, :error, :fatal
  #
  # thread_name
  #   Name of the thread in which the logging call was called
  #
  # name
  #   Class name supplied to the logging instance
  #
  # message
  #   Text message to be logged
  #
  # payload
  #   Optional Hash or Ruby Exception object to be logged
  #
  # time
  #   The time at which the log entry was created
  #
  # duration
  #   The time taken to complete a measure call
  #
  # tags
  #   Any tags active on the thread when the log call was made
  #
  # level_index
  #   Internal index of the log level
  #
  # exception
  #   Ruby Exception object to log
  #
  # metric [Object]
  #   Object supplied when measure_x was called
  #
  # backtrace [Array<String>]
  #   The backtrace captured at source when the log level >= SemanticLogger.backtrace_level
  #
  # metric_amount [Numeric]
  #   Used for numeric or counter metrics.
  #   For example, the number of inquiries or, the amount purchased etc.
  #
  # context [Hash]
  #   Named contexts that were captured when the log entry was created.
  class Log
    attr_accessor :level, :level_index, :name, :message, :time, :duration,
                  :payload, :exception, :thread_name, :backtrace,
                  :tags, :named_tags, :context,
                  :metric, :metric_amount, :dimensions

    def initialize(name, level, index = nil)
      @level       = level
      @thread_name = Thread.current.name
      @name        = name
      @time        = Time.now
      @tags        = SemanticLogger.tags
      @named_tags  = SemanticLogger.named_tags
      @level_index = index.nil? ? Levels.index(level) : index
    end

    # Assign named arguments to this log entry, supplying defaults where applicable
    #
    # Returns [true|false] whether this log entry should be logged
    #
    # Example:
    #   logger.info(name: 'value')
    def assign(message: nil,
               payload: nil,
               min_duration: 0.0,
               exception: nil,
               metric: nil,
               metric_amount: nil,
               duration: nil,
               backtrace: nil,
               log_exception: :full,
               on_exception_level: nil,
               dimensions: nil)
      # Elastic logging: Log when :duration exceeds :min_duration
      # Except if there is an exception when it will always be logged
      if duration
        self.duration = duration
        return false if (duration < min_duration) && exception.nil?
      end

      self.message = message
      if payload && payload.is_a?(Hash)
        self.payload = payload
      elsif payload
        self.message = message.nil? ? payload.to_s : "#{message} -- #{payload}"
        self.payload = nil
      end

      if exception
        case log_exception
        when :full
          self.exception = exception
        when :partial
          self.message = "#{message} -- Exception: #{exception.class}: #{exception.message}"
        when nil, :none
          # Log the message without the exception that was raised
          nil
        else
          raise(ArgumentError, "Invalid value:#{log_exception.inspect} for argument :log_exception")
        end
        # On exception change the log level
        if on_exception_level
          self.level       = on_exception_level
          self.level_index = Levels.index(level)
        end
      end

      if backtrace
        self.backtrace = Utils.extract_backtrace(backtrace)
      elsif level_index >= SemanticLogger.backtrace_level_index
        self.backtrace = Utils.extract_backtrace
      end

      if metric
        self.metric        = metric
        self.metric_amount = metric_amount
        self.dimensions    = dimensions
      end

      true
    end

    # Assign positional arguments to this log entry, supplying defaults where applicable
    #
    # Returns [true|false] whether this log entry should be logged
    #
    # Example:
    #   logger.info('value', :debug, 0, "hello world")
    def assign_positional(message = nil, payload = nil, exception = nil)
      # Exception being logged?
      # Under JRuby a java exception is not a Ruby Exception
      #   Java::JavaLang::ClassCastException.new.is_a?(Exception) => false
      if exception.nil? && payload.nil? && message.respond_to?(:backtrace) && message.respond_to?(:message)
        exception = message
        message   = nil
      elsif exception.nil? && payload && payload.respond_to?(:backtrace) && payload.respond_to?(:message)
        exception = payload
        payload   = nil
      elsif payload && !payload.is_a?(Hash)
        message = message.nil? ? payload : "#{message} -- #{payload}"
        payload = nil
      end

      # Add result of block as message or payload if not nil
      if block_given? && (result = yield)
        if result.is_a?(String)
          message = message.nil? ? result : "#{message} -- #{result}"
          assign(message: message, payload: payload, exception: exception)
        elsif message.nil? && result.is_a?(Hash) && %i[message payload exception].any? { |k| result.key? k }
          assign(result)
        elsif payload&.respond_to?(:merge)
          assign(message: message, payload: payload.merge(result), exception: exception)
        else
          assign(message: message, payload: result, exception: exception)
        end
      else
        assign(message: message, payload: payload, exception: exception)
      end
    end

    MAX_EXCEPTIONS_TO_UNWRAP = 5
    # Call the block for exception and any nested exception
    def each_exception
      # With thanks to https://github.com/bugsnag/bugsnag-ruby/blob/6348306e44323eee347896843d16c690cd7c4362/lib/bugsnag/notification.rb#L81
      depth      = 0
      exceptions = []
      ex         = exception
      while !ex.nil? && !exceptions.include?(ex) && exceptions.length < MAX_EXCEPTIONS_TO_UNWRAP
        exceptions << ex
        yield(ex, depth)

        depth += 1
        ex    =
          if ex.respond_to?(:cause) && ex.cause
            ex.cause
          elsif ex.respond_to?(:continued_exception) && ex.continued_exception
            ex.continued_exception
          elsif ex.respond_to?(:original_exception) && ex.original_exception
            ex.original_exception
          end
      end
    end

    # Returns [String] the exception backtrace including all of the child / caused by exceptions
    def backtrace_to_s
      trace = ''
      each_exception do |exception, i|
        if i.zero?
          trace = (exception.backtrace || []).join("\n")
        else
          trace << "\nCause: #{exception.class.name}: #{exception.message}\n#{(exception.backtrace || []).join("\n")}"
        end
      end
      trace
    end

    # Returns [String] duration of the log entry as a string
    # Returns nil if their is no duration
    # Java time precision does not include microseconds
    if Formatters::Base::PRECISION == 3
      def duration_to_s
        "#{duration.to_i}ms" if duration
      end
    else
      def duration_to_s
        return unless duration
        duration < 10.0 ? "#{format('%.3f', duration)}ms" : "#{format('%.1f', duration)}ms"
      end
    end

    # Returns [String] the duration in human readable form
    def duration_human
      return nil unless duration
      seconds = duration / 1000
      if seconds >= 86_400.0 # 1 day
        "#{(seconds / 86_400).to_i}d #{Time.at(seconds).strftime('%-Hh %-Mm')}"
      elsif seconds >= 3600.0 # 1 hour
        Time.at(seconds).strftime('%-Hh %-Mm')
      elsif seconds >= 60.0 # 1 minute
        Time.at(seconds).strftime('%-Mm %-Ss')
      elsif seconds >= 1.0 # 1 second
        "#{format('%.3f', seconds)}s"
      else
        duration_to_s
      end
    end

    # Returns [String] single character upper case log level
    def level_to_s
      level.to_s[0..0].upcase
    end

    # Returns [String] the available process info
    # Example:
    #    18934:thread 23 test_logging.rb:51
    def process_info(thread_name_length = 30)
      file, line = file_name_and_line(true)
      file_name  = " #{file}:#{line}" if file

      "#{$$}:#{format("%.#{thread_name_length}s", thread_name)}#{file_name}"
    end

    CALLER_REGEXP = /^(.*):(\d+).*/

    # Extract the filename and line number from the last entry in the supplied backtrace
    def extract_file_and_line(stack, short_name = false)
      match = CALLER_REGEXP.match(stack.first)
      [short_name ? File.basename(match[1]) : match[1], match[2].to_i]
    end

    # Returns [String, String] the file_name and line_number from the backtrace supplied
    # in either the backtrace or exception
    def file_name_and_line(short_name = false)
      stack = backtrace || exception&.backtrace
      extract_file_and_line(stack, short_name) if stack&.size&.positive?
    end

    # Strip the standard Rails colorizing from the logged message
    def cleansed_message
      message.to_s.gsub(/(\e(\[([\d;]*[mz]?))?)?/, '').strip
    end

    # Return the payload in text form
    # Returns nil if payload is missing or empty
    def payload_to_s
      payload.inspect if payload?
    end

    # Returns [true|false] whether the log entry has a payload
    def payload?
      !(payload.nil? || (payload.respond_to?(:empty?) && payload.empty?))
    end

    # DEPRECATED
    alias has_payload? payload?

    # DEPRECATED
    def formatted_time
      time.strftime(Formatters::Base.build_time_format)
    end

    DeprecatedLogger = Struct.new(:host, :application)

    # DEPRECATED: Use SemanticLogger::Formatters::Raw
    def to_h(host = SemanticLogger.host, application = SemanticLogger.application)
      logger = DeprecatedLogger.new(host, application)
      SemanticLogger::Formatters::Raw.new.call(self, logger)
    end

    # Lazy initializes the context hash and assigns a key value pair.
    def set_context(key, value)
      (self.context ||= {})[key] = value
    end

    # A metric only event has a metric but no message or exception.
    def metric_only?
      metric && message.nil? && exception.nil?
    end
  end
end


# frozen-string-literal: true

module DidYouMean
  # The +DidYouMean::VerboseFormatter+ uses extra empty lines to make the
  # suggestion stand out more in the error message.
  #
  # In order to activate the verbose formatter,
  #
  # @example
  #
  #   OBject
  #   # => NameError: uninitialized constant OBject
  #   #    Did you mean?  Object
  #
  #   require 'did_you_mean/verbose'
  #
  #   OBject
  #   # => NameError: uninitialized constant OBject
  #   #
  #   #        Did you mean? Object
  #   #
  #
  class VerboseFormatter

    # Returns a human readable string that contains +corrections+. This
    # formatter is designed to be less verbose to not take too much screen
    # space while being helpful enough to the user.
    #
    # @example
    #
    #   formatter = DidYouMean::PlainFormatter.new
    #
    #   puts formatter.message_for(["methods", "method"])
    #
    #
    #       Did you mean? methods
    #                     method
    #
    #   # => nil
    #
    def message_for(corrections)
      return "" if corrections.empty?

      output = "\n\n    Did you mean? ".dup
      output << corrections.join("\n                  ")
      output << "\n "
    end
  end
end


module DidYouMean
  module Levenshtein # :nodoc:
    # This code is based directly on the Text gem implementation
    # Copyright (c) 2006-2013 Paul Battley, Michael Neumann, Tim Fletcher.
    #
    # Returns a value representing the "cost" of transforming str1 into str2
    def distance(str1, str2)
      n = str1.length
      m = str2.length
      return m if n.zero?
      return n if m.zero?

      d = (0..m).to_a
      x = nil

      # to avoid duplicating an enumerable object, create it outside of the loop
      str2_codepoints = str2.codepoints

      str1.each_codepoint.with_index(1) do |char1, i|
        j = 0
        while j < m
          cost = (char1 == str2_codepoints[j]) ? 0 : 1
          x = min3(
            d[j+1] + 1, # insertion
            i + 1,      # deletion
            d[j] + cost # substitution
          )
          d[j] = i
          i = x

          j += 1
        end
        d[m] = x
      end

      x
    end
    module_function :distance

    private

    # detects the minimum value out of three arguments. This method is
    # faster than `[a, b, c].min` and puts less GC pressure.
    # See https://github.com/yuki24/did_you_mean/pull/1 for a performance
    # benchmark.
    def min3(a, b, c)
      if a < b && a < c
        a
      elsif b < c
        b
      else
        c
      end
    end
    module_function :min3
  end
end


module Flor

  # ganger | ˈɡaNGər | \ noun British \ the foreman of a gang of laborers.
  #
  # The ganger receives the tasks from the flor executor, decides what
  # tasker will be invoked and hands it the task.
  #
  class Ganger

    # NB: tasker configuration entries start with "gan_"

    #RESERVED_NAMES = %w[ tag ]

    attr_reader :unit

    def initialize(unit)

      @unit = unit
    end

    def shutdown
    end

    # Used by flor when it looks up for a variable and finds nothing.
    # The last step is to ask the ganger if it knows about a tasker under
    # the given (domain and) name.
    #
    # If it returns true, flor knows there is a tasker under that name.
    #
    def has_tasker?(exid, name)

      #return false if RESERVED_NAMES.include?(name)

      d = Flor.domain(exid)

      !! (
        @unit.loader.tasker(d, 'ganger') ||
        @unit.loader.tasker(d, 'tasker') ||
        @unit.loader.tasker(d, name))
    end

    # Called by Flor::Scheduler. The ganger then has to hand the task
    # (the message) to the proper tasker.
    #
    def task(executor, message)

      domain = message['exid'].split('-', 2).first
      tname = message['tasker']

      tconf =
        ( ! message['routed'] &&
         (@unit.loader.tasker(domain, 'ganger', message) ||
          @unit.loader.tasker(domain, 'tasker', message))) ||
        @unit.loader.tasker(domain, tname, message)

      fail ArgumentError.new(
        "tasker #{tname.inspect} not found"
      ) unless tconf

      if tconf.is_a?(Array)

        points = [ nil, message['point'] ]
        points << 'detask' if points.include?('cancel')

        tconf = tconf.find { |h| points.include?(h['point']) }
      end

      message['tconf'] = tconf unless tconf['include_tconf'] == false

      message['vars'] = gather_vars(executor, tconf, message)

      m = dup_message(message)
        #
        # the tasker gets a copy of the message (and it can play with it
        # to its heart content), meanwhile the message is handed to the
        # "post" notifiers.

      @unit.caller.call(self, tconf, m)
        #
        # might return a re-routing message,
        # especially if it's a domain tasker
    end

    # Called by the tasker implementations when they're done with a task
    # and want to hand it back to flor. It might be a failure message.
    #
    def return(message)

      @unit.return(message)
    end

    protected

    def dup_message(m)

      tc = m.delete('tconf')
      m1 = Flor.dup(m)
      m1['tconf'] = tc.inject({}) { |h, (k, v)|
        h[k] = k == 'class' ? v : Flor.dup(v); h } \
          if tc

      m1
    end

    def var_match(k, filter)

      filter.each { |f| return true if (f.is_a?(String) ? k == f : f.match(k)) }
      false
    end

    def expand_filter(f)

      return f unless f.is_a?(Array)

      f.collect { |e|
        if e.is_a?(String)
          e
        elsif e.is_a?(Array) && e[0] == '_rxs' && e[1].is_a?(String)

          s = e[1]
          li, ri = s.index('/'), s.rindex('/')
          tail = s[ri + 1..-1]
          ops =
            (tail.index('i') ? Regexp::IGNORECASE : 0) |
            (tail.index('x') ? Regexp::EXTENDED : 0)
          Regexp.new(s[li + 1..ri - 1], ops)
        else
          nil
        end
      }.compact
    end

    # By default, taskers don't see the flor variables in the execution.
    # If 'include_vars' or 'exclude_vars' is present in the configuration
    # of the tasker, some or all of the variables are passed.
    #
    def gather_vars(executor, tconf, message)

      # try to return before a potentially costly call to executor.vars(nid)

      return nil if (tconf.keys & %w[ include_vars exclude_vars ]).empty?
        # default behaviour, don't pass variables to taskers

      iv = expand_filter(tconf['include_vars'])
      return nil if iv == false

      ev = expand_filter(tconf['exclude_vars'])
      return {} if ev == true

      vars = executor.vars(message['nid'])

      return vars if iv == true

      vars = vars.select { |k, v| var_match(k, iv) } if iv
      vars = vars.reject { |k, v| var_match(k, ev) } if ev

      vars
    end
  end
end


module Flor

  class Hooker

    # NB: logger configuration entries start with "hok_"

    def initialize(unit)

      @unit = unit

      class << unit

        def hook(*args, &block)

          @hooker.add(*args, &block)
        end
      end

      @hooks = []
    end

    def shutdown

      @hooks.each do |n, o, hook, b|

        hook.shutdown if hook.respond_to?(:shutdown)
      end
    end

    def [](name)

      h = @hooks.find { |n, _, _, _| n == name }

      h ? h[2] || h[3] : nil
    end

    def wlist

      @wlist ||= self['wlist']
    end

    def add(*args, &block)

      name = nil
      hook = nil
      opts = {}

      args.each do |arg|
        case arg
        when String then name = arg
        when Hash then opts = arg
        else hook = arg
        end
      end

      hook = Flor::Hook.instantiate(@unit, hook) if hook.is_a?(Class)

      @hooks << [ name, opts, hook, block ]
    end

    def notify(executor, message)

      (
        @hooks + executor.traps_and_hooks
      )
        .inject([]) { |a, (_, opts, hook, block)|
          # name of hook is piped into "_" oblivion

          a.concat(
            if ! match?(executor, hook, opts, message)
              []
            elsif hook.is_a?(Flor::Trap)
              executor.trigger_trap(hook, message)
            elsif hook
              executor.trigger_hook(hook, message)
            else
              executor.trigger_block(block, opts, message)
            end) }
    end

    protected

    def o(opts, *keys)

      array = false
      array = keys.pop if keys.last == []

      r = nil
      keys.each { |k| break r = opts[k] if opts.has_key?(k) }

      return nil if r == nil
      array ? Array(r) : r
    end

    def includes?(arr, value)

      Array(value)
        .find { |v| arr.find { |e| e.is_a?(Regexp) ? e.match(v) : e == v } }
    end

    def match?(executor, hook, opts, message)

      opts = hook.opts if hook.respond_to?(:opts) && opts.empty?

      c = o(opts, :consumed, :c)
      return false if c == true && ! message['consumed']
      return false if c == false && message['consumed']

      if hook.is_a?(Flor::Trap)
        return false if message['point'] == 'trigger'
        return false if hook.within_itself?(executor, message)
      end

      ps = o(opts, :point, :p, [])
      return false if ps && ! ps.include?(message['point'])

      if nid = o(opts, :nid)
        return false \
          unless nid.include?(message['nid'])
      end

      if exi = o(opts, :exid)
        return false \
          unless message['exid'] == exi
      end

      dm = Flor.domain(message['exid'])

      if dm && ds = o(opts, :domain, :d, [])
        return false \
          unless ds.find { |d| d.is_a?(Regexp) ? (!! d.match(dm)) : (d == dm) }
      end

      if dm && sds = o(opts, :subdomain, :sd, [])
        return false \
          unless sds.find do |sd|
            dm[0, sd.length] == sd
          end
      end

      if ts = o(opts, :tag, :t, [])
        return false unless %w[ entered left ].include?(message['point'])
        return false unless includes?(ts, message['tags'])
      end

      if ns = o(opts, :name, :n)
        name = message['name']
        return false \
          unless ns.find { |n|
            case n
            when String then name == n
            when Array then name.match(Flor.to_regex(n))
            else false
            end }
      end

      node = nil

      if hook.is_a?(Flor::Trap) && o(opts, :subnid)
        if node = executor.node(message['nid'], true)
          return false unless node.descendant_of?(hook.bnid, true)
          node = node.h
        else
          return false if hook.bnid != '0'
        end
      end

      if hps = o(opts, :heap, :hp, [])
        return false unless node ||= executor.node(message['nid'])
        return false unless includes?(hps, node['heap'])
      end

      if hts = o(opts, :heat, :ht, [])
        return false unless node ||= executor.node(message['nid'])
        return false unless includes?(hts, node['heat0'])
      end

      true
    end
  end
end


module Flor

  class Scheduler

    attr_reader :conf, :env

    attr_reader :hooker, :storage, :loader, :ganger, :caller
    attr_reader :logger

    attr_reader :thread_status

    attr_reader :archive

    def initialize(conf={}, over_conf={})

      @conf = Flor::Conf.prepare(conf, over_conf)

      fail ArgumentError.new(
        "invalid domain name #{@conf['domain']}"
      ) if @conf['domain'] && ! Flor.potential_domain_name?(@conf['domain'])

      @env = @conf['env'] ||= 'dev'

      @env = (Kernel.const_get(@env) rescue @env) if @env.match(/\A[A-Z]+\z/)
        # when env is "RAILS_ENV" for example...

      @loader =
        (Flor::Conf.get_class(@conf, 'loader') || Flor::Loader).new(self)
      @caller =
        (Flor::Conf.get_class(@conf, 'caller') || Flor::Caller).new(self)
      @hooker =
        (Flor::Conf.get_class(@conf, 'hooker') || Flor::Hooker).new(self)
      @storage =
        (Flor::Conf.get_class(@conf, 'storage') || Flor::Storage).new(self)
      @ganger =
        (Flor::Conf.get_class(@conf, 'ganger') || Flor::Ganger).new(self)

      @logger =
        (Flor::Conf.get_class(@conf, 'logger') || Flor::Logger).new(self)
      @wlist =
        (Flor::Conf.get_class(@conf, 'wlist') || Flor::WaitList).new(self)

      @spooler =
        (Flor::Conf.get_class(@conf, 'spooler') || Flor::Spooler).new(self)

      @heart_rate = @conf[:sch_heart_rate] || 0.3
      @reload_after = @conf[:sch_reload_after] || 60
        #
      @wake_up = true
      @next_time = nil
      @reloaded_at = Time.now

      @msg_max_res_time = @conf[:sch_msg_max_res_time] || 10 * 60

      @idle_count = 0

      @max_executors = @conf[:sch_max_executors] || 1
        #
      @executors = []

      c = @conf['constant']
        #
      Kernel.const_set(c, self) if c

      @archive = @conf['archive'] ? {} : nil # used, so far, only for testing
    end

    def name

      @conf['unit'] || @conf['uni_name'] || 'u0'
    end

    def storage_mutex

      @storage.mutex
    end

    def identifier

      @identifier ||=
        begin
          ai =
            Socket.ip_address_list.find { |a| a.ipv4_private? } ||
            Socket.ip_address_list.find { |a| a.ip_address != '::1' }
          ip =
            (ai ? ai.ip_address : '::1').split('%').first
          [
            'sch', self.name,
            'i' + ip,
            'p' + Process.pid.to_s,
            'o' + (self.object_id % 100_000).to_s(32)
          ].join('-')
        end
    end

    def has_tasker?(exid, tname)

      @ganger.has_tasker?(exid, tname)
    end

    def shutdown

      @thread_status = :shutdown
      @thread = nil

      @executors.each(&:shutdown)

      @hooker.shutdown
      @storage.shutdown
      @ganger.shutdown
    end

    def on_start_exc(e)

      io = StringIO.new

      head, kind =
        e.is_a?(StandardError) ? [ '=sch', 'error' ] : [ '!sch', 'exception' ]
      thr = Thread.current

      t = head[0, 2] + Time.now.to_f.to_s.split('.').last
      io.puts ' /' + t + ' ' + head * 17
      io.puts " |#{t} + in #{self.class}#start"
      io.puts " |#{t} db: #{@storage.db.class} #{@storage.db.object_id}"
      io.puts " |#{t} thread: t#{thr.object_id} #{thr.inspect}"
      io.puts " |#{t} #{kind}: #{e.inspect}"
      io.puts " |#{t} backtrace:"
      e.backtrace.each { |l| io.puts "|#{t} #{l}" }
      io.puts ' \\' + t + ' ' + (head * 17) + ' .'

      io.string
    end

    def start

      # TODO heartbeat, every x minutes, when idle, log something

      fail(
        "database not ready, " +
        "db ver: #{@storage.db_version.inspect}, " +
        "mig ver: #{@storage.migration_version}"
      ) if !! @conf['sto_migration_check'] && @storage.ready?

      @thread_status = :running

      @thread =
        if defined?(@thread) && @thread
          @thread.run
        else
          Thread.new do
            loop do
              Thread.stop if @thread_status == :stop
              break if @thread_status == :shutdown
              tick
            end
          end
        end

      self
    end

    def stop

      @thread_status = :stop
    end

    def running?; @thread_status == :running; end
    def stopped?; ! running?; end

    def join

      @thread.join
    end

    def launch(source_or_path, opts={})

      source, domain, flow_name =
        if df = Flor.split_flow_name(source_or_path)
          [ source_or_path, opts[:domain] || df[0], df[1] ]
        else
          [ source_or_path, opts[:domain] || @conf['domain'] || 'domain0', nil ]
        end

      fail ArgumentError.new(
        "invalid domain name #{domain.inspect}"
      ) unless Flor.potential_domain_name?(domain)

      if flow_name

        source_path, source = @loader.library(source_or_path)

        opts[:fname] = source_path

        # TODO variables
        #        loaded as needed, via the loader
        # TODO payload
        #        yes, still has to be done
      end

      fail ArgumentError.new(
        "flow not found in #{Flor.truncate_string(source_or_path, 35).inspect}"
      ) unless source # will anyway fail badly if src is a tree (array of ...)

      @archive ||= {} if opts[:archive]
        # all subsequent launches will be `archive: true` ...

      @logger.log_src(source, opts)

      unit = opts[:unit] || self.name

      exid = Flor.generate_exid(domain, unit)
      msg = Flor.make_launch_msg(exid, source, opts)

      @logger.log_tree(msg['tree'])

      return [ msg, opts ] if opts[:nolaunch]
        # for testing purposes

      queue(msg, opts)
    end

    def queue(message, opts={})

      @storage.put_message(message)

      if opts[:wait]
        wait(message['exid'], opts)
      else
        message['exid']
      end
    end

    def return(message)

      m =
        if message['point'] == 'failed'
          message
        else
          message
            .select { |k, _| %w[ exid nid payload tasker cause ].include?(k) }
            .merge!('point' => 'return')
        end

      queue(m)

      nil
    end

    def cancel(exid, *as)

      msg, opts = prepare_message('cancel', [ exid, *as ])
      msg['nid'] ||= '0'

      queue(msg, opts)
    end

    def kill(exid, *as)

      msg, opts = prepare_message('kill', [ exid, *as ])

      msg['point'] = 'cancel'
      msg['flavour'] = 'kill'
      msg['nid'] ||= '0'

      queue(msg, opts)
    end

    def signal(name, h={})

      h[:payload] ||= {}
      h[:name] ||= name

      msg, opts = prepare_message('signal', [ h ])

      fail ArgumentError.new('missing :name string key') \
        unless msg['name'].is_a?(String)

      queue(msg, opts)
    end

    def re_apply(exid, *as)

      msg, opts = prepare_message('cancel', [ exid, *as ])

      msg['on_receive_last'] = prepare_re_apply_messages(msg, opts)

      queue(msg, opts)
    end
    alias reapply re_apply

    def add_branches(exid, *as)

      msg, opts = prepare_message('add-branches', [ exid, *as ])

      msg['point'] = 'add'
      msg['trees'] = prepare_trees(opts)

      msg['tnid'] = tnid =
        opts.delete(:tnid) || msg.delete('nid')
      msg['nid'] =
        msg.delete('nid') || opts.delete(:pnid) || Flor.parent_nid(tnid)

      exe = @storage.executions[exid: msg['exid']]
      pnid = msg['nid']
      ptree = exe.lookup_tree(pnid)

      fail ArgumentError.new(
        "parent #{pnid} is a leaf, cannot add branch at #{tnid}"
      ) unless ptree[1].is_a?(Array)
        #
        # not likely to happen, since leaves reply immediately

      size = ptree[1].size
      tnid = (msg['tnid'] ||= Flor.make_child_nid(pnid, size))

      cid = Flor.child_id(tnid)

      tide, tcid = nil
        (0..size - 1).reverse_each do |i|
          tcid = Flor.make_child_nid(pnid, i)
          next unless exe.nodes[tcid]
          tide = i; break
        end

      fail ArgumentError.new(
        "target #{tnid} too low, execution has already reached #{tcid}"
      ) if tide && cid < tide

      fail ArgumentError.new(
        "target #{tnid} is off by #{cid - size}, " +
        "node #{pnid} has #{size} branch#{size == 1 ? '' : 'es'}"
      ) if cid > size

      queue(msg, opts)
    end
    alias add_branch add_branches

    def add_iterations(exid, *as)

      msg, opts = prepare_message('add-iterations', [ exid, *as ])

      msg['point'] = 'add'
      msg['elements'] = prepare_elements(opts)
      msg['nid'] = msg.delete('nid') || opts.delete(:pnid)

      exe = @storage.executions[exid: msg['exid']]
      nid = msg['nid']

      fail ArgumentError.new(
        "cannot add iteration to missing execution #{msg['exid'].inspect}"
      ) unless exe

      fail ArgumentError.new(
        "missing nid: or pnid:"
      ) unless nid

      fail ArgumentError.new(
        "cannot add iteration to missing node #{nid.inspect}"
      ) unless exe.lookup_tree(nid)

      queue(msg, opts)
    end
    alias add_iteration add_iterations

    def schedule(message)

      @storage.put_timer(message)
    end

    def wake_up

      @wake_up = true
    end

    def notify(executor, o)

      if executor
        @hooker.notify(executor, o)
      else
        @hooker.wlist.notify(nil, o)
      end

    rescue => err
      puts '-sch' * 19
      puts "+ error in #{self.class}#notify"
      p err
      puts err.backtrace
      puts ('-sch' * 19) + ' .'
    end

    def trap(node, tra)

      @storage.put_trap(node, tra)
    end

    def archive_node(exid, node)

      (@archive[exid] ||= {})[node['nid']] = Flor.dup(node) if @archive
    end

    def archived_node(exid, nid)

      (@archive[exid] || {})[nid]
    end

    def executor(exid)

      @executors.find { |x| x.exid == exid }
    end

    # Given an exid, returns the execution, if currently executing.
    #
    def execution(exid)

      ex = executor(exid)
      ex ? ex.execution : nil
    end

    DUMP_KEYS = %w[ timestamp executions timers traps pointers ]

    # Dumps all or some of the executions to a JSON string.
    # See Scheduler#load for importing.
    #
    # unit.dump -> string  # returns a JSON string of all executions
    # unit.dump(io) -> io  # dumps the JSON to the given IO instance
    #
    # unit.dump(exid: i)  # dumps only the given execution
    # unit.dump(exids: [ i0, i1 ])  # dumps only the givens executions
    # unit.dump(domain: d)            # dumps exes from domains,
    # unit.dump(domains: [ d0, d1 ])  # and their subdomains
    # unit.dump(sdomain: d)            # dumps strictly from given domains,
    # unit.dump(sdomains: [ d0, d1 ])  # doesn't look at subdomains
    #
    # unit.dump() { |h| ... }  # modify the has right before it's turned to JSON
    #
    def dump(io=nil, opts=nil, &block)

      io, opts = nil, io if io.is_a?(Hash)
      opts ||= {}

      o = lambda { |k| v = opts[k] || opts["#{k}s".to_sym]; v ? Array(v) : nil }
        #
      exis = o[:exid]
      doms = o[:domain]
      sdms = o[:strict_domain] || o[:sdomain]
        #
      filter = lambda { |q|
        q = q.where(
          exid: exis) if exis
        q = q.where {
          Sequel.|(*doms
            .inject([]) { |a, d|
              a.concat([
                { domain: d },
                Sequel.like(:domain, d + '.%') ]) }) } if doms
        q = q.where(
          domain: sdms) if sdms
        q }

      exs, tms, tps, pts =
        storage.db.transaction {
          [ filter[executions].collect(&:to_h),
            filter[timers].collect(&:to_h),
            filter[traps].collect(&:to_h),
            filter[pointers].collect(&:to_h) ] }

      o = io ? io : StringIO.new

      h = {
        timestamp: Flor.tstamp,
        executions: exs,
        timers: tms,
        traps: tps,
        pointers: pts }

      block.call(h) if block

      JSON.dump(h, o)

      io ? io : o.string
    end

    # Read a previous JSON dump and loads it into the storage.
    # Can be useful when testing, dumping once and reloading multiple times
    # to test variants.
    #
    # load(string) -> h  # load all executions from given JSON string
    #                    # returns object inserted stat hash
    # load(io)  # load all executions from the given IO
    # load(io, close: true)  # load from the given IO and close it after read
    #
    # load(x, exid: i)            # load only given executions,
    # load(x, exids: [ i0, i1 ])  # ignore the rest of the data in the source
    # load(x, domain: d)            # load only exes from given domains,
    # load(x, domains: [ d0, d1 ])  # and their subdomains
    # load(x, sdomain: d)            # load only exes from strict domains,
    # load(x, sdomains: [ d0, d1 ])  # ignores exes in their subdomains
    #
    def load(string_or_io, opts={}, &block)

      s = string_or_io
      s = s.read if s.respond_to?(:read)
      string_or_io.close if string_or_io.respond_to?(:close) && opts[:close]
      h = JSON.load(s)

      mks = DUMP_KEYS - h.keys
      fail Flor::FlorError.new("missing keys #{mks.inspect}") if mks.any?

      o = lambda { |k| v = opts[k] || opts["#{k}s".to_sym]; v ? Array(v) : nil }
        #
      exis = o[:exid]
      doms = o[:domain]
      sdms = o[:strict_domain] || o[:sdomain]
        #
      doms = doms.collect { |d| /\A#{d}(\.#{Flor::NAME_REX})*\z/ } if doms

      counts = { executions: 0, timers: 0, traps: 0, pointers: 0, total: 0 }

      storage.db.transaction do

        (DUMP_KEYS - %w[ timestamp ]).each do |k|

          y = k.to_sym
          cla = storage.send(k)
          cols = cla.columns

          rows = h[k]
            .inject([]) { |a, hh|

              next a if exis && ! exis.include?(hh['exid'])
              next a if doms && ! doms.find { |d| d.match(hh['domain']) }
              next a if sdms && ! sdms.include?(hh['domain'])

              counts[y] += 1
              counts[:total] += 1

              vals = cla.from_h(hh)
              a << cols.collect { |c| vals[c] } }

          cla.import(cols, rows) if rows.any?
        end

        block.call(h) if block
      end

      counts
    end

    protected

    def tick

      t0 = Time.now

      spool

      if should_wake_up?

        unreserve_messages

        trigger_timers
        trigger_executions

        reload_next_time
        reload_wake_up
        @reloaded_at = Time.now

      elsif @executors.empty?

        @idle_count += 1
        notify(nil, make_idle_message)
      end

      if @idle_count < 1
        sleep 0.001
      else
        sleep([ @heart_rate - (Time.now - t0), 0.001 ].max)
      end

    rescue Exception => ex

      puts on_start_exc(ex)
    end

    def prepare_message(point, args)

      h = args
        .inject({}) { |hh, a|
          if a.is_a?(Hash) then a.each { |k, v| hh[k.to_s] = v }
          elsif ! hh.has_key?('exid') then hh['exid'] = a
          elsif ! hh.has_key?('nid') then hh['nid'] = a
          end
          hh }

      msg = { 'point' => point }
      opts = {}

      h.each do |k, v|
        if %w[ exid name nid payload on_receive_last ].include?(k)
          msg[k] = v
        else
          opts[k.to_sym] = v
        end
      end

      fail ArgumentError.new('missing :exid key') \
        unless msg['exid'].is_a?(String)

      [ msg, opts ]
    end

    def prepare_trees(opts)

      fname = Flor.caller_fname

      ts = (opts[:trees] || [ opts[:tree] ].compact)
        .collect { |t| Flor.parse(t, fname, {}) }

      fail ArgumentError.new('missing trees: or tree:') if ts.empty?

      ts
    end

    def prepare_elements(opts)

      elts =
        opts[:elements] || opts[:elts] ||
        opts[:element] || opts[:elt]

      fail ArgumentError.new('missing elements: or element:') unless elts

      elts = [ elts ] unless elts.is_a?(Array)

      elts
    end

    def prepare_re_apply_messages(msg, opts)

      pl = msg['payload']

      fail ArgumentError.new("missing 'payload' to re_apply") unless pl

      t = Flor.parse(opts[:tree], Flor.caller_fname, {})

      [ { 'point' => 'execute',
          'exid' => msg['exid'], 'nid' => msg['nid'],
          'from' => 'parent',
          'tree' => t,
          'payload' => pl } ]
    end

    def make_idle_message

      m = {}
      m['point'] = 'idle'
      m['idle_count'] = @idle_count
      m['consumed'] = true

      m
    end

    def reload_after

      @idle_count < 5 ? 1 : @reload_after
    end

    def should_wake_up?

      return true if @wake_up
      return true if Time.now - @reloaded_at >= reload_after

      @next_time && @next_time <= Flor.tstamp.split('.').first
    end

    def unreserve_messages

      c = @storage.unreserve_messages(@msg_max_res_time)

      @logger.info(
        "#{self.class}#unreserve_messages", "#{c} message#{c > 1 ? 's' : ''}"
      ) if c > 0
    end

    def spool

      count = @spooler.spool
      @wake_up = true if count > 0
    end

    def trigger_timers

      @storage.trigger_timers
    end

    def trigger_executions

      @executors.select! { |e| e.alive? }
        # drop done executors

      free_executor_count = @max_executors - @executors.size

      return if free_executor_count < 1

      messages = @storage.load_messages(free_executor_count)

      messages.each do |exid, ms|

        next unless @storage.reserve_all_messages(ms)

        @idle_count = 0

        @executors << UnitExecutor.new(self, ms).run
      end
    end

    def reload_next_time

      @next_time = @storage.fetch_next_time
    end

    def reload_wake_up

      @wake_up = @storage.any_message?
    end
  end
end


module Flor

  # A loader which keeps everything in a Hash, while the traditional/default
  # Flor::Loader reads from a file tree.
  #
  class HashLoader

    # NB: tasker configuration entries start with "loa_", like for Flor::Loader

    attr_reader :environment

    def environment=(h)

      @environment = Flor.to_string_keyed_hash(h)
        .inject({}) { |r, (cat, v)|
          r[cat] = recompose(v)
          r }
    end

    def initialize(unit)

      @unit = unit

      class << @unit; include Flor::HashLoader::UnitAdders; end

      self.environment = (@unit.conf['lod_environment'] || {})
    end

    def shutdown
    end

    HCATS = {
      'v' => 'variables',
      'var' => 'variables',
      'variable' => 'variables',
      'flo' => 'libraries',
      'flow' => 'libraries',
      'lib' => 'libraries',
      'library' => 'libraries',
      'sub' => 'sublibraries',
      'sublib' => 'sublibraries',
      'sublibrary' => 'sublibraries',
      'tasker' => 'taskers',
      'hook' => 'hooks' }
    CATS =
      HCATS.values.uniq

    def add(cat, path, value, &block)

      c = recat(cat)

      path = path.to_s
      path = path + '.' if c == 'hooks' && path.length > 0 && path[-1, 1] != '.'

      value = block ?
        block_to_class(c, block) :
        Flor.to_string_keyed_hash(value)

      e = (@environment[c] ||= [])
      e << [ *split(path), value ]
      e.sort_by! { |pa, _, _| pa.count('.') }
    end

    def remove(cat, path)

      c = recat(cat)

      e = @environment[c]
      return [] unless e

      pa, ke = split(path)

      e.reject! { |epa, eke, _| epa == pa && eke == ke }
    end

    def variables(domain)

      entries('variables', domain)
        .inject({}) { |h, (_, k, v)| h[k] = v; h }
    end

    #def procedures(path)
    #
    #  # TODO
    #  # TODO work with Flor.load_procedures
    #end

    # If found, returns [ source_path, path ]
    #
    def library(domain, name=nil, opts={})

      path, key = split(domain, name)

      libs = entries('libraries', path)
      if opts[:subflows] # used by "graft"/"import"
        libs += entries('sublibraries', path)
        libs = libs.sort_by { |pa, _, _| pa.count('.') }
      end

      libs
        .each { |pa, ke, va|
          next unless ke == key
          return [ [ pa, ke ].join('.'), va ] }

      nil
    end

    def tasker(domain, name, message={})

      path, key = split(domain, name)

      entries('taskers', path)
        .reverse # FIXME
        .each { |pa, ke, va|
          next unless ke == key
          return to_conf_h('taskers', pa, va, message) }

      nil
    end

    def hooks(domain)

      entries('hooks', domain)
        .collect { |_, _, va| to_conf_h('hooks', domain, va, {}) }
        .flatten(1)
    end

    def load_hooks(exid)

      hooks(Flor.domain(exid))
        .collect { |h| Flor::Hook.new(@unit, exid, h) }
    end

    protected

    def recompose(h)

      deflate(h, {}, nil)
        .sort_by { |k, _| k.count('.') }
        .collect { |k, v| [ *split(k), v ] }
    end

    def deflate(h, out, path)

      h
        .inject(out) { |r, (k, v)|
          pathk = path ? [ path, k ].join('.') : k
          if v.is_a?(Hash)
            deflate(v, r, pathk)
          else
            r[pathk] = v
          end
          r }
    end

    def recat(cat)

      c = cat.to_s
      c = HCATS[c] || c

      fail ArgumentError.new("unknown category #{cat.to_s.inspect}") \
        unless CATS.include?(c)

      c
    end

    def split(path, key=nil)

      return [ path, key ] if key

      i = path.rindex('.') || 0

      [ path[0, i], path[(i == 0 ? i : i + 1)..-1] ]
    end

    def entries(cat, domain)

      (@environment[cat.to_s] || [])
        .select { |path, _, _| Flor.sub_domain?(path, domain) }
    end

    def to_conf_h(cat, path, value, context)

      is_array = value.is_a?(Array)

      a = (is_array ? value : [ value ])
        .collect { |v|
          h =
            case v
            when String then eval(path, v, context)
            when Class then { 'class' => v }
            when Hash then v
            else { 'instance' => v }
            end
          h['_path'] = path
          h['root'] = nil
          h }

      is_array ? a : a.first
    end

    def eval(path, code, context)

      code = Flor::ConfExecutor.load(code)

      Flor::ConfExecutor.interpret(path, code, context)
    end

    def block_to_class(cat, block)

      c =
        cat == 'taskers' ?
        Class.new(Flor::BasicTasker) :
        Class.new

      class << c; attr_accessor :source_location; end
      c.source_location = block.source_location
      c.send(:define_method, :on_message, &block)

      c
    end

    module UnitAdders

      def add_variable(path, value)
        self.loader.add(:variable, path, value)
      end
      def add_library(path, value)
        self.loader.add(:library, path, value)
      end
      def add_sublibrary(path, value)
        self.loader.add(:sublibrary, path, value)
      end
      def add_tasker(path, value=nil, &block)
        self.loader.add(:tasker, path, value, &block)
      end
      def add_hook(path, value=nil, &block)
        self.loader.add(:hook, path, value, &block)
      end

      alias add_var add_variable
      alias add_lib add_library
      alias add_sub add_sublibrary
      alias add_sub_lib add_sublibrary

      # to remove: unit.loader.remove(:tasker, path)
    end
  end
end


module Flor

  class UnitExecutor < Flor::Executor

    attr_reader :exid
    attr_reader :consumed

    def initialize(unit, messages)

      @exid = messages.first[:exid]

      super(
        unit,
        unit.loader.load_hooks(@exid),
        unit.storage.fetch_traps(@exid),
        unit.storage.load_execution(@exid))

      @messages = messages
        .collect { |m|
          Flor::Storage
            .from_blob(m[:content])
            .tap { |mm| mm['mid'] = m[:id].to_i } }
      @consumed = []

      @alive = true
      @shutdown = false

      @thread = nil
    end

    def alive?; @alive; end

    def run

      @thread = Thread.new { do_run }
#p [ :unit_executor, :thread, @thread.object_id ]

      self
    end

    def shutdown

      @shutdown = true
      @thread.join
    end

    protected

    CLOSING_POINTS = %w[ task terminated ceased ]
      #
      # point for messages that, after consumption, are conserved
      # in the execution's "closing_messages" array

    def do_run

      @unit.logger.log_run_start(self)

      counter_next('runs')

      t0 = Time.now

      (@unit.conf['exe_max_messages'] || 77).times do |i|

        break if @shutdown

        m = @messages.shift
        break unless m

        m = (@messages << m).shift \
          if m['point'] == 'terminated' && @messages.any?
            #
            # handle 'terminated' messages last

        ms = process(m)

        @consumed << m

        ims, oms = ms.partition { |mm| mm['exid'] == @exid }
          # qui est "in", qui est "out"?

        counter_add('omsgs', oms.size)
          # keep track of "out" messages, messages to other executions

        @messages.concat(ims)
        @unit.storage.put_messages(oms)
      end

      @alive = false

      @execution.merge!(
        closing_messages: @consumed.select { |m|
          CLOSING_POINTS.include?(m['point']) })

      @unit.storage.put_execution(@execution)
      @unit.storage.consume(@consumed)

      @unit.storage.put_messages(@messages)

      du = Time.now - t0
      t0 = Flor.tstamp(t0)

      @unit.logger.log_run_end(self, t0, du)
      @unit.hooker.notify(self, make_end_message(t0, du, @execution['size']))

      @consumed.clear

    rescue Exception => exc

# TODO eventually, have a dump dir
      fn =
        [
          'flor',
          @unit.conf['env'], @unit.identifier, @exid,
          'r' + counter('runs').to_s
        ].collect(&:to_s).join('_') + '.dump'

      @unit.logger.error(
        "#{self.class}#do_run()", exc, "(dumping to #{fn})")

      File.open(fn, 'wb') do |f|
        f.puts(Flor.to_pretty_s({
          execution: @execution,
          messages: @messages,
          consumed: @consumed,
          traps: @traps.collect(&:to_h),
          exid: @exid,
          alive: @alive,
          shutdown: @shutdown,
          thread: [ @thread.object_id, @thread.to_s ]
        }))
        f.puts('-' * 80)
        f.puts(on_do_run_exc(exc))
      end

      #puts on_do_run_exc(exc)
        # dump notification above
    end

    def task(message)

      return error_reply(
        node(message['nid']),
        message,
        "don't know how to apply #{message['tasker'].inspect}"
      ) if message['routed'] == false
        #
        # use an error message similar to what the core executor would emit

      @unit.ganger.task(self, message)
    end
    alias detask task

    def return(message)

      n = @execution['nodes'][message['nid']] || {}
      m = n['message'] || {}
      c = m['cause']

      rm = message.dup
      rm['point'] = 'receive'
      rm['cause'] = c if c # preserve 'cause' for routing

      [ rm ]
    end

    def schedule(message)

      @unit.schedule(message)

      []
    end

    def trigger(message)

      m = message['message']
      m['cause'] = message['cause']

      m['nid'] = Flor.sub_nid(m['nid'], counter_next('subs')) \
        if m['point'] == 'execute'

      [ m ]
    end

    def add(message)

      apply(@execution['nodes'][message['nid']], message)
    end

    def on_do_run_exc(e)

      io = StringIO.new
      thr = Thread.current

      head, kind =
        e.is_a?(StandardError) ? [ '=exe', 'error' ] : [ '!exe', 'exception' ]

      t = head[0, 2] + Time.now.to_f.to_s.split('.').last
      io.puts ' /' + t + ' ' + head * 17
      io.puts " |#{t} + in #{self.class}#do_run"
      io.puts " |#{t} #{kind}: #{e.inspect}"
      io.puts " |#{t} db:  #{@unit.storage.db.class} #{@unit.storage.db.hash}"
      io.puts " |#{t} thread: t#{thr.object_id} #{thr.inspect}"
      if @execution
        io.puts " |#{t} exe:"
        io.puts " |#{t}   exid: #{@execution['exid'].inspect}"
        io.puts " |#{t}   counters: #{@execution['counters'].inspect}"
      end
      if @messages
        io.puts " |#{t} messages:"
        io.puts " |#{t}   #{@messages.map { |m| [ m['mid'], m['point'] ] }.inspect}"
      end
      if is = Thread.current[:sto_errored_items]
        io.puts " |#{t} storage errored items/models:"
        is.each do |i|
          io.puts " |#{t}   * #{i.inspect}"
        end
      end
      io.puts " |#{t} #{kind}: #{e.inspect}"
      io.puts " |#{t} backtrace:"
      e.backtrace.each { |l| io.puts "|#{t} #{l}" }
      io.puts ' \\' + t + ' ' + (head * 17) + ' .'

      io.string
    end

    def make_end_message(start, duration, execution_size)

      m = {}
      m['point'] = 'end'
      m['exid'] = @exid
      m['start'] = start
      m['duration'] = "#{duration}s"
      m['consumed'] = @consumed.size
      m['counters'] = Flor.dup(@execution['counters'])
      m['nodes'] = @execution['nodes'].size
      m['execution_size'] = execution_size
      m['archive_size'] = @unit.archive[@exid].size if @unit.archive
      m['er'] = @execution['counters']['runs'] # "emitting run"
      m['pr'] = m['er'] # "processing run"

      m
    end
  end
end


module Flor

  class BasicTasker

    attr_reader :ganger, :conf, :message

    def initialize(ganger, conf, message)

      @ganger = ganger
      @conf = conf
      @message = message
    end

    protected

    def exid; @message['exid']; end
    def nid; @message['nid']; end

    def payload; @message['payload']; end
    alias fields payload

    def attd; @message['attd']; end
    def attl; @message['attl']; end

    def tasker; @message['tasker']; end
    def taskname; @message['taskname']; end
    alias task_name taskname

    def vars; @message['vars']; end

    def execution

      @ganger.unit.execution(exid)
    end

    # For domain taskers
    #
    def route(name)

      if name.is_a?(String)

        [ Flor.dup_and_merge(
            @message,
            'tasker' => name, 'original_tasker' => @message['tasker'],
            'routed' => true) ]

      else

        [ Flor.dup_and_merge(
            @message,
            'routed' => !! name) ]
      end
    end

    def reply(message=@message, force=false)

      fail ArgumentError.new(
        "argument to reply must be a Hash but is #{message.class}"
      ) unless message.is_a?(Hash)

      @ganger.return(derive_message(message)) if force || @ganger

      [] # very important, return no further messages
    end

    def reply_with_error(error)

      reply(
        Flor.to_error_message(@message, error))
    end

    # So that #reply may be called with
    # ```
    # reply
    # reply(@message)
    # reply(payload: {})
    # reply(ret: 123)
    # reply(ret: 123, set: { a: 1 }, unset: [ :b ])
    # ```
    #
    def derive_message(m)

      exid = m['exid']
      nid = m['nid']
      pl = m['payload']

      return m if Flor.is_exid?(exid) && Flor.is_nid?(nid) && pl.is_a?(Hash)

      m = Flor.to_string_keyed_hash(m)
      h = Flor.dup(@message)
      ks = m.keys

      if ks == [ 'payload' ]

        h['payload'] = m['payload']

      elsif (ks & %w[ ret set unset ]).size > 0

        pl = (h['payload'] ||= {})

        pl['ret'] = m['ret'] if m.has_key?('ret')
        (m['set'] || {}).each { |k, v| pl[k] = v }
        (m['unset'] || []).each { |k| pl.delete(k.to_s) }

      else

        h['payload'] = m
      end

      h
    end
  end
end


module Flor

  # Used by Flor::Loader to prepare hooks from hooks.json files.
  #
  class Hook

    def initialize(unit, exid, h)

      @unit = unit
      @exid = exid
      @h = h
    end

    def to_hook

      if @h['class']
        class_to_hook
      elsif @h['instance']
        instance_to_hook
      else
        classical_to_hook
      end
    end

    def notify(executor, message)

      @unit.caller.call(executor, @h, message)
    end

  # Avoid the proc/cancel problem upstreams in ConfExecutor, by ignoring
  # non-core procedures, keeping this around for now
  #
#    protected
#
#    def correct_points(opts)
#
#      pts = opts[:point]; return unless pts
#
#      opts[:point] =
#        Flor.is_tree?(pts) ?
#        correct_point(pts) :
#        pts.collect { |pt| correct_point(pt) }
#    end
#
#    def correct_point(point)
#
#      return point[1]['proc'] if point.is_a?(Array) && point[0] == '_proc'
#      point
#    end

    def extract_filters(h)

      r = {}
      r[:consumed] = h['consumed']
      r[:point] = Flor.h_fetch_a(h, 'points', 'point', nil)
      r[:nid] = Flor.h_fetch_a(h, 'nids', 'nid', nil)
      r[:heap] = Flor.h_fetch_a(h, 'heaps', 'heap', nil)
      r[:heat] = Flor.h_fetch_a(h, 'heats', 'heat', nil)
      #opts[:name] = data['names']

      r
    end

    def class_to_hook

      i = Flor::Hook.instantiate(@unit, @h['class'])

      h = @h.dup
      h.merge!(Flor.to_string_keyed_hash(i.opts)) if i.respond_to?(:opts)
      opts = extract_filters(h)

      [ "hooc#{object_id}", opts, i, nil ]
    end

    def instance_to_hook

      i = @h['instance']

      h = @h.dup
      h.merge!(Flor.to_string_keyed_hash(i.opts)) if i.respond_to?(:opts)
      opts = extract_filters(h)

      [ "hooi#{object_id}", opts, i, nil ]
    end

    def classical_to_hook

      opts = extract_filters(@h)

      #correct_points(opts)
        #
        # Necessary since "cancel" gets interpreted as
        # [ '_proc', { 'proc' => 'cancel' }, @line ]
        # ...

      [ "hook#{object_id}", opts, self, nil ]
    end

    class << self

      def instantiate(unit, hook_class)

        c =
          case hook_class
          when String then Flor.const_get(hook_class)
          else hook_class
          end
        a =
          case i = c.instance_method(:initialize).arity
          when 0, 1 then [ unit ][0, i]
          else []
          end

        c.new(*a)
      end
    end
  end
end