module Flor

  class TransientExecutor < Executor

    class TransientUnit

      attr_accessor :conf, :opts
      attr_reader :loader, :logger
      attr_reader :journal
      attr_accessor :archive

      def initialize(conf)

        @conf = conf
        @opts = {}
        @logger = TransientLogger.new(self)
        @journal = []
        @archive = nil
      end

      def notify(executor, msg)

        return [] if msg['consumed']

        @logger.notify(executor, msg)
        @journal << msg

        []
      end

      def archive_node(exid, node)

        (@archive[exid] ||= {})[node['nid']] = Flor.dup(node) if @archive
      end

      def has_tasker?(exid, tname)

        false
      end
    end

    class TransientLogger

      def initialize(unit)

        @unit = unit

        @out = Flor::Logger::Out.prepare(unit)
      end

      def notify(executor, message)

        return if message['point'] == 'end'
        return unless @unit.conf['log_msg']

        @out.puts(Flor.message_to_one_line_s(executor, message, out: @out))
      end

      def log_err(executor, message, opts={})

        return unless @unit.conf['log_err']

        @out.puts(
          Flor.msg_to_detail_s(executor, message, opts.merge(flag: true)))
      end

      def log_src(source, opts, log_opts={})

        return unless @unit.conf['log_src']

        @out.puts(Flor.src_to_s(source, opts, log_opts))
      end

      def log_tree(tree, nid='0', opts={})

        return unless @unit.conf['log_tree']

        @out.puts(Flor.tree_to_s(tree, nid, opts.merge(out: @out)))
      end
    end

    def initialize(conf={})

      conf = Flor::Conf.prepare(conf, {})

      super(
        TransientUnit.new(conf),
        [], # no hooks
        [], # no traps
        {
          'exid' => Flor.generate_exid('eval', 'u0'),
          'nodes' => {}, 'counters' => {}, 'start' => Flor.tstamp
        })

      @unit.archive = {} if conf['archive']
    end

    def journal; @unit.journal; end
    def archive; @unit.archive[exid]; end

    def launch(tree, opts={})

      @unit.opts = opts
      @unit.archive ||= {} if opts[:archive]

      @unit.logger.log_src(tree, opts)

      messages = [ Flor.make_launch_msg(@execution['exid'], tree, opts) ]

      @unit.logger.log_tree(messages.first['tree'], '0', opts)

      walk(messages, opts)
    end

    def walk(messages, opts={})

      loop do

        message = messages.shift
        return nil unless message

        if message['point'] == 'terminated' && messages.any?
          #
          # try to handle 'terminated' last
          #
          messages << message
          message = messages.shift
        end

        msgs = process(message)

        messages.concat(msgs)

        return messages \
          if message_match?(message, opts[:until_after])
        return messages \
          if message_match?(messages, opts[:until])
            #
            # Walk is suspended if options :until_after or :until
            # are satisfied.
            # Returns the remaining messages.

        return message \
          if message['point'] == 'terminated'
        return message \
          if message['point'] == 'failed' && message['on_error'] == nil
            #
            # Walk exits when execution terminates or fails (without on_error).
            # Returns the last message
      end
    end

    def step(message)

      process(message)
    end

    # Used in specs when testing multiple message arrival order on
    # a "suite" of transient executors
    #
    def clone

      c = TransientExecutor.allocate

      c.instance_variable_set(:@unit, @unit)
      c.instance_variable_set(:@traps, []) # not useful for a TransientEx clone
      c.instance_variable_set(:@execution, Flor.dup(@execution))

      c
    end

    protected

    # TODO eventually merge with Waiter.parse_serie
    #
    def message_match?(msg_s, ountil)

      return false unless ountil

      ms = msg_s; ms = [ ms ] if ms.is_a?(Hash)

      nid, point = ountil.split(' ')

      ms.find { |m| m['nid'] == nid && m['point'] == point }
    end
  end

  class ConfExecutor < TransientExecutor

    class << self

      def load(path)

        src =
          if path.match(/[\r\n]/)
            path.strip
          else
            File.readlines(path)
              .reject { |l| l.strip[0, 1] == '#' }
              .join("\n")
              .strip
          end

        az = "#{src[0, 1]}#{src[-1, 1]}"

        if az == '{}' || az == '[]'
          src
        elsif src.match(/[^\r\n{]+:/) || src == ''
          "{\n#{src}\n}"
        else
          "[\n#{src}\n]"
        end
      end

      def interpret(path, source, context)

        path ||= '.'

        fs = context['payload'] || {}

        vs = Hash.new { |h, k| k }
          #
        vs.merge!(context['vars'] || {})
        vs['root'] = determine_root(path)
        vs['ruby_version'] = RUBY_VERSION
        vs['ruby_platform'] = RUBY_PLATFORM
          #
        class << vs
          def has_key?(k)
            prc = Flor::Procedure[k]
            ( ! prc) || ( ! prc.core?) # ignore non-core procedures
          end
        end

        c = ! (ENV['FLOR_DEBUG'] || '').match(/conf/)
        e = self.new('conf' => c)
        r = e.launch(source, payload: fs, vars: vs)

        unless r['point'] == 'terminated'
          ae = ArgumentError.new(
            "error while reading conf: #{r['error']['msg']}")
          ae.set_backtrace(r['error']['trc'])
          fail ae
        end

        o = Flor.dup(r['payload']['ret'])

        if o.is_a?(Hash)
          o['_path'] = path
          o['root'] ||= Flor.relativize_path(vs['root'])
        elsif o.is_a?(Array)
          o.each { |ee| ee['_path'] = path }
        end

        o
      end

      def interpret_path(path, context=nil)

        interpret(path, load(path), context || {})
      end

      def interpret_source(source, context=nil)

        interpret(nil, source, context || {})
      end

      def interpret_path_or_source(s, context=nil)

        if s.index("\n")
          interpret_source(load(s), context)
        else
          interpret_path(s, context)
        end
      end

      # Used by "flosh" the flor shell
      #
      def interpret_line(s)

        r = interpret_source(s)
        r.delete('root') if r.is_a?(Hash)

        r
      end

      def determine_root(path)

        dir = File.absolute_path(File.dirname(path))
        ps = dir.split(File::SEPARATOR)

        ps.last == 'etc' ? File.absolute_path(File.join(dir, '..')) : dir
      end
    end
  end
end


module Flor

  class Loader

    # NB: tasker configuration entries start with "loa_"

    def initialize(unit)

      @unit = unit

      @cache = {}
      @mutex = Mutex.new

      @root = File.absolute_path(@unit.conf['lod_path'] || @unit.conf['root'])
    end

    def shutdown
    end

    def variables(domain)

      Dir[File.join(@root, '**/*.json')]
        .select { |f| f.index('/etc/variables/') }
        .collect { |pa| [ pa, expose_d(pa, {}) ] }
        .select { |pa, d| Flor.sub_domain?(d, domain) }
        .sort_by { |pa, d| d.count('.') }
        .inject({}) { |vars, (pa, _)| vars.merge!(eval(pa, {})) }
    end

    #def procedures(path)
    #
    #  # TODO
    #  # TODO work with Flor.load_procedures
    #end

    # If found, returns [ source_path, path ]
    #
    def library(domain, name=nil, opts={})

      domain, name, opts = [ domain, nil, name ] if name.is_a?(Hash)
      domain, name = split_dn(domain, name)

      if m = name.match(/(\.flor?)\z/)
        name = name[0..m[1].length - 1]
      end

      path, _, _ = (Dir[File.join(@root, '**/*.{flo,flor}')])
        .select { |f| f.index('/lib/') }
        .collect { |pa| [ pa, *expose_dn(pa, opts) ] }
        .select { |pa, d, n| n == name && Flor.sub_domain?(d, domain) }
        .sort_by { |pa, d, n| d.count('.') }
        .last

      path ? [ Flor.relativize_path(path), File.read(path) ] : nil
    end

    def tasker(domain, name, message={})

      # NB: do not relativize path, because Ruby load path != cwd,
      # stay absolute for `require` and `load`

      domain, name = split_dn(domain, name)

      pat, _, nam = Dir[File.join(@root, '**/*.{json,rb}')]
        .select { |pa| pa.index('/lib/taskers/') }
        .collect { |pa| [ pa, *expose_dn(pa, {}) ] }
        .select { |pa, d, n|
          Flor.sub_domain?([ d, n ].join('.'), domain) ||
          (n == name && Flor.sub_domain?(d, domain)) }
        .sort_by { |pa, d, n| d.count('.') }
        .last

      return nil unless pat

      conf = eval(pat, message)

      return conf if nam == name

      conf = conf[name]

      return nil unless conf

      (conf.is_a?(Array) ? conf : [ conf ])
        .each { |h| h['_path'] = pat }

      conf
    end

    def hooks(domain)

      # NB: do not relativize path, because Ruby load path != cwd,
      # stay absolute for `require` and `load`

      Dir[File.join(@root, '**/*.json')]
        .select { |f| f.index('/lib/hooks/') }
        .collect { |pa| [ pa, expose_d(pa, {}) ] }
        .select { |pa, d| Flor.sub_domain?(d, domain) }
        .sort_by { |pa, d| d.count('.') }
        .collect { |pa, d|
          eval(pa, {}).each_with_index { |h, i|
            h['_path'] = pa + ":#{i}" } }
        .flatten(1)
    end

    def load_hooks(exid)

      hooks(Flor.domain(exid))
        .collect { |h| Flor::Hook.new(@unit, exid, h) }
    end

    protected

    def split_dn(domain, name)

      if name
        [ domain, name ]
      else
        elts = domain.split('.')
        [ elts[0..-2].join('.'), elts[-1] ]
      end
    end

    def expose_d(path, opts)

      pa = path[@root.length..-1]
      pa = pa[5..-1] if pa[0, 5] == '/usr/'

      libregex =
        opts[:subflows] ?
        /\/lib\/(subflows|flows|hooks|taskers)\// :
        /\/lib\/(flows|hooks|taskers)\//

      pa
        .sub(/\/etc\/variables\//, '/')
        .sub(libregex, '/')
        .sub(/\/\z/, '')
        .sub(/\/(flo|flor|dot|hooks)\.json\z/, '')
        .sub(/\.(flo|flor|json|rb)\z/, '')
        .sub(/\A\//, '')
        .gsub(/\//, '.')
    end

    def expose_dn(path, opts)

      pa = expose_d(path, opts)

      if ri = pa.rindex('.')
        [ pa[0..ri - 1], pa[ri + 1..-1] ]
      else
        [ '', pa ]
      end
    end

    def eval(path, context)

      ext =
        File.extname(path)

      src =
        @mutex.synchronize do

          mt1 = File.mtime(path)
          val, mt0 = @cache[path]

          if val && mt1 == mt0
            val
          elsif ext == '.rb'
            (@cache[path] = [ File.read(path), mt1 ]).first
          else
            (@cache[path] = [ Flor::ConfExecutor.load(path), mt1 ]).first
          end
        end

      case ext
      when '.rb' then eval_ruby(path, src, context)
      else eval_json(path, src, context)
      end
    end

    def eval_json(path, src, context)

      Flor::ConfExecutor.interpret(path, src, context)
    end

    def eval_ruby(path, src, context)

      ks = context.keys.select { |k| k.match(/\A[a-z][a-zA-Z0-9_]+\z/) }

      s = StringIO.new
      s << "lambda {\n"
      ks.each { |k| s << k << " = " << context[k].inspect << "\n" }
      s << src
      s << "\n}.call\n"

      r = Kernel.module_eval(s.string, path, - ks.size)

      r = JSON.parse(JSON.dump(r)) #if r.keys.find { |k| k.is_a?(Symbol) }
        #
        # so that symbols may be used in the .rb file, but plain JSON-like
        # string keys are on the output

      r.merge!('_path' => path)
    end
  end
end


module Flor

  class Timer < FlorModel

    #create_table :flor_timers do
    #
    #  primary_key :id, type: :Integer
    #  String :domain, null: false
    #  String :exid, null: false
    #  String :nid, null: false
    #  String :type, null: false # 'at', 'in', 'cron', 'every', ...
    #  String :schedule, null: false # '20141128.103239' or '00 23 * * *'
    #  String :ntime # next time
    #  File :content # JSON msg to trigger
    #  Integer :count, null: false
    #  String :status, null: false
    #  String :ctime, null: false
    #  String :mtime, null: false
    #  String :cunit
    #  String :munit
    #  String :onid, null: false
    #  String :bnid, null: false
    #
    #  index :exid
    #  index [ :exid, :nid ]
    #end

    def to_trigger_message

      d = self.data(false)

      m = d['message']
      m['timer_id'] = self.id

      sm = d['m']

      { 'point' => 'trigger',
        'exid' => self.exid,
        'nid' => self.onid,
        'bnid' => self.nid,
        'type' => self.type,
        'schedule' => self.schedule,
        'timer_id' => self.id,
        'message' => m,
        'sm' => sm }
    end

    def ntime_t

      @ntime_t ||= Time.parse(ntime)
    end
  end
end


module Flor

  class Waiter

    def initialize(exid, opts)

      serie, timeout, on_timeout =
        expand_args(opts)

      # TODO fail if the serie mixes msg_waiting with row_waiting...

      @exid = exid
      @serie = serie
      @timeout = timeout
      @on_timeout = on_timeout

      @queue = []
      @mutex = Mutex.new
      @var = ConditionVariable.new

      @executor = nil
    end

    ROW_PSEUDO_POINTS = %w[ status tag tasker var variable ]
      # "tasker", not "task", since "task" is already a message point

    def row_waiter?

      @serie.find { |_, points|
        points.find { |po|
          pos = po.split(':')
          pos.length > 1 && ROW_PSEUDO_POINTS.include?(pos[0]) } }
    end

    def msg_waiter?

      @serie.find { |_, points|
        points.find { |po|
          ! ROW_PSEUDO_POINTS.include?(po.split(':').first) } }
    end

    def to_s

      "#{super[0..-2]}#{{ exid: @exid, timeout: @timeout }.inspect}>"
    end

    def notify(executor, message)

      @executor = executor
        # could be handy

      @mutex.synchronize do

        return false unless match?(message)

        @serie.shift
        return false if @serie.any?

        @queue << [ executor, message ]
        @var.signal
      end

      true # serie over, remove me
    end

    def check(unit, rs)

      @mutex.synchronize do

        row = nil

        loop do

          break if @serie.empty?

          row = row_match?(unit, rs)
          return false unless row

          @serie.shift
        end

        @queue << [ unit, row ]
        @var.signal
      end

      true # serie over, remove me

    rescue => err

#puts "!" * 80; p err
      unit.logger.warn(
        "#{self.class}#check()", err, '(returning true, aka remove me)')

      true # remove me
    end

    def wait

      @mutex.synchronize do

        if @queue.empty?

          @var.wait(@mutex, @timeout)
            # will wait "in aeternum" if @timeout is nil

          if @queue.empty?
            fail RuntimeError.new(
              "timeout for #{self.to_s}, " +
              "msg_waiter? #{ !! msg_waiter?}, row_waiter? #{ !! row_waiter?}"
            ) if @on_timeout == 'fail'
            return { 'exid' => @exid, 'timed_out' => @on_timeout }
          end
        end

        @queue.shift[1]
      end
    end

    def to_query_hashes

      @serie
        .inject([ [], [] ]) { |a, (nid, points)|

          points.each do |point|

            ss = point.split(':')

            h = {}
            h[:exid] = @exid if @exid
            h[:nid] = nid if nid

            case ss[0]
            when 'status'
              h[:status] = ss[1]
              a[0] << h
            when 'tag', 'tasker', 'var', 'variable'
              t = ss[0]; t = 'var' if t == 'variable'
              h[:type] = t
              h[:name] = ss[1]
              h[:value] = ss[2] if ss[2]
              a[1] << h
            else
              fail ArgumentError, "cannot turn to query_hash, #{self.inspect}"
            end
          end

          a }
    end

    protected

    def match?(message)

      mpoint = message['point']

      return false if @exid && @exid != message['exid'] && mpoint != 'idle'

      nid, points = @serie.first
      mnid = message['nid']

      return false if nid && mnid && nid != mnid

      return false unless points.find { |point|
        ps = point.split(':')
        next false if ps[0] != mpoint
        next false if ps[1] && ! message['tags'].include?(ps[1])
        true }

      true
    end

    def row_match?(unit, rs)

      nid, points = @serie.first

      row = nil

      points.find { |point|
        ps = point.split(':')
        row = send("row_match_#{ps[0]}?", unit, rs, nid, ps[1..-1]) }

      row
    end

    def row_match_status?(unit, rs, _, cdr)

      rs[0].find { |exe|
        (@exid == nil || exe.exid == @exid) &&
        exe.status == cdr.first }
    end

    def row_match_tag?(unit, rs, nid, (name, value))

      rs[1].find { |ptr|
        ptr.type == 'tag' &&
        (@exid == nil || ptr.exid == @exid) &&
        (nid == nil || ptr.nid == nid) &&
        (name == nil || ptr.name == name) &&
        (value == nil || ptr.value == value) }
    end

    def row_match_var?(unit, rs, nid, (name, value))

      rs[1].find { |ptr|
        ptr.type == 'var' &&
        (@exid == nil || ptr.exid == @exid) &&
        (nid == nil || ptr.nid == nid) &&
        (name == nil || ptr.name == name) &&
        (value == nil || ptr.value == value.to_s) }
    end
    alias row_match_variable?  row_match_var?

    def row_match_tasker?(unit, rs, nid, (name, value))

      rs[1].find { |ptr|
        ptr.type == 'tasker' &&
        (@exid == nil || ptr.exid == @exid) &&
        (nid == nil || ptr.nid == nid) &&
        (name == nil || ptr.name == name) &&
        (value == nil || ptr.value == value) }
    end

    def expand_args(opts)

      owait = opts[:wait]
      otimeout = opts[:timeout]
      oontimeout = opts[:on_timeout] || opts[:ontimeout] || 'fail'

      case owait
      when nil, true
        [ [ [ nil, %w[ failed terminated ] ] ], # serie
          otimeout,
          oontimeout ]
      when Numeric
        [ [ [ nil, %w[ failed terminated ] ] ], # serie
          owait, # timeout
          oontimeout ]
      when String, Array
        [ parse_serie(owait), # serie
          otimeout,
          oontimeout ]
      else
        fail ArgumentError.new(
          "don't know how to deal with #{owait.inspect} (#{owait.class})")
      end
    end

    WAIT_REX =
      %r{
        \A
        ([0-9_\-]+)?[ ]*
        (
          [a-z]+(?::[^:|;,\s]+){0,2}
          (?:[|, ][a-z]+(:[^:|;,\s]+){0,2})*
        )\z
      }x

    def parse_serie(s)

      return s if s.is_a?(Array) && s.collect(&:class).uniq == [ Array ]

      (s.is_a?(String) ? s.split(';') : s)
        .collect { |ss|
          m = ss.strip.match(WAIT_REX)
          fail ArgumentError.new(
            "cannot parse #{ss.strip.inspect} wait directive") unless m
          ni, pt = m[1, 2]
          [ ni, pt.split(/[|,]/).collect(&:strip) ] }
    end
  end
end


require 'zlib'

module Flor

  class Storage

    attr_reader :unit, :db, :models

    attr_reader :mutex
      # might be useful for some implementations

    attr_accessor :archive

    def initialize(unit)

      @unit = unit

      @models = {}
      @archive = @unit.conf['sto_archive']
      @mutex = @unit.conf['sto_sync'] ? Mutex.new : nil

      connect
    end

    def shutdown

      @db.disconnect
#p [ :disconnected, @db.object_id ]
    end

    def db_version

      (@db[:schema_info].first rescue {})[:version]
    end

    def migration_version

      Dir[File.join(File.dirname(__FILE__), '../migrations/*.rb')]
        .inject([]) { |a, fn|
          m = File.basename(fn).match(/^(\d{4})_/)
          a << m[1].to_i if m
          a
        }
        .max
    end

    def ready?

      db_version == migration_version
    end

    def synchronize(on=true, &block)

      Thread.current[:sto_errored_items] = nil if on

      if @mutex && on
        @mutex.synchronize(&block)
      else
        block.call
      end
    end

    def transync(on=true, &block)

      Thread.current[:sto_errored_items] = nil if on

      if @mutex && on
        @mutex.synchronize { @db.transaction(&block) }
      else
        block.call
      end
    end

    def migrate(to=nil, from=nil, opts=nil)

      opts = [ to, from, opts ].find { |e| e.is_a?(Hash) } || {}
      opts[:target] ||= to if to.is_a?(Integer)
      opts[:current] ||= from if from.is_a?(Integer)

      opts[:table] = (
        @unit.conf['db_migration_table'] ||
        @unit.conf['sto_migration_table'] ||
        :schema_info).to_sym
      opts[:column] = (
        @unit.conf['db_migration_column'] ||
        @unit.conf['sto_migration_column'] ||
        :version).to_sym
          #
          # defaults for the migration version table:
          # { table: :schema_info,
          #   column: :version }

      skip =
        @unit.conf['db_sparse_migrations'] ||
        @unit.conf['sto_sparse_migrations'] ||
        opts[:sparse_migrations]
      if skip && ! opts.has_key?(:allow_missing_migration_files)
        opts[:allow_missing_migration_files] = true
      end

      dir =
        @unit.conf['db_migrations'] ||
        @unit.conf['db_migration_dir'] ||
        @unit.conf['sto_migrations'] ||
        @unit.conf['sto_migration_dir'] ||
        opts[:migrations] ||
        opts[:migration_dir] ||
        Flor.migration_dir

      synchronize do

        Sequel::Migrator.run(@db, dir, opts)
      end
    end

    # Delete tables in the storage database that begin with "flor_"
    # and have more than 2 columns (the Sequel schema_info table has 1 column
    # as of this writing)
    #
    def delete_tables

      @db.tables.each { |t|
        @db[t].delete \
          if t.to_s.match(/^flor_/) && @db[t].columns.size > 2 }
    end

    def load_execution(exid)

      synchronize do

        e = @db[:flor_executions]
          .select(:id, :content)
          .first(exid: exid) # status active or terminated doesn't matter

        return {
          'exid' => exid, 'nodes' => {}, 'counters' => {},
          'start' => Flor.tstamp, 'size' => 0
        } unless e

        ex = from_blob(e[:content])

        fail("couldn't parse execution (db id #{e[:id].to_i})") unless ex

        ex['id'] = e[:id].to_i
        ex['size'] = e[:content].size

        ex
      end
    end

    def put_execution(exe)

      status =
        exe['nodes'].find { |_, n| n['status'].last['status'] != 'ended' } ?
        'active' :
        'terminated'

      id = exe['id']

      if id

        exe['end'] ||= Flor.tstamp \
          if status == 'terminated'
        exe['duration'] = Time.parse(exe['end']) - Time.parse(exe['start']) \
          if exe['end']
      end

      data = to_blob(exe)
      exe['size'] = data.size

      u = @unit.identifier

      transync do

        now = Flor.tstamp

        if id

          @db[:flor_executions]
            .where(id: id.to_i)
            .update(
              content: data,
              status: status,
              mtime: now,
              munit: u)

        else

          exe['id'] =
            @db[:flor_executions]
              .insert(
                domain: Flor.domain(exe['exid']),
                exid: exe['exid'],
                content: data,
                status: status,
                ctime: now,
                mtime: now,
                cunit: u,
                munit: u)
              .to_i
        end

        remove_nodes(exe, status, now)
        update_pointers(exe, status, now)
      end

      exe
        # return the execution hash

    rescue => err

      Thread.current[:sto_errored_items] = [ exe ]
      raise err
    end

    def load_messages(exe_count)

      exe_count += 2
        # load two more, could prove useful if they vanish like "petits pains"

      synchronize do

        _exids_being_processed =
          @db[:flor_messages]
            .select(:exid)
            .exclude(status: %w[ created consumed ])
        _exids =
          @db[:flor_messages]
            .select(:exid)
            .exclude(exid: _exids_being_processed)
            .exclude(status: %w[ reserved consumed ])
            .limit(exe_count)
        @db[:flor_messages]
          .where(exid: _exids, status: 'created')
          .inject({}) { |h, m| (h[m[:exid]] ||= []) << m; h }
      end

    rescue => err

      @unit.logger.warn(
        "#{self.class}#load_messages()", err, '(returning {})')

      {}
    end

    def reserve_all_messages(messages)

      now = Flor.tstamp
      count = 0

      transync do

        messages.each do |m|

          c = @db[:flor_messages]
            .where(
              id: m[:id].to_i, status: 'created',
              mtime: m[:mtime], munit: m[:munit])
            .update(
              status: 'reserved', mtime: now, munit: @unit.identifier)

          raise Sequel::Rollback if c != 1

          count += 1
        end
      end

      count == messages.size
        # true means success: all the messages could be reserved,
        # executor is clear to work on the execution

    rescue => err

      @unit.logger.warn(
        "#{self.class}#reserve_all_messages()", err, '(returning false)')

      false
        # failure
    end

    def any_message?

      synchronize do

        @db[:flor_messages].where(status: 'created').count > 0
      end

    rescue => err

      @unit.logger.warn(
        "#{self.class}#any_message?()", err, '(returning false)')

      false
    end

    def fetch_traps(exid)

      synchronize do

        traps
          .where(status: 'active')
          .where(domain: split_domain(exid))
          .all
      end

    rescue => err

      @unit.logger.warn(
        "#{self.class}#fetch_traps()", err, '(returning [])')

      []
    end

    POINTS_TO_ARCHIVE = %w[ terminated failed ceased ]

    def consume(messages)

      if @archive
        consume_and_archive(messages)
      else
        consume_and_discard(messages)
      end

    rescue => err

      Thread.current[:sto_errored_items] = messages
      raise err
    end

    def put_messages(ms, syn=true)

      return if ms.empty?

      n = Flor.tstamp
      u = @unit.identifier

      synchronize(syn) do

        stored, unstored = ms.partition { |m| m['mid'] }

        #
        # de-reserve any previously stored message, might happen
        # for "terminated" messages that got queued back to let
        # other messages get processed

        @db[:flor_messages]
          .where(id: stored.collect { |m| m['mid'] })
          .update(status: 'created', mtime: n, munit: u) \
            if stored.any?

        #
        # store new messages

        @db[:flor_messages]
          .import(
            [ :domain, :exid, :point, :content,
              :status, :ctime, :mtime, :cunit, :munit ],
            unstored.map { |m|
              [ Flor.domain(m['exid']), m['exid'], m['point'], to_blob(m),
                'created', n, n, u, u ] }) \
                  if unstored.any?
      end

      @unit.wake_up

    rescue => err

      Thread.current[:sto_errored_items] = ms
      raise err
    end

    def put_message(m)

      put_messages([ m ])
    end

    def unreserve_messages(max_sec)

      tstamp = Flor.tstamp(Time.now - max_sec)
      tstamp = tstamp[0..tstamp.rindex('.')]

      synchronize do

        @db[:flor_messages]
          .where(status: 'reserved')
          .where { mtime < tstamp }
          .update(status: 'created')
      end

    rescue => err

      @unit.logger.warn(
        "#{self.class}#unreserve_messages(#{max_sec})", err, '(returning nil)')

      -1 # not zero, to indicate a problem
    end

    def put_timer(message)

      type, string = determine_type_and_schedule(message)

      next_time = compute_next_time(type, string)

      now = Flor.tstamp
      u = @unit.identifier

      synchronize do

        @db[:flor_timers]
          .insert(
            domain: Flor.domain(message['exid']),
            exid: message['exid'],
            nid: message['nid'],
            onid: message['onid'] || message['nid'],
            bnid: message['nid'],
            type: type,
            schedule: string,
            ntime: next_time,
            content: to_blob(message),
            count: 0,
            status: 'active',
            ctime: now,
            mtime: now,
            cunit: u,
            munit: u)
      end

      @unit.wake_up

    rescue => err

      Thread.current[:sto_errored_items] = [ message ]
      raise err
    end

    def trigger_timers

      synchronize do

        load_timers.each do |t|

          @db.transaction do

            next unless reschedule_timer(t) == 1

            trigger_timer(t)
          end
        end
      end
    end

    def put_trap(node, tra)

      exid = node['exid']
      dom = Flor.domain(exid)
      now = Flor.tstamp
      u = @unit.identifier

      id =
        synchronize do

    #points = att_a('point', 'points', nil)  ### TODO
    #tags = att_a('tag', 'tags', nil)        #
    #heats = att_a('heat', 'heats', nil)     #
    #heaps = att_a('heap', 'heaps', nil)     #
    #names = att_a('name', 'names', nil)     #
        #
      #opts[:heap] = theaps.split(',') if theaps
      #opts[:heat] = theats.split(',') if theats
            #
          @db[:flor_traps]
            .insert(
              domain: dom,
              exid: exid,
              nid: tra['nid'],
              onid: tra['onid'] || tra['nid'],
              bnid: tra['bnid'],
              trange: tra['range'],
              tpoints: commaify(tra['points']),
              ttags: commaify(tra['tags']),
              theats: commaify(tra['heats']),
              theaps: commaify(tra['heaps']),
              content: to_blob(tra),
              status: 'active',
              ctime: now,
              mtime: now,
              cunit: u,
              munit: u)
        end

      traps[id]

    rescue => err

      Thread.current[:sto_errored_items] = [ node, tra ]
      raise err
    end

    def trace(exid, nid, tracer, text)

      text = text.is_a?(String) ? text : JSON.dump(text)

      synchronize do

        @db[:flor_traces]
          .insert(
            domain: Flor.domain(exid),
            exid: exid,
            nid: nid,
            tracer: tracer,
            text: text,
            ctime: Flor.tstamp,
            cunit: @unit.identifier)
      end
    end

    def put_task_pointer(msg, tname, tconf)

      exid = msg['exid']
      dom = Flor.domain(exid)

      synchronize do

        @db[:flor_pointers]
          .insert(
            domain: dom,
            exid: exid,
            nid: msg['nid'],
            type: 'tasker',
            name: tname,
            ctime: Flor.tstamp,
            cunit: @unit.identifier)
      end
    end

    def fetch_next_time

      t =
        synchronize do
          @db[:flor_timers]
            .select(:ntime)
            .order(:ntime)
            .first(status: 'active')
        end

      t ? t[:ntime].split('.').first : nil

    rescue => err

      @unit.logger.warn(
        "#{self.class}#fetch_next_time()", err, '(returning nil)')

      nil
    end

    protected

    def _commaify(o)

      if Flor.is_regex_tree?(o)
        o[1].to_s
      else #if o.is_a?(String)
        o.split(/\s*,\s*/).join(',')
      end
    end

    def commaify(o)

      return nil unless o

      o = [ o ] if Flor.is_regex_tree?(o)
      o = [ o ] unless o.is_a?(Array)

      o.collect { |e| _commaify(e) }.join(',')
    end

    def consume_and_archive(messages)

      transync do

        n = Flor.tstamp
        u = @unit.identifier

        @db[:flor_messages]
          .where(
            id: messages.collect { |m| m['mid'] }.compact)
          .update(
            status: 'consumed', mtime: n, munit: u)

        @db[:flor_messages]
          .import(
            [ :domain, :exid, :point, :content,
              :status, :ctime, :mtime, :cunit, :munit ],
            messages
              .select { |m|
                ! m['mid'] && POINTS_TO_ARCHIVE.include?(m['point']) }
              .map { |m|
                [ Flor.domain(m['exid']), m['exid'], m['point'], to_blob(m),
                  'consumed', n, n, u, u ] })
      end
    end

    def consume_and_discard(messages)

      synchronize do

        @db[:flor_messages]
          .where(
            id: messages.collect { |m| m['mid'] }.compact)
          .delete
      end
    end

    def load_timers

      now = Flor.tstamp
      no = now[0, now.rindex('.')]

      timers
        .where(status: 'active')
        .where { ntime <= no }
        .order(:ntime)
        .all

    rescue => err

      @unit.logger.warn("#{self.class}#load_timers()", err, '(returning [])')

      []
    end

    def trigger_timer(t)

      put_messages([ t.to_trigger_message ], false)
    end

    def reschedule_timer(t)

      w = { id: t.id.to_i, status: 'active', mtime: t.mtime, munit: t.munit }

      if t.type != 'at' && t.type != 'in'

        @db[:flor_timers]
          .where(w)
          .update(
            count: t.count.to_i + 1,
            status: 'active',
            ntime: compute_next_time(t.type, t.schedule, t.ntime_t),
            mtime: Flor.tstamp,
            munit: @unit.identifier)

      elsif @archive

        @db[:flor_timers]
          .where(w)
          .update(
            count: t.count.to_i + 1,
            status: 'triggered',
            mtime: Flor.tstamp,
            munit: @unit.identifier)

      else

        @db[:flor_timers]
          .where(w)
          .delete
      end
    end

    def remove_nodes(exe, status, now)

      exid = exe['exid']

      x = (status == 'terminated') ? {} : { nid: exe['nodes'].keys }
        # if 'terminated' include all nodes

      if @archive
        @db[:flor_timers].where(exid: exid).exclude(x).update(status: 'removed')
        @db[:flor_traps].where(exid: exid).exclude(x).update(status: 'removed')
      else
        @db[:flor_timers].where(exid: exid).exclude(x).delete
        @db[:flor_traps].where(exid: exid).exclude(x).delete
      end

      #@db[:flor_pointers].where(exid: exid).exclude(x).delete
        # done in update_pointers
    end

    def update_pointers(exe, status, now)

# Q Should we archive old pointers?
#   Well, it might be better to only archive the execution and leave
#   in there enough information...

      exid = exe['exid']

      if status == 'terminated'
        @db[:flor_pointers].where(exid: exid).delete
        return
      end

      @db[:flor_pointers]
        .where(exid: exid)
        .where(Sequel.|({ type: %w[ var ] }, Sequel.~(nid: exe['nodes'].keys)))
        .delete
          #
          # Delete all pointer to vars, their value might have changed,
          # let's reinsert them.
          # Delete pointers to gone nodes.

      dom = Flor.domain(exid)
      u = @unit.identifier

      pointers = exe['nodes']
        .inject([]) { |a, (nid, node)|

          ts = node['tags']
          ts.each { |t|
            a << [ dom, exid, nid, 'tag', t, nil, now, u ] } if ts

          vs = nid == '0' ? node['vars'] : nil
          vs.each { |k, v|
            case v; when Numeric, String, TrueClass, FalseClass, NilClass
              a << [ dom, exid, '0', 'var', k, v.to_s, now, u ]
            when Array, Hash
              s = '(array)'; s = '(object)' if v.is_a?(Hash)
              a << [ dom, exid, '0', 'var', k, s, now, u ]
            else
              a << [ dom, exid, '0', 'var', k, nil, now, u ]
            end } if vs

          #ta = node['heap'] == 'task' ? node['task'] : nil
          ta = node['task']
          a << [ dom, exid, nid, 'tasker', ta['tasker'], ta['name'], now, u ] \
            if ta

          a }

      cps = @db[:flor_pointers] # current pointers
        .where(exid: exid)
        .select(:nid, :type, :name)
        .all
      pointers.reject! { |_, _, ni, ty, na, _, _, _|
        cps.find { |cp| cp[:nid] == ni && cp[:type] == ty && cp[:name] == na } }
          #
          # don't insert when already inserted

      @db[:flor_pointers]
        .import(
          [ :domain, :exid, :nid, :type, :name, :value, :ctime, :cunit ],
          pointers)
    end

    def determine_type_and_schedule(message)

      t, s = message['type'], message['string']
      return [ t, s ] if t

      t = Fugit.determine_type(s)
      return [ t, s ] if t

      s = "every #{s}"
      return [ 'cron', s ] if Fugit.parse_nat(s)

      nil
    end

    def compute_next_time(type, string, from=nil)

      f =
        case type
        when 'cron' then Fugit.parse_cron(string) || Fugit.parse_nat(string)
        when 'at' then Fugit.parse_at(string)
        when 'in' then Fugit.parse_duration(string)
        #when 'every' then Fugit.parse_duration(string)
        else Fugit.parse(string)
        end

      nt = f.is_a?(Time) ? f : f.next_time(from || Time.now) # local...

      Flor.tstamp(nt.utc)
    end

    def split_domain(exid)

      Flor.domain(exid)
        .split('.')
        .inject([]) { |a, elt| a << [ a.last, elt ].compact.join('.'); a }
    end

    class DbLogger

      def initialize(unit); @unit = unit; end

      def info(msg); @unit.logger.db_log(:info, msg); end
      def error(msg); @unit.logger.db_log(:error, msg); end
    end

    def connect

      uri = @unit.conf['sto_uri']

      #uri = DB.uri if uri == 'DB' && defined?(DB)
      uri = (Kernel.const_get(uri).uri rescue uri) if uri.match(/\A[A-Z]+\z/)
        # for cases where `sto_uri: "DB"`

      @db = Sequel.connect(uri)

      class << @db; attr_accessor :flor_unit; end
      @db.flor_unit = @unit

      if cv = @unit.conf['sto_connection_validation']

        to = cv.is_a?(Numeric) || cv.is_a?(String) ? cv.to_i : -1

        @db.extension(:connection_validator)
        @db.pool.connection_validation_timeout = to
          # NB: -1 means "check at every use"
      end

      @db_logger = DbLogger.new(@unit)
      @db.loggers << @db_logger
    end

    class << self

      def to_blob(h)

        h ? Sequel.blob(Zlib::Deflate.deflate(JSON.dump(h))) : nil
#rescue => e; pp h; raise e
      end

      def from_blob(content)

        content ? JSON.parse(Zlib::Inflate.inflate(content)) : nil
      end
    end

    def to_blob(h); self.class.to_blob(h); end
    def from_blob(content); self.class.from_blob(content); end
  end

  # module Flor

  class << self

    def to_blob(h); ::Flor::Storage.to_blob(h); end
    def from_blob(content); ::Flor::Storage.from_blob(content); end
  end
end


module Flor

  class DummySequelAdapter < Sequel::Dataset

    class Db
      def supports_schema_parsing?; false; end
      def transaction(opts={}); yield; end
    end

    def initialize(opts)
      @opts = opts
      @db = Db.new
      @cache = {} # necessary for Sequel >= 4.42.0
    end

    def fetch_rows(sql); yield([]); end
    #DJV
    # add missing methods from dummy adaptor
    def typecast_value_boolean(opts={});true;end
    def test_connection();true;end
    #DJV

    DB = Sequel.connect(:adapter => Flor::DummySequelAdapter)
  end

  class FlorModel < Sequel::Model(DummySequelAdapter::DB)

    self.require_valid_table = false

    class << self

      attr_accessor :unit
    end

    def unit; self.class.unit; end
    def storage; unit.storage; end

    # Return a Flor::Execution instance linked to this model
    #
    def execution(reload=false)

      exid = @values[:exid]; return nil unless exid

      @flor_model_cache_execution = nil if reload

      @flor_model_cache_execution ||= unit.executions[exid: exid]
    end

    # Returns the node hash linked to this model
    #
    def node(reload=false)

      nid = @values[:nid]; return nil unless nid
      exe = execution(reload); return nil unless exe

      nodes = exe.data['nodes']; return nil unless nodes
      nodes[nid]
    end

    def payload(reload=false)

      nod = node(reload)
      nod ? nod['payload'] : nil
    end

    def data(cache=true)

      cache ? (@flor_model_cache_data = _data) : _data
    end

    def refresh

      instance_variables
        .each do |k|
          instance_variable_set(k, nil) \
            if k.to_s.start_with?('@flor_model_cache_')
        end

      super
    end

    def to_h

      values.inject({}) do |h, (k, v)|
        if k == :content
          h[:data] = data
        else
          h[k] = v
        end
        h
      end
    end

    class << self

      def from_h(h)

        cols = columns

        h
          .inject({}) { |r, (k, v)|
            k = k.to_sym
            if k == :data
              r[:content] = Flor.to_blob(v)
            elsif cols.include?(k)
              r[k] = v
            end
            r }
      end
    end

    protected

    def _data

      d = Flor::Storage.from_blob(content)
      d['id'] = id if d.is_a?(Hash)

      d
    end
  end

  MODELS = [ :executions, :timers, :traces, :traps, :pointers, :messages ]

  dir = File.dirname(__FILE__)
  MODELS.each { |m| require File.join(dir, 'models', "#{m.to_s[0..-2]}.rb") }

  def self.add_model(key, parent_module=Flor, table_prefix='flor_')

    Flor::Storage.send(:define_method, key) do

      s = self
      c = Flor.to_camel_case(key.to_s[0..-2])

      @models[key] ||=
        parent_module.const_set(
          "#{c}#{@db.object_id.to_s.gsub('-', 'M')}",
          Class.new(parent_module.const_get(c)) do
            self.dataset = s.db["#{table_prefix}#{key}".to_sym]
            self.unit = s.unit
          end)
    end

    Flor::Scheduler.send(:define_method, key) do

      @storage.send(key)
    end
  end

  MODELS.each do |k|

    add_model(k)
  end
end


module Flor

  class Executor

    attr_reader :unit
    attr_reader :execution
    attr_reader :hooks
    attr_reader :traps

    def initialize(unit, hooks, traps, execution)

      @unit = unit
      @execution = execution

      @hooks = hooks # raw hooks if any, fresh from the loader
      @traps = traps # array of Trap instances

      @htraps = nil
    end

    def conf; @unit.conf; end
    def exid; @execution['exid']; end

    def node(msg_or_nid, node_instance=false)

      return nil unless msg_or_nid

      nid = msg_or_nid
      msg = msg_or_nid
      #
      if nid.is_a?(String)
        msg = nil
      else
        nid = msg['nid']
      end

      n = @execution['nodes'][nid]

      return nil unless n
      node_instance ? Flor::Node.new(self, n, msg) : n
    end

    def counter(key)

      @execution['counters'][key.to_s] || -1
    end

    def counter_add(key, count)

      k = key.to_s

      @execution['counters'][k] ||= 0
      @execution['counters'][k] += count
    end

    def counter_next(key)

      counter_add(key, 1)
    end

    def trigger_trap(trap, message)

      del, msgs = trap.trigger(self, message)
      @traps.delete(trap) if del

      msgs
    end

    def trigger_hook(hook, message)

      m =
        case
        when hook.respond_to?(:notify) then :notify
        when hook.respond_to?(:on_message) then :on_message
        else :on
        end
      as =
        case hook.method(m).arity
        when 3 then [ @unit, self, message ]
        when 2 then [ self, message ]
        else [ message ]
        end

      r = hook.send(m, *as)

      Flor.is_array_of_messages?(r) ? r : []
    end

    def trigger_block(block, opts, message)

      r =
        case block.arity
        when 1 then block.call(message)
        when 2 then block.call(message, opts)
        else block.call(self, message, opts)
        end

      r.is_a?(Array) && r.all? { |e| e.is_a?(Hash) } ? r : []
        # be lenient with block hooks, help them return an array
    end

    # Given a nid, returns a copy of all the var the node sees at that point.
    #
    def vars(nid, vs={})

      n = node(nid); return vs unless n

      (n['vars'] || {})
        .each { |k, v| vs[k] = Flor.dup(v) unless vs.has_key?(k) }

      pnid = n['parent']

      if @unit.loader && pnid == nil && n['vdomain'] != false

        @unit.loader.variables(n['vdomain'] || Flor.domain(@exid))
          .each { |k, v| vs[k] = Flor.dup(v) unless vs.has_key?(k) }
      end

      if cn = n['cnid']; vars(cn, vs); end
      vars(pnid, vs) if pnid

      vs
    end

    def traps_and_hooks

      @htraps = nil if @htraps && @htraps.size != @traps.size

      @htraps ||= @traps.collect(&:to_hook)
      @hhooks ||= @hooks.collect(&:to_hook)

      @htraps + @hhooks
    end

    protected

    def make_node(message)

      nid = message['nid']

      n = Flor.tstamp

      node = {
        'nid' => nid,
        'parent' => message['from'],
        'payload' => message['payload'],
        'status' => [ { 'status' => nil, 'point' => 'execute', 'ctime' => n } ],
        'ctime' => n,
        'mtime' => n }

      %w[ vars vdomain cnid dbg ].each do |k|
        v = message[k]
        node[k] = v if v != nil
      end
        #
        # vars: variables
        # vdomain: variable domain (used in conjuction with the loader)
        # cnid: closure nid
        # dbg: used to debug messages (useful @node['dbg'] when 'receive')

      %w[ error cancel timeout ]
        .each { |k|
          h = message["on_#{k}_handler"]
          node["on_#{k}"] = [ h ] if h }

      @execution['nodes'][nid] = node
    end

    def determine_heat(message)

      nid = message['nid']

      return unless nid

      node =
        message['point'] == 'execute' ?
        make_node(message) :
        @execution['nodes'][nid]

      return if node.nil? || node['heat']

      n = Flor::Node.new(self, node, message)

      mt = message['tree']
      mt = [ 'noeval', [ 0 ], mt[2] ] \
        if mt[0] == '_' && Flor.is_array_of_trees?(mt[1])

      nt = n.lookup_tree(nid)

      node['tree'] = mt if mt && (mt != nt)
      tree = node['tree'] || nt

      node['heat0'] = t0 = tree[0]
      node['heat'] = heat = n.deref(t0)

      if heat == nil && ! message['accept_symbol']

        node['heat'] = '(none)'

        fail FlorError.new("cannot find #{t0.inspect}", n) if tree[1].empty?
        fail FlorError.new("don't know how to apply #{t0.inspect}", n)
# TODO how about _ref and letting that procedure fail
      end

      node['heap'] = heap = n.reheap(tree, heat)

      # "exceptions"

      if heat == nil #&& message['accept_symbol']
        #
        # tag: et al

        node['tree'] = message['tree'] = t = [ '_sqs', tree[0], tree[2] ]

        node['heat0'] = t[0]
        node['heat'] = h = n.deref(t[0])
        node['heap'] = n.reheap(t, h)

      elsif heap == 'task' && heat[0] == '_tasker'
        #
        # rewrite `alpha` into `task alpha`

        l = message['tree'][2]

        message['otree'] = Flor.dup(message['tree'])

        message['tree'][0] =
          'task'
        message['tree'][1].unshift(
          [ '_att', [ [ '_sqs', heat[1]['tasker'], l ] ], l ])
      end
    end

    def execute(message)

      apply(@execution['nodes'][message['nid']], message)
    end

    def apply(node, message)

      heap = node['heap']

      heac = Flor::Procedure[heap]
unless heac
  puts "v" * 80
  puts "===node:"
  p node['nid']
  p heap
  puts "."
  pp node
  puts "===message:"
  p message['point']
  puts "."
  pp message
  puts "." * 80
end
      fail NameError.new("unknown procedure #{heap.inspect}") unless heac

      head = heac.new(self, node, message)

      return [ head.rewrite ] if head.is_a?(Flor::Macro)

      point = message['point']
      point = 'kill' if message['flavour'] == 'kill'

      messages = head.send("do_#{point}")

      fail StandardError.new(
        "#{heap}/#{message['point']} did not return an Array"
      ) unless messages.is_a?(Array)

      messages
    end

    def toc_messages(message)

      return [] if message['flavour'] == 'flank'

      m = message.select { |k, v| %w[ exid nid from payload ].include?(k) }
      m['sm'] = message['m']
      m['point'] = message['from'] == '0' ? 'terminated' : 'ceased'
      m['cause'] = message['cause'] if message.has_key?('cause')

      [ m ]
    end

    def receive(message)

      messages = leave_node(message)

      nid = message['nid']

      return messages + toc_messages(message) unless nid
        # 'terminated' or 'ceased'

      node = @execution['nodes'][nid]

      return messages unless node
        # node gone...

      messages + apply(node, message)
    end

    def leave_node(message)

      return [] if %w[ flank part ].include?(message['flavour'])

      fnid = message['from']; return [] unless fnid
      fnode = @execution['nodes'][fnid]; return [] unless fnode

      remove_node(message, fnode) +
      leave_tags(message, fnode)
    end

    def remove_node(message, node)

      nid = node['nid']
      cls = node['closures']

      pro = Flor::Procedure.make(self, node, message)
      pro.end

      cancels = pro.send(:wrap_cancel_children, 'cancel_trailing' => true)

      return cancels if cls && cls.any?
        # don't remove the node if it's a closure for some other nodes

      return cancels if nid == '0'
        # don't remove if it's the "root" node

      @unit.archive_node(message['exid'], node)
        # archiving is only active during testing

      #update_parent_node_tree(node)

      @execution['nodes'].delete(nid)

      cancels
    end

  # This saves the modified trees in the parent when the node is removed
  # it works ok except for 3 (2017-05-9) failing specs.
  #
  # Introducing 3 exceptions for this is not interesting.
  #
#    def update_parent_node_tree(node)
#
#      t = node['tree']; return unless t
##return if t[0] == '_apply'
#      pnode = @execution['nodes'][node['parent']]; return unless pnode
#
#      pt =
#        pnode['tree'] ||
#        Flor::Node.new(self, pnode, nil).lookup_tree(pnode['nid'])
#      cid =
#        Flor.child_id(node['nid'])
#
#      if cid == pt[1].size # head "exception"
#        return if pt[0] == t
#        pt[0] = t
#        pnode['tree'] = pt
#        return
#      end
#
#      pt[1][cid] = t
#      pnode['tree'] = pt
#    end

    def leave_tags(message, node)

      ts = node['tags']; return [] unless ts && ts.any?

      [ { 'point' => 'left',
          'tags' => ts,
          'exid' => exid,
          'nid' => node['nid'],
          'payload' => message['payload'] } ]
    end

    def error_reply(node, message, err)

      m = message
        .select { |k, v| %w[ sm exid nid from payload tree ].include?(k) }

      m['point'] = 'failed'
      m['fpoint'] = message['point']
      m['error'] = Flor.to_error(err)

      @unit.logger.log_err(self, m, flag: true)

      #if m['error']['msg'].match(/\AToo many open files in system/)
      #  puts "=" * 80 + ' ...'
      #  system(`lsof #{Process.pid}`)
      #  puts "=" * 80 + ' .'
      #end
        #
        # can't seem to provoke that error, so keeping the trap
        # around but commented out...

      [ m ]
    end

    def cancel(message)

      n = @execution['nodes'][message['nid']]
      return [] unless n # node gone

      apply(n, message)
    end

    def stack_cause(message)

      pt = message['point']
      fl = message['flavour']

      cause = pt # trigger or cancel
      cause = fl if %w[ kill timeout ].include?(fl) # only for cancel, hopefully

      last = (message['cause'] ||= [])[0]

      c = { 'cause' => cause, 'at' => last && last['at'] }
      %w[ m sm nid type ].each { |k| c[k] = message[k] }

      return if c == last

      # argh, the causes in the messages go most recent first
      # while the statuses in the nodes go most recent last

      message['cause'] =
        [ c.tap { |h| h['at'] = Flor.tstamp } ] +
        message['cause']
    end

    def process(message)

      fail ArgumentError.new("incoming message has non nil or Hash payload") \
        unless message['payload'] == nil || message['payload'].is_a?(Hash)
          #
          # weed out messages with non-conforming payloads

      begin

        message['m'] = counter_next('msgs') # number messages
        message['pr'] = counter('runs') # "processing run"

        stack_cause(message) \
          if %w[ trigger cancel ].include?(message['point'])

        determine_heat(message)

        ms = []
        ms += @unit.notify(self, message) # pre

        ms += send(message['point'], message)

        message['payload'] = message.delete('pld') if message.has_key?('pld')
        message['consumed'] = Flor.tstamp

        ms += @unit.notify(self, message) # post

        ms.each { |m| m['er'] = counter('runs') } # "emitting run"

      rescue => e
        error_reply(nil, message, e)
      rescue ScriptError => se
        error_reply(nil, message, se)
      end
    end

    def trap(message)

      exid = message['exid']
      nid = message['nid']
      trap = message['trap']

      nd = node(nid)
      nd['exid'] = exid

      @traps << @unit.trap(nd, trap)

      []
    end

    def terminated(message)

      message['vars'] = @execution['nodes']['0']['vars']
        # especially useful for debugging

      []
    end

    def failed(message)

      n = node(message['nid'])

      fail RuntimeError.new(
        "node #{message['nid']} is gone, cannot flag it as failed"
      ) unless n

      n['failure'] = Flor.dup(message)

      if oep = lookup_on_error_parent(message)
        #
        # There is a parent with an 'on_error', trigger it that parent
        # with its 'on_error' turned on.
        #
        oep.trigger_on_error
        #
      else
        #
        # Simply log and don't add further messages ([]) to execute
        #
        @unit.logger.log_err(self, message)
        []
      end
    end

    def signal(message); []; end
    def entered(message); []; end
    def left(message); []; end
    def ceased(message); []; end
      #
      # Return an empty array of new messages. No direct effect.
      #
      # Some trap, hook, and/or waiter might lie in wait though.

    def lookup_on_error_parent(message)

      nd = Flor::Node.new(self, nil, message).on_error_parent
      nd ? nd.to_procedure_node : nil
    end
  end
end


module Flor

  class Trap < FlorModel

    #create_table :flor_traps do
    #
    #  primary_key :id, type: :Integer
    #  String :domain, null: false
    #  String :exid, null: false
    #  String :onid, null: false
    #  String :nid, null: false
    #  #
    #  TrueClass :tconsumed, null: false, default: false
    #  String :trange, null: false
    #  String :tpoints, null: true
    #  String :ttags, null: true
    #  String :theats, null: true
    #  String :theaps, null: true
    #  #
    #  File :content # JSON msg to trigger
    #  #
    #  String :status, null: false
    #  String :ctime, null: false
    #  String :mtime, null: false
    #
    #  String :cunit
    #  String :munit
    #
    #  String :bnid, null: false
    #
    #  index :exid
    #  index [ :exid, :nid ]
    #end

    def to_hook

      opts = {}

      opts[:consumed] = tconsumed

      opts[:point] = tpoints.split(',') if tpoints
      opts[:tag] = do_split(ttags) if ttags
      opts[:heap] = do_split(theaps) if theaps
      opts[:heat] = do_split(theats) if theats

      opts[:name] = data['names']

      case trange
      when 'execution'
        opts[:exid] = exid
      when 'subdomain'
        opts[:subdomain] = Flor.domain(exid)
      when 'domain'
        opts[:domain] = Flor.domain(exid)
      else #'subnid' # default
        opts[:exid] = exid
        opts[:subnid] = true
      end

      [ "trap#{id}", opts, self, nil ]
    end

    def trigger(executor, message)

      [ decrement, [ to_trigger_message(executor, message) ] ]
    end

    def within_itself?(executor, message)

      return false if message['exid'] != exid

      n = executor.node(message['from'] || message['nid'], true)
      return false if n == nil

      n.descendant_of?(onid)
    end

    protected

    # returns true if the trap should be removed from the execution's list
    # of traps
    #
    def decrement

      c = data['count']
      return false unless c

      c = c - 1
      data['count'] = c
      self[:status] = s = (c > 0) ? 'active' : 'consumed'

      self.update(
        content: Flor::Storage.to_blob(@flor_model_cache_data),
        status: s)

      c < 1
    end

    def to_trigger_message(executor, message)

      dat = self.data(false)
      msg = dat['message']

      msg['trap_id'] = self.id

      args = msg['arguments'] = [ [ 'msg', message ] ]

      if sig = (message['point'] == 'signal' && message['name'])
        args << [ 'sig', sig ]
      end

      case pl = dat['pl']
      when 'event'
        args << [ 'payload', msg['payload'] ]
        msg['payload'] = Flor.dup(message['payload'])
      #when 'trap'
      when Hash
        msg['payload'] = Flor.dup(pl)
      #else
      end

      { 'point' => 'trigger',
        'exid' => self.exid,
        'nid' => self.nid,
        'type' => 'trap',
        'trap' => to_hash,
        'trap_id' => self.id,
        'message' => msg,
        'sm' => message['m'] }
#.tap { |m| pp m }
#.tap { |m| pp m['message'] }
            #'dbg' => xx }
    end

    def to_hash

      values
        .inject({}) { |h, (k, v)| h[k.to_s] = v if k != :content; h }
    end

    def do_split(v)

      v
        .split(',')
        .collect { |e| Flor.is_regex_string?(e) ? Flor.to_regex(e) : e }
    end
  end
end


#
# Author:: Adam Jacob (<adam@chef.io>)
# Copyright:: Copyright (c) 2008-2016 Chef Software, Inc.
# License:: Apache License, Version 2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

require "optparse"

module Mixlib

  # == Mixlib::CLI
  # Adds a DSL for defining command line options and methods for parsing those
  # options to the including class.
  #
  # Mixlib::CLI does some setup in #initialize, so the including class must
  # call `super()` if it defines a custom initializer.
  #
  # === DSL
  # When included, Mixlib::CLI also extends the including class with its
  # ClassMethods, which define the DSL. The primary methods of the DSL are
  # ClassMethods#option, which defines a command line option, and
  # ClassMethods#banner, which defines the "usage" banner.
  #
  # === Parsing
  # Command line options are parsed by calling the instance method
  # #parse_options. After calling this method, the attribute #config will
  # contain a hash of `:option_name => value` pairs.
  module CLI

    module InheritMethods
      def inherited(receiver)
        receiver.options = deep_dup(options)
        receiver.extend(Mixlib::CLI::InheritMethods)
      end

      # object:: Instance to clone
      # This method will return a "deep clone" of the provided
      # `object`. If the provided `object` is an enumerable type the
      # contents will be iterated and cloned as well.
      def deep_dup(object)
        cloned_object = object.respond_to?(:dup) ? object.dup : object
        if cloned_object.kind_of?(Enumerable)
          if cloned_object.kind_of?(Hash)
            new_hash = cloned_object.class.new
            cloned_object.each do |key, value|
              cloned_key = deep_dup(key)
              cloned_value = deep_dup(value)
              new_hash[cloned_key] = cloned_value
            end
            cloned_object.replace(new_hash)
          else
            cloned_object.map! do |shallow_instance|
              deep_dup(shallow_instance)
            end
          end
        end
        cloned_object
      rescue TypeError
        # Symbol will happily provide a `#dup` method even though
        # attempts to clone it will result in an exception (atoms!).
        # So if we run into an issue of TypeErrors, just return the
        # original object as we gave our "best effort"
        object
      end

    end

    module ClassMethods
      # When this setting is set to +true+, default values supplied to the
      # mixlib-cli DSL will be stored in a separate Hash
      def use_separate_default_options(true_or_false)
        @separate_default_options = true_or_false
      end

      def use_separate_defaults?
        @separate_default_options ||= false
      end

      # Add a command line option.
      #
      # === Parameters
      # name<Symbol>:: The name of the option to add
      # args<Hash>:: A hash of arguments for the option, specifying how it should be parsed.
      # === Returns
      # true:: Always returns true.
      def option(name, args)
        @options ||= {}
        raise(ArgumentError, "Option name must be a symbol") unless name.kind_of?(Symbol)
        @options[name.to_sym] = args
      end

      # Get the hash of current options.
      #
      # === Returns
      # @options<Hash>:: The current options hash.
      def options
        @options ||= {}
        @options
      end

      # Set the current options hash
      #
      # === Parameters
      # val<Hash>:: The hash to set the options to
      #
      # === Returns
      # @options<Hash>:: The current options hash.
      def options=(val)
        raise(ArgumentError, "Options must recieve a hash") unless val.kind_of?(Hash)
        @options = val
      end

      # Change the banner.  Defaults to:
      #   Usage: #{0} (options)
      #
      # === Parameters
      # bstring<String>:: The string to set the banner to
      #
      # === Returns
      # @banner<String>:: The current banner
      def banner(bstring = nil)
        if bstring
          @banner = bstring
        else
          @banner ||= "Usage: #{$0} (options)"
          @banner
        end
      end
    end

    # Gives the command line options definition as configured in the DSL. These
    # are used by #parse_options to generate the option parsing code. To get
    # the values supplied by the user, see #config.
    attr_accessor :options

    # A Hash containing the values supplied by command line options.
    #
    # The behavior and contents of this Hash vary depending on whether
    # ClassMethods#use_separate_default_options is enabled.
    # ==== use_separate_default_options *disabled*
    # After initialization, +config+ will contain any default values defined
    # via the mixlib-config DSL. When #parse_options is called, user-supplied
    # values (from ARGV) will be merged in.
    # ==== use_separate_default_options *enabled*
    # After initialization, this will be an empty hash. When #parse_options is
    # called, +config+ is populated *only* with user-supplied values.
    attr_accessor :config

    # If ClassMethods#use_separate_default_options is enabled, this will be a
    # Hash containing key value pairs of `:option_name => default_value`
    # (populated during object initialization).
    #
    # If use_separate_default_options is disabled, it will always be an empty
    # hash.
    attr_accessor :default_config

    # Any arguments which were not parsed and placed in "config"--the leftovers.
    attr_accessor :cli_arguments

    # Banner for the option parser. If the option parser is printed, e.g., by
    # `puts opt_parser`, this string will be used as the first line.
    attr_accessor :banner

    # Create a new Mixlib::CLI class.  If you override this, make sure you call super!
    #
    # === Parameters
    # *args<Array>:: The array of arguments passed to the initializer
    #
    # === Returns
    # object<Mixlib::Config>:: Returns an instance of whatever you wanted :)
    def initialize(*args)
      @options = Hash.new
      @config  = Hash.new
      @default_config = Hash.new
      @opt_parser = nil

      # Set the banner
      @banner = self.class.banner

      # Dupe the class options for this instance
      klass_options = self.class.options
      klass_options.keys.inject(@options) { |memo, key| memo[key] = klass_options[key].dup; memo }

      # If use_separate_defaults? is on, default values go in @default_config
      defaults_container = if self.class.use_separate_defaults?
                             @default_config
                           else
                             @config
                           end

      # Set the default configuration values for this instance
      @options.each do |config_key, config_opts|
        config_opts[:on] ||= :on
        config_opts[:boolean] ||= false
        config_opts[:required] ||= false
        config_opts[:proc] ||= nil
        config_opts[:show_options] ||= false
        config_opts[:exit] ||= nil
        config_opts[:in] ||= nil

        if config_opts.key?(:default)
          defaults_container[config_key] = config_opts[:default]
        end
      end

      super(*args)
    end

    # Parses an array, by default ARGV, for command line options (as configured at
    # the class level).
    # === Parameters
    # argv<Array>:: The array of arguments to parse; defaults to ARGV
    #
    # === Returns
    # argv<Array>:: Returns any un-parsed elements.
    def parse_options(argv = ARGV)
      argv = argv.dup
      opt_parser.parse!(argv)

      # Deal with any required values
      options.each do |opt_key, opt_value|
        if opt_value[:required] && !config.key?(opt_key)
          reqarg = opt_value[:short] || opt_value[:long]
          puts "You must supply #{reqarg}!"
          puts @opt_parser
          exit 2
        end
        if opt_value[:in]
          unless opt_value[:in].kind_of?(Array)
            raise(ArgumentError, "Options config key :in must receive an Array")
          end
          if config[opt_key] && !opt_value[:in].include?(config[opt_key])
            reqarg = opt_value[:short] || opt_value[:long]
            puts "#{reqarg}: #{config[opt_key]} is not included in the list ['#{opt_value[:in].join("', '")}'] "
            puts @opt_parser
            exit 2
          end
        end
      end

      @cli_arguments = argv
      argv
    end

    # The option parser generated from the mixlib-cli DSL. +opt_parser+ can be
    # used to print a help message including the banner and any CLI options via
    # `puts opt_parser`.
    # === Returns
    # opt_parser<OptionParser>:: The option parser object.
    def opt_parser
      @opt_parser ||= OptionParser.new do |opts|
        # Set the banner
        opts.banner = banner

        # Create new options
        options.sort { |a, b| a[0].to_s <=> b[0].to_s }.each do |opt_key, opt_val|
          opt_args = build_option_arguments(opt_val)

          opt_method = case opt_val[:on]
                       when :on
                         :on
                       when :tail
                         :on_tail
                       when :head
                         :on_head
                       else
                         raise ArgumentError, "You must pass :on, :tail, or :head to :on"
                       end

          parse_block =
            Proc.new() do |c|
              config[opt_key] = if opt_val[:proc]
                                  if opt_val[:proc].arity == 2
                                    # New hotness to allow for reducer-style procs.
                                    opt_val[:proc].call(c, config[opt_key])
                                  else
                                    # Older single-argument proc.
                                    opt_val[:proc].call(c)
                                  end
                                else
                                  # No proc.
                                  c
                                end
              puts opts if opt_val[:show_options]
              exit opt_val[:exit] if opt_val[:exit]
            end

          full_opt = [ opt_method ]
          opt_args.inject(full_opt) { |memo, arg| memo << arg; memo }
          full_opt << parse_block
          opts.send(*full_opt)
        end
      end
    end

    def build_option_arguments(opt_setting)
      arguments = Array.new

      arguments << opt_setting[:short] if opt_setting.key?(:short)
      arguments << opt_setting[:long] if opt_setting.key?(:long)
      if opt_setting.key?(:description)
        description = opt_setting[:description].dup
        description << " (required)" if opt_setting[:required]
        description << " (included in ['#{opt_setting[:in].join("', '")}'])" if opt_setting[:in]
        opt_setting[:description] = description
        arguments << description
      end

      arguments
    end

    def self.included(receiver)
      receiver.extend(Mixlib::CLI::ClassMethods)
      receiver.extend(Mixlib::CLI::InheritMethods)
    end

  end
end


# -*- coding: utf-8 -*-
require_relative 'queue'
require_relative 'conn_adapter'

module QC
  # A Worker object can process jobs from one or many queues.
  class Worker

    attr_accessor :queues, :running

    # Creates a new worker but does not start the worker. See Worker#start.
    # This method takes a single hash argument. The following keys are read:
    # fork_worker:: Worker forks each job execution.
    # wait_interval:: Time to wait between failed lock attempts
    # connection:: PG::Connection object.
    # q_name:: Name of a single queue to process.
    # q_names:: Names of queues to process. Will process left to right.
    # top_bound:: Offset to the head of the queue. 1 == strict FIFO.
    def initialize(args={})
      @fork_worker = args[:fork_worker] || QC.fork_worker?
      @wait_interval = args[:wait_interval] || QC.wait_time

      if args[:connection]
        @conn_adapter = ConnAdapter.new(args[:connection])
      else
        @conn_adapter = QC.default_conn_adapter
      end

      @queues = setup_queues(@conn_adapter,
        (args[:q_name] || QC.queue),
        (args[:q_names] || QC.queues),
        (args[:top_bound] || QC.top_bound))
      log(args.merge(:at => "worker_initialized"))
      @running = true
    end

    # Commences the working of jobs.
    # start() spins on @running –which is initialized as true.
    # This method is the primary entry point to starting the worker.
    # The canonical example of starting a worker is as follows:
    # QC::Worker.new.start
    def start
      QC.unlock_jobs_of_dead_workers

      while @running
        @fork_worker ? fork_and_work : work
      end
    end

    # Signals the worker to stop taking new work.
    # This method has no immediate effect. However, there are
    # two loops in the worker (one in #start and another in #lock_job)
    # which check the @running variable to determine if further progress
    # is desirable. In the case that @running is false, the aforementioned
    # methods will short circuit and cause the blocking call to #start
    # to unblock.
    def stop
      @running = false
    end

    # Calls Worker#work but after the current process is forked.
    # The parent process will wait on the child process to exit.
    def fork_and_work
      cpid = fork {setup_child; work}
      log(:at => :fork, :pid => cpid)
      Process.wait(cpid)
    end

    # Blocks on locking a job, and once a job is locked,
    # it will process the job.
    def work
      queue, job = lock_job
      if queue && job
        QC.log_yield(:at => "work", :job => job[:id]) do
          process(queue, job)
        end
      end
    end

    # Attempt to lock a job in the queue's table.
    # If a job can be locked, this method returns an array with
    # 2 elements. The first element is the queue from which the job was locked
    # and the second is a hash representation of the job.
    # If a job is returned, its locked_at column has been set in the
    # job's row. It is the caller's responsibility to delete the job row
    # from the table when the job is complete.
    def lock_job
      log(:at => "lock_job")
      job = nil
      while @running
        @queues.each do |queue|
          if job = queue.lock
            return [queue, job]
          end
        end
        @conn_adapter.wait(@wait_interval, *@queues.map {|q| q.name})
      end
    end

    # A job is processed by evaluating the target code.
    # if the job is evaluated with no exceptions
    # then it is deleted from the queue.
    # If the job has raised an exception the responsibility of what
    # to do with the job is delegated to Worker#handle_failure.
    # If the job is not finished and an INT signal is trapped,
    # this method will unlock the job in the queue.
    def process(queue, job)
      start = Time.now
      finished = false
      begin
        call(job).tap do
          queue.delete(job[:id])
          finished = true
        end
      rescue => e
        handle_failure(job, e)
        finished = true
      ensure
        if !finished
          queue.unlock(job[:id])
        end
        ttp = Integer((Time.now - start) * 1000)
        QC.measure("time-to-process=#{ttp} source=#{queue.name}")
      end
    end

    # Each job includes a method column. We will use ruby's eval
    # to grab the ruby object from memory. We send the method to
    # the object and pass the args.
    def call(job)
      args = job[:args]
      receiver_str, _, message = job[:method].rpartition('.')
      receiver = eval(receiver_str)
      receiver.send(message, *args)
    end

    # This method will be called when an exception
    # is raised during the execution of the job.
    def handle_failure(job,e)
      $stderr.puts("count#qc.job-error=1 job=#{job} error=#{e.inspect} at=#{e.backtrace.first}")
    end

    # This method should be overriden if
    # your worker is forking and you need to
    # re-establish database connections
    def setup_child
      log(:at => "setup_child")
    end

    def log(data)
      QC.log(data)
    end

    private

    def setup_queues(adapter, queue, queues, top_bound)
      names = queues.length > 0 ? queues : [queue]
      names.map do |name|
        QC::Queue.new(name, top_bound).tap do |q|
          q.conn_adapter = adapter
        end
      end
    end

  end
end


require_relative 'conn_adapter'
require 'json'
require 'time'

module QC
  # The queue class maps a queue abstraction onto a database table.
  class Queue

    attr_reader :name, :top_bound
    def initialize(name, top_bound=nil)
      @name = name
      @top_bound = top_bound || QC.top_bound
    end

    def conn_adapter=(a)
      @adapter = a
    end

    def conn_adapter
      @adapter ||= QC.default_conn_adapter
    end

    # enqueue(m,a) inserts a row into the jobs table and trigger a notification.
    # The job's queue is represented by a name column in the row.
    # There is a trigger on the table which will send a NOTIFY event
    # on a channel which corresponds to the name of the queue.
    # The method argument is a string encoded ruby expression. The expression
    # will be separated by a `.` character and then `eval`d.
    # Examples of the method argument include: `puts`, `Kernel.puts`,
    # `MyObject.new.puts`.
    # The args argument will be encoded as JSON and stored as a JSON datatype
    # in the row. (If the version of PG does not support JSON,
    # then the args will be stored as text.
    # The args are stored as a collection and then splatted inside the worker.
    # Examples of args include: `'hello world'`, `['hello world']`,
    # `'hello', 'world'`.
    # This method returns a hash with the id of the enqueued job.
    def enqueue(method, *args)
      QC.log_yield(:measure => 'queue.enqueue') do
        s = "INSERT INTO #{QC.table_name} (q_name, method, args) VALUES ($1, $2, $3) RETURNING id"
        conn_adapter.execute(s, name, method, JSON.dump(args))
      end
    end

    # enqueue_at(t,m,a) inserts a row into the jobs table representing a job
    # to be executed not before the specified time.
    # The time argument must be a Time object or a float timestamp. The method
    # and args argument must be in the form described in the documentation for
    # the #enqueue method.
    # This method returns a hash with the id of the enqueued job.
    def enqueue_at(timestamp, method, *args)
      offset = Time.at(timestamp).to_i - Time.now.to_i
      enqueue_in(offset, method, *args)
    end

    # enqueue_in(t,m,a) inserts a row into the jobs table representing a job
    # to be executed not before the specified time offset.
    # The seconds argument must be an integer. The method and args argument
    # must be in the form described in the documentation for the #enqueue
    # method.
    # This method returns a hash with the id of the enqueued job.
    def enqueue_in(seconds, method, *args)
      QC.log_yield(:measure => 'queue.enqueue') do
        s = "INSERT INTO #{QC.table_name} (q_name, method, args, scheduled_at)
             VALUES ($1, $2, $3, now() + interval '#{seconds.to_i} seconds')
             RETURNING id"
        conn_adapter.execute(s, name, method, JSON.dump(args))
      end
    end

    def lock
      QC.log_yield(:measure => 'queue.lock') do
        s = "SELECT * FROM lock_head($1, $2)"
        if r = conn_adapter.execute(s, name, top_bound)
          {}.tap do |job|
            job[:id] = r["id"]
            job[:q_name] = r["q_name"]
            job[:method] = r["method"]
            job[:args] = JSON.parse(r["args"])
            if r["scheduled_at"]
              job[:scheduled_at] = Time.parse(r["scheduled_at"])
              ttl = Integer((Time.now - job[:scheduled_at]) * 1000)
              QC.measure("time-to-lock=#{ttl}ms source=#{name}")
            end
          end
        end
      end
    end

    def unlock(id)
      QC.log_yield(:measure => 'queue.unlock') do
        s = "UPDATE #{QC.table_name} SET locked_at = NULL WHERE id = $1"
        conn_adapter.execute(s, id)
      end
    end

    def delete(id)
      QC.log_yield(:measure => 'queue.delete') do
        conn_adapter.execute("DELETE FROM #{QC.table_name} WHERE id = $1", id)
      end
    end

    def delete_all
      QC.log_yield(:measure => 'queue.delete_all') do
        s = "DELETE FROM #{QC.table_name} WHERE q_name = $1"
        conn_adapter.execute(s, name)
      end
    end

    def count
      QC.log_yield(:measure => 'queue.count') do
        s = "SELECT COUNT(*) FROM #{QC.table_name} WHERE q_name = $1"
        r = conn_adapter.execute(s, name)
        r["count"].to_i
      end
    end

  end
end


require "net_http_hacked"

module Rack

  # Wraps the hacked net/http in a Rack way.
  class HttpStreamingResponse
    attr_accessor :use_ssl
    attr_accessor :verify_mode
    attr_accessor :read_timeout
    attr_accessor :ssl_version

    def initialize(request, host, port = nil)
      @request, @host, @port = request, host, port
    end

    def body
      self
    end

    def code
      response.code.to_i
    end
    # #status is deprecated
    alias_method :status, :code

    def headers
      h = Utils::HeaderHash.new

      response.to_hash.each do |k, v|
        h[k] = v
      end

      h
    end

    # Can be called only once!
    def each(&block)
      response.read_body(&block)
    ensure
      session.end_request_hacked
      session.finish
    end

    def to_s
      @body ||= begin
        lines = []

        each do |line|
          lines << line
        end

        lines.join
      end
    end

    protected

    # Net::HTTPResponse
    def response
      @response ||= session.begin_request_hacked(@request)
    end

    # Net::HTTP
    def session
      @session ||= begin
        http = Net::HTTP.new @host, @port
        http.use_ssl = self.use_ssl
        http.verify_mode = self.verify_mode
        http.read_timeout = self.read_timeout
        http.ssl_version = self.ssl_version if self.use_ssl
        http.start
      end
    end

  end

end


module LazyHighCharts
  class HighChart
    include LayoutHelper

    SERIES_OPTIONS = %w(data index legendIndex name stack type xAxis yAxis)

    attr_accessor :series_data, :options, :placeholder, :html_options
    alias :canvas :placeholder
    alias :canvas= :placeholder=

    def initialize(canvas = nil, html_opts = {})

      self.tap do |high_chart|
        high_chart.series_data ||= []
        high_chart.options ||= {}
        high_chart.defaults_options
        high_chart.html_options ||= html_opts
        high_chart.canvas = (canvas ? canvas : random_canvas_id)
        yield high_chart if block_given?
      end
    end

    #	title:		legend: 		xAxis: 		yAxis: 		tooltip: 	credits:  :plotOptions
    def defaults_options
      self.title({:text => nil})
      self.legend({:layout => "vertical", :style => {}})
      self.xAxis({})
      self.yAxis({:title => {:text => nil}, :labels => {}})
      self.tooltip({:enabled => true})
      self.credits({:enabled => false})
      self.plotOptions({:areaspline => {}})
      self.chart({:defaultSeriesType => "line", :renderTo => nil})
      self.subtitle({})
    end

    # Pass other methods through to the javascript high_chart object.
    #
    # For instance: <tt>high_chart.grid(:color => "#699")</tt>
    def method_missing(meth, opts = {})
      if meth.to_s == 'to_ary'
        super
      end

      if meth.to_s.end_with? '!'
        deep_merge_options meth[0..-2].to_sym, opts
      else
        merge_options meth, opts
      end
    end

    # Add a simple series to the graph:
    #
    #   data = [[0,5], [1,5], [2,5]]
    #   @high_chart.series :name=>'Updated', :data=>data
    #   @high_chart.series :name=>'Updated', :data=>[5, 1, 6, 1, 5, 4, 9]
    def series(opts = {})
      if not opts.empty?
        @series_data << OptionsKeyFilter.filter(opts.merge(:name => opts[:name], :data => opts[:data]))
      end
    end

    # Pre-processes and returns full set of options relevant to the chart. Identical to what happens in the high_charts
    # view helper.
    #
    # @return [Hash] options JSON options hash
    def full_options
      options_collection_as_string self
    end

    def to_json
      data = self.options.clone
      data[:series] = self.series_data.clone
      data
    end

    private

    def random_canvas_id
      canvas_id_length = 11
      # Don't use SecureRandom.urlsafe_base64; it gives invalid characters.
      ('a'..'z').to_a.shuffle.take(canvas_id_length).join
    end

    def series_options
      @options.reject { |k, _| SERIES_OPTIONS.include?(k.to_s) == false }
    end

    def merge_options(name, opts)
      @options.merge! name => opts
    end

    def deep_merge_options(name, opts)
      @options.deep_merge! name => opts
    end

    def arguments_to_options(args)
      if args.blank?
        {:show => true}
      elsif args.is_a? Array
        args.first
      else
        args
      end
    end

  end
end


module RackCAS
  module ActiveRecordStore
    class Session < ActiveRecord::Base
    end

    def self.destroy_session_by_cas_ticket(cas_ticket)
      affected = Session.where(cas_ticket: cas_ticket).delete_all
      affected == 1
    end

    def self.prune(after = nil)
      after ||= Time.now - 2592000 # 30 days ago
      Session.where('updated_at < ?', after).delete_all
    end

    private

    # Rack 2.0 method
    def find_session(env, sid)
      if sid.nil?
        sid = generate_sid
        data = nil
      else
        unless session = Session.where(session_id: sid).first
          session = {}
          # force generation of new sid since there is no associated session
          sid = generate_sid
        end
        data = unpack(session['data'])
      end

      [sid, data]
    end

    # Rack 2.0 method
    def write_session(req, sid, session_data, options)
      cas_ticket = (session_data['cas']['ticket'] unless session_data['cas'].nil?)

      session = if ActiveRecord.respond_to?(:version) && ActiveRecord.version >= Gem::Version.new('4.0.0')
        Session.where(session_id: sid).first_or_initialize
      else
        Session.find_or_initialize_by_session_id(sid)
      end
      session.data = pack(session_data)
      session.cas_ticket = cas_ticket
      success = session.save

      success ? session.session_id : false
    end

    # Rack 2.0 method
    def delete_session(req, sid, options)
      Session.where(session_id: sid).delete_all

      options[:drop] ? nil : generate_sid
    end

    # Rack 1.* method
    alias get_session find_session

    # Rack 1.* method
    def set_session(env, sid, session_data, options) # rack 1.x compatibilty
      write_session(Rack::Request.new(env), sid, session_data, options)
    end

    # Rack 1.* method
    def destroy_session(env, sid, options) # rack 1.x compatibilty
      delete_session(Rack::Request.new(env), sid, options)
    end

    def pack(data)
      ::Base64.encode64(Marshal.dump(data)) if data
    end

    def unpack(data)
      Marshal.load(::Base64.decode64(data)) if data
    end
  end
end


module RackCAS
  module RedisStore
    class Session
      @client = nil

      def self.client
        @client ||= (RackCAS.config.redis_options? ? Redis.new(RackCAS.config.redis_options) : Redis.new)
        return @client
      end

      def self.find_by_id(session_id)
        session = self.client.get("rack_cas_session:#{session_id}")
        session ? {'sid' => session_id, 'data' => session} : session
      end

      def self.write(session_id:, data:, cas_ticket: )
        #create a row with the session_id and the data
        #create a row with the cas_ticket acting as a reverse index
        results = self.client.pipelined do
          self.client.set("rack_cas_session:#{session_id}",data)
          self.client.expireat("rack_cas_session:#{session_id}",30.days.from_now.to_i)
          self.client.set("rack_cas_ticket:#{cas_ticket}","rack_cas_session:#{session_id}")
          self.client.expireat("rack_cas_ticket:#{cas_ticket}",30.days.from_now.to_i)
        end

        results == ["OK",true,"OK",true] ? session_id : false
      end

      def self.destroy_by_cas_ticket(cas_ticket)
        session_id = self.client.get("rack_cas_ticket:#{cas_ticket}")
        results = self.client.pipelined do
          self.client.del("rack_cas_ticket:#{cas_ticket}")
          self.client.del(session_id)
        end
        return results[1]
      end

      def self.delete(session_id)
        self.client.del("rack_cas_session:#{session_id}")
      end
    end

    def self.destroy_session_by_cas_ticket(cas_ticket)
      affected = Session.destroy_by_cas_ticket(cas_ticket)
      affected == 1
    end

    #we don't need to prune because the keys expire automatically
    def self.prune(after = nil)
    end

    private

    # Rack 2.0 method
    def find_session(env, sid)
      if sid.nil?
        sid = generate_sid
        data = nil
      else
        unless session = Session.find_by_id(sid)
          session = {}
          # force generation of new sid since there is no associated session
          sid = generate_sid
        end
        data = unpack(session['data'])
      end

      [sid, data]
    end

    # Rack 2.0 method
    def write_session(env, sid, session_data, options)
      cas_ticket = (session_data['cas']['ticket'] unless session_data['cas'].nil?)

      success = Session.write(session_id: sid, data: pack(session_data), cas_ticket: cas_ticket)

      success ? sid : false
    end

    # Rack 2.0 method
    def delete_session(env, sid, options)
      Session.delete(sid)

      options[:drop] ? nil : generate_sid
    end

    # Rack 1.* method
    alias get_session find_session

    # Rack 1.* method
    def set_session(env, sid, session_data, options) # rack 1.x compatibilty
      write_session(Rack::Request.new(env), sid, session_data, options)
    end

    # Rack 1.* method
    def destroy_session(env, sid, options) # rack 1.x compatibilty
      delete_session(Rack::Request.new(env), sid, options)
    end

    def pack(data)
      ::Base64.encode64(Marshal.dump(data)) if data
    end

    def unpack(data)
      Marshal.load(::Base64.decode64(data)) if data
    end
  end
end


require 'addressable/uri'

module RackCAS
  class URL < Addressable::URI
    def self.parse(uri)
      # I know this looks stupid but this seems to be the best way to get
      # Addressable to replace + spaces with %20 spaces. Standardizing on %20
      # should prevent service lookup issues due to encoding differences.
      super.tap do |u|
        u.query_values = u.query_values
      end
    end

    def append_path(path)
      self.tap do |u|
        u.path = (u.path.split('/') + [path]).join('/')
      end
    end

    def add_params(params)
      self.tap do |u|
        u.query_values = (u.query_values || {}).tap do |qv|
          params.each do |key, value|
            qv[key] = value
          end
        end
      end
    end

    def remove_param(param)
      remove_params(Array(param))
    end

    # params can be an array or a hash
    def remove_params(params)
      self.tap do |u|
        u.query_values = (u.query_values || {}).tap do |qv|
          params.each do |key, value|
            qv.delete key
          end
        end
        if u.query_values.empty?
          u.query_values = nil
        end
      end
    end

    def dup
      RackCAS::URL.new(super.to_hash)
    end
  end
end


module RackCAS
  class ServiceValidationResponse
    class AuthenticationFailure < StandardError; end
    class RequestInvalidError < AuthenticationFailure; end
    class TicketInvalidError < AuthenticationFailure; end
    class ServiceInvalidError < AuthenticationFailure; end

    REQUEST_HEADERS = { 'Accept' => '*/*' }

    def initialize(url)
      @url = URL.parse(url)
    end

    def user
      if success?
        xml.at('//serviceResponse/authenticationSuccess/user').text
      else
        case failure_code
        when 'INVALID_REQUEST'
          raise RequestInvalidError, failure_message
        when 'INVALID_TICKET'
          raise TicketInvalidError, failure_message
        when 'INVALID_SERVICE'
          raise ServiceInvalidError, failure_message
        else
          raise AuthenticationFailure, failure_message
        end
      end
    end

    def extra_attributes
      attrs = {}

      raise AuthenticationFailure, failure_message unless success?

      # Jasig style
      if attr_node = xml.at('//serviceResponse/authenticationSuccess/attributes')
        attrs = parse_user_info(attr_node)

      # RubyCas-Server style
      else
        xml.at('//serviceResponse/authenticationSuccess').children.each do |node|
          if node.is_a? Nokogiri::XML::Element
            if !node.namespace || !node.namespace.prefix == 'cas'
              # TODO: support JSON encoding
              attrs[node.name] = YAML.load node.text.strip
            end
          end
        end
      end

      attrs
    end

    protected

    def success?
      @success ||= !!xml.at('//serviceResponse/authenticationSuccess')
    end

    def authentication_failure
      @authentication_failure ||= xml.at('//serviceResponse/authenticationFailure')
    end

    def failure_message
      if authentication_failure
        authentication_failure.text.strip
      end
    end

    def failure_code
      if authentication_failure
        authentication_failure['code']
      end
    end

    def response
      require 'net/http'
      return @response unless @response.nil?

      http = Net::HTTP.new(@url.host, @url.inferred_port)
      if @url.scheme == 'https'
        http.use_ssl = true
        http.verify_mode = RackCAS.config.verify_ssl_cert ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
      end

      http.start do |conn|
        @response = conn.get(@url.request_uri, REQUEST_HEADERS)
      end

      @response
    end

    def xml
      return @xml unless @xml.nil?

      @xml = Nokogiri::XML(response.body).remove_namespaces!
    end

    # initially borrowed from omniauth-cas
    def parse_user_info(node)
      return nil if node.nil?
      {}.tap do |hash|
        node.children.each do |e|
          unless e.kind_of?(Nokogiri::XML::Text) || e.name == 'proxies'
            # There are no child elements
            if e.element_children.count == 0
              if hash.has_key?(e.name)
                hash[e.name] = [hash[e.name]] if hash[e.name].is_a? String
                hash[e.name] << e.content
              else
                hash[e.name] = e.content
              end
            elsif e.element_children.count
              # JASIG style extra attributes
              if e.name == 'attributes'
                hash.merge!(parse_user_info(e))
              else
                hash[e.name] = [] if hash[e.name].nil?
                hash[e.name] = [hash[e.name]] if hash[e.name].is_a? String
                hash[e.name].push(parse_user_info(e))
              end
            end
          end
        end
      end
    end
  end
end


module RackCAS
  module MongoidStore
    class Session
      include Mongoid::Document
      include Mongoid::Timestamps

      field :_id, type: String
      if defined? Moped::BSON
        # Mongoid < 4
        field :data, type: Moped::BSON::Binary, default: Moped::BSON::Binary.new(:generic, Marshal.dump({}))
      else
        # Mongoid 4
        field :data, type: BSON::Binary, default: BSON::Binary.new(Marshal.dump({}))
      end
      field :cas_ticket, type: String
    end

    def self.destroy_session_by_cas_ticket(cas_ticket)
      affected = Session.where(cas_ticket: cas_ticket).delete
      affected == 1
    end

    def self.prune(after = nil)
      after ||= Time.now - 2592000 # 30 days ago
      Session.where(:updated_at.lte => after).delete
    end

    private

    # Rack 2.0 method
    def find_session(env, sid)
      if sid.nil?
        sid = generate_sid
        data = nil
      else
        unless session = Session.where(_id: sid).first
          session = {}
          # force generation of new sid since there is no associated session
          sid = generate_sid
        end
        data = unpack(session['data'])
      end

      [sid, data]
    end

    # Rack 2.0 method
    def write_session(env, sid, session_data, options)
      cas_ticket = (session_data['cas']['ticket'] unless session_data['cas'].nil?)

      session = Session.find_or_initialize_by(_id: sid)
      success = session.update_attributes(data: pack(session_data), cas_ticket: cas_ticket)

      success ? session.id : false
    end

    # Rack 2.0 method
    def delete_session(env, sid, options)
      Session.where(_id: sid).delete

      options[:drop] ? nil : generate_sid
    end

    # Rack 1.* method
    alias get_session find_session

    # Rack 1.* method
    def set_session(env, sid, session_data, options) # rack 1.x compatibilty
      write_session(Rack::Request.new(env), sid, session_data, options)
    end

    # Rack 1.* method
    def destroy_session(env, sid, options) # rack 1.x compatibilty
      delete_session(Rack::Request.new(env), sid, options)
    end

    def pack(data)
      if defined? Moped::BSON
        Moped::BSON::Binary.new(:generic, Marshal.dump(data))
      else
        BSON::Binary.new(Marshal.dump(data))
      end
    end

    def unpack(packed)
      return nil unless packed

      if defined? Moped::BSON
        Marshal.load(StringIO.new(packed.to_s))
      else
        Marshal.load(packed.data)
      end
    end
  end
end


module ForemanRemoteExecution
  module HostExtensions
    # rubocop:disable Metrics/BlockLength
    def self.prepended(base)
      base.instance_eval do
        has_many :targeting_hosts, :dependent => :destroy, :foreign_key => 'host_id'
        has_many :template_invocations, :dependent => :destroy, :foreign_key => 'host_id'
        has_one :execution_status_object, :class_name => 'HostStatus::ExecutionStatus', :foreign_key => 'host_id', :dependent => :destroy
        has_many :run_host_job_tasks, :through => :template_invocations

        scoped_search :relation => :run_host_job_tasks, :on => :result, :rename => 'job_invocation.result',
                      :ext_method => :search_by_job_invocation,
                      :only_explicit => true,
                      :complete_value => TemplateInvocation::TaskResultMap::REVERSE_MAP

        scoped_search :relation => :template_invocations, :on => :job_invocation_id,
                      :rename => 'job_invocation.id', :only_explicit => true, :ext_method => :search_by_job_invocation

        scoped_search :relation => :execution_status_object, :on => :status, :rename => :execution_status,
                      :complete_value => { :ok => HostStatus::ExecutionStatus::OK, :error => HostStatus::ExecutionStatus::ERROR }

        def search_by_job_invocation(key, operator, value)
          if key == 'job_invocation.result'
            operator = operator == '=' ? 'IN' : 'NOT IN'
            value = TemplateInvocation::TaskResultMap.status_to_task_result(value)
          end

          mapping = {
            'job_invocation.id'     => %(#{TemplateInvocation.table_name}.job_invocation_id #{operator} ?),
            'job_invocation.result' => %(#{ForemanTasks::Task.table_name}.result #{operator} (?))
          }
          {
            :conditions => sanitize_sql_for_conditions([mapping[key], value_to_sql(operator, value)]),
            :joins => { :template_invocations => [:run_host_job_task] }
          }
        end
      end
    end

    def execution_status(options = {})
      @execution_status ||= get_status(HostStatus::ExecutionStatus).to_status(options)
    end

    def execution_status_label(options = {})
      @execution_status_label ||= get_status(HostStatus::ExecutionStatus).to_label(options)
    end

    def host_params_hash
      params = super
      keys = remote_execution_ssh_keys
      source = 'global'
      if keys.present?
        params['remote_execution_ssh_keys'] = {:value => keys, :safe_value => keys, :source => source}
      end
      [:remote_execution_ssh_user, :remote_execution_effective_user_method,
       :remote_execution_connect_by_ip].each do |key|
        value = Setting[key]
        params[key.to_s] = {:value => value, :safe_value => value, :source => source} unless params.key?(key.to_s)
      end
      params
    end

    def execution_interface
      get_interface_by_flag(:execution)
    end

    # rubocop:disable Metrics/PerceivedComplexity
    def remote_execution_proxies(provider, authorized = true)
      proxies = {}
      proxies[:subnet]   = execution_interface.subnet.remote_execution_proxies.with_features(provider) if execution_interface && execution_interface.subnet
      proxies[:fallback] = smart_proxies.with_features(provider) if Setting[:remote_execution_fallback_proxy]

      if Setting[:remote_execution_global_proxy]
        proxy_scope = if Taxonomy.enabled_taxonomies.any? && User.current.present?
                        ::SmartProxy.with_taxonomy_scope_override(location, organization)
                      else
                        ::SmartProxy.unscoped
                      end

        proxy_scope = proxy_scope.authorized if authorized
        proxies[:global] = proxy_scope.with_features(provider)
      end

      proxies
    end

    def remote_execution_ssh_keys
      remote_execution_proxies('SSH', false).values.flatten.uniq.map { |proxy| proxy.pubkey }.compact.uniq
    end

    def drop_execution_interface_cache
      @execution_interface = nil
    end

    def becomes(*args)
      became = super(*args)
      became.drop_execution_interface_cache if became.respond_to? :drop_execution_interface_cache
      became
    end

    def reload(*args)
      drop_execution_interface_cache
      super(*args)
    end

    private

    def build_required_interfaces(attrs = {})
      super(attrs)
      self.primary_interface.execution = true if self.execution_interface.nil?
    end
  end
end