/*
 *
 *  *  Copyright 2010-2016 OrientDB LTD (http://orientdb.com)
 *  *
 *  *  Licensed under the Apache License, Version 2.0 (the "License");
 *  *  you may not use this file except in compliance with the License.
 *  *  You may obtain a copy of the License at
 *  *
 *  *       http://www.apache.org/licenses/LICENSE-2.0
 *  *
 *  *  Unless required by applicable law or agreed to in writing, software
 *  *  distributed under the License is distributed on an "AS IS" BASIS,
 *  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  *  See the License for the specific language governing permissions and
 *  *  limitations under the License.
 *  *
 *  * For more information: http://orientdb.com
 *
 */
package com.orientechnologies.orient.distributed.impl;

import com.orientechnologies.common.concur.OOfflineNodeException;
import com.orientechnologies.common.concur.lock.OSimpleLockManager;
import com.orientechnologies.common.concur.lock.OSimpleLockManagerImpl;
import com.orientechnologies.common.exception.OException;
import com.orientechnologies.common.log.OLogManager;
import com.orientechnologies.common.profiler.OAbstractProfiler;
import com.orientechnologies.common.profiler.OProfiler;
import com.orientechnologies.common.util.OCallable;
import com.orientechnologies.orient.core.Orient;
import com.orientechnologies.orient.core.command.OCommandDistributedReplicateRequest;
import com.orientechnologies.orient.core.config.OGlobalConfiguration;
import com.orientechnologies.orient.core.db.ODatabaseDocumentInternal;
import com.orientechnologies.orient.core.db.ODatabaseRecordThreadLocal;
import com.orientechnologies.orient.core.db.record.OIdentifiable;
import com.orientechnologies.orient.core.exception.OSecurityAccessException;
import com.orientechnologies.orient.core.id.ORID;
import com.orientechnologies.orient.core.id.ORecordId;
import com.orientechnologies.orient.core.storage.ORawBuffer;
import com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OLogSequenceNumber;
import com.orientechnologies.orient.server.OServer;
import com.orientechnologies.orient.server.OSystemDatabase;
import com.orientechnologies.orient.server.distributed.*;
import com.orientechnologies.orient.server.distributed.ODistributedServerLog.DIRECTION;
import com.orientechnologies.orient.distributed.impl.task.ODistributedLockTask;
import com.orientechnologies.orient.distributed.impl.task.OWaitForTask;
import com.orientechnologies.orient.server.distributed.task.OAbstractRemoteTask;
import com.orientechnologies.orient.server.distributed.task.ODistributedOperationException;
import com.orientechnologies.orient.server.distributed.task.ODistributedRecordLockedException;
import com.orientechnologies.orient.server.distributed.task.ORemoteTask;
import com.orientechnologies.orient.distributed.hazelcast.OHazelcastPlugin;

import java.io.File;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;

import static com.orientechnologies.orient.core.config.OGlobalConfiguration.DISTRIBUTED_ATOMIC_LOCK_TIMEOUT;

/**
 * Distributed database implementation. There is one instance per database. Each node creates own instance to talk with each
 * others.
 *
 * @author Luca Garulli (l.garulli--at--orientechnologies.com)
 */
public class ODistributedDatabaseImpl implements ODistributedDatabase {
  public static final  String                                    DISTRIBUTED_SYNC_JSON_FILENAME = "distributed-sync.json";
  private static final HashSet<Integer>                          ALL_QUEUES                     = new HashSet<Integer>();
  protected final      ODistributedAbstractPlugin                manager;
  protected final      ODistributedMessageServiceImpl            msgService;
  protected final      String                                    databaseName;
  protected            ODistributedDatabaseRepairer              repairer;
  protected            ODistributedSyncConfiguration             syncConfiguration;
  protected            ConcurrentHashMap<ORID, ODistributedLock> lockManager                    = new ConcurrentHashMap<ORID, ODistributedLock>(
      256);

  protected       ConcurrentHashMap<ODistributedRequestId, ODistributedTxContext> activeTxContexts = new ConcurrentHashMap<ODistributedRequestId, ODistributedTxContext>(
      64);
  protected final List<ODistributedWorker>                                        workerThreads    = new ArrayList<ODistributedWorker>();
  protected       ODistributedWorker                                              lockThread;
  protected       ODistributedWorker                                              nowaitThread;

  private          AtomicLong                            totalSentRequests     = new AtomicLong();
  private          AtomicLong                            totalReceivedRequests = new AtomicLong();
  private          TimerTask                             txTimeoutTask         = null;
  private          CountDownLatch                        waitForOnline         = new CountDownLatch(1);
  private volatile boolean                               running               = true;
  private          AtomicBoolean                         parsing               = new AtomicBoolean(true);
  private final    AtomicReference<ODistributedMomentum> filterByMomentum      = new AtomicReference<ODistributedMomentum>();

  private final String                     localNodeName;
  private final OSimpleLockManager<ORID>   recordLockManager;
  private final OSimpleLockManager<Object> indexKeyLockManager;

  public OSimpleLockManager<ORID> getRecordLockManager() {
    return recordLockManager;
  }

  public OSimpleLockManager<Object> getIndexKeyLockManager() {
    return indexKeyLockManager;
  }

  public class ODistributedLock {
    protected final    ODistributedRequestId reqId;
    protected final    CountDownLatch        lock;
    protected final    long                  acquiredOn;
    protected volatile ORawBuffer            record;

    private ODistributedLock(final ODistributedRequestId reqId) {
      this.reqId = reqId;
      this.lock = new CountDownLatch(1);
      this.acquiredOn = System.currentTimeMillis();
    }
  }

  public ODistributedDatabaseImpl(final OHazelcastPlugin manager, final ODistributedMessageServiceImpl msgService,
      final String iDatabaseName, final ODistributedConfiguration cfg, OServer server) {
    this.manager = manager;
    this.msgService = msgService;
    this.databaseName = iDatabaseName;
    this.localNodeName = manager.getLocalNodeName();

    // SELF REGISTERING ITSELF HERE BECAUSE IT'S NEEDED FURTHER IN THE CALL CHAIN
    final ODistributedDatabaseImpl prev = msgService.databases.put(iDatabaseName, this);
    if (prev != null) {
      // KILL THE PREVIOUS ONE
      prev.shutdown();
    }

    startAcceptingRequests();

    if (iDatabaseName.equals(OSystemDatabase.SYSTEM_DB_NAME)) {
      recordLockManager = null;
      indexKeyLockManager = null;
      return;
    }

    startTxTimeoutTimerTask();

    repairer = new OConflictResolverDatabaseRepairer(manager, databaseName);

    Orient.instance().getProfiler()
        .registerHookValue("distributed.db." + databaseName + ".msgSent", "Number of replication messages sent from current node",
            OProfiler.METRIC_TYPE.COUNTER, new OAbstractProfiler.OProfilerHookValue() {
              @Override
              public Object getValue() {
                return totalSentRequests.get();
              }
            }, "distributed.db.*.msgSent");

    Orient.instance().getProfiler().registerHookValue("distributed.db." + databaseName + ".msgReceived",
        "Number of replication messages received from external nodes", OProfiler.METRIC_TYPE.COUNTER,
        new OAbstractProfiler.OProfilerHookValue() {
          @Override
          public Object getValue() {
            return totalReceivedRequests.get();
          }
        }, "distributed.db.*.msgReceived");

    Orient.instance().getProfiler()
        .registerHookValue("distributed.db." + databaseName + ".activeContexts", "Number of active distributed transactions",
            OProfiler.METRIC_TYPE.COUNTER, new OAbstractProfiler.OProfilerHookValue() {
              @Override
              public Object getValue() {
                return (long) activeTxContexts.size();
              }
            }, "distributed.db.*.activeContexts");

    Orient.instance().getProfiler()
        .registerHookValue("distributed.db." + databaseName + ".workerThreads", "Number of worker threads",
            OProfiler.METRIC_TYPE.COUNTER, new OAbstractProfiler.OProfilerHookValue() {
              @Override
              public Object getValue() {
                return (long) workerThreads.size();
              }
            }, "distributed.db.*.workerThreads");

    Orient.instance().getProfiler().registerHookValue("distributed.db." + databaseName + ".recordLocks", "Number of record locked",
        OProfiler.METRIC_TYPE.COUNTER, new OAbstractProfiler.OProfilerHookValue() {
          @Override
          public Object getValue() {
            return (long) lockManager.size();
          }
        }, "distributed.db.*.recordLocks");

    long timeout = manager.getServerInstance().getContextConfiguration().getValueAsLong(DISTRIBUTED_ATOMIC_LOCK_TIMEOUT);
    recordLockManager = new OSimpleLockManagerImpl<>(timeout);
    indexKeyLockManager = new OSimpleLockManagerImpl<>(timeout);
  }

  public OLogSequenceNumber getLastLSN(final String server) {
    if (server == null)
      return null;
    return getSyncConfiguration().getLastLSN(server);
  }

  @Override
  public void waitForOnline() {
    try {
      if (!databaseName.equalsIgnoreCase(OSystemDatabase.SYSTEM_DB_NAME))
        waitForOnline.await();
    } catch (InterruptedException e) {
      Thread.currentThread().interrupt();
      // IGNORE IT
    }
  }

  public void reEnqueue(final int senderNodeId, final long msgSequence, final String databaseName, final ORemoteTask payload,
      int retryCount) {

    Orient.instance().scheduleTask(new TimerTask() {
      @Override
      public void run() {
        processRequest(new ODistributedRequest(getManager(), senderNodeId, msgSequence, databaseName, payload), false);
      }
    }, 10 * retryCount, 0);
  }

  /**
   * Distributed requests against the available workers by using one queue per worker. This guarantee the sequence of the operations
   * against the same record cluster.
   */
  public void processRequest(final ODistributedRequest request, final boolean waitForAcceptingRequests) {
    if (!running) {
      throw new ODistributedException("Server is going down or is removing the database:'" + getDatabaseName() + "' discarding");
    }

    final ORemoteTask task = request.getTask();

    if (waitForAcceptingRequests) {
      waitIsReady(task);

      if (!running) {
        throw new ODistributedException("Server is going down or is removing the database:'" + getDatabaseName() + "' discarding");
      }
    }

    totalReceivedRequests.incrementAndGet();

    // final ODistributedMomentum lastMomentum = filterByMomentum.get();
    // if (lastMomentum != null && task instanceof OAbstractReplicatedTask) {
    // final OLogSequenceNumber taskLastLSN = ((OAbstractReplicatedTask) task).getLastLSN();
    //
    // final String sourceServer = manager.getNodeNameById(request.getId().getNodeId());
    // final OLogSequenceNumber lastLSNFromMomentum = lastMomentum.getLSN(sourceServer);
    //
    // if (taskLastLSN != null && lastLSNFromMomentum != null && taskLastLSN.compareTo(lastLSNFromMomentum) < 0) {
    // // SKIP REQUEST BECAUSE CONTAINS AN OLD LSN
    // final String msg = String.format("Skipped request %s on database '%s' because %s < current %s", request, databaseName,
    // taskLastLSN, lastLSNFromMomentum);
    // ODistributedServerLog.info(this, localNodeName, null, DIRECTION.NONE, msg);
    // ODistributedWorker.sendResponseBack(this, manager, request, new ODistributedException(msg));
    // return;
    // }
    // }

    final int[] partitionKeys = task.getPartitionKey();

    if (ODistributedServerLog.isDebugEnabled())
      ODistributedServerLog
          .debug(this, localNodeName, task.getNodeSource(), DIRECTION.IN, "Request %s on database '%s' partitionKeys=%s task=%s",
              request, databaseName, Arrays.toString(partitionKeys), task);

    if (partitionKeys.length > 1 || partitionKeys[0] == -1) {

      final Set<Integer> involvedWorkerQueues;
      if (partitionKeys.length > 1)
        involvedWorkerQueues = getInvolvedQueuesByPartitionKeys(partitionKeys);
      else
        // LOCK ALL THE QUEUES
        involvedWorkerQueues = ALL_QUEUES;

      // if (ODistributedServerLog.isDebugEnabled())
      ODistributedServerLog
          .debug(this, localNodeName, null, DIRECTION.NONE, "Request %s on database '%s' involvedQueues=%s", request, databaseName,
              involvedWorkerQueues);

      if (involvedWorkerQueues.size() == 1)
        // JUST ONE QUEUE INVOLVED: PROCESS IT IMMEDIATELY
        processRequest(involvedWorkerQueues.iterator().next(), request);
      else {
        // INVOLVING MULTIPLE QUEUES

        // if (ODistributedServerLog.isDebugEnabled())
        ODistributedServerLog.debug(this, localNodeName, null, DIRECTION.NONE,
            "Request %s on database '%s' waiting for all the previous requests to be completed", request, databaseName);

        // WAIT ALL THE INVOLVED QUEUES ARE FREE AND SYNCHRONIZED
        final CountDownLatch syncLatch = new CountDownLatch(involvedWorkerQueues.size());
        final ODistributedRequest syncRequest = new ODistributedRequest(null, request.getId().getNodeId(), -1, databaseName,
            new OSynchronizedTaskWrapper(syncLatch));
        for (int queue : involvedWorkerQueues) {
          ODistributedWorker worker = workerThreads.get(queue);
          worker.processRequest(syncRequest);
        }

        // Make infinite timeout everytime
        long taskTimeout = 0;
        try {
          if (taskTimeout <= 0)
            syncLatch.await();
          else {
            // WAIT FOR COMPLETION. THE TIMEOUT IS MANAGED IN SMALLER CYCLES TO PROPERLY RECOGNIZE WHEN THE DB IS REMOVED
            final long start = System.currentTimeMillis();
            final long cycleTimeout = Math.min(taskTimeout, 2000);

            boolean locked = false;
            do {
              if (syncLatch.await(cycleTimeout, TimeUnit.MILLISECONDS)) {
                // DONE
                locked = true;
                break;
              }

              if (this.workerThreads.size() == 0)
                // DATABASE WAS SHUTDOWN
                break;

            } while (System.currentTimeMillis() - start < taskTimeout);

            if (!locked) {
              final String msg = String.format(
                  "Cannot execute distributed request (%s) because all worker threads (%d) are busy (pending=%d timeout=%d)",
                  request, workerThreads.size(), syncLatch.getCount(), taskTimeout);
              ODistributedWorker.sendResponseBack(this, manager, request, new ODistributedOperationException(msg));
              return;
            }
          }
        } catch (InterruptedException e) {
          // IGNORE
          Thread.currentThread().interrupt();
          final String msg = String
              .format("Cannot execute distributed request (%s) because all worker threads (%d) are busy", request,
                  workerThreads.size());
          ODistributedWorker.sendResponseBack(this, manager, request, new ODistributedOperationException(msg));
          return;
        }

        // PUT THE TASK TO EXECUTE ONLY IN THE FIRST QUEUE AND PUT WAIT-FOR TASKS IN THE OTHERS. WHEN THE REAL TASK IS EXECUTED,
        // ALL THE OTHER TASKS WILL RETURN, SO THE QUEUES WILL BE BUSY DURING THE EXECUTION OF THE TASK. THIS AVOID CONCURRENT
        // EXECUTION FOR THE SAME PARTITION
        final CountDownLatch queueLatch = new CountDownLatch(1);

        int i = 0;
        for (int queue : involvedWorkerQueues) {
          final ODistributedRequest req;
          if (i++ == 0) {
            // USE THE FIRST QUEUE TO PROCESS THE REQUEST
            final String senderNodeName = manager.getNodeNameById(request.getId().getNodeId());
            request.setTask(new OSynchronizedTaskWrapper(queueLatch, senderNodeName, task));
            req = request;
          } else
            req = new ODistributedRequest(manager, request.getId().getNodeId(), -1, databaseName, new OWaitForTask(queueLatch));

          workerThreads.get(queue).processRequest(req);
        }
      }
    } else if (partitionKeys.length == 1 && partitionKeys[0] == -2) {
      // ANY PARTITION: USE THE FIRST EMPTY IF ANY, OTHERWISE THE FIRST IN THE LIST
      boolean found = false;

      for (ODistributedWorker q : workerThreads) {
        if (q.isWaitingForNextRequest() && q.localQueue.isEmpty()) {
          q.processRequest(request);
          found = true;
          break;
        }
      }

      if (!found)
        // ALL THE THREADS ARE BUSY, SELECT THE FIRST EMPTY ONE
        for (ODistributedWorker q : workerThreads) {
          if (q.localQueue.isEmpty()) {
            q.processRequest(request);
            found = true;
            break;
          }
        }

      if (!found)
        // EXEC ON THE FIRST QUEUE
        workerThreads.get(0).processRequest(request);

    } else if (partitionKeys.length == 1 && partitionKeys[0] == -3) {
      // SERVICE - LOCK
      ODistributedServerLog.debug(this, localNodeName, request.getTask().getNodeSource(), DIRECTION.IN,
          "Request %s on database '%s' dispatched to the lock worker", request, databaseName);

      lockThread.processRequest(request);

    } else if (partitionKeys.length == 1 && partitionKeys[0] == -4) {
      // SERVICE - FAST_NOLOCK
      ODistributedServerLog.debug(this, localNodeName, request.getTask().getNodeSource(), DIRECTION.IN,
          "Request %s on database '%s' dispatched to the nowait worker", request, databaseName);

      nowaitThread.processRequest(request);

    } else {
      processRequest(partitionKeys[0], request);
    }
  }

  public void waitIsReady(ORemoteTask task) {
    if (task.isNodeOnlineRequired())
      if (!parsing.get()) {
        // WAIT FOR PARSING REQUESTS
        while (!parsing.get()) {
          try {
            Thread.sleep(300);
          } catch (InterruptedException e) {
            break;
          }
        }
      }
  }

  protected Set<Integer> getInvolvedQueuesByPartitionKeys(final int[] partitionKeys) {
    final Set<Integer> involvedWorkerQueues = new HashSet<Integer>(partitionKeys.length);
    for (int pk : partitionKeys) {
      if (pk >= 0)
        involvedWorkerQueues.add(pk % workerThreads.size());
    }
    return involvedWorkerQueues;
  }

  protected void processRequest(final int partitionKey, final ODistributedRequest request) {
    if (workerThreads.isEmpty())
      throw new ODistributedException("There are no worker threads to process request " + request);

    final int partition = partitionKey % workerThreads.size();

    ODistributedServerLog.debug(this, localNodeName, request.getTask().getNodeSource(), DIRECTION.IN,
        "Request %s on database '%s' dispatched to the worker %d", request, databaseName, partition);

    workerThreads.get(partition).processRequest(request);
  }

  @Override
  public ODistributedResponse send2Nodes(final ODistributedRequest iRequest, final Collection<String> iClusterNames,
      Collection<String> iNodes, final ODistributedRequest.EXECUTION_MODE iExecutionMode, final Object localResult,
      final OCallable<Void, ODistributedRequestId> iAfterSentCallback,
      final OCallable<Void, ODistributedResponseManager> endCallback) {
    return send2Nodes(iRequest, iClusterNames, iNodes, iExecutionMode, localResult, iAfterSentCallback, endCallback,
        (iRequest1, iNodes1, endCallback1, task, nodesConcurToTheQuorum, availableNodes, expectedResponses, quorum, groupByResponse, waitLocalNode) -> {
          return new ODistributedResponseManagerImpl(manager, iRequest, iNodes, nodesConcurToTheQuorum, expectedResponses, quorum,
              waitLocalNode, adjustTimeoutWithLatency(iNodes, task.getSynchronousTimeout(expectedResponses), iRequest.getId()),
              adjustTimeoutWithLatency(iNodes, task.getTotalTimeout(availableNodes), iRequest.getId()), groupByResponse,
              endCallback);
        });
  }

  public ODistributedResponse send2Nodes(final ODistributedRequest iRequest, final Collection<String> iClusterNames,
      Collection<String> iNodes, final ODistributedRequest.EXECUTION_MODE iExecutionMode, final Object localResult,
      final OCallable<Void, ODistributedRequestId> iAfterSentCallback,
      final OCallable<Void, ODistributedResponseManager> endCallback, ODistributedResponseManagerFactory responseManagerFactory) {
    boolean afterSendCallBackCalled = false;
    try {
      checkForServerOnline(iRequest);

      final String databaseName = iRequest.getDatabaseName();

      if (iNodes.isEmpty()) {
        ODistributedServerLog
            .error(this, localNodeName, null, DIRECTION.OUT, "No nodes configured for database '%s' request: %s", databaseName,
                iRequest);
        throw new ODistributedException("No nodes configured for partition '" + databaseName + "' request: " + iRequest);
      }

      final ODistributedConfiguration cfg = manager.getDatabaseConfiguration(databaseName);

      final ORemoteTask task = iRequest.getTask();

      final boolean checkNodesAreOnline = task.isNodeOnlineRequired();

      final Set<String> nodesConcurToTheQuorum = manager.getDistributedStrategy()
          .getNodesConcurInQuorum(manager, cfg, iRequest, iNodes, databaseName, localResult);

      // AFTER COMPUTED THE QUORUM, REMOVE THE OFFLINE NODES TO HAVE THE LIST OF REAL AVAILABLE NODES
      final int availableNodes = checkNodesAreOnline ?
          manager.getNodesWithStatus(iNodes, databaseName, ODistributedServerManager.DB_STATUS.ONLINE,
              ODistributedServerManager.DB_STATUS.BACKUP, ODistributedServerManager.DB_STATUS.SYNCHRONIZING) :
          iNodes.size();

      final int expectedResponses = localResult != null ? availableNodes + 1 : availableNodes;

      final int quorum = calculateQuorum(task.getQuorumType(), iClusterNames, cfg, expectedResponses, nodesConcurToTheQuorum.size(),
          checkNodesAreOnline, localNodeName);

      final boolean groupByResponse = task.getResultStrategy() != OAbstractRemoteTask.RESULT_STRATEGY.UNION;

      final boolean waitLocalNode = waitForLocalNode(cfg, iClusterNames, iNodes);

      // CREATE THE RESPONSE MANAGER
      final ODistributedResponseManager currentResponseMgr = responseManagerFactory
          .newResponseManager(iRequest, iNodes, endCallback, task, nodesConcurToTheQuorum, availableNodes, expectedResponses,
              quorum, groupByResponse, waitLocalNode);

      if (localResult != null && currentResponseMgr.setLocalResult(localNodeName, localResult)) {
        // COLLECT LOCAL RESULT ONLY
        return currentResponseMgr.getFinalResponse();
      }

      // SORT THE NODE TO GUARANTEE THE SAME ORDER OF DELIVERY
      if (!(iNodes instanceof List))
        iNodes = new ArrayList<String>(iNodes);
      if (iNodes.size() > 1)
        Collections.sort((List<String>) iNodes);

      msgService.registerRequest(iRequest.getId().getMessageId(), currentResponseMgr);

      if (ODistributedServerLog.isDebugEnabled())
        ODistributedServerLog.debug(this, localNodeName, iNodes.toString(), DIRECTION.OUT, "Sending request %s...", iRequest);

      for (String node : iNodes) {
        // CATCH ANY EXCEPTION LOG IT AND IGNORE TO CONTINUE SENDING REQUESTS TO OTHER NODES
        try {
          final ORemoteServerController remoteServer = manager.getRemoteServer(node);

          remoteServer.sendRequest(iRequest);

        } catch (Exception e) {
          currentResponseMgr.removeServerBecauseUnreachable(node);

          String reason = e.getMessage();
          if (e instanceof ODistributedException && e.getCause() instanceof IOException) {
            // CONNECTION ERROR: REMOVE THE CONNECTION
            reason = e.getCause().getMessage();

          } else if (e instanceof OSecurityAccessException) {
            // THE CONNECTION COULD BE STALE, CREATE A NEW ONE AND RETRY
            try {
              final ORemoteServerController remoteServer = manager.getRemoteServer(node);
              remoteServer.sendRequest(iRequest);
              continue;

            } catch (Exception ex) {
              // IGNORE IT BECAUSE MANAGED BELOW
            }
          }

          if (!manager.isNodeAvailable(node))
            // NODE IS NOT AVAILABLE
            ODistributedServerLog.debug(this, localNodeName, node, ODistributedServerLog.DIRECTION.OUT,
                "Error on sending distributed request %s. The target node is not available. Active nodes: %s", e, iRequest,
                manager.getAvailableNodeNames(databaseName));
          else
            ODistributedServerLog.error(this, localNodeName, node, ODistributedServerLog.DIRECTION.OUT,
                "Error on sending distributed request %s (err=%s). Active nodes: %s", iRequest, reason,
                manager.getAvailableNodeNames(databaseName));
        }
      }

      if (currentResponseMgr.getExpectedNodes().isEmpty())
        // NO SERVER TO SEND A MESSAGE
        throw new ODistributedException(
            "No server active for distributed request (" + iRequest + ") against database '" + databaseName + (
                iClusterNames != null ? "." + iClusterNames : "") + "' to nodes " + iNodes);

      if (ODistributedServerLog.isDebugEnabled())
        ODistributedServerLog.debug(this, localNodeName, iNodes.toString(), DIRECTION.OUT, "Sent request %s", iRequest);

      totalSentRequests.incrementAndGet();

      afterSendCallBackCalled = true;

      if (iAfterSentCallback != null)
        iAfterSentCallback.call(iRequest.getId());

      if (iExecutionMode == ODistributedRequest.EXECUTION_MODE.RESPONSE)
        return waitForResponse(iRequest, currentResponseMgr);

      return null;

    } catch (RuntimeException e) {
      throw e;
    } catch (Exception e) {
      throw OException.wrapException(new ODistributedException(
          "Error on executing distributed request (" + iRequest + ") against database '" + databaseName + (iClusterNames != null ?
              "." + iClusterNames :
              "") + "' to nodes " + iNodes), e);
    } finally {
      if (iAfterSentCallback != null && !afterSendCallBackCalled)
        iAfterSentCallback.call(iRequest.getId());
    }
  }

  public ODistributedResponseManager newResponseManager(ODistributedRequest iRequest, Collection<String> iNodes,
      OCallable<Void, ODistributedResponseManager> endCallback, ORemoteTask task, Set<String> nodesConcurToTheQuorum,
      int availableNodes, int expectedResponses, int quorum, boolean groupByResponse, boolean waitLocalNode) {
    return new ODistributedResponseManagerImpl(manager, iRequest, iNodes, nodesConcurToTheQuorum, expectedResponses, quorum,
        waitLocalNode, adjustTimeoutWithLatency(iNodes, task.getSynchronousTimeout(expectedResponses), iRequest.getId()),
        adjustTimeoutWithLatency(iNodes, task.getTotalTimeout(availableNodes), iRequest.getId()), groupByResponse, endCallback);
  }

  private long adjustTimeoutWithLatency(final Collection<String> iNodes, final long timeout,
      final ODistributedRequestId requestId) {
    long delta = 0;
    if (iNodes != null)
      for (String n : iNodes) {
        // UPDATE THE TIMEOUT WITH THE CURRENT SERVER LATENCY
        final long l = msgService.getCurrentLatency(n);
        delta = Math.max(delta, l);
      }

    if (delta > 500)
      ODistributedServerLog.debug(this, localNodeName, iNodes.toString(), DIRECTION.OUT,
          "Adjusted timeouts by adding +%dms because this is the maximum latency recorded against servers %s (reqId=%s)", delta,
          iNodes, requestId);

    return timeout + delta;
  }

  @Override
  public void setOnline() {
    ODistributedServerLog
        .info(this, localNodeName, null, DIRECTION.NONE, "Publishing ONLINE status for database %s.%s...", localNodeName,
            databaseName);

    // SET THE NODE.DB AS ONLINE
    manager.setDatabaseStatus(localNodeName, databaseName, ODistributedServerManager.DB_STATUS.ONLINE);

    waitForOnline.countDown();
  }

  @Override
  public ORawBuffer getRecordIfLocked(final ORID rid) {
    final ODistributedLock currentLock = lockManager.get(rid);
    if (currentLock != null)
      return currentLock.record;
    return null;
  }

  @Override
  public void replaceRecordContentIfLocked(final ORID rid, final byte[] bytes) {
    final ODistributedLock currentLock = lockManager.get(rid);
    if (currentLock != null && currentLock.record != null)
      currentLock.record.buffer = bytes;
  }

  @Override
  public boolean lockRecord(final ORID rid, final ODistributedRequestId requestId, final long timeout) {
    final ODistributedLock lock = new ODistributedLock(requestId);

    ORawBuffer currentRecord = null;
    boolean newLock = true;

    ODistributedLock currentLock = lockManager.putIfAbsent(rid, lock);
    if (currentLock != null) {
      currentRecord = currentLock.record;

      if (requestId.equals(currentLock.reqId)) {
        // SAME ID, ALREADY LOCKED
        ODistributedServerLog.debug(this, localNodeName, null, DIRECTION.NONE,
            "Distributed transaction: %s locked record %s in database '%s' owned by %s (thread=%d)", requestId, rid, databaseName,
            currentLock.reqId, Thread.currentThread().getId());
        currentLock = null;
        newLock = false;
      } else {
        // TRY TO RE-LOCK IT UNTIL TIMEOUT IS EXPIRED
        final long startTime = System.currentTimeMillis();
        do {
          try {
            if (timeout > 0) {
              if (!currentLock.lock.await(timeout, TimeUnit.MILLISECONDS))
                continue;
            } else
              currentLock.lock.await();

            currentLock = lockManager.putIfAbsent(rid, lock);

          } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            break;
          }
        } while (currentLock != null && (timeout == 0 || System.currentTimeMillis() - startTime < timeout));
      }
    }

    if (currentLock == null) {
      // SAVE CURRENT RECORD IN RAM FOR READ-ONLY ACCESS WHILE IT IS LOCKED
      if (currentRecord != null)
        // USE THE EXISTENT
        lock.record = currentRecord;
      else if (rid.isPersistent()) {
        final ODatabaseDocumentInternal db = ODatabaseRecordThreadLocal.instance().getIfDefined();
        if (db != null)
          lock.record = db.getStorage().getUnderlying().readRecord((ORecordId) rid, null, false, false, null).getResult();
      }

      if (ODistributedServerLog.isDebugEnabled())
        ODistributedServerLog
            .debug(this, localNodeName, null, DIRECTION.NONE, "Locked record %s in database '%s' (reqId=%s thread=%d)", rid,
                databaseName, requestId, Thread.currentThread().getId());
    } else {
      if (ODistributedServerLog.isDebugEnabled())
        ODistributedServerLog.debug(this, localNodeName, null, DIRECTION.NONE,
            "Cannot lock record %s in database '%s' owned by %s (reqId=%s thread=%d)", rid, databaseName, currentLock.reqId,
            requestId, Thread.currentThread().getId());
    }

    if (currentLock != null)
      throw new ODistributedRecordLockedException(manager.getLocalNodeName(), rid, currentLock.reqId, timeout);

    return newLock;
  }

  @Override
  public void unlockRecord(final OIdentifiable iRecord, final ODistributedRequestId requestId) {
    if (requestId == null)
      return;

    final ODistributedLock owner = lockManager.get(iRecord.getIdentity());
    if (owner != null) {
      if (!owner.reqId.equals(requestId)) {
        ODistributedServerLog.debug(this, localNodeName, null, DIRECTION.NONE,
            "Distributed transaction: cannot unlock record %s in database '%s' because owner %s <> current %s (thread=%d)", iRecord,
            databaseName, owner.reqId, requestId, Thread.currentThread().getId());
        return;
      }

      lockManager.remove(iRecord.getIdentity());

      // NOTIFY ANY WAITERS
      owner.lock.countDown();
    }

    if (ODistributedServerLog.isDebugEnabled())
      ODistributedServerLog.debug(this, localNodeName, null, DIRECTION.NONE,
          "Distributed transaction: %s unlocked record %s in database '%s' (owner=%s, thread=%d)", requestId, iRecord, databaseName,
          owner != null ? owner.reqId : "null", Thread.currentThread().getId());
  }

  @Override
  public boolean forceLockRecord(final ORID rid, final ODistributedRequestId requestId) {
    final ODistributedLock lock = new ODistributedLock(requestId);

    ORawBuffer currentRecord = null;

    boolean newLock = true;
    ODistributedLock currentLock = lockManager.put(rid, lock);
    if (currentLock != null) {
      currentRecord = currentLock.record;

      if (requestId.equals(currentLock.reqId)) {
        // SAME ID, ALREADY LOCKED
        ODistributedServerLog.debug(this, localNodeName, null, DIRECTION.NONE,
            "Distributed transaction: rid %s was already locked by %s in database '%s' owned by %s (thread=%d)", rid, requestId,
            databaseName, currentLock.reqId, Thread.currentThread().getId());
        currentLock = null;
        newLock = false;
      } else {
        if (currentLock.reqId.getNodeId() == requestId.getNodeId())
          // BOTH REQUESTS COME FROM THE SAME SERVER, AVOID TO CANCEL THE REQUEST BECAUSE IS THE ONLY CASE WHEN IT SHOULD WAIT FOR COMPLETION
          return lockRecord(rid, requestId, 0);

        ODistributedServerLog
            .debug(this, localNodeName, null, DIRECTION.NONE, "Canceling request %s in database '%s' (reqId=%s thread=%d)",
                currentLock.reqId, databaseName, requestId, Thread.currentThread().getId());

        // WAKE UP WAITERS OF PREVIOUS LOCK
        currentLock.lock.countDown();

        // CANCEL REQ-ID/TX THAT OWNED THE LOCK
        final ODistributedTxContext lockedCtx = activeTxContexts.get(currentLock.reqId);
        if (lockedCtx != null) {
          // CANCEL THE ENTIRE TX/CONTEXT/REQ-ID

          lockedCtx.cancel(manager, ODatabaseRecordThreadLocal.instance().get());

        } else {
          // ABORT SINGLE REQUEST
          final ODistributedResponseManager respMgr = manager.getMessageService().getResponseManager(requestId);
          if (respMgr != null) {
            respMgr.cancel();
          }
        }
      }
    }

    if (currentLock == null) {
      // SAVE CURRENT RECORD IN RAM FOR READ-ONLY ACCESS WHILE IT IS LOCKED
      if (currentRecord != null)
        // USE THE EXISTENT
        lock.record = currentRecord;
      else if (rid.isPersistent()) {
        // RELOAD IT
        final ODatabaseDocumentInternal db = ODatabaseRecordThreadLocal.instance().getIfDefined();
        if (db != null)
          lock.record = db.getStorage().getUnderlying().readRecord((ORecordId) rid, null, false, false, null).getResult();
      }

      if (newLock && ODistributedServerLog.isDebugEnabled())
        ODistributedServerLog
            .debug(this, localNodeName, null, DIRECTION.NONE, "Locked rid %s in database '%s' (reqId=%s thread=%d)", rid,
                databaseName, requestId, Thread.currentThread().getId());
    } else {
      if (ODistributedServerLog.isDebugEnabled())
        ODistributedServerLog.debug(this, localNodeName, null, DIRECTION.NONE,
            "Forced locking of rid %s in database '%s' owned by %s (reqId=%s thread=%d)", rid, databaseName, currentLock.reqId,
            requestId, Thread.currentThread().getId());
    }

    return newLock;
  }

  @Override
  public void unlockResourcesOfServer(final ODatabaseDocumentInternal database, final String serverName) {
    throw new UnsupportedOperationException(" no valid anymore");
  }

  @Override
  public ODistributedTxContext registerTxContext(final ODistributedRequestId reqId) {
    return registerTxContext(reqId, new ODistributedTxContextImpl(this, reqId));
  }

  @Override
  public ODistributedTxContext registerTxContext(final ODistributedRequestId reqId, ODistributedTxContext ctx) {
    final ODistributedTxContext prevCtx = activeTxContexts.putIfAbsent(reqId, ctx);
    if (prevCtx != null) {
      // ALREADY EXISTENT
      ODistributedServerLog.debug(this, localNodeName, null, DIRECTION.NONE,
          "Distributed transaction: repeating request %s in database '%s' (thread=%d)", reqId, databaseName,
          Thread.currentThread().getId());
      ctx = prevCtx;
    } else
      // REGISTERED
      ODistributedServerLog.debug(this, localNodeName, null, DIRECTION.NONE,
          "Distributed transaction: registered request %s in database '%s' (thread=%d)", reqId, databaseName,
          Thread.currentThread().getId());

    return ctx;
  }

  @Override
  public ODistributedTxContext popTxContext(final ODistributedRequestId requestId) {
    final ODistributedTxContext ctx = activeTxContexts.remove(requestId);
    ODistributedServerLog
        .debug(this, localNodeName, null, DIRECTION.NONE, "Distributed transaction: pop request %s for database %s -> %s",
            requestId, databaseName, ctx);
    return ctx;
  }

  @Override
  public ODistributedTxContext getTxContext(final ODistributedRequestId requestId) {
    final ODistributedTxContext ctx = activeTxContexts.get(requestId);
    ODistributedServerLog
        .debug(this, localNodeName, null, DIRECTION.NONE, "Distributed transaction: pop request %s for database %s -> %s",
            requestId, databaseName, ctx);
    return ctx;
  }

  @Override
  public ODistributedServerManager getManager() {
    return manager;
  }

  public boolean exists() {
    return manager.getServerInstance().existsDatabase(databaseName);
  }

  public ODistributedSyncConfiguration getSyncConfiguration() {
    if (syncConfiguration == null) {
      final String path = manager.getServerInstance().getDatabaseDirectory() + databaseName + "/" + DISTRIBUTED_SYNC_JSON_FILENAME;
      final File cfgFile = new File(path);
      try {
        syncConfiguration = new ODistributedSyncConfiguration(manager, databaseName, cfgFile);
      } catch (IOException e) {
        throw OException
            .wrapException(new ODistributedException("Cannot open database distributed sync configuration file: " + cfgFile), e);
      }
    }

    return syncConfiguration;
  }

  public void filterBeforeThisMomentum(final ODistributedMomentum momentum) {
    this.filterByMomentum.set(momentum);
  }

  @Override
  public void handleUnreachableNode(final String nodeName) {
  }

  @Override
  public String getDatabaseName() {
    return databaseName;
  }

  @Override
  public ODatabaseDocumentInternal getDatabaseInstance() {
    return manager.getServerInstance().openDatabase(databaseName, "internal", "internal", null, true);
  }

  @Override
  public long getReceivedRequests() {
    return totalReceivedRequests.get();
  }

  @Override
  public long getProcessedRequests() {
    long total = 0;

    if (lockThread != null)
      total += lockThread.getProcessedRequests();

    if (nowaitThread != null)
      total += nowaitThread.getProcessedRequests();

    for (ODistributedWorker workerThread : workerThreads) {
      if (workerThread != null)
        total += workerThread.getProcessedRequests();
    }

    return total;
  }

  public void shutdown() {
    running = false;

    try {
      if (txTimeoutTask != null)
        txTimeoutTask.cancel();

      if (repairer != null)
        repairer.shutdown();

      // SEND THE SHUTDOWN TO ALL THE WORKER THREADS
      if (lockThread != null)
        lockThread.sendShutdown();
      if (nowaitThread != null)
        nowaitThread.sendShutdown();
      for (ODistributedWorker workerThread : workerThreads) {
        if (workerThread != null)
          workerThread.sendShutdown();
      }

      // WAIT A BIT FOR PROPER SHUTDOWN
      if (lockThread != null)
        try {
          lockThread.join(2000);
        } catch (InterruptedException e) {
        }

      if (nowaitThread != null)
        try {
          nowaitThread.join(2000);
        } catch (InterruptedException e) {
        }

      for (ODistributedWorker workerThread : workerThreads) {
        if (workerThread != null) {
          try {
            workerThread.join(2000);
          } catch (InterruptedException e) {
          }
        }
      }
      lockThread = null;
      nowaitThread = null;
      workerThreads.clear();

      // SAVE SYNC CONFIGURATION
      try {
        getSyncConfiguration().save();
      } catch (IOException e) {
        ODistributedServerLog
            .warn(this, localNodeName, null, DIRECTION.NONE, "Error on saving distributed LSN table for database '%s'",
                databaseName);
      }
      syncConfiguration = null;

      ODistributedServerLog.info(this, localNodeName, null, DIRECTION.NONE,
          "Shutting down distributed database manager '%s'. Pending objects: txs=%d locks=%d", databaseName,
          activeTxContexts.size(), lockManager.size());

      lockManager.clear();
      activeTxContexts.clear();

      Orient.instance().getProfiler().unregisterHookValue("distributed.db." + databaseName + ".msgSent");
      Orient.instance().getProfiler().unregisterHookValue("distributed.db." + databaseName + ".msgReceived");
      Orient.instance().getProfiler().unregisterHookValue("distributed.db." + databaseName + ".activeContexts");
      Orient.instance().getProfiler().unregisterHookValue("distributed.db." + databaseName + ".workerThreads");
      Orient.instance().getProfiler().unregisterHookValue("distributed.db." + databaseName + ".recordLocks");

    } finally {

      final ODistributedServerManager.DB_STATUS serverStatus = manager.getDatabaseStatus(manager.getLocalNodeName(), databaseName);

      if (serverStatus == ODistributedServerManager.DB_STATUS.ONLINE
          || serverStatus == ODistributedServerManager.DB_STATUS.SYNCHRONIZING) {
        try {
          manager.setDatabaseStatus(manager.getLocalNodeName(), databaseName, ODistributedServerManager.DB_STATUS.NOT_AVAILABLE);
        } catch (Exception e) {
          // IGNORE IT
        }
      }

    }
  }

  protected void checkForServerOnline(final ODistributedRequest iRequest) throws ODistributedException {
    final ODistributedServerManager.NODE_STATUS srvStatus = manager.getNodeStatus();
    if (srvStatus == ODistributedServerManager.NODE_STATUS.OFFLINE
        || srvStatus == ODistributedServerManager.NODE_STATUS.SHUTTINGDOWN) {
      ODistributedServerLog
          .error(this, localNodeName, null, DIRECTION.OUT, "Local server is not online (status='%s'). Request %s will be ignored",
              srvStatus, iRequest);
      throw new OOfflineNodeException(
          "Local server is not online (status='" + srvStatus + "'). Request " + iRequest + " will be ignored");
    }
  }

  protected boolean waitForLocalNode(final ODistributedConfiguration cfg, final Collection<String> iClusterNames,
      final Collection<String> iNodes) {
    boolean waitLocalNode = false;
    if (iNodes.contains(localNodeName))
      if (iClusterNames == null || iClusterNames.isEmpty()) {
        // DEFAULT CLUSTER (*)
        if (cfg.isReadYourWrites(null))
          waitLocalNode = true;
      } else
        // BROWSE FOR ALL CLUSTER TO GET THE FIRST 'waitLocalNode'
        for (String clName : iClusterNames) {
          if (cfg.isReadYourWrites(clName)) {
            waitLocalNode = true;
            break;
          }
        }
    return waitLocalNode;
  }

  protected int calculateQuorum(final OCommandDistributedReplicateRequest.QUORUM_TYPE quorumType, Collection<String> clusterNames,
      final ODistributedConfiguration cfg, final int totalServers, final int totalMasterServers, final boolean checkNodesAreOnline,
      final String localNodeName) {

    int quorum = 1;

    if (clusterNames == null || clusterNames.isEmpty()) {
      clusterNames = new ArrayList<String>(1);
      clusterNames.add(null);
    }

    int totalServerInQuorum = totalServers;
    for (String cluster : clusterNames) {
      int clusterQuorum = 0;
      switch (quorumType) {
      case NONE:
        // IGNORE IT
        break;
      case READ:
        clusterQuorum = cfg.getReadQuorum(cluster, totalServers, localNodeName);
        break;
      case WRITE:
        clusterQuorum = cfg.getWriteQuorum(cluster, totalMasterServers, localNodeName);
        totalServerInQuorum = totalMasterServers;
        break;
      case ALL:
        clusterQuorum = totalServers;
        break;
      }

      quorum = Math.max(quorum, clusterQuorum);
    }

    if (quorum < 0)
      quorum = 0;

    if (checkNodesAreOnline && quorum > totalServerInQuorum)
      throw new ODistributedException(
          "Quorum (" + quorum + ") cannot be reached on server '" + localNodeName + "' database '" + databaseName
              + "' because it is major than available nodes (" + totalServerInQuorum + ")");

    return quorum;
  }

  protected ODistributedResponse waitForResponse(final ODistributedRequest iRequest,
      final ODistributedResponseManager currentResponseMgr) throws InterruptedException {
    final long beginTime = System.currentTimeMillis();

    // WAIT FOR THE MINIMUM SYNCHRONOUS RESPONSES (QUORUM)
    if (!currentResponseMgr.waitForSynchronousResponses()) {
      final long elapsed = System.currentTimeMillis() - beginTime;

      if (elapsed > currentResponseMgr.getSynchTimeout()) {

        ODistributedServerLog.warn(this, localNodeName, null, DIRECTION.IN,
            "Timeout (%dms) on waiting for synchronous responses from nodes=%s responsesSoFar=%s request=(%s)", elapsed,
            currentResponseMgr.getExpectedNodes(), currentResponseMgr.getRespondingNodes(), iRequest);
      }
    }

    return currentResponseMgr.getFinalResponse();
  }

  @Override
  public void checkNodeInConfiguration(final ODistributedConfiguration cfg, final String serverName) {
    manager.executeInDistributedDatabaseLock(databaseName, 20000, cfg != null ? cfg.modify() : null,
        new OCallable<Void, OModifiableDistributedConfiguration>() {
          @Override
          public Void call(final OModifiableDistributedConfiguration lastCfg) {
            // GET LAST VERSION IN LOCK
            final List<String> foundPartition = lastCfg.addNewNodeInServerList(serverName);
            if (foundPartition != null) {
              ODistributedServerLog
                  .info(this, localNodeName, null, DIRECTION.NONE, "Adding node '%s' in partition: %s db=%s v=%d", serverName,
                      foundPartition, databaseName, lastCfg.getVersion());
            }
            return null;
          }
        });
  }

  protected String getLocalNodeName() {
    return localNodeName;
  }

  private void startAcceptingRequests() {
    // START ALL THE WORKER THREADS (CONFIGURABLE)
    int totalWorkers = OGlobalConfiguration.DISTRIBUTED_DB_WORKERTHREADS.getValueAsInteger();
    if (totalWorkers < 0)
      throw new ODistributedException("Cannot create configured distributed workers (" + totalWorkers + ")");
    else if (totalWorkers == 0) {
      // AUTOMATIC
      final int totalDatabases = manager.getManagedDatabases().size() + 1;

      final int cpus = Runtime.getRuntime().availableProcessors();

      if (cpus > 1)
        totalWorkers = cpus / totalDatabases;

      if (totalWorkers == 0)
        totalWorkers = 1;
    }

    lockThread = new ODistributedWorker(this, databaseName, -3, false) {
      protected void handleError(final ODistributedRequest iRequest, final Object responsePayload) {
        // CANNOT SEND MSG BACK, UNLOCK IT
        final ODistributedLockTask task = (ODistributedLockTask) iRequest.getTask();
        task.undo(manager);
      }
    };
    lockThread.start();

    nowaitThread = new ODistributedWorker(this, databaseName, -4, true);
    nowaitThread.start();

    for (int i = 0; i < totalWorkers; ++i) {
      final ODistributedWorker workerThread = new ODistributedWorker(this, databaseName, i, true);
      workerThreads.add(workerThread);
      workerThread.start();

      ALL_QUEUES.add(i);
    }
  }

  @Override
  public void setLSN(final String sourceNodeName, final OLogSequenceNumber taskLastLSN, final boolean updateLastOperationTimestamp)
      throws IOException {
    if (taskLastLSN == null)
      return;

    final ODistributedSyncConfiguration cfg = getSyncConfiguration();
    cfg.setLastLSN(sourceNodeName, taskLastLSN, updateLastOperationTimestamp);
  }

  @Override
  public ODistributedDatabaseRepairer getDatabaseRepairer() {
    return repairer;
  }

  private void startTxTimeoutTimerTask() {
    txTimeoutTask = new TimerTask() {
      @Override
      public void run() {
        ODatabaseDocumentInternal database = null;
        try {
          final long now = System.currentTimeMillis();
          final long timeout = OGlobalConfiguration.DISTRIBUTED_TX_EXPIRE_TIMEOUT.getValueAsLong();

          final Set<ORecordId> rids2Repair = new HashSet<ORecordId>();

          for (final Iterator<ODistributedTxContext> it = activeTxContexts.values().iterator(); it.hasNext(); ) {
            if (!isRunning())
              break;

            final ODistributedTxContext ctx = it.next();
            if (ctx != null) {
              final long started = ctx.getStartedOn();
              final long elapsed = now - started;
              if (elapsed > timeout) {
                // TRANSACTION EXPIRED, ROLLBACK IT

                if (database == null)
                  // GET THE DATABASE THE FIRST TIME
                  database = getDatabaseInstance();

                ODistributedServerLog.debug(this, localNodeName, null, DIRECTION.NONE,
                    "Distributed transaction %s on database '%s' is expired after %dms", ctx.getReqId(), databaseName, elapsed);

                if (database != null)
                  database.activateOnCurrentThread();

                try {
                  rids2Repair.addAll(ctx.cancel(manager, database));

                  if (ctx.getReqId().getNodeId() == manager.getLocalNodeId())
                    // REQUEST WAS ORIGINATED FROM CURRENT SERVER
                    msgService.timeoutRequest(ctx.getReqId().getMessageId());

                } catch (Exception t) {
                  ODistributedServerLog.info(this, localNodeName, null, DIRECTION.NONE,
                      "Error on rolling back distributed transaction %s on database '%s' (err=%s)", ctx.getReqId(), databaseName,
                      t);
                } finally {
                  it.remove();
                }
              }
            }
          }

          // CHECK INDIVIDUAL LOCKS TOO
          for (final Iterator<Map.Entry<ORID, ODistributedLock>> it = lockManager.entrySet().iterator(); it.hasNext(); ) {
            final Map.Entry<ORID, ODistributedLock> entry = it.next();

            final ODistributedLock lock = entry.getValue();
            if (lock != null) {
              final long elapsed = now - lock.acquiredOn;
              if (elapsed > timeout) {
                // EXPIRED
                ODistributedServerLog.debug(this, localNodeName, null, DIRECTION.NONE,
                    "Distributed lock on database '%s' record %s is expired after %dms", databaseName, entry.getKey(), elapsed);

                it.remove();
              }
            }
          }

          getDatabaseRepairer().enqueueRepairRecords(rids2Repair);

        } catch (Exception t) {
          // CATCH EVERYTHING TO AVOID THE TIMER IS CANCELED
          ODistributedServerLog.info(this, localNodeName, null, DIRECTION.NONE,
              "Error on checking for expired distributed transaction on database '%s'", databaseName);
        } finally {
          if (database != null) {
            database.activateOnCurrentThread();
            database.close();
          }
        }
      }
    };

//    Orient.instance().scheduleTask(txTimeoutTask, OGlobalConfiguration.DISTRIBUTED_TX_EXPIRE_TIMEOUT.getValueAsLong(),
//        OGlobalConfiguration.DISTRIBUTED_TX_EXPIRE_TIMEOUT.getValueAsLong() / 2);
  }

  private boolean isRunning() {
    return running;
  }

  public void suspend() {
    if (this.parsing.get()) {
      // RESET THE DATABASE
      if (lockThread != null)
        lockThread.reset();
      if (nowaitThread != null)
        nowaitThread.reset();
      for (ODistributedWorker w : workerThreads) {
        if (w != null)
          w.reset();
      }
    }

    this.parsing.set(false);
  }

  public void resume() {
    this.parsing.set(true);
  }

  @Override
  public String dump() {
    final StringBuilder buffer = new StringBuilder(1024);

    buffer.append("\n\nDATABASE '" + databaseName + "' ON SERVER '" + manager.getLocalNodeName() + "'");

    buffer.append("\n- " + ODistributedOutput.formatRecordLocks(manager, databaseName));

    buffer.append("\n- MESSAGES IN QUEUES");
    buffer.append(" (" + (workerThreads != null ? workerThreads.size() : 0) + " WORKERS):");

    if (lockThread != null) {
      final ODistributedRequest processing = lockThread.getProcessing();
      final ArrayBlockingQueue<ODistributedRequest> queue = lockThread.localQueue;

      if (processing != null || !queue.isEmpty()) {
        buffer.append("\n - QUEUE LOCK EXECUTING: " + processing);
        int i = 0;
        for (ODistributedRequest m : queue) {
          if (m != null)
            buffer.append("\n  - " + i + " = " + m.toString());
        }
      }
    }

    if (nowaitThread != null) {
      final ODistributedRequest processing = nowaitThread.getProcessing();
      final ArrayBlockingQueue<ODistributedRequest> queue = nowaitThread.localQueue;

      if (processing != null || !queue.isEmpty()) {
        buffer.append("\n - QUEUE UNLOCK EXECUTING: " + processing);
        int i = 0;
        for (ODistributedRequest m : queue) {
          if (m != null)
            buffer.append("\n  - " + i + " = " + m.toString());
        }
      }
    }

    if (workerThreads != null) {
      for (ODistributedWorker t : workerThreads) {
        final ODistributedRequest processing = t.getProcessing();
        final ArrayBlockingQueue<ODistributedRequest> queue = t.localQueue;

        if (processing != null || !queue.isEmpty()) {
          buffer.append("\n  - QUEUE " + t.id + " EXECUTING: " + processing);
          int i = 0;
          for (ODistributedRequest m : queue) {
            if (m != null)
              buffer.append("\n   - " + (i++) + " = " + m.toString());
          }
        }
      }
    }

    return buffer.toString();
  }

}


/*
 *
 *  *  Copyright 2010-2016 OrientDB LTD (http://orientdb.com)
 *  *
 *  *  Licensed under the Apache License, Version 2.0 (the "License");
 *  *  you may not use this file except in compliance with the License.
 *  *  You may obtain a copy of the License at
 *  *
 *  *       http://www.apache.org/licenses/LICENSE-2.0
 *  *
 *  *  Unless required by applicable law or agreed to in writing, software
 *  *  distributed under the License is distributed on an "AS IS" BASIS,
 *  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  *  See the License for the specific language governing permissions and
 *  *  limitations under the License.
 *  *
 *  * For more information: http://orientdb.com
 *
 */
package com.orientechnologies.orient.core;

import com.orientechnologies.common.directmemory.OByteBufferPool;
import com.orientechnologies.common.directmemory.ODirectMemoryAllocator;
import com.orientechnologies.common.io.OFileUtils;
import com.orientechnologies.common.listener.OListenerManger;
import com.orientechnologies.common.log.OLogManager;
import com.orientechnologies.common.parser.OSystemVariableResolver;
import com.orientechnologies.common.profiler.OAbstractProfiler;
import com.orientechnologies.common.profiler.OProfiler;
import com.orientechnologies.common.profiler.OProfilerStub;
import com.orientechnologies.common.thread.OThreadPoolExecutorWithLogging;
import com.orientechnologies.common.util.OClassLoaderHelper;
import com.orientechnologies.orient.core.cache.OLocalRecordCacheFactory;
import com.orientechnologies.orient.core.cache.OLocalRecordCacheFactoryImpl;
import com.orientechnologies.orient.core.command.script.OScriptManager;
import com.orientechnologies.orient.core.config.OGlobalConfiguration;
import com.orientechnologies.orient.core.conflict.ORecordConflictStrategyFactory;
import com.orientechnologies.orient.core.db.ODatabaseLifecycleListener;
import com.orientechnologies.orient.core.db.ODatabaseThreadLocalFactory;
import com.orientechnologies.orient.core.db.OrientDBEmbedded;
import com.orientechnologies.orient.core.db.OrientDBInternal;
import com.orientechnologies.orient.core.engine.OEngine;
import com.orientechnologies.orient.core.record.ORecordFactoryManager;
import com.orientechnologies.orient.core.security.OSecuritySystem;
import com.orientechnologies.orient.core.shutdown.OShutdownHandler;
import com.orientechnologies.orient.core.storage.OStorage;

import java.lang.ref.ReferenceQueue;
import java.lang.ref.WeakReference;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.PriorityQueue;
import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;

public class Orient extends OListenerManger<OOrientListener> {
  public static final String ORIENTDB_HOME = "ORIENTDB_HOME";
  public static final String URL_SYNTAX    = "<engine>:<db-type>:<db-name>[?<db-param>=<db-value>[&]]*";

  private static volatile Orient instance;
  private static final    Lock   initLock = new ReentrantLock();

  private static volatile boolean registerDatabaseByPath = false;

  private final ConcurrentMap<String, OEngine> engines = new ConcurrentHashMap<String, OEngine>();

  private final Map<ODatabaseLifecycleListener, ODatabaseLifecycleListener.PRIORITY> dbLifecycleListeners          = new LinkedHashMap<ODatabaseLifecycleListener, ODatabaseLifecycleListener.PRIORITY>();
  private final OScriptManager                                                       scriptManager                 = new OScriptManager();
  private final ThreadGroup                                                          threadGroup;
  private final ReadWriteLock                                                        engineLock                    = new ReentrantReadWriteLock();
  private final ORecordConflictStrategyFactory                                       recordConflictStrategy        = new ORecordConflictStrategyFactory();
  private final ReferenceQueue<OOrientStartupListener>                               removedStartupListenersQueue  = new ReferenceQueue<OOrientStartupListener>();
  private final ReferenceQueue<OOrientShutdownListener>                              removedShutdownListenersQueue = new ReferenceQueue<OOrientShutdownListener>();
  private final Set<OOrientStartupListener>                                          startupListeners              = Collections
      .newSetFromMap(new ConcurrentHashMap<OOrientStartupListener, Boolean>());
  private final Set<WeakHashSetValueHolder<OOrientStartupListener>>                  weakStartupListeners          = Collections
      .newSetFromMap(new ConcurrentHashMap<WeakHashSetValueHolder<OOrientStartupListener>, Boolean>());
  private final Set<WeakHashSetValueHolder<OOrientShutdownListener>>                 weakShutdownListeners         = Collections
      .newSetFromMap(new ConcurrentHashMap<WeakHashSetValueHolder<OOrientShutdownListener>, Boolean>());

  private final PriorityQueue<OShutdownHandler> shutdownHandlers = new PriorityQueue<OShutdownHandler>(11,
      new Comparator<OShutdownHandler>() {
        @Override
        public int compare(OShutdownHandler handlerOne, OShutdownHandler handlerTwo) {
          if (handlerOne.getPriority() > handlerTwo.getPriority())
            return 1;

          if (handlerOne.getPriority() < handlerTwo.getPriority())
            return -1;

          return 0;
        }
      });

  private final OLocalRecordCacheFactory localRecordCache = new OLocalRecordCacheFactoryImpl();

  private Set<OrientDBEmbedded> factories = Collections.newSetFromMap(new ConcurrentHashMap<>());

  private Set<OrientDBInternal> runningInstances = new HashSet<>();

  private final String os;

  private volatile Timer                       timer;
  private volatile ORecordFactoryManager       recordFactoryManager = new ORecordFactoryManager();
  private          OrientShutdownHook          shutdownHook;
  private volatile OAbstractProfiler           profiler;
  private          ODatabaseThreadLocalFactory databaseThreadFactory;
  private volatile boolean                     active               = false;
  private          ThreadPoolExecutor          workers;
  private          OSignalHandler              signalHandler;
  private volatile OSecuritySystem             security;
  private          boolean                     runningDistributed   = false;

  /**
   * Indicates that engine is initialized inside of web application container.
   */
  private volatile boolean insideWebContainer;

  /**
   * Prevents duplications because of recursive initialization.
   */
  private static boolean initInProgress = false;

  private static class WeakHashSetValueHolder<T> extends WeakReference<T> {
    private final int hashCode;

    private WeakHashSetValueHolder(T referent, ReferenceQueue<? super T> q) {
      super(referent, q);
      this.hashCode = referent.hashCode();
    }

    @Override
    public int hashCode() {
      return hashCode;
    }

    @Override
    public boolean equals(Object o) {
      if (this == o)
        return true;

      if (o == null || getClass() != o.getClass())
        return false;

      WeakHashSetValueHolder that = (WeakHashSetValueHolder) o;

      if (hashCode != that.hashCode)
        return false;

      final T thisObject = get();
      final Object thatObject = that.get();

      if (thisObject == null && thatObject == null)
        return super.equals(that);
      else if (thisObject != null && thatObject != null)
        return thisObject.equals(thatObject);

      return false;
    }
  }

  Orient(boolean insideWebContainer) {
    super(true);
    this.insideWebContainer = insideWebContainer;
    this.os = System.getProperty("os.name").toLowerCase(Locale.ENGLISH);
    threadGroup = new ThreadGroup("OrientDB");
    threadGroup.setDaemon(false);
  }

  public boolean isInsideWebContainer() {
    return insideWebContainer;
  }

  public static Orient instance() {
    if (instance != null)
      return instance;

    return startUp(false);
  }

  public static Orient startUp(boolean insideWebContainer) {
    initLock.lock();
    try {
      if (initInProgress) {
        return null;
      }

      initInProgress = true;
      if (instance != null)
        return instance;

      final Orient orient = new Orient(false);
      orient.startup();

      instance = orient;
    } finally {
      initInProgress = false;
      initLock.unlock();
    }

    return instance;
  }

  public static String getHomePath() {
    String v = System.getProperty("orient.home");

    if (v == null)
      v = OSystemVariableResolver.resolveVariable(ORIENTDB_HOME);

    return OFileUtils.getPath(v);
  }

  public static String getTempPath() {
    return OFileUtils.getPath(System.getProperty("java.io.tmpdir") + "/orientdb/");
  }

  /**
   * Tells if to register database by path. Default is false. Setting to true allows to have multiple databases in different path
   * with the same name.
   *
   * @see #setRegisterDatabaseByPath(boolean)
   */
  public static boolean isRegisterDatabaseByPath() {
    return registerDatabaseByPath;
  }

  /**
   * Register database by path. Default is false. Setting to true allows to have multiple databases in different path with the same
   * name.
   */
  public static void setRegisterDatabaseByPath(final boolean iValue) {
    registerDatabaseByPath = iValue;
  }

  public ORecordConflictStrategyFactory getRecordConflictStrategy() {
    return recordConflictStrategy;
  }

  public Orient startup() {
    engineLock.writeLock().lock();
    try {
      if (active)
        // ALREADY ACTIVE
        return this;

      if (timer == null)
        timer = new Timer(true);

      profiler = new OProfilerStub(false);

      shutdownHook = new OrientShutdownHook();
      if (signalHandler == null) {
        signalHandler = new OSignalHandler();
        signalHandler.installDefaultSignals();
      }

      final int cores = Runtime.getRuntime().availableProcessors();

      workers = new OThreadPoolExecutorWithLogging(cores, cores * 3, 10, TimeUnit.SECONDS,
          new LinkedBlockingQueue<Runnable>(cores * 500) {
            @Override
            public boolean offer(Runnable e) {
              // turn offer() and add() into a blocking calls (unless interrupted)
              try {
                put(e);
                return true;
              } catch (InterruptedException ignore) {
                Thread.currentThread().interrupt();
              }
              return false;
            }
          });

      registerEngines();

      if (OGlobalConfiguration.ENVIRONMENT_DUMP_CFG_AT_STARTUP.getValueAsBoolean())
        OGlobalConfiguration.dumpConfiguration(System.out);

      active = true;

      for (OOrientStartupListener l : startupListeners)
        try {
          if (l != null)
            l.onStartup();
        } catch (Exception e) {
          OLogManager.instance().error(this, "Error on startup", e);
        }

      purgeWeakStartupListeners();
      for (final WeakHashSetValueHolder<OOrientStartupListener> wl : weakStartupListeners)
        try {
          if (wl != null) {
            final OOrientStartupListener l = wl.get();
            if (l != null)
              l.onStartup();
          }

        } catch (Exception e) {
          OLogManager.instance().error(this, "Error on startup", e);
        }

      initShutdownQueue();
      registerWeakOrientStartupListener(profiler);
    } finally {
      engineLock.writeLock().unlock();
    }

    return this;
  }

  /**
   * Add handler which will be executed during {@link #shutdown()} call.
   *
   * @param shutdownHandler Shutdown handler instance.
   */
  public void addShutdownHandler(OShutdownHandler shutdownHandler) {
    engineLock.writeLock().lock();
    try {
      shutdownHandlers.add(shutdownHandler);
    } finally {
      engineLock.writeLock().unlock();
    }
  }

  /**
   * Adds shutdown handlers in order which will be used during execution of shutdown.
   */
  private void initShutdownQueue() {
    addShutdownHandler(new OShutdownWorkersHandler());
    addShutdownHandler(new OShutdownOrientDBInstancesHandler());
    addShutdownHandler(new OShutdownPendingThreadsHandler());
    addShutdownHandler(new OShutdownProfilerHandler());
    addShutdownHandler(new OShutdownCallListenersHandler());
  }

  /**
   * Shutdown whole OrientDB ecosystem. Usually is called during JVM shutdown by JVM shutdown handler. During shutdown all handlers
   * which were registered by the call of {@link #addShutdownHandler(OShutdownHandler)} are called together with pre-registered
   * system shoutdown handlers according to their priority.
   *
   * @see OShutdownWorkersHandler
   * @see
   */
  private void registerEngines() {
    ClassLoader classLoader = Orient.class.getClassLoader();

    Iterator<OEngine> engines = OClassLoaderHelper.lookupProviderWithOrientClassLoader(OEngine.class, classLoader);

    OEngine engine = null;
    while (engines.hasNext()) {
      try {
        engine = engines.next();
        registerEngine(engine);
      } catch (IllegalArgumentException e) {
        if (engine != null)
          OLogManager.instance().debug(this, "Failed to replace engine " + engine.getName(), e);
      }
    }
  }

  public Orient shutdown() {
    engineLock.writeLock().lock();
    try {
      if (!active)
        return this;

      active = false;

      OLogManager.instance().info(this, "Orient Engine is shutting down...");
      for (OShutdownHandler handler : shutdownHandlers) {
        try {
          OLogManager.instance().debug(this, "Shutdown handler %s is going to be called", handler);
          handler.shutdown();
          OLogManager.instance().debug(this, "Shutdown handler %s completed", handler);
        } catch (Exception e) {
          OLogManager.instance().error(this, "Exception during calling of shutdown handler %s", e, handler);
        }
      }

      shutdownHandlers.clear();

      OLogManager.instance().info(this, "Clearing byte buffer pool");
      OByteBufferPool.instance(null).clear();

      OByteBufferPool.instance(null).checkMemoryLeaks();
      ODirectMemoryAllocator.instance().checkMemoryLeaks();

      OLogManager.instance().info(this, "OrientDB Engine shutdown complete");
      OLogManager.instance().flush();
    } finally {
      try {
        removeShutdownHook();
      } finally {
        try {
          removeSignalHandler();
        } finally {
          engineLock.writeLock().unlock();
        }
      }
    }

    return this;
  }

  public TimerTask scheduleTask(final Runnable task, final long delay, final long period) {
    engineLock.readLock().lock();
    try {
      final TimerTask timerTask = new TimerTask() {
        @Override
        public void run() {
          try {
            task.run();
          } catch (Exception e) {
            OLogManager.instance().error(this, "Error during execution of task " + task.getClass().getSimpleName(), e);
          } catch (Error e) {
            OLogManager.instance().error(this, "Error during execution of task " + task.getClass().getSimpleName(), e);
            throw e;
          }
        }
      };

      if (active) {
        if (period > 0) {
          timer.schedule(timerTask, delay, period);
        } else {
          timer.schedule(timerTask, delay);
        }
      } else {
        OLogManager.instance().warn(this, "OrientDB engine is down. Task will not be scheduled.");
      }

      return timerTask;
    } finally {
      engineLock.readLock().unlock();
    }
  }

  public TimerTask scheduleTask(final Runnable task, final Date firstTime, final long period) {
    engineLock.readLock().lock();
    try {
      final TimerTask timerTask = new TimerTask() {
        @Override
        public void run() {
          try {
            task.run();
          } catch (Exception e) {
            OLogManager.instance().error(this, "Error during execution of task " + task.getClass().getSimpleName(), e);
          } catch (Error e) {
            OLogManager.instance().error(this, "Error during execution of task " + task.getClass().getSimpleName(), e);
            throw e;
          }
        }
      };

      if (active) {
        if (period > 0) {
          timer.schedule(timerTask, firstTime, period);
        } else {
          timer.schedule(timerTask, firstTime);
        }
      } else {
        OLogManager.instance().warn(this, "OrientDB engine is down. Task will not be scheduled.");
      }

      return timerTask;
    } finally {
      engineLock.readLock().unlock();
    }
  }

  public boolean isActive() {
    return active;
  }

  /**
   * @deprecated This method is not thread safe. Use {@link #submit(java.util.concurrent.Callable)} instead.
   */
  @Deprecated
  public ThreadPoolExecutor getWorkers() {
    return workers;
  }

  public Future<?> submit(final Runnable runnable) {
    engineLock.readLock().lock();
    try {
      if (active)
        return workers.submit(runnable);
      else {
        OLogManager.instance().warn(this, "OrientDB engine is down. Task will not be submitted.");
        throw new IllegalStateException("OrientDB engine is down. Task will not be submitted.");
      }
    } finally {
      engineLock.readLock().unlock();
    }
  }

  public <V> Future<V> submit(final Callable<V> callable) {
    engineLock.readLock().lock();
    try {
      if (active)
        return workers.submit(callable);
      else {
        OLogManager.instance().warn(this, "OrientDB engine is down. Task will not be submitted.");
        throw new IllegalStateException("OrientDB engine is down. Task will not be submitted.");
      }
    } finally {
      engineLock.readLock().unlock();
    }
  }

  public boolean isWindowsOS() {
    return os.contains("win");
  }

  public void registerEngine(final OEngine iEngine) throws IllegalArgumentException {
    OEngine oEngine = engines.get(iEngine.getName());

    if (oEngine != null) {
      if (!oEngine.getClass().isAssignableFrom(iEngine.getClass())) {
        throw new IllegalArgumentException("Cannot replace storage " + iEngine.getName());
      }
    }
    engines.put(iEngine.getName(), iEngine);
  }

  /**
   * Returns the engine by its name.
   *
   * @param engineName Engine name to retrieve
   *
   * @return OEngine instance of found, otherwise null
   */
  public OEngine getEngine(final String engineName) {
    engineLock.readLock().lock();
    try {
      return engines.get(engineName);
    } finally {
      engineLock.readLock().unlock();
    }
  }

  /**
   * Obtains an {@link OEngine engine} instance with the given {@code engineName}, if it is {@link OEngine#isRunning() running}.
   *
   * @param engineName the name of the engine to obtain.
   *
   * @return the obtained engine instance or {@code null} if no such engine known or the engine is not running.
   */
  public OEngine getEngineIfRunning(final String engineName) {
    engineLock.readLock().lock();
    try {
      final OEngine engine = engines.get(engineName);
      return engine == null || !engine.isRunning() ? null : engine;
    } finally {
      engineLock.readLock().unlock();
    }
  }

  /**
   * Obtains a {@link OEngine#isRunning() running} {@link OEngine engine} instance with the given {@code engineName}. If engine is
   * not running, starts it.
   *
   * @param engineName the name of the engine to obtain.
   *
   * @return the obtained running engine instance, never {@code null}.
   *
   * @throws IllegalStateException if an engine with the given is not found or failed to start.
   */
  public OEngine getRunningEngine(final String engineName) {
    engineLock.readLock().lock();
    try {
      OEngine engine = engines.get(engineName);
      if (engine == null)
        throw new IllegalStateException("Engine '" + engineName + "' is not found.");

      if (!engine.isRunning() && !startEngine(engine))
        throw new IllegalStateException("Engine '" + engineName + "' is failed to start.");

      return engine;
    } finally {
      engineLock.readLock().unlock();
    }
  }

  public Set<String> getEngines() {
    engineLock.readLock().lock();
    try {
      return Collections.unmodifiableSet(engines.keySet());
    } finally {
      engineLock.readLock().unlock();
    }
  }

  public Collection<OStorage> getStorages() {
    List<OStorage> storages = new ArrayList<>();
    for (OrientDBEmbedded factory : factories) {
      storages.addAll(factory.getStorages());
    }
    return storages;
  }

  public void removeShutdownHook() {
    if (shutdownHook != null) {
      shutdownHook.cancel();
      shutdownHook = null;
    }
  }

  public OSignalHandler getSignalHandler() {
    return signalHandler;
  }

  public void removeSignalHandler() {
    if (signalHandler != null) {
      signalHandler.cancel();
      signalHandler = null;
    }
  }

  public boolean isSelfManagedShutdown() {
    return shutdownHook != null;
  }

  public Iterator<ODatabaseLifecycleListener> getDbLifecycleListeners() {
    return new HashSet<ODatabaseLifecycleListener>(dbLifecycleListeners.keySet()).iterator();
  }

  public void addDbLifecycleListener(final ODatabaseLifecycleListener iListener) {
    final Map<ODatabaseLifecycleListener, ODatabaseLifecycleListener.PRIORITY> tmp = new LinkedHashMap<ODatabaseLifecycleListener, ODatabaseLifecycleListener.PRIORITY>(
        dbLifecycleListeners);
    if (iListener.getPriority() == null)
      throw new IllegalArgumentException("Priority of DatabaseLifecycleListener '" + iListener + "' cannot be null");

    tmp.put(iListener, iListener.getPriority());
    dbLifecycleListeners.clear();
    for (ODatabaseLifecycleListener.PRIORITY p : ODatabaseLifecycleListener.PRIORITY.values()) {
      for (Map.Entry<ODatabaseLifecycleListener, ODatabaseLifecycleListener.PRIORITY> e : tmp.entrySet()) {
        if (e.getValue() == p)
          dbLifecycleListeners.put(e.getKey(), e.getValue());
      }
    }
  }

  public void removeDbLifecycleListener(final ODatabaseLifecycleListener iListener) {
    dbLifecycleListeners.remove(iListener);
  }

  public ThreadGroup getThreadGroup() {
    return threadGroup;
  }

  public ODatabaseThreadLocalFactory getDatabaseThreadFactory() {
    return databaseThreadFactory;
  }

  public ORecordFactoryManager getRecordFactoryManager() {
    return recordFactoryManager;
  }

  public void setRecordFactoryManager(final ORecordFactoryManager iRecordFactoryManager) {
    recordFactoryManager = iRecordFactoryManager;
  }

  public OProfiler getProfiler() {
    return profiler;
  }

  public void setProfiler(final OAbstractProfiler iProfiler) {
    profiler = iProfiler;
  }

  public OSecuritySystem getSecurity() {
    return this.security;
  }

  public void setSecurity(final OSecuritySystem security) {
    this.security = security;
  }

  public void registerThreadDatabaseFactory(final ODatabaseThreadLocalFactory iDatabaseFactory) {
    databaseThreadFactory = iDatabaseFactory;
  }

  public OScriptManager getScriptManager() {
    return scriptManager;
  }

  @Override
  public void registerListener(OOrientListener listener) {
    if (listener instanceof OOrientStartupListener)
      registerOrientStartupListener((OOrientStartupListener) listener);

    super.registerListener(listener);
  }

  @Override
  public void unregisterListener(OOrientListener listener) {
    if (listener instanceof OOrientStartupListener)
      unregisterOrientStartupListener((OOrientStartupListener) listener);

    super.unregisterListener(listener);
  }

  public void registerOrientStartupListener(OOrientStartupListener listener) {
    startupListeners.add(listener);
  }

  public void registerWeakOrientStartupListener(OOrientStartupListener listener) {
    purgeWeakStartupListeners();
    weakStartupListeners.add(new WeakHashSetValueHolder<OOrientStartupListener>(listener, removedStartupListenersQueue));
  }

  public void unregisterOrientStartupListener(OOrientStartupListener listener) {
    startupListeners.remove(listener);
  }

  public void unregisterWeakOrientStartupListener(OOrientStartupListener listener) {
    purgeWeakStartupListeners();
    weakStartupListeners.remove(new WeakHashSetValueHolder<OOrientStartupListener>(listener, null));
  }

  public void registerWeakOrientShutdownListener(OOrientShutdownListener listener) {
    purgeWeakShutdownListeners();
    weakShutdownListeners.add(new WeakHashSetValueHolder<OOrientShutdownListener>(listener, removedShutdownListenersQueue));
  }

  public void unregisterWeakOrientShutdownListener(OOrientShutdownListener listener) {
    purgeWeakShutdownListeners();
    weakShutdownListeners.remove(new WeakHashSetValueHolder<OOrientShutdownListener>(listener, null));
  }

  @Override
  public void resetListeners() {
    super.resetListeners();

    weakShutdownListeners.clear();

    startupListeners.clear();
    weakStartupListeners.clear();
  }

  public OLocalRecordCacheFactory getLocalRecordCache() {
    return localRecordCache;
  }

  private void purgeWeakStartupListeners() {
    synchronized (removedStartupListenersQueue) {
      WeakHashSetValueHolder<OOrientStartupListener> ref = (WeakHashSetValueHolder<OOrientStartupListener>) removedStartupListenersQueue
          .poll();
      while (ref != null) {
        weakStartupListeners.remove(ref);
        ref = (WeakHashSetValueHolder<OOrientStartupListener>) removedStartupListenersQueue.poll();
      }

    }
  }

  private void purgeWeakShutdownListeners() {
    synchronized (removedShutdownListenersQueue) {
      WeakHashSetValueHolder<OOrientShutdownListener> ref = (WeakHashSetValueHolder<OOrientShutdownListener>) removedShutdownListenersQueue
          .poll();
      while (ref != null) {
        weakShutdownListeners.remove(ref);
        ref = (WeakHashSetValueHolder<OOrientShutdownListener>) removedShutdownListenersQueue.poll();
      }

    }
  }

  private boolean startEngine(OEngine engine) {
    final String name = engine.getName();

    try {
      engine.startup();
      return true;
    } catch (Exception e) {
      OLogManager.instance().error(this, "Error during initialization of engine '%s', engine will be removed", e, name);

      try {
        engine.shutdown();
      } catch (Exception se) {
        OLogManager.instance().error(this, "Error during engine shutdown", se);
      }

      engines.remove(name);
    }

    return false;
  }

  /**
   * Closes all storages and shutdown all engines.
   */
  public class OShutdownOrientDBInstancesHandler implements OShutdownHandler {
    @Override
    public int getPriority() {
      return SHUTDOWN_ENGINES_PRIORITY;
    }

    @Override
    public void shutdown() throws Exception {
      for (OrientDBInternal internal : runningInstances) {
        internal.internalClose();
      }
      runningInstances.clear();
    }

    @Override
    public String toString() {
      return getClass().getSimpleName();
    }
  }

  /**
   * Shutdown thread group which is used in methods {@link #submit(Callable)} and {@link #submit(Runnable)}.
   */
  public class OShutdownWorkersHandler implements OShutdownHandler {
    @Override
    public int getPriority() {
      return SHUTDOWN_WORKERS_PRIORITY;
    }

    @Override
    public void shutdown() throws Exception {
      workers.shutdown();
      try {
        workers.awaitTermination(2, TimeUnit.MINUTES);
      } catch (InterruptedException e) {
        OLogManager.instance().error(this, "Shutdown was interrupted", e);
      }
    }

    @Override
    public String toString() {
      return getClass().getSimpleName();
    }
  }

  /**
   * Interrupts all threads in OrientDB thread group and stops timer is used in methods {@link #scheduleTask(Runnable, Date, long)}
   * and {@link #scheduleTask(Runnable, long, long)}.
   */
  private class OShutdownPendingThreadsHandler implements OShutdownHandler {
    @Override
    public int getPriority() {
      return SHUTDOWN_PENDING_THREADS_PRIORITY;
    }

    @Override
    public void shutdown() throws Exception {
      if (threadGroup != null)
        // STOP ALL THE PENDING THREADS
        threadGroup.interrupt();

      if (timer != null) {
        timer.cancel();
        timer = null;
      }
    }

    @Override
    public String toString() {
      //it is strange but windows defender block compilation if we get class name programmatically using Class instance
      return "OShutdownPendingThreadsHandler";
    }
  }

  /**
   * Shutdown OrientDB profiler.
   */
  private class OShutdownProfilerHandler implements OShutdownHandler {
    @Override
    public int getPriority() {
      return SHUTDOWN_PROFILER_PRIORITY;
    }

    @Override
    public void shutdown() throws Exception {
      // NOTE: DON'T REMOVE PROFILER TO AVOID NPE AROUND THE CODE IF ANY THREADS IS STILL WORKING
      profiler.shutdown();
    }

    @Override
    public String toString() {
      return getClass().getSimpleName();
    }
  }

  /**
   * Calls all shutdown listeners.
   */
  private class OShutdownCallListenersHandler implements OShutdownHandler {
    @Override
    public int getPriority() {
      return SHUTDOWN_CALL_LISTENERS;
    }

    @Override
    public void shutdown() throws Exception {
      purgeWeakShutdownListeners();
      for (final WeakHashSetValueHolder<OOrientShutdownListener> wl : weakShutdownListeners)
        try {
          if (wl != null) {
            final OOrientShutdownListener l = wl.get();
            if (l != null) {
              l.onShutdown();
            }
          }

        } catch (Exception e) {
          OLogManager.instance().error(this, "Error during orient shutdown", e);
        }

      // CALL THE SHUTDOWN ON ALL THE LISTENERS
      for (OOrientListener l : browseListeners()) {
        if (l != null)
          try {
            l.onShutdown();
          } catch (Exception e) {
            OLogManager.instance().error(this, "Error during orient shutdown", e);
          }

      }

      System.gc();
    }

    @Override
    public String toString() {
      return getClass().getSimpleName();
    }
  }

  public boolean isRunningDistributed() {
    return runningDistributed;
  }

  public void setRunningDistributed(final boolean runningDistributed) {
    this.runningDistributed = runningDistributed;
  }

  public void onEmbeddedFactoryInit(OrientDBEmbedded embeddedFactory) {
    OEngine memory = engines.get("memory");
    if (!memory.isRunning())
      memory.startup();
    OEngine disc = engines.get("plocal");
    if (!disc.isRunning())
      disc.startup();
    factories.add(embeddedFactory);
  }

  public void onEmbeddedFactoryClose(OrientDBEmbedded embeddedFactory) {
    factories.remove(embeddedFactory);
    if (factories.isEmpty()) {
      OEngine memory = engines.get("memory");
      if (memory.isRunning())
        memory.shutdown();
      OEngine disc = engines.get("plocal");
      if (disc.isRunning())
        disc.shutdown();
    }
  }

  public void addOrientDB(OrientDBInternal internal) {
    engineLock.writeLock().lock();
    try {
      runningInstances.add(internal);
    } finally {
      engineLock.writeLock().unlock();
    }

  }

  public void removeOrientDB(OrientDBInternal internal) {
    engineLock.writeLock().lock();
    try {
      runningInstances.remove(internal);
    } finally {
      engineLock.writeLock().unlock();
    }
  }

}


/* Generated By:JJTree: Do not edit this line. OProfileStorageStatement.java Version 4.3 */
/* JavaCCOptions:MULTI=true,NODE_USES_PARSER=false,VISITOR=true,TRACK_TOKENS=true,NODE_PREFIX=O,NODE_EXTENDS=,NODE_FACTORY=,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */
package com.orientechnologies.orient.core.sql.parser;

import com.orientechnologies.common.listener.OProgressListener;
import com.orientechnologies.orient.core.command.OCommandContext;
import com.orientechnologies.orient.core.db.ODatabaseDocumentInternal;
import com.orientechnologies.orient.core.db.ODatabaseInternal;
import com.orientechnologies.orient.core.record.impl.ODocument;
import com.orientechnologies.orient.core.sql.executor.OInternalResultSet;
import com.orientechnologies.orient.core.sql.executor.OResultInternal;
import com.orientechnologies.orient.core.sql.executor.OResultSet;
import com.orientechnologies.orient.core.sql.query.OSQLAsynchQuery;
import com.orientechnologies.orient.core.sql.query.OSQLSynchQuery;
import com.orientechnologies.orient.core.storage.OStorage;
import com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage;
import com.orientechnologies.orient.core.storage.impl.local.statistic.OSessionStoragePerformanceStatistic;

import java.util.Map;

public class OProfileStorageStatement extends OSimpleExecStatement {

  protected boolean on;

  public static final String KEYWORD_PROFILE = "PROFILE";

  public OProfileStorageStatement(int id) {
    super(id);
  }

  public OProfileStorageStatement(OrientSql p, int id) {
    super(p, id);
  }

  //new execution logic
  @Override public OResultSet executeSimple(OCommandContext ctx) {
    OResultInternal result = new OResultInternal();
    result.setProperty("operation", "optimize database");

    OStorage storage = ((ODatabaseInternal) ctx.getDatabase()).getStorage();

    if (on) {
      // activate the profiler
      ((OAbstractPaginatedStorage) storage).startGatheringPerformanceStatisticForCurrentThread();
      result.setProperty("value", "on");
    } else {
      // stop the profiler and return the stats
      final OSessionStoragePerformanceStatistic performanceStatistic = ((OAbstractPaginatedStorage) storage)
          .completeGatheringPerformanceStatisticForCurrentThread();

      result.setProperty("value", "off");
      if (performanceStatistic != null) {
        result.setProperty("result", performanceStatistic.toDocument());
      } else {
        result.setProperty("result", "error");
        result.setProperty("errorMessage", "profiling of storage was not started");
      }
    }

    OInternalResultSet rs = new OInternalResultSet();
    rs.add(result);
    return rs;
  }

  //old execution logic
  @Override public Object execute(OSQLAsynchQuery<ODocument> request, OCommandContext context, OProgressListener progressListener) {
    try {
      ODatabaseDocumentInternal db = getDatabase();

      final OStorage storage = db.getStorage();
      if (on) {
        // activate the profiler
        ((OAbstractPaginatedStorage) storage).startGatheringPerformanceStatisticForCurrentThread();
        ODocument result = new ODocument();
        result.field("result", "OK");
        request.getResultListener().result(result);
      } else {
        // stop the profiler and return the stats
        final OSessionStoragePerformanceStatistic performanceStatistic = ((OAbstractPaginatedStorage) storage)
            .completeGatheringPerformanceStatisticForCurrentThread();

        if (performanceStatistic != null)
          request.getResultListener().result(performanceStatistic.toDocument());
        else {
          ODocument result = new ODocument();
          result.field("result", "Error: profiling of storage was not started.");
          request.getResultListener().result(result);
        }

      }
      return getResult(request);
    } finally {
      if (request.getResultListener() != null) {
        request.getResultListener().end();
      }
    }
  }

  protected Object getResult(OSQLAsynchQuery<ODocument> request) {
    if (request instanceof OSQLSynchQuery)
      return ((OSQLSynchQuery<ODocument>) request).getResult();

    return null;
  }

  public void toString(Map<Object, Object> params, StringBuilder builder) {
    builder.append("PROFILE STORAGE ");
    builder.append(on ? "ON" : "OFF");
  }

  @Override public OProfileStorageStatement copy() {
    OProfileStorageStatement result = new OProfileStorageStatement(-1);
    result.on = on;
    return result;
  }

  @Override public boolean equals(Object o) {
    if (this == o)
      return true;
    if (o == null || getClass() != o.getClass())
      return false;

    OProfileStorageStatement that = (OProfileStorageStatement) o;

    if (on != that.on)
      return false;

    return true;
  }

  @Override public int hashCode() {
    return (on ? 1 : 0);
  }
}

/* JavaCC - OriginalChecksum=645887712797ae14a17820bfa944f78e (do not edit this line) */