/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.nc2.util.cache;

import ucar.nc2.dataset.DatasetUrl;
import ucar.nc2.time.CalendarDate;
import ucar.nc2.time.CalendarDateFormatter;
import ucar.nc2.util.CancelTask;
import ucar.nc2.util.Misc;

import javax.annotation.concurrent.GuardedBy;
import javax.annotation.concurrent.ThreadSafe;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;

/**
 * Keep cache of open FileCacheable objects, for example NetcdfFile.
 * The FileCacheable object typically contains a RandomAccessFile object that wraps a system resource like a file handle.
 * These are left open when the FileCacheable is in the cache. The maximum number of these is bounded, though not strictly.
 * A cleanup routine reduces cache size to a minimum number. This cleanup is called periodically in a background thread,
 * and also when the maximum cache size is reached.
 *

 *
The FileCacheable object must not be modified.
 *
The hashKey must uniquely define the FileCacheable object.
 *
The location must be usable in a FileFactory.open().
 *
If the FileCacheable is acquired from the cache (ie already open), getLastModified() is used to see if it has changed,
 *     and is discarded if so.
 *
Make sure you call shutdown() when exiting the program, in order to shut down the cleanup thread.
 *

 *

 * Normal usage is through the NetcdfDataset interface:
 *

 * NetcdfDataset.initNetcdfFileCache(...); // on application startup
 * ...
 * NetcdfFile ncfile = null;
 * try {
 *   ncfile = NetcdfDataset.acquireFile(location, cancelTask);
 *   ...
 * } finally {
 *   if (ncfile != null) ncfile.close();
 * }
 * ...
 * NetcdfDataset.shutdown();  // when terminating the application
 *

 * All methods are thread safe.
 * Cleanup is done automatically in a background thread, using LRU algorithm.
 *
 * @author caron
 * @since May 30, 2008
 */

@ThreadSafe
public class FileCache implements FileCacheIF {
  static protected final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(FileCache.class);
  static protected final org.slf4j.Logger cacheLog = org.slf4j.LoggerFactory.getLogger("cacheLogger");

  // static private ScheduledExecutorService exec;
  static final boolean debug = false;
  static final boolean debugPrint = false;
  static final boolean debugCleanup = false;

  // Shared mutable data. Access to it is limited to the following 3 synchronized methods.
  static private Timer timer;

  /**
   * You must call shutdown() to shut down the background threads in order to get a clean process shutdown.
   */
  public static synchronized void shutdown() {
    if (timer != null) {
      timer.cancel();
      System.out.printf("FileCache.shutdown called%n");
    }
    timer = null;
  }

  private static synchronized void scheduleAtFixedRate(TimerTask task, long delay, long period) {
    if (timer == null) {
      timer = new Timer("FileCache");
    }
    timer.scheduleAtFixedRate(task, delay, period);
  }

  private static synchronized void schedule(TimerTask task, long delay) {
    if (timer == null) {
      timer = new Timer("FileCache");
    }
    timer.schedule(task, delay);
  }

  /////////////////////////////////////////////////////////////////////////////////////////

  protected String name;
  protected final int softLimit, minElements, hardLimit;
  protected final long period; // msecs

  private final AtomicBoolean disabled = new AtomicBoolean(false);  // cache is disabled
  protected final AtomicBoolean hasScheduled = new AtomicBoolean(false); // a cleanup is scheduled

  protected final ConcurrentHashMap<Object, CacheElement> cache; // unique files (by key, often = filename)
  protected final ConcurrentHashMap<FileCacheable, CacheElement.CacheFile> files; // list of all files in the cache

  // debugging and stats
  protected final AtomicInteger cleanups = new AtomicInteger();  // how many cleanups
  protected final AtomicInteger hits = new AtomicInteger();
  protected final AtomicInteger miss = new AtomicInteger();
  protected ConcurrentHashMap<Object, Tracker> track;
  protected boolean trackAll = false;

  /**
   * Constructor.
   *
   * @param minElementsInMemory keep this number in the cache
   * @param maxElementsInMemory trigger a cleanup if it goes over this number.
   * @param period              (secs) do periodic cleanups every this number of seconds.
   */
  public FileCache(int minElementsInMemory, int maxElementsInMemory, int period) {
    this("", minElementsInMemory, maxElementsInMemory, -1, period);
  }

  /**
   * Constructor.
   *
   * @param minElementsInMemory keep this number in the cache
   * @param softLimit           trigger a cleanup if it goes over this number.
   * @param hardLimit           if > 0, never allow more than this many elements. This causes a cleanup to be done in the calling thread.
   * @param period              if > 0, do periodic cleanups every this number of seconds.
   */
  public FileCache(int minElementsInMemory, int softLimit, int hardLimit, int period) {
    this("", minElementsInMemory, softLimit, hardLimit, period);
  }

  /**
   * Constructor.
   *
   * @param name                of file cache
   * @param minElementsInMemory keep this number in the cache
   * @param softLimit           trigger a cleanup if it goes over this number.
   * @param hardLimit           if > 0, never allow more than this many elements. This causes a cleanup to be done in the calling thread.
   * @param period              if > 0, do periodic cleanups every this number of seconds.
   */
  public FileCache(String name, int minElementsInMemory, int softLimit, int hardLimit, int period) {
    this.name = name;
    this.minElements = minElementsInMemory;
    this.softLimit = softLimit;
    this.hardLimit = hardLimit;
    this.period = (long) 1000 * period;

    cache = new ConcurrentHashMap<>(2 * softLimit, 0.75f, 8);
    files = new ConcurrentHashMap<>(4 * softLimit, 0.75f, 8);
    boolean wantsCleanup = period > 0;

    if (wantsCleanup) {
      scheduleAtFixedRate(new CleanupTask(), this.period, this.period);

      if (cacheLog.isDebugEnabled())
        cacheLog.debug("FileCache " + name + " cleanup every " + period + " secs");
    }

    if (trackAll)
      track = new ConcurrentHashMap<>(5000);

  }

  /**
   * Disable the cache, and force release all files.
   * You must still call shutdown() before exiting the application.
   */
  @Override
  public void disable() {
    this.disabled.set(true);
    clearCache(true);
  }

  /**
   * Enable the cache, with the current set of parameters.
   */
  @Override
  public void enable() {
    this.disabled.set(false);
  }

  /**
   * Acquire a FileCacheable, and lock it so no one else can use it.
   * call FileCacheable.close when done.
   *
   * @param factory    use this factory to open the file; may not be null
   * @param durl   file location, also used as the cache name, will be passed to the NetcdfFileFactory
   * @param cancelTask user can cancel, ok to be null.
   * @return NetcdfFile corresponding to location.
   * @throws IOException on error
   */
  public FileCacheable acquire(FileFactory factory, DatasetUrl durl, ucar.nc2.util.CancelTask cancelTask) throws IOException {
    return acquire(factory, durl.trueurl, durl, -1, cancelTask, null);
  }

  @Override
  public FileCacheable acquire(FileFactory factory, DatasetUrl durl) throws IOException {
    return acquire(factory, durl.trueurl, durl, -1, null, null);
  }

  /**
   * Acquire a FileCacheable from the cache, and lock it so no one else can use it.
   * If not already in cache, open it the FileFactory, and put in cache.
   *

   * App should  call FileCacheable.close when done, and the file is then released instead of closed.
   *

   * If cache size goes over maxElement, then immediately (actually in 100 msec) schedule a cleanup in a background thread.
   * This means that the cache should never get much larger than maxElement, unless you have them all locked.
   *
   * @param factory     use this factory to open the file if not in the cache; may not be null
   * @param hashKey     unique key for this file. If null, the location will be used
   * @param location    file location, may also used as the cache name, will be passed to the NetcdfFileFactory
   * @param buffer_size RandomAccessFile buffer size, if <= 0, use default size
   * @param cancelTask  user can cancel, ok to be null.
   * @param spiObject   passed to the factory if object needs to be recreated
   * @return FileCacheable corresponding to location.
   * @throws IOException on error
   */
  @Override
  public FileCacheable acquire(FileFactory factory, Object hashKey, DatasetUrl location,
                               int buffer_size, CancelTask cancelTask, Object spiObject) throws IOException {

    if (null == hashKey) hashKey = location.trueurl;
    if (null == hashKey) throw new IllegalArgumentException();

    Tracker t = null;
    if (trackAll) {
      t = new Tracker(hashKey);
      Tracker prev = track.putIfAbsent(hashKey, t);
      if (prev != null) t = prev;
    }

    FileCacheable ncfile = acquireCacheOnly(hashKey);
    if (ncfile != null) {
      hits.incrementAndGet();
      if (t != null) t.hit++;
      return ncfile;
    }
    miss.incrementAndGet();
    if (t != null) t.miss++;

    // open the file
    ncfile = factory.open(location, buffer_size, cancelTask, spiObject);
    if (cacheLog.isDebugEnabled())
      cacheLog.debug("FileCache " + name + " acquire " + hashKey + " " + ncfile.getLocation());
    if (debugPrint) System.out.println("  FileCache " + name + " acquire " + hashKey + " " + ncfile.getLocation());

    // user may have canceled
    if ((cancelTask != null) && (cancelTask.isCancel())) {
      if (ncfile != null) ncfile.close();       // LOOK ??
      return null;
    }

    if (disabled.get()) return ncfile;

    // see if cache element already exists
    // cant use putIfAbsent, because we cant create the CacheElement until we know if doesnt exist
    CacheElement elem;
    synchronized (cache) {
      elem = cache.get(hashKey);
      if (elem == null) cache.put(hashKey, new CacheElement(ncfile, hashKey)); // new element
    }

    // already exists, add to list
    if (elem != null) {
      synchronized (elem) {
        elem.addFile(ncfile); // add to existing list
      }
    }

    // increment the number of files in the cache
    //int count = counter.incrementAndGet();

    // do we need a cleanup ??
    boolean needHard = false;
    boolean needSoft = false;
    synchronized (hasScheduled) {
      if (!hasScheduled.get()) {
        int count = files.size();
        if ((count > hardLimit) && (hardLimit > 0)) {
          needHard = true;
          hasScheduled.getAndSet(true); // tell other threads not to schedule another cleanup

        } else if ((count > softLimit) && (softLimit > 0)) {   // && wantsCleanup) { //
          hasScheduled.getAndSet(true); // tell other threads not to schedule another cleanup
          needSoft = true;
        }
      }
    }

    if (needHard) {
      if (debugCleanup)
        System.out.println("CleanupTask due to hard limit time=" + new Date().getTime()); // +" Thread="+Thread.currentThread().hashCode()
      cleanup(hardLimit);

    } else if (needSoft) {
      schedule(new CleanupTask(), 100); // immediate cleanup in 100 msec

      if (debugCleanup) System.out.println("CleanupTask scheduled due to soft limit time=" + new Date());
    }

    return ncfile;
  }

  /**
   * Try to find a file in the cache.
   *
   * @param hashKey used as the key.
   * @return file if its in the cache, null otherwise.
   */
  private FileCacheable acquireCacheOnly(Object hashKey) {
    if (disabled.get()) return null;

    // see if its in the cache
    CacheElement wantCacheElem = cache.get(hashKey);
    if (wantCacheElem == null) return null;  // not found in cache

    CacheElement.CacheFile want = null;
    synchronized (wantCacheElem) { // synch in order to traverse the list
      for (CacheElement.CacheFile file : wantCacheElem.list) {
        if (file.isLocked.compareAndSet(false, true)) {
          want = file;
          break;
        }
      }
    }
    if (want == null) return null; // no unlocked file in cache

    // check if modified, remove if so
    if (want.ncfile != null) {
      long lastModified = want.ncfile.getLastModified();
      boolean changed = lastModified != want.lastModified;

      if (cacheLog.isDebugEnabled() && changed)
        cacheLog.debug("FileCache " + name + ": acquire from cache " + hashKey + " " + want.ncfile.getLocation() + " was changed; discard");
      if (changed) {
        remove(want);
      }
    }

    if (want.ncfile != null) {
      try {
        want.ncfile.reacquire(); // rehydrate
      } catch (IOException ioe) {
        if (cacheLog.isDebugEnabled())
          cacheLog.debug("FileCache " + name + " acquire from cache " + hashKey + " " + want.ncfile.getLocation() +
                         " failed: " + ioe.getMessage());
        remove(want);  // failed
      }
    }

    if (debugPrint && want.ncfile != null) {
      System.out.printf("  FileCache %s found in cache %s (countLocks %d)%n", name, hashKey, countLocked());
    }

    return want.ncfile;
  }

  // LOOK should you remove the entire CacheElement ?
  private void remove(CacheElement.CacheFile want) {
     want.remove();
     files.remove(want.ncfile);
     try {
       want.ncfile.setFileCache(null); // unhook the caching
       want.ncfile.close();
     } catch (IOException e) {
       log.error("close failed on "+want.ncfile.getLocation(), e);
     }
     want.ncfile = null;
  }

  /**
   * Remove all instances of object from the cache
   * @param hashKey the object
   */
  @Override
  public void eject(Object hashKey) {
     if (disabled.get()) return;

     // see if its in the cache
     CacheElement wantCacheElem = cache.get(hashKey);
     if (wantCacheElem == null) return;

     synchronized (wantCacheElem) { // synch in order to traverse the list
       for (CacheElement.CacheFile want : wantCacheElem.list) {
          // LOOK can we use remove(want);  ??
          files.remove(want.ncfile);
          try {
            want.ncfile.setFileCache(null); // unhook the caching
            want.ncfile.close();  // really close the file
            log.debug("close "+want.ncfile.getLocation());
          } catch (IOException e) {
            log.error("close failed on "+want.ncfile.getLocation(), e);
          }
          want.ncfile = null;
         if (debugPrint) System.out.println("  FileCache " + name + " eject " + hashKey);
       }
       wantCacheElem.list.clear();
     }
    cache.remove(hashKey);
   }

  /**
   * Release the file. This unlocks it, updates its lastAccessed date.
   * Normally applications need not call this, just close the file as usual.
   * The FileCacheable has to do tricky stuff.
   *
   * @param ncfile release this file.
   * @return true if file was in cache, false if it was not
   */
  @Override
  public boolean release(FileCacheable ncfile) throws IOException {
    if (ncfile == null) return false;

    if (disabled.get()) {
      ncfile.setFileCache(null); // prevent infinite loops
      ncfile.close();
      return false;
    }

    // find it in the file cache
    CacheElement.CacheFile file = files.get(ncfile); // using hashCode of the FileCacheable
    if (file != null) {
      if (!file.isLocked.get()) {
        cacheLog.warn("FileCache " + name + " release " + ncfile.getLocation() + " not locked; hash= "+ncfile.hashCode());
      }
      file.lastAccessed = System.currentTimeMillis();
      file.countAccessed++;
      file.isLocked.set(false);
      file.ncfile.release();

      if (cacheLog.isDebugEnabled()) cacheLog.debug("FileCache " + name + " release " + ncfile.getLocation()+"; hash= "+ncfile.hashCode());
      if (debugPrint) System.out.printf("  FileCache %s release %s lock=%s count=%d%n", name, ncfile.getLocation(), file.isLocked.get(), countLocked());
      return true;
    }
    return false;
    // throw new IOException("FileCache " + name + " release does not have file in cache = " + ncfile.getLocation());
  }

  private int countLocked() {
    int count = 0;
    for (CacheElement.CacheFile file : files.values())
      if (file.isLocked.get()) count++;
    return count;
  }

  // debug
  public String getInfo(FileCacheable ncfile) throws IOException {
    if (ncfile == null) return "";

    // find it in the file cache
    CacheElement.CacheFile file = files.get(ncfile);
    if (file != null) {
      return "File is in cache= " + file;
    }
    return "File not in cache";
  }

  // debug
  Map<Object, CacheElement> getCache() {
    return cache;
  }

  /**
   * Remove all cache entries.
   *
   * @param force if true, remove them even if they are currently locked.
   */
  public synchronized void clearCache(boolean force) {
    List<CacheElement.CacheFile> deleteList = new ArrayList<>(2 * cache.size());

    if (force) {
      cache.clear(); // deletes everything from the cache
      deleteList.addAll(files.values());  // add everything to the delete list
      files.clear();
      // counter.set(0);

    } else {

      // add unlocked files to the delete list, remove from files hash
      Iterator<CacheElement.CacheFile> iter = files.values().iterator();
      while (iter.hasNext()) {
        CacheElement.CacheFile file = iter.next();
        if (file.isLocked.compareAndSet(false, true)) {
          file.remove(); // remove from the containing CacheElement
          deleteList.add(file);
          iter.remove();
        }
      }

      // remove empty cache elements
      synchronized (cache) {
        for (CacheElement elem : cache.values()) {
          synchronized (elem) {
            if (elem.list.size() == 0)
              cache.remove(elem.hashKey);
          }
        }
      }
    }

    // close all files in deleteList
    for (CacheElement.CacheFile file : deleteList) {
      if (force && file.isLocked.get())
        cacheLog.warn("FileCache " + name + " force close locked file= " + file);
      //counter.decrementAndGet();

      try {
        if(file == null || file.ncfile == null) {
          log.error(String.format("FileCache %s: null file or null ncfile",name));
        } else {
          file.ncfile.setFileCache(null);
          file.ncfile.close();
        }
        if (file != null) file.ncfile = null; // help the gc
      } catch (IOException e) {
        log.error("FileCache " + name + " close failed on " + file);
      }
    }
    if (cacheLog.isDebugEnabled())
      cacheLog.debug("*FileCache " + name + " clearCache force= " + force + " deleted= " + deleteList.size() + " left=" + files.size());
    //System.out.println("\n*NetcdfFileCache.clearCache force= " + force + " deleted= " + deleteList.size() + " left=" + counter.get());
  }

  /**
   * Show individual cache entries, add to formatter.
   *
   * @param format add to this
   */
  @Override
  public void showCache(Formatter format) {
    ArrayList<CacheElement.CacheFile> allFiles = new ArrayList<>(files.size());
    for (CacheElement elem : cache.values()) {
      synchronized (elem) {
        allFiles.addAll(elem.list);
      }
    }
    Collections.sort(allFiles); // sort so oldest are on top

    format.format("%nFileCache %s (min=%d softLimit=%d hardLimit=%d scour=%d secs):%n", name, minElements, softLimit, hardLimit, period/1000);
    format.format(" isLocked  accesses lastAccess                   location %n");
    for (CacheElement.CacheFile file : allFiles) {
      String loc = file.ncfile != null ? file.ncfile.getLocation() : "null";
      format.format("%8s %9d %s == %s %n", file.isLocked, file.countAccessed, CalendarDateFormatter.toDateTimeStringISO(file.lastAccessed), loc);
    }
    showStats(format);
  }

  @Override
  public List<String> showCache() {
    List<CacheElement.CacheFile> allFiles = new ArrayList<>(files.size());
    for (CacheElement elem : cache.values()) {
      synchronized (elem) {
        allFiles.addAll(elem.list);
      }
    }
    Collections.sort(allFiles); // sort so oldest are on top

    List<String> result = new ArrayList<>(allFiles.size());
    for (CacheElement.CacheFile file : allFiles) {
      result.add(file.toString());
    }

    return result;
  }

  /**
   * Add stat report (hits, misses, etc) to formatter.
   *
   * @param format add to this
   */
  public void showStats(Formatter format) {
    format.format("  hits= %d miss= %d nfiles= %d elems= %d%n", hits.get(), miss.get(), files.size(), cache.values().size());
  }

  public void showTracking(Formatter format) {
    if (track == null) return;
    List<Tracker> all = new ArrayList<>(track.size());
    for (Tracker val : track.values()) all.add(val);
    Collections.sort(all);
    int seq = 0;
    int countAll = 0;
    int countHits = 0;
    int countMiss = 0;
    format.format("%nTracking All files in cache %s%n", name);
    format.format("    #    accum       hit    miss  file%n");
    for (Tracker t : all) {
      seq++;
      countAll += t.hit + t.miss;
      countHits += t.hit;
      countMiss += t.miss;
      format.format("%6d  %7d : %6d %6d %s%n", seq, countAll, t.hit, t.miss, t.key);
    }
    float r = (countAll == 0) ? 0 : ((float)countHits) / countAll;
    format.format("  total=%7d : %6d %6d hit ratio=%f%n", countAll, countHits, countMiss, r);
  }

  @Override
  public void resetTracking() {
    track = new ConcurrentHashMap<>(5000);
    trackAll = true;
  }

  private static class Tracker implements Comparable<Tracker> {
    Object key;
    int hit, miss;

    private Tracker(Object key) {
      this.key = key;
    }

    @Override
    public boolean equals(Object o) {
      if (this == o) return true;
      if (o == null || getClass() != o.getClass()) return false;
      Tracker tracker = (Tracker) o;
      return key.equals(tracker.key);
    }

    @Override
    public int hashCode() {
      return key.hashCode();
    }

    @Override
    public int compareTo(Tracker o) {
      return Misc.compare(hit+miss, o.hit+o.miss);
    }
  }

  /**
   * Cleanup the cache, bringing it down to minimum number.
   * Will close the LRU (least recently used) ones first. Will not close locked files.
   * Normally this is done in a background thread, you dont need to call.
   *

   * We have to synchronize because of clearCache()
   */
  synchronized void cleanup(int maxElements) {

    try {
      /* int size = counter.get();
      int fsize = files.size();
      if (debug && (size != fsize)) {
        log.warn("FileCache " + name + " counter " + size + " doesnt match files().size=" + fsize);
      } */

      int size = files.size();
      if (size <= minElements) return;

      if (cacheLog.isDebugEnabled()) cacheLog.debug("FileCache {} cleanup started at {} for maxElements={}", name, CalendarDate.present(), maxElements);
      if (debugCleanup)
        System.out.printf(" FileCache %s cleanup started at %s for maxElements=%d%n", name, CalendarDate.present(), maxElements);

      cleanups.incrementAndGet();

      // add unlocked files to the all list
      List<CacheElement.CacheFile> allFiles = new ArrayList<>(size + 10);
      for (CacheElement.CacheFile file : files.values()) {
        if (!file.isLocked.get()) allFiles.add(file);
      }
      Collections.sort(allFiles); // sort so oldest are on top

      // take oldest ones and put on delete list
      int need2delete = size - minElements;
      int minDelete = size - maxElements;
      List<CacheElement.CacheFile> deleteList = new ArrayList<>(need2delete);

      int count = 0;
      Iterator<CacheElement.CacheFile> iter = allFiles.iterator();
      while (iter.hasNext() && (count < need2delete)) {
        CacheElement.CacheFile file = iter.next();
        if (file.isLocked.compareAndSet(false, true)) { // lock it so it isnt used anywhere else
          file.remove(); // remove from the containing element
          deleteList.add(file);
          count++;
        }
      }
      if (count < minDelete) {
        cacheLog.warn("FileCache " + name + " cleanup couldnt remove enough to keep under the maximum= " + maxElements + " due to locked files; currently at = " + (size - count));
        if (debugCleanup)
          System.out.println("FileCache " + name + " cleanup couldnt remove enough to keep under the maximum= " + maxElements + " due to locked files; currently at = " + (size - count));
      }

      // remove empty cache elements
      synchronized (cache) {
        for (CacheElement elem : cache.values()) {
          synchronized (elem) {
            if (elem.list.size() == 0)
              cache.remove(elem.hashKey);
          }
        }
      }

      // now actually close the files
      long start = System.currentTimeMillis();
      for (CacheElement.CacheFile file : deleteList) {
        //counter.decrementAndGet();
        if (null == files.remove(file.ncfile)) {
          if (cacheLog.isDebugEnabled()) cacheLog.debug(" FileCache {} cleanup failed to remove {}%n", name, file.ncfile.getLocation());
        }
        try {
          file.ncfile.setFileCache(null);
          file.ncfile.close();
          file.ncfile = null; // help the gc
        } catch (IOException e) {
          log.error("FileCache " + name + " close failed on " + file.getCacheName());
        }
      }

      long took = System.currentTimeMillis() - start;
      if (cacheLog.isDebugEnabled()) cacheLog.debug(" FileCache {} cleanup had={} removed={} took={} msecs%n", name, size,deleteList.size(), took);
      if (debugCleanup)
        System.out.printf(" FileCache %s cleanup had=%d removed=%d took=%d msecs%n", name, size,deleteList.size(), took);

    } finally {
      // allow scheduling again
      hasScheduled.set(false);
    }
  }

  // not private for testing
  class CacheElement {
    @GuardedBy("this")
    final List<CacheFile> list = new LinkedList<>(); // may have multiple copies of the same file opened
    final Object hashKey;

    CacheElement(FileCacheable ncfile, Object hashKey) {
      this.hashKey = hashKey;
      CacheFile file = new CacheFile(ncfile);
      list.add(file);
      if (debug) {
        if (files.get(ncfile) != null)
          cacheLog.error("files already has " + hashKey + " " + name);
      }
      files.put(ncfile, file);
      if (cacheLog.isDebugEnabled()) cacheLog.debug("CacheElement add to cache " + hashKey + " " + name);
    }

    CacheFile addFile(FileCacheable ncfile) {
      CacheFile file = new CacheFile(ncfile);
      synchronized (this) {
        list.add(file);
      }
      if (debug) {
        if (files.get(ncfile) != null)
          cacheLog.error("files (2) already has " + hashKey + " " + name);
      }
      files.put(ncfile, file);
      return file;
    }

    public String toString() {
      return hashKey + " count=" + list.size();
    }

    class CacheFile implements Comparable<CacheFile> {
      FileCacheable ncfile; // actually final, but we null it out for gc
      final AtomicBoolean isLocked = new AtomicBoolean(true);
      int countAccessed = 0;
      long lastModified = 0;
      long lastAccessed = 0;

      private CacheFile(FileCacheable ncfile) {
        this.ncfile = ncfile;
        this.lastModified = ncfile.getLastModified();
        this.lastAccessed = System.currentTimeMillis();

        ncfile.setFileCache(FileCache.this);

        if (cacheLog.isDebugEnabled()) cacheLog.debug("FileCache " + name + " add to cache " + hashKey);
        if (debugPrint) System.out.printf("  FileCache %s add to cache %s (hash %d)%n", name, hashKey, this.hashCode());
      }

      String getCacheName() {
        return ncfile.getLocation();
      }

      void remove() {
        synchronized (CacheElement.this) {
          if (!list.remove(this))
            cacheLog.warn("FileCache " + name + " could not remove " + ncfile.getLocation());
        }
        if (cacheLog.isDebugEnabled()) cacheLog.debug("FileCache " + name + " remove " + ncfile.getLocation());
        if (debugPrint) System.out.printf("  FileCache %s remove %s%n", name, ncfile.getLocation());
      }

      public String toString() {
        String name = ncfile == null ? "ncfile is null" : ncfile.getLocation();
        return isLocked + " " + countAccessed + " " + CalendarDateFormatter.toDateTimeStringISO(lastAccessed) + "   " + name;
      }

      public int compareTo(CacheFile o) {
        return Long.compare(lastAccessed, o.lastAccessed);
      }
    }
  }

  private class CleanupTask extends TimerTask {
    public void run() {
      if (disabled.get()) return;
      cleanup(softLimit);
    }
  }

}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.nc2.iosp.bufr;

import java.io.IOException;
import ucar.unidata.io.RandomAccessFile;

/**
 * A class that contains static methods for converting multiple
 * bytes into one float or integer.
 */

final public class BufrNumbers {

  // used to check missing values when value is packed with all 1's
  static private final long missing_value[] = new long[65];

  static {
    long accum = 0;
    for (int i = 0; i < 65; i++) {
      missing_value[i] = accum;
      //System.out.printf("BufrNumbers %2d : %20d = %s %n", i, accum, Long.toBinaryString(accum));
      accum = accum * 2 + 1;
    }
  }

  static public boolean isMissing(long raw, int bitWidth) {
    return (raw == BufrNumbers.missing_value[bitWidth]);
  }

  static long missingValue(int bitWidth) {
    return BufrNumbers.missing_value[bitWidth];
  }

  /**
   * if missing value is not defined use this value.
   */
  public static final int UNDEFINED = -9999;

  /**
   * Convert 2 bytes into a signed integer.
   *
   * @param raf
   * @return integer value
   * @throws IOException
   */
  public static int int2(RandomAccessFile raf) throws IOException {
    int a = raf.read();
    int b = raf.read();

    return int2(a, b);
  }

  /**
   * convert 2 bytes to a signed integer.
   *
   * @param a
   * @param b
   * @return int
   */
  public static int int2(int a, int b) {
    //System.out.println( "a=" + a );
    //System.out.println( "b=" + b );
    if ((a == 0xff && b == 0xff)) // all bits set to one
      return UNDEFINED;

    return (1 - ((a & 128) >> 6)) * ((a & 127) << 8 | b);
  }

  /**
   * Convert 3 bytes into a signed integer.
   *
   * @param raf
   * @return integer value
   * @throws IOException
   */
  public static int int3(RandomAccessFile raf) throws IOException {
    int a = raf.read();
    int b = raf.read();
    int c = raf.read();

    return int3(a, b, c);
  }

  /**
   * Convert 3 bytes to signed integer.
   *
   * @param a
   * @param b
   * @param c
   * @return int
   */
  private static int int3(int a, int b, int c) {
    return (1 - ((a & 128) >> 6)) * ((a & 127) << 16 | b << 8 | c);
  }

  /**
   * Convert 4 bytes into a signed integer.
   *
   * @param raf
   * @return integer value
   * @throws IOException
   */
  public static int int4(RandomAccessFile raf) throws IOException {
    int a = raf.read();
    int b = raf.read();
    int c = raf.read();
    int d = raf.read();

    return int4(a, b, c, d);
  }

  /**
   * Convert 4 bytes into a signed integer.
   *
   * @param a
   * @param b
   * @param c
   * @param d
   * @return int
   */
  private static int int4(int a, int b, int c, int d) {
    // all bits set to ones
    if (a == 0xff && b == 0xff && c == 0xff && d == 0xff)
      return UNDEFINED;

    return (1 - ((a & 128) >> 6)) * ((a & 127) << 24 | b << 16 | c << 8 | d);
  } // end int4

  /**
   * Convert 2 bytes into an unsigned integer.
   *
   * @param raf
   * @return integer value
   * @throws IOException
   */
  public static int uint2(RandomAccessFile raf) throws IOException {
    int a = raf.read();
    int b = raf.read();

    return uint2(a, b);
  }

  /**
   * convert 2 bytes to an unsigned integer.
   *
   * @param a
   * @param b
   * @return unsigned int
   */
  private static int uint2(int a, int b) {
    return a << 8 | b;
  }

  /**
   * Convert 3 bytes into an unsigned integer.
   *
   * @param raf
   * @return integer
   * @throws IOException
   */
  public static int uint3(RandomAccessFile raf) throws IOException {
    int a = raf.read();
    int b = raf.read();
    int c = raf.read();

    return uint3(a, b, c);
  }

  /**
   * Convert 3 bytes into an unsigned int.
   *
   * @param a
   * @param b
   * @param c
   * @return unsigned integer
   */
  public static int uint3(int a, int b, int c) {
    return a << 16 | b << 8 | c;
  }

// --Commented out by Inspection START (12/5/05 4:21 PM):
//  /**
//    * Convert 4 bytes into an unsigned integer.
//    * @param raf
//    * @return integer value
//    * @throws IOException
//    */
//   public static int uint4(RandomAccessFile raf) throws IOException
//   {
//     int a = raf.read();
//     int b = raf.read();
//     int c = raf.read();
//     int d = raf.read();
//
//     return uint4(a, b, c, d);
//   }
// --Commented out by Inspection STOP (12/5/05 4:21 PM)

// --Commented out by Inspection START (12/5/05 4:21 PM):
//    /**
//     * Convert 4 bytes to  an unsigned int.
//     * @param a
//     * @param b
//     * @param c
//     * @param d
//     * @return unsigned int
//     */
//   private static int uint4(int a, int b, int c, int d)
//   {
//      return  a << 32 | b << 16 | c << 8 | d;
//   }
// --Commented out by Inspection STOP (12/5/05 4:21 PM)

  /**
   * Convert 4 bytes into a float value.
   *
   * @param raf
   * @return float value
   * @throws IOException
   */
  public static float float4(RandomAccessFile raf) throws IOException {
    int a = raf.read();
    int b = raf.read();
    int c = raf.read();
    int d = raf.read();

    return float4(a, b, c, d);
  }

  /**
   * Convert 4 bytes to a float.
   *
   * @param a
   * @param b
   * @param c
   * @param d
   * @return float
   */
  private static float float4(int a, int b, int c, int d) {
    int sgn, mant, exp;

    mant = b << 16 | c << 8 | d;
    if (mant == 0) return 0.0f;

    sgn = -(((a & 128) >> 6) - 1);
    exp = (a & 127) - 64;

    return (float) (sgn * Math.pow(16.0, exp - 6) * mant);
  }

  /**
   * Convert 8 bytes into a signed long.
   *
   * @param raf RandomAccessFile
   * @return long value
   * @throws IOException
   */
  public static long int8(RandomAccessFile raf) throws IOException {
    int a = raf.read();
    int b = raf.read();
    int c = raf.read();
    int d = raf.read();
    int e = raf.read();
    int f = raf.read();
    int g = raf.read();
    int h = raf.read();

    return (1 - ((a & 128) >> 6)) * ((long)(a & 127) << 56 | (long) b << 48 | (long) c << 40 | (long) d << 32 | e << 24 | f << 16 | g << 8 | h);

   }
}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.util.cache;

import ucar.nc2.dataset.DatasetUrl;
import ucar.nc2.time.CalendarDate;
import ucar.nc2.time.CalendarDateFormatter;
import ucar.nc2.util.CancelTask;
import ucar.nc2.util.Misc;

import javax.annotation.concurrent.ThreadSafe;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;

/**
  ARC cach algorithm
 not complete.
 */
@ThreadSafe
public class FileCacheARC implements FileCacheIF {
  static protected final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(FileCacheARC.class);
  static protected final org.slf4j.Logger cacheLog = org.slf4j.LoggerFactory.getLogger("cacheLogger");
  static boolean debug = false;
  static boolean debugPrint = false;
  static boolean trackAll = true;

  /////////////////////////////////////////////////////////////////////////////////////////

  protected String name;
  protected final int softLimit, minElements, hardLimit, period;

  private final AtomicBoolean disabled = new AtomicBoolean(false);  // cache is disabled

  protected final ConcurrentSkipListMap<CacheElement, CacheElement> shadowCache; // this is used to decide which file to release
                                                                                // this can be biggger than the cache, eg ARC uses 2c, where c = cache size
  protected final ConcurrentHashMap<Object, CacheElement> cache;                // collection of all unique files in the cache, keyed by hashKey, typically filename
                                                                                // cache has open file handle / object in memory
  protected final ConcurrentHashMap<Integer, CacheElement.CacheFile> files;     // collection of all files in the cache, keyed by FileCacheable hashcode (object id)
                                                                                // this is needed for release

  // debugging and global stats
  protected final AtomicInteger hits = new AtomicInteger();
  protected final AtomicInteger miss = new AtomicInteger();
  protected ConcurrentHashMap<Object, Tracker> track;

  /**
   * Constructor.
   *
   * @param name                of file cache
   * @param minElementsInMemory keep this number in the cache
   * @param softLimit           trigger a cleanup if it goes over this number.
   * @param hardLimit           if > 0, never allow more than this many elements. This causes a cleanup to be done in the calling thread.
   * @param period              if > 0, do periodic cleanups every this number of seconds.
   */
  public FileCacheARC(String name, int minElementsInMemory, int softLimit, int hardLimit, int period) {
    this.name = name;
    this.minElements = minElementsInMemory;
    this.softLimit = softLimit;
    this.hardLimit = hardLimit;
    this.period = period;

    shadowCache = new ConcurrentSkipListMap<>(new CacheElementComparator());
    cache = new ConcurrentHashMap<>(2 * softLimit, 0.75f, 8);
    files = new ConcurrentHashMap<>(4 * softLimit, 0.75f, 8);

    if (trackAll)
      track = new ConcurrentHashMap<>(5000);
  }

  /**
   * Disable the cache, and force release all files.
   * You must still call shutdown() before exiting the application.
   */
  @Override
  public void disable() {
    this.disabled.set(true);
    clearCache(true);
  }

  /**
   * Enable the cache, with the current set of parameters.
   */
  @Override
  public void enable() {
    this.disabled.set(false);
  }

  /**
   * Acquire a FileCacheable, and lock it so no one else can use it.
   * call FileCacheable.close() when done.
   *
   * @param factory    use this factory to open the file; may not be null
   * @param location   file location, also used as the cache name, will be passed to the NetcdfFileFactory
   * @return NetcdfFile corresponding to location.
   * @throws IOException on error
   */
  @Override
  public FileCacheable acquire(FileFactory factory, DatasetUrl location) throws IOException {
    return acquire(factory, location.trueurl, location, -1, null, null);
  }

  /**
   * Acquire a FileCacheable from the cache, and lock it so no one else can use it.
   * If not already in cache, open it with FileFactory, and put in cache.
   *

   * Call FileCacheable.close() when done, (rather than FileCacheIF.release() directly) and the file is then released instead of closed.
   *

   * If cache size goes over maxElement, then immediately (actually in 100 msec) schedule a cleanup in a background thread.
   * This means that the cache should never get much larger than maxElement, unless you have them all locked.
   *
   * @param factory     use this factory to open the file if not in the cache; may not be null
   * @param hashKey     unique key for this file. If null, the location will be used
   * @param location    file location, may also used as the cache name, will be passed to the NetcdfFileFactory
   * @param buffer_size RandomAccessFile buffer size, if <= 0, use default size
   * @param cancelTask  user can cancel, ok to be null.
   * @param spiObject   sent to iosp.setSpecial() if not null
   * @return FileCacheable corresponding to location.
   * @throws IOException on error
   */
  @Override
  public FileCacheable acquire(FileFactory factory, Object hashKey, DatasetUrl location,
                               int buffer_size, CancelTask cancelTask, Object spiObject) throws IOException {

    if (null == hashKey) hashKey = location.trueurl;
    if (null == hashKey) throw new IllegalArgumentException();

    Tracker t = null;
    if (trackAll) {
      t = new Tracker(hashKey);
      Tracker prev = track.putIfAbsent(hashKey, t);
      if (prev != null) t = prev;
    }

    FileCacheable ncfile = acquireCacheOnly(hashKey);
    if (ncfile != null) {
      hits.incrementAndGet();
      if (t != null) t.hit++;
      return ncfile;
    }
    miss.incrementAndGet();
    if (t != null) t.miss++;

    // open the file
    ncfile = factory.open(location, buffer_size, cancelTask, spiObject);
    if (cacheLog.isDebugEnabled())
      cacheLog.debug("FileCacheARC " + name + " acquire " + hashKey + " " + ncfile.getLocation());
    if (debugPrint) System.out.println("  FileCacheARC " + name + " acquire " + hashKey + " " + ncfile.getLocation());

    // user may have canceled
    if ((cancelTask != null) && (cancelTask.isCancel())) {
      if (ncfile != null) ncfile.close();
      return null;
    }

    if (disabled.get()) return ncfile;

    addToCache(hashKey, ncfile);

    return ncfile;
  }

  /**
   * Try to find a file in the cache.
   *
   * @param hashKey used as the key.
   * @return file if its in the cache, null otherwise.
   */
  private FileCacheable acquireCacheOnly(Object hashKey) {
    if (disabled.get()) return null;

    // see if its in the cache
    CacheElement wantCacheElem  = cache.get(hashKey);
    if (wantCacheElem == null) return null;  // not found in cache

    CacheElement.CacheFile want = null;
    for (CacheElement.CacheFile file : wantCacheElem.list) {
      if (file.isLocked.compareAndSet(false, true)) {
        want = file;
        break;
      }
    }
    if (want == null) return null; // no unlocked file in cache

    // check if modified, remove if so
    if (want.ncfile != null) {
      long lastModified = want.ncfile.getLastModified();
      boolean changed = lastModified != wantCacheElem.lastModified.get();

      if (changed) { // underlying file was modified
        if (cacheLog.isDebugEnabled())
          cacheLog.debug("FileCacheARC " + name + ": acquire from cache " + hashKey + " " + want.ncfile.getLocation() + " was changed; discard");

        expireFromCache(wantCacheElem);
        return null;
      }
    }

    updateInCache(wantCacheElem);
    return want.ncfile;
  }

  // get CacheElement specified by hashKey. If found, update lastUsed in shadowCache.
  private CacheElement updateInCache(CacheElement elem) {
    if (shadowCache.firstKey() == elem) return elem;

    elem.updateAccessed();
    CacheElement prev = shadowCache.put(elem, elem); // faster if we could just insert at the top of the list. maybe we need to use LinkedList ?
    if (prev != null && (elem != prev)) {
      CacheElementComparator cc = new CacheElementComparator();
      System.out.printf("elem != prev compare=%d%n", cc.compare(elem, prev));
      System.out.printf("hash elem =%d prev=%d%n", elem.hashCode(), prev.hashCode());
    }
    return elem;
  }

  private AtomicInteger cacheSize = new AtomicInteger();

  private void expireFromCache(CacheElement elem) {
    for (CacheElement.CacheFile cacheFile : elem.list) {
      cacheFile.ncfile.setFileCache(null);        // next time it closes, it will really close LOOK race condition ??
      cacheSize.getAndDecrement();
    }
    cache.remove(elem.hashKey);
    shadowCache.remove(elem);
  }

  /** LOOK copied from FileCache, probably wrong
    * Remove all instances of object from the cache
    * @param hashKey the object
    */
   @Override
   public void eject(Object hashKey) {
   }

  /*
      if (disabled.get()) return;

      // see if its in the cache
      CacheElement wantCacheElem = cache.get(hashKey);
      if (wantCacheElem == null) return;

      synchronized (wantCacheElem) { // synch in order to traverse the list
        for (CacheElement.CacheFile want : wantCacheElem.list) {
           files.remove(want.ncfile);
           want.ncfile.setFileCache(null); // unhook the caching
           try {
             want.ncfile.close();  // really close the file
             log.debug("close "+want.ncfile.getLocation());
           } catch (IOException e) {
             log.error("close failed on "+want.ncfile.getLocation(), e);
           }
           want.ncfile = null;
        }
        wantCacheElem.list.clear();
      }
     cache.remove(hashKey);
    } */

  private void addToCache(Object hashKey, FileCacheable ncfile) {
    CacheElement newCacheElem = new CacheElement(hashKey);
    CacheElement previous = cache.putIfAbsent(hashKey, newCacheElem); // add new element if doesnt exist
    CacheElement elem = (previous != null) ? previous : newCacheElem;  // use previous if it exists

    elem.addFile(ncfile);                                 // add to existing list
    shadowCache.put(newCacheElem, newCacheElem);

    int size = cacheSize.getAndIncrement();
    if (size > softLimit) {
      removeFromCache(size - softLimit);
    }
  }

  private void removeFromCache(int count) {
    int done = 0;
    while (count > done) {
      CacheElement elem = shadowCache.lastKey();
      done += elem.list.size();
      expireFromCache(elem);
    }
  }

  /**
   * Release the file. This unlocks it, updates its lastAccessed date.
   * FileCacheable.close() needs to call this instead of actually closing.
   *
   * @param ncfile release the lock on this FileCacheable object.
   * @throws IOException if file not in cache.
   */
  @Override
  public boolean release(FileCacheable ncfile) throws IOException {
    if (ncfile == null) return false;

    if (disabled.get()) {
      ncfile.setFileCache(null); // prevent infinite loops
      ncfile.close();
      return false;
    }

    // find it in the file cache
    int hashcode = System.identityHashCode(ncfile);    // using Object hashCode of the FileCacheable
    CacheElement.CacheFile file = files.get(hashcode);
    if (file != null) {
      if (!file.isLocked.get()) {
        Exception e = new Exception("Stack trace");
        cacheLog.warn("FileCacheARC " + name + " release " + ncfile.getLocation() + " not locked; hash= "+ncfile.hashCode(), e);
      }
      //file.lastAccessed = System.currentTimeMillis();
      //file.countAccessed++;
      file.isLocked.set(false);
      if (cacheLog.isDebugEnabled()) cacheLog.debug("FileCacheARC " + name + " release " + ncfile.getLocation()+"; hash= "+ncfile.hashCode());
      if (debugPrint) System.out.println("  FileCacheARC " + name + " release " + ncfile.getLocation());
      return true;
    }
    return false;
    // throw new IOException("FileCacheARC " + name + " release does not have file in cache = " + ncfile.getLocation());
  }

  // not private for testing
  class CacheElement {
    final Object hashKey;
    final List<CacheFile> list = new CopyOnWriteArrayList<>(); // may have multiple copies of the same file opened.

    final AtomicLong lastModified = new AtomicLong();
    final AtomicLong  lastAccessed = new AtomicLong();
    final AtomicInteger countAccessed = new AtomicInteger();

    CacheElement(Object hashKey) {
      this.hashKey = hashKey;
    }

    CacheFile addFile(FileCacheable ncfile) {
      CacheFile file = new CacheFile(ncfile);
      list.add(file);

      this.lastModified.set(ncfile.getLastModified());
      this.lastAccessed.set(System.currentTimeMillis());
      this.countAccessed.incrementAndGet();

      int hashcode = System.identityHashCode(ncfile);    // using Object hashCode of the FileCacheable
      if (debug) {
        if (files.get(hashcode) != null)
          cacheLog.error("files (2) already has " + hashKey + " " + name);
      }

      files.put(hashcode, file);
      if (cacheLog.isDebugEnabled()) cacheLog.debug("CacheElement add to cache " + hashKey + " " + name);
      return file;
    }

    public long getLastAccessed() {
      return lastAccessed.get();
    }

    public void updateAccessed() {
      lastAccessed.set(System.currentTimeMillis());
      this.countAccessed.incrementAndGet();
    }

    public String toString() {
      return hashKey + " count=" + list.size()+ " countAccessed=" + countAccessed + " lastAccessed=" + new Date(getLastAccessed());
    }

    @Override
    public boolean equals(Object o) {
      if (this == o) return true;
      if (o == null || getClass() != o.getClass()) return false;
      CacheElement that = (CacheElement) o;
      return that.hashCode() == hashCode();
    }

    @Override
    public int hashCode() {
      return hashKey.hashCode();
    }

    class CacheFile implements Comparable<CacheFile> {
      FileCacheable ncfile; // actually final, but we null it out for gc
      final AtomicBoolean isLocked = new AtomicBoolean(true);

      private CacheFile(FileCacheable ncfile) {
        this.ncfile = ncfile;
        ncfile.setFileCache(FileCacheARC.this);

        if (cacheLog.isDebugEnabled()) cacheLog.debug("FileCacheARC " + name + " add to cache " + hashKey);
        if (debugPrint) System.out.println("  FileCacheARC " + name + " add to cache " + hashKey);
      }

      public long getLastAccessed() {
        return lastAccessed.get();
      }

      public int getCountAccessed() {
         return countAccessed.get();
       }

      void remove() {
        int hashcode = System.identityHashCode(ncfile);    // using Object hashCode of the FileCacheable
        if (null == files.remove(hashcode))
          cacheLog.warn("FileCacheARC {} could not remove {} from files", name, ncfile.getLocation());
        if (!list.remove(this))
          cacheLog.warn("FileCacheARC {} could not remove {} from list", name, ncfile.getLocation());
        close();
      }

      void close() {
        // really close the file
        ncfile.setFileCache(null);
        try {
          ncfile.close();
        } catch (IOException e) {
          log.error("close failed on "+ncfile.getLocation(), e);
        }

        if (cacheLog.isDebugEnabled()) cacheLog.debug("FileCacheARC " + name + " remove " + ncfile.getLocation());
        if (debugPrint) System.out.println("  FileCacheARC " + name + " remove " + ncfile.getLocation());
        ncfile = null;
      }

      public String toString() {
        return isLocked + " " + ncfile.getLocation();
      }

      public int compareTo(CacheFile o) {
        return Misc.compare(lastAccessed.get(), o.getLastAccessed());
      }
    }
  }

  static private class CacheElementComparator implements
            Comparator<CacheElement> {
    @Override
    public int compare(CacheElement o1, CacheElement o2) {
      return Misc.compare(o1.getLastAccessed(), o2.getLastAccessed());
    }
  }

  /*
  case I:  if (x in T1 + T2)    // cache hit
    x -> MRU in T2

  case II: if (x in B1)         // cache miss
    adapt1(p)
    replace(x,p)
    move x from B1 -> MRU in T2

  case III: if (x in B2)        // cache miss
    adapt2(p)
    replace(x,p)
    move x from B2 -> MRU in T2

  case IV: not in cache         // cache miss
    case A: if (size L1 == c)
      if (size T1 < c)
        delete LRU in B1
        replace(x,p)
      else            // B1 is empty
        delete LRU in T1
    case B:
      if (size L1 + L2 >= c)
         if (size L1 + L2 == 2c)
           delete LRU in B2
         replace(x,p)

    x -> MRU in T1

   */

  // debug
  public String getInfo(FileCacheable ncfile) throws IOException {
    if (ncfile == null) return "";

    // find it in the file cache
    int hashcode = System.identityHashCode(ncfile);
    CacheElement.CacheFile file = files.get(hashcode);
    if (file != null) {
      return "File is in cache= " + file;
    }
    return "File not in cache";
  }

  // debug
  Map<Object, CacheElement> getCache() {
    return cache;
  }

  /**
   * Remove all cache entries.
   *
   * @param force if true, remove them even if they are currently locked.
   */
  public synchronized void clearCache(boolean force) {
    List<CacheElement.CacheFile> deleteList = new ArrayList<>(2 * cache.size());

    if (force) {
      cache.clear(); // deletes everything from the cache
      deleteList.addAll(files.values());  // add everything to the delete list
      files.clear();
      // counter.set(0);

    } else {

      // add unlocked files to the delete list, remove from files hash
      Iterator<CacheElement.CacheFile> iter = files.values().iterator();
      while (iter.hasNext()) {
        CacheElement.CacheFile file = iter.next();
        if (file.isLocked.compareAndSet(false, true)) {
          file.remove(); // remove from the containing CacheElement
          deleteList.add(file);
          iter.remove();
        }
      }

      // remove empty cache elements
      for (CacheElement elem : cache.values()) {
        if (elem.list.size() == 0)
          cache.remove(elem.hashKey);
      }
    }

    // close all files in deleteList
    for (CacheElement.CacheFile file : deleteList) {
      if (force && file.isLocked.get())
        cacheLog.warn("FileCacheARC " + name + " force close locked file= " + file);
      //counter.decrementAndGet();

      if (file.ncfile == null) continue;

      try {
        file.ncfile.setFileCache(null);
        file.ncfile.close();
        file.ncfile = null; // help the gc
      } catch (IOException e) {
        log.error("FileCacheARC " + name + " close failed on " + file);
      }
    }
    if (cacheLog.isDebugEnabled())
      cacheLog.debug("*FileCacheARC " + name + " clearCache force= " + force + " deleted= " + deleteList.size() + " left=" + files.size());
    //System.out.println("\n*NetcdfFileCache.clearCache force= " + force + " deleted= " + deleteList.size() + " left=" + counter.get());
  }

  /**
   * Show individual cache entries, add to formatter.
   *
   * @param format add to this
   */
  @Override
  public void showCache(Formatter format) {
    ArrayList<CacheElement.CacheFile> allFiles = new ArrayList<>(files.size());
    for (CacheElement elem : cache.values()) {
      allFiles.addAll(elem.list);
    }
    Collections.sort(allFiles); // sort so oldest are on top

    format.format("%nFileCacheARC %s (min=%d softLimit=%d hardLimit=%d scour=%d):%n", name, minElements, softLimit, hardLimit, period);
    format.format("isLocked  accesses lastAccess                   location %n");
    for (CacheElement.CacheFile file : allFiles) {
      String loc = file.ncfile != null ? file.ncfile.getLocation() : "null";
      CalendarDate cd = CalendarDate.of(file.getLastAccessed());
      format.format("%8s %9d %s %s %n", file.isLocked, file.getCountAccessed(), CalendarDateFormatter.toTimeUnits(cd), loc);
    }
    showStats(format);
  }

  @Override
  public List<String> showCache() {
    List<CacheElement.CacheFile> allFiles = new ArrayList<>(files.size());
    for (CacheElement elem : cache.values()) {
      allFiles.addAll(elem.list);
    }
    Collections.sort(allFiles); // sort so oldest are on top

    List<String> result = new ArrayList<>(allFiles.size());
    for (CacheElement.CacheFile file : allFiles) {
      result.add(file.toString());
    }

    return result;
  }

  /**
   * Add stat report (hits, misses, etc) to formatter.
   *
   * @param format add to this
   */
  public void showStats(Formatter format) {
    format.format("  hits= %d miss= %d nfiles= %d elems= %d shadow=%d%n", hits.get(), miss.get(), files.size(), cache.values().size(), shadowCache.size());
  }

  ///////////////////////////////////////////////////////////////

  public void showTracking(Formatter format) {
    if (track == null) return;
    List<Tracker> all = new ArrayList<>(track.size());
    for (Tracker val : track.values()) all.add(val);
    Collections.sort(all);       // LOOK what should we sort by ??
    int count = 0;
    int countAll = 0;
    format.format("%nTrack of all files in FileCacheARC%n");
    format.format("   seq  accum   hit   miss  file%n");
    for (Tracker t : all) {
      count++;
      countAll += t.hit + t.miss;
      format.format("%6d  %6d : %5d %5d %s%n", count, countAll, t.hit, t.miss, t.key);
    }
    format.format("%n");
  }

  @Override
  public void resetTracking() {
    track = new ConcurrentHashMap<>(5000);
  }

  private static class Tracker implements Comparable<Tracker> {
    Object key;
    int hit, miss;

    private Tracker(Object key) {
      this.key = key;
    }

    @Override
    public boolean equals(Object o) {
      if (this == o) return true;
      if (o == null || getClass() != o.getClass()) return false;
      Tracker tracker = (Tracker) o;
      return key.equals(tracker.key);   // maybe == ?? LOOK
    }

    @Override
    public int hashCode() {
      return key.hashCode();
    }

    @Override
    public int compareTo(Tracker o) {
      return Misc.compare(hit + miss, o.hit + o.miss);
    }
  }
}