/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.unidata.io;

import ucar.nc2.constants.CDM;
import ucar.nc2.dataset.DatasetUrl;
import ucar.nc2.util.CancelTask;
import ucar.nc2.util.cache.FileCache;
import ucar.nc2.util.cache.FileCacheIF;
import ucar.nc2.util.cache.FileCacheable;
import ucar.nc2.util.cache.FileFactory;
import ucar.unidata.util.StringUtil2;

import javax.annotation.concurrent.NotThreadSafe;
import java.io.*;
import java.nio.ByteOrder;
import java.nio.channels.WritableByteChannel;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;

/**
 * A buffered drop-in replacement for java.io.RandomAccessFile.
 * Instances of this class realise substantial speed increases over
 * java.io.RandomAccessFile through the use of buffering. This is a
 * subclass of Object, as it was not possible to subclass
 * java.io.RandomAccessFile because many of the methods are
 * final. However, if it is necessary to use RandomAccessFile and
 * java.io.RandomAccessFile interchangeably, both classes implement the
 * DataInput and DataOutput interfaces.
 *

 *
 By Russ Rew, based on
 * BufferedRandomAccessFile by Alex McManus, based on Sun's source code
 * for java.io.RandomAccessFile.  For Alex McManus version from which
 * this derives, see his
 * Freeware Java Classes.
 *

 * Must be thread confined - that is, can only be used by a single thread at a time..
 *
 * @author Alex McManus
 * @author Russ Rew
 * @author john caron
 * @see java.io.DataInput
 * @see java.io.DataOutput
 * @see java.io.RandomAccessFile
 */

@NotThreadSafe
public class RandomAccessFile implements DataInput, DataOutput, FileCacheable, Closeable {

  static public final int BIG_ENDIAN = 0;
  static public final int LITTLE_ENDIAN = 1;

  static protected final int defaultBufferSize = 8092;  // The default buffer size, in bytes.

  ///////////////////////////////////////////////////////////////////////
  // debug leaks - keep track of open files
  static protected boolean debugLeaks = false;
  static protected boolean debugAccess = false;
  static protected Set<String> allFiles = null;
  static protected List<String> openFiles = Collections.synchronizedList(new ArrayList<>());   // could keep map on file hashcode
  static private AtomicLong count_openFiles = new AtomicLong();
  static private AtomicInteger maxOpenFiles = new AtomicInteger();
  static private AtomicInteger debug_nseeks = new AtomicInteger();
  static private AtomicLong debug_nbytes = new AtomicLong();

  static protected boolean showOpen = false;
  static protected boolean showRead = false;

  /**
   * Debugging, do not use.
   *
   * @return true if debugLeaks is on
   */
  static public boolean getDebugLeaks() {
    return debugLeaks;
  }

  /**
   * Debugging, do not use in production.
   * Set counters to zero, set debugging on
   * @param b set true to track java.io.RandomAccessFile
   */
  static public void setDebugLeaks(boolean b) {
    if (b) {
      count_openFiles.set(0);
      maxOpenFiles.set(0);
      allFiles = new HashSet<>(1000);
    }
    debugLeaks = b;
  }

  /**
   * Debugging, do not use.
   *
   * @return list of open files.
   */
  static public List<String> getOpenFiles() {
    return Collections.unmodifiableList(openFiles);
  }

  static public long getOpenFileCount() {
    return count_openFiles.get();
  }

  static public int getMaxOpenFileCount() {
    return maxOpenFiles.get();
  }

  /**
   * Debugging, do not use.
   *
   * @return list of all files used.
   */
  static public List<String> getAllFiles() {
    if (null == allFiles) return null;
    List<String> result = new ArrayList<>();
    result.addAll(allFiles);
    Collections.sort(result);
    return result;
  }

  /**
   * Debugging, do not use.
   *
   * @param b to debug file reading
   */
  static public void setDebugAccess(boolean b) {
    debugAccess = b;
    if (b) {
      debug_nseeks = new AtomicInteger();
      debug_nbytes = new AtomicLong();
    }
  }

  /**
   * Debugging, do not use.
   *
   * @return number of seeks
   */
  static public int getDebugNseeks() {
    return (debug_nseeks == null) ? 0 : debug_nseeks.intValue();
  }

  /**
   * Debugging, do not use.
   *
   * @return number of bytes read
   */
  static public long getDebugNbytes() {
    return (debug_nbytes == null) ? 0 : debug_nbytes.longValue();
  }

  /////////////////////////////////////////////////////////////////////////////////////////////
  // internal File Caching. this allows a global pool of OS files.
  // note read only

  static private final ucar.nc2.util.cache.FileFactory factory = new FileFactory() {
    public FileCacheable open(DatasetUrl durl, int buffer_size, CancelTask cancelTask, Object iospMessage) throws IOException {
      String location = StringUtil2.replace(durl.trueurl, "\\", "/"); // canonicalize the name
      RandomAccessFile result = new RandomAccessFile(location, "r", buffer_size);
      result.cacheState = 1;  // in use
      return result;
    }
  };

  static private FileCacheIF cache = null;

  static public synchronized void enableDefaultGlobalFileCache() {
    if (cache != null) cache.disable();
    cache = new FileCache("RandomAccessFile", 200, 300, 400, 60 * 60); // default; override for higher performance, or set to null for no caching;
  }

  static public synchronized void setGlobalFileCache(FileCacheIF _cache) {
    if (cache != null) cache.disable();
    cache = _cache;
   }

  static public synchronized FileCacheIF getGlobalFileCache() {
    return cache;
  }

  static public RandomAccessFile acquire(String location) throws IOException {
    if (cache == null)
      return new RandomAccessFile(location, "r");
    else
      return (RandomAccessFile) cache.acquire(factory, new DatasetUrl(null, location));
  }

  static public RandomAccessFile acquire(String location, int buffer_size) throws IOException {
    if (cache == null)
      return new RandomAccessFile(location, "r", buffer_size);
    else
      return (RandomAccessFile) cache.acquire(factory, location, new DatasetUrl(null, location), buffer_size, null, null);
  }

  static public void eject(String location) {
    if (cache != null) cache.eject(location);
  }

  static public void shutdown() {
    if (cache != null) cache.clearCache(true);
  }

  /////////////////////////////////////////////////////////////////////////////////////////////

  /**
   * File location
   */
  protected String location;
  private int cacheState = 0;  // 0 - not in cache, 1 = in cache && in use, 2 = in cache but not in use

  /**
   * The underlying java.io.RandomAccessFile.
   */
  protected java.io.RandomAccessFile file;
  protected java.nio.channels.FileChannel fileChannel;

  /**
   * The offset in bytes from the file start, of the next read or
   * write operation.
   */
  protected long filePosition;

  /**
   * The buffer used for reading the data.
   */
  protected byte buffer[];

  /**
   * The offset in bytes of the start of the buffer, from the start of the file.
   */
  protected long bufferStart;

  /**
   * The offset in bytes of the end of the data in the buffer, from
   * the start of the file. This can be calculated from
   * bufferStart + dataSize, but it is cached to speed
   * up the read( ) method.
   */
  protected long dataEnd;

  /**
   * The size of the data stored in the buffer, in bytes. This may be
   * less than the size of the buffer.
   */
  protected int dataSize;

  /**
   * True if we are at the end of the file.
   */
  protected boolean endOfFile;

  /**
   * The access mode of the file.
   */
  protected boolean readonly;

  /**
   * The current endian (big or little) mode of the file.
   */
  protected boolean bigEndian;

  /**
   * True if the data in the buffer has been modified.
   */
  boolean bufferModified = false;

  /**
   * make sure file is at least this long when closed
   */
  private long minLength = 0;

  /**
   * STUPID extendMode for truncated, yet valid files. old netcdf C library code allowed NOFILL to do this
   */
  private boolean extendMode = false;

  /**
   * Constructor, for subclasses
   *
   * @param bufferSize size of read buffer
   */
  protected RandomAccessFile(int bufferSize) {
    file = null;
    readonly = true;
    init(bufferSize);
  }

  /**
   * Constructor, default buffer size.
   *
   * @param location location of the file
   * @param mode     same as for java.io.RandomAccessFile, usually "r" or "rw"
   * @throws IOException on open error
   */
  public RandomAccessFile(String location, String mode) throws IOException {
    this(location, mode, defaultBufferSize);
    this.location = location;
  }

  /**
   * Constructor.
   *
   * @param location   location of the file
   * @param mode       same as for java.io.RandomAccessFile
   * @param bufferSize size of buffer to use.
   * @throws IOException on open error
   */
  public RandomAccessFile(String location, String mode, int bufferSize) throws IOException {
    if (bufferSize < 0) bufferSize = defaultBufferSize;
    this.location = location;
    if (debugLeaks) {
      allFiles.add(location);
    }

    try {
      this.file = new java.io.RandomAccessFile(location, mode);
    } catch (IOException ioe) {
      if (ioe.getMessage().equals("Too many open files")) {
        System.out.printf("RandomAccessFile %s%n", ioe);
        try {
          Thread.currentThread().sleep(100);
        } catch (InterruptedException e) {
          e.printStackTrace();
        }
        this.file = new java.io.RandomAccessFile(location, mode); // Windows having troublke keeping up ??
      } else {
        throw ioe;
      }
    }

    this.readonly = mode.equals("r");
    init(bufferSize);

    if (debugLeaks) {
      openFiles.add(location);
      int max = Math.max(openFiles.size(), maxOpenFiles.get());
      maxOpenFiles.set(max);
      count_openFiles.getAndIncrement();
      if (showOpen) System.out.println(" DebugRAF open " + location);
      //if (openFiles.size() > 1000)
      //  System.out.println("RandomAccessFile debugLeaks");
    }
  }

  /**
   * Allow access to the underlying java.io.RandomAccessFile.
   * WARNING! BROKEN ENCAPSOLATION, DO NOT USE. May change implementation in the future.
   *
   * @return the underlying java.io.RandomAccessFile.
   */
  public java.io.RandomAccessFile getRandomAccessFile() {
    return this.file;
  }

  private void init(int bufferSize) {
    // Initialise the buffer
    bufferStart = 0;
    dataEnd = 0;
    dataSize = 0;
    filePosition = 0;
    buffer = new byte[bufferSize];
    endOfFile = false;
  }

  /**
   * Set the buffer size.
   * If writing, call flush() first.
   *
   * @param bufferSize length in bytes
   */
  public void setBufferSize(int bufferSize) {
    init(bufferSize);
  }

  /**
   * Get the buffer size
   *
   * @return bufferSize length in bytes
   */
  public int getBufferSize() {
    return buffer.length;
  }

  /**
   * Close the file, and release any associated system resources.
   *
   * @throws IOException if an I/O error occurrs.
   */
  public synchronized void close() throws IOException {
    if (cache != null) {
      if (cacheState > 0) {
        if (cacheState == 1) {
          cacheState = 2;
          if (cache.release(this))  // return true if in the cache, otherwise was opened regular, so must be closed regular
            return;
          cacheState = 0; // release failed, bail out
        } else {
          return; // close has been called more than once - ok
        }
      }
    }

    if (debugLeaks) {
      openFiles.remove(location);
      if (showOpen) System.out.println("  close " + location);
    }

    if (file == null)
      return;

    // If we are writing and the buffer has been modified, flush the contents of the buffer.
    flush();

    // may need to extend file, in case no fill is being used
    // may need to truncate file in case overwriting a longer file
    // use only if minLength is set (by N3iosp)
    long fileSize = file.length();
    if (!readonly && (minLength != 0) && (minLength != fileSize)) {
      file.setLength(minLength);
      // System.out.println("TRUNCATE!!! minlength="+minLength);
    }

    // Close the underlying file object.
    file.close();
    file = null;  // help the gc
  }

  @Override
  public void release() {  // one to one with java.io.RandomAccessFile
    cacheState = 2;
  }

  @Override
  public void reacquire() {
    cacheState = 1;
  }

  @Override
  public synchronized void setFileCache(FileCacheIF fileCache) {
    if (fileCache == null)
      cacheState = 0;
  }

  @Override
  public long getLastModified() {
    File file = new File(getLocation());
    return file.lastModified();
  }

  /**
   * Return true if file pointer is at end of file.
   *
   * @return true if file pointer is at end of file
   */
  public boolean isAtEndOfFile() {
    return endOfFile;
  }

  /**
   * Set the position in the file for the next read or write.
   *
   * @param pos the offset (in bytes) from the start of the file.
   * @throws IOException if an I/O error occurrs.
   */
  public void seek(long pos) throws IOException {
    if (pos < 0)
      throw new java.io.IOException("Negative seek offset");

    // If the seek is into the buffer, just update the file pointer.
    if ((pos >= bufferStart) && (pos < dataEnd)) {
      filePosition = pos;
      return;
    }

    // need new buffer, starting at pos
    readBuffer(pos);
  }

  protected void readBuffer(long pos) throws IOException {
    // If the current buffer is modified, write it to disk.
    if (bufferModified) {
      flush();
    }

    bufferStart = pos;
    filePosition = pos;

    dataSize = read_(pos, buffer, 0, buffer.length);

    if (dataSize <= 0) {
      dataSize = 0;
      endOfFile = true;
    } else {
      endOfFile = false;
    }

    // Cache the position of the buffer end.
    dataEnd = bufferStart + dataSize;
  }

  /**
   * Returns the current position in the file, where the next read or
   * write will occur.
   *
   * @return the offset from the start of the file in bytes.
   * @throws IOException if an I/O error occurrs.
   */
  public long getFilePointer() throws IOException {
    return filePosition;
  }

  /**
   * Get the file location, or name.
   *
   * @return file location
   */
  public String getLocation() {
    return location;
  }

  /**
   * Get the length of the file. The data in the buffer (which may not
   * have been written the disk yet) is taken into account.
   *
   * @return the length of the file in bytes.
   * @throws IOException if an I/O error occurrs.
   */
  public long length() throws IOException {
    long fileLength = (file == null) ? -1L : file.length(); // GRIB has closed the data raf
    if (fileLength < dataEnd) {
      return dataEnd;
    } else {
      return fileLength;
    }
  }

  /**
   * Change the current endian mode. Subsequent reads of short, int, float, double, long, char will
   * use this. Does not currently affect writes.
   * Default values is BIG_ENDIAN.
   *
   * @param endian RandomAccessFile.BIG_ENDIAN or RandomAccessFile.LITTLE_ENDIAN
   */
  public void order(int endian) {
    if (endian < 0) return;
    this.bigEndian = (endian == BIG_ENDIAN);
  }

  public void order(ByteOrder bo) {
    if (bo == null) return;
    this.bigEndian = bo.equals(ByteOrder.BIG_ENDIAN);
  }

  /**
   * Copy the contents of the buffer to the disk.
   *
   * @throws IOException if an I/O error occurs.
   */
  public void flush() throws IOException {
    if (bufferModified) {
      file.seek(bufferStart);
      file.write(buffer, 0, dataSize);
      //System.out.println("--flush at "+bufferStart+" dataSize= "+dataSize+ " filePosition= "+filePosition);
      bufferModified = false;
    }

    /* check min length
    if (!readonly && (minLength != 0) && (minLength != file.length())) {
      file.setLength(minLength);
    } */
  }

  /**
   * Make sure file is at least this long when its closed.
   * needed when not using fill mode, and not all data is written.
   *
   * @param minLength minimum length of the file.
   */
  public synchronized void setMinLength(long minLength) {
    this.minLength = minLength;
  }

  /**
   * Set extendMode for truncated, yet valid files - old NetCDF code allowed this
   * when NOFILL on, and user doesnt write all variables.
   */
  public void setExtendMode() {
    this.extendMode = true;
  }

  //////////////////////////////////////////////////////////////////////////////////////////////
  // Read primitives.
  //

  /**
   * Read a byte of data from the file, blocking until data is
   * available.
   *
   * @return the next byte of data, or -1 if the end of the file is
   *         reached.
   * @throws IOException if an I/O error occurrs.
   */
  public int read() throws IOException {

    // If the file position is within the data, return the byte...
    if (filePosition < dataEnd) {
      int pos = (int) (filePosition - bufferStart);
      filePosition++;
      return (buffer[pos] & 0xff);

      // ...or should we indicate EOF...
    } else if (endOfFile) {
      return -1;

      // ...or seek to fill the buffer, and try again.
    } else {
      seek(filePosition);
      return read();
    }
  }

  /**
   * Read up to len bytes into an array, at a specified
   * offset. This will block until at least one byte has been read.
   *
   * @param b   the byte array to receive the bytes.
   * @param off the offset in the array where copying will start.
   * @param len the number of bytes to copy.
   * @return the actual number of bytes read, or -1 if there is not
   *         more data due to the end of the file being reached.
   * @throws IOException if an I/O error occurrs.
   */
  public int readBytes(byte b[], int off, int len) throws IOException {

    // Check for end of file.
    if (endOfFile) {
      return -1;
    }

    // See how many bytes are available in the buffer - if none,
    // seek to the file position to update the buffer and try again.
    int bytesAvailable = (int) (dataEnd - filePosition);
    if (bytesAvailable < 1) {
      seek(filePosition);
      return readBytes(b, off, len);
    }

    // Copy as much as we can.
    int copyLength = (bytesAvailable >= len)
            ? len
            : bytesAvailable;
    System.arraycopy(buffer, (int) (filePosition - bufferStart), b, off, copyLength);
    filePosition += copyLength;

    // If there is more to copy...
    if (copyLength < len) {
      int extraCopy = len - copyLength;

      // If the amount remaining is more than a buffer's length, read it
      // directly from the file.
      if (extraCopy > buffer.length) {
        extraCopy = read_(filePosition, b, off + copyLength, len - copyLength);

        // ...or read a new buffer full, and copy as much as possible...
      } else {
        seek(filePosition);
        if (!endOfFile) {
          extraCopy = (extraCopy > dataSize)
                  ? dataSize
                  : extraCopy;
          System.arraycopy(buffer, 0, b, off + copyLength, extraCopy);
        } else {
          extraCopy = -1;
        }
      }

      // If we did manage to copy any more, update the file position and
      // return the amount copied.
      if (extraCopy > 0) {
        filePosition += extraCopy;
        return copyLength + extraCopy;
      }
    }

    // Return the amount copied.
    return copyLength;
  }

  /**
   * Read nbytes bytes, at the specified file offset, send to a WritableByteChannel.
   * This will block until all bytes are read.
   * This uses the underlying file channel directly, bypassing all user buffers.
   *
   * @param dest   write to this WritableByteChannel.
   * @param offset the offset in the file where copying will start.
   * @param nbytes the number of bytes to read.
   * @return the actual number of bytes read and transfered
   * @throws IOException if an I/O error occurs.
   */
  public long readToByteChannel(WritableByteChannel dest, long offset, long nbytes) throws IOException {

    if (fileChannel == null)
      fileChannel = file.getChannel();

    long need = nbytes;
    while (need > 0) {
      long count = fileChannel.transferTo(offset, need, dest);
      //if (count == 0) break;  // LOOK not sure what the EOF condition is
      need -= count;
      offset += count;
    }
    return nbytes - need;
  }

  /**
   * Read directly from file, without going through the buffer.
   * All reading goes through here or readToByteChannel;
   *
   * @param pos    start here in the file
   * @param b      put data into this buffer
   * @param offset buffer offset
   * @param len    this number of bytes
   * @return actual number of bytes read
   * @throws IOException on io error
   */
  protected int read_(long pos, byte[] b, int offset, int len) throws IOException {
    file.seek(pos);
    int n = file.read(b, offset, len);
    if (debugAccess) {
      if (showRead)
        System.out.println(" **read_ " + location + " = " + len + " bytes at " + pos + "; block = " + (pos / buffer.length));
      debug_nseeks.incrementAndGet();
      debug_nbytes.addAndGet(len);
    }

    if (extendMode && (n < len)) {
      //System.out.println(" read_ = "+len+" at "+pos+"; got = "+n);
      n = len;
    }
    return n;
  }

  /**
   * Read up to len bytes into an array, at a specified
   * offset. This will block until at least one byte has been read.
   *
   * @param b   the byte array to receive the bytes.
   * @param off the offset in the array where copying will start.
   * @param len the number of bytes to copy.
   * @return the actual number of bytes read, or -1 if there is not
   *         more data due to the end of the file being reached.
   * @throws IOException if an I/O error occurrs.
   */
  public int read(byte b[], int off, int len) throws IOException {
    return readBytes(b, off, len);
  }

  /**
   * Read up to b.length( ) bytes into an array. This
   * will block until at least one byte has been read.
   *
   * @param b the byte array to receive the bytes.
   * @return the actual number of bytes read, or -1 if there is not
   *         more data due to the end of the file being reached.
   * @throws IOException if an I/O error occurrs.
   */
  public int read(byte b[]) throws IOException {
    return readBytes(b, 0, b.length);
  }

  /**
   * Read fully count number of bytes
   *
   * @param count how many bytes tp read
   * @return a byte array of length count, fully read in
   * @throws IOException if an I/O error occurrs.
   */
  public byte[] readBytes(int count) throws IOException {
    byte[] b = new byte[count];
    readFully(b);
    return b;
  }

  /**
   * Reads b.length bytes from this file into the byte
   * array. This method reads repeatedly from the file until all the
   * bytes are read. This method blocks until all the bytes are read,
   * the end of the stream is detected, or an exception is thrown.
   *
   * @param b the buffer into which the data is read.
   * @throws EOFException if this file reaches the end before reading
   *                      all the bytes.
   * @throws IOException  if an I/O error occurs.
   */
  public final void readFully(byte b[]) throws IOException {
    readFully(b, 0, b.length);
  }

  /**
   * Reads exactly len bytes from this file into the byte
   * array. This method reads repeatedly from the file until all the
   * bytes are read. This method blocks until all the bytes are read,
   * the end of the stream is detected, or an exception is thrown.
   *
   * @param b   the buffer into which the data is read.
   * @param off the start offset of the data.
   * @param len the number of bytes to read.
   * @throws EOFException if this file reaches the end before reading
   *                      all the bytes.
   * @throws IOException  if an I/O error occurs.
   */
  public final void readFully(byte b[], int off, int len) throws IOException {
    int n = 0;
    while (n < len) {
      int count = this.read(b, off + n, len - n);
      if (count < 0) {
        throw new EOFException("Reading "+location+" at "+filePosition+" file length = "+length());
      }
      n += count;
    }
  }

  /**
   * Skips exactly n bytes of input.
   * This method blocks until all the bytes are skipped, the end of
   * the stream is detected, or an exception is thrown.
   *
   * @param n the number of bytes to be skipped.
   * @return the number of bytes skipped, which is always n.
   * @throws EOFException if this file reaches the end before skipping
   *                      all the bytes.
   * @throws IOException  if an I/O error occurs.
   */
  public int skipBytes(int n) throws IOException {
    seek(getFilePointer() + n);
    return n;
  }

  public long skipBytes(long n) throws IOException {
    seek(getFilePointer() + n);
    return n;
  }

  /* public void skipToMultiple( int multipleOfBytes) throws IOException {
   long pos = getFilePointer();
   int pad = (int) (pos % multipleOfBytes);
   if (pad != 0) pad = multipleOfBytes - pad;
   if (pad > 0) skipBytes(pad);
 } */

  /**
   * Unread the last byte read.
   * This method should not be used more than once
   * between reading operations, or strange things might happen.
   */
  public void unread() {
    filePosition--;
  }

  //
  // Write primitives.
  //

  /**
   * Write a byte to the file. If the file has not been opened for
   * writing, an IOException will be raised only when an attempt is
   * made to write the buffer to the file.
   *

   * Caveat: the effects of seek( )ing beyond the end of the file are
   * undefined.
   *
   * @param b write this byte
   * @throws IOException if an I/O error occurrs.
   */
  public void write(int b) throws IOException {

    // If the file position is within the block of data...
    if (filePosition < dataEnd) {
      int pos = (int) (filePosition - bufferStart);
      buffer[pos] = (byte) b;
      bufferModified = true;
      filePosition++;

      // ...or (assuming that seek will not allow the file pointer
      // to move beyond the end of the file) get the correct block of
      // data...
    } else {

      // If there is room in the buffer, expand it...
      if (dataSize != buffer.length) {
        int pos = (int) (filePosition - bufferStart);
        buffer[pos] = (byte) b;
        bufferModified = true;
        filePosition++;
        dataSize++;
        dataEnd++;

        // ...or do another seek to get a new buffer, and start again...
      } else {
        seek(filePosition);
        write(b);
      }
    }
  }

  /**
   * Write len bytes from an array to the file.
   *
   * @param b   the array containing the data.
   * @param off the offset in the array to the data.
   * @param len the length of the data.
   * @throws IOException if an I/O error occurrs.
   */
  public void writeBytes(byte b[], int off, int len) throws IOException {
    // If the amount of data is small (less than a full buffer)...
    if (len < buffer.length) {

      // If any of the data fits within the buffer...
      int spaceInBuffer = 0;
      int copyLength = 0;
      if (filePosition >= bufferStart) {
        spaceInBuffer = (int) ((bufferStart + buffer.length) - filePosition);
      }

      if (spaceInBuffer > 0) {
        // Copy as much as possible to the buffer.
        copyLength = (spaceInBuffer > len) ? len : spaceInBuffer;
        System.arraycopy(b, off, buffer, (int) (filePosition - bufferStart), copyLength);
        bufferModified = true;
        long myDataEnd = filePosition + copyLength;
        dataEnd = (myDataEnd > dataEnd) ? myDataEnd : dataEnd;
        dataSize = (int) (dataEnd - bufferStart);
        filePosition += copyLength;
        ///System.out.println("--copy to buffer "+copyLength+" "+len);
      }

      // If there is any data remaining, move to the new position and copy to
      // the new buffer.
      if (copyLength < len) {
        //System.out.println("--need more "+copyLength+" "+len+" space= "+spaceInBuffer);
        seek(filePosition);   // triggers a flush
        System.arraycopy(b, off + copyLength, buffer, (int) (filePosition - bufferStart), len - copyLength);
        bufferModified = true;
        long myDataEnd = filePosition + (len - copyLength);
        dataEnd = (myDataEnd > dataEnd) ? myDataEnd : dataEnd;
        dataSize = (int) (dataEnd - bufferStart);
        filePosition += (len - copyLength);
      }

      // ...or write a lot of data...
    } else {

      // Flush the current buffer, and write this data to the file.
      if (bufferModified) {
        flush();
      }
      file.seek(filePosition);  // moved per Steve Cerruti; Jan 14, 2005
      file.write(b, off, len);
      //System.out.println("--write at "+filePosition+" "+len);

      filePosition += len;
      bufferStart = filePosition;  // an empty buffer
      dataSize = 0;
      dataEnd = bufferStart + dataSize;
    }
  }

  /**
   * Writes b.length bytes from the specified byte array
   * starting at offset off to this file.
   *
   * @param b the data.
   * @throws IOException if an I/O error occurs.
   */
  public void write(byte b[]) throws IOException {
    writeBytes(b, 0, b.length);
  }

  /**
   * Writes len bytes from the specified byte array
   * starting at offset off to this file.
   *
   * @param b   the data.
   * @param off the start offset in the data.
   * @param len the number of bytes to write.
   * @throws IOException if an I/O error occurs.
   */
  public void write(byte b[], int off, int len) throws IOException {
    writeBytes(b, off, len);
  }

  //
  // DataInput methods.
  //

  /**
   * Reads a boolean from this file. This method reads a
   * single byte from the file. A value of 0 represents
   * false. Any other value represents true.
   * This method blocks until the byte is read, the end of the stream
   * is detected, or an exception is thrown.
   *
   * @return the boolean value read.
   * @throws EOFException if this file has reached the end.
   * @throws IOException  if an I/O error occurs.
   */
  public final boolean readBoolean() throws IOException {
    int ch = this.read();
    if (ch < 0) {
      throw new EOFException();
    }
    return (ch != 0);
  }

  /**
   * Reads a signed 8-bit value from this file. This method reads a
   * byte from the file. If the byte read is b, where
   * 0 <= b <= 255,
   * then the result is:
   *

   * (byte)(b)
   *

   *

   * This method blocks until the byte is read, the end of the stream
   * is detected, or an exception is thrown.
   *
   * @return the next byte of this file as a signed 8-bit
   *         byte.
   * @throws EOFException if this file has reached the end.
   * @throws IOException  if an I/O error occurs.
   */
  public final byte readByte() throws IOException {
    int ch = this.read();
    if (ch < 0) {
      throw new EOFException();
    }
    return (byte) (ch);
  }

  /**
   * Reads an unsigned 8-bit number from this file. This method reads
   * a byte from this file and returns that byte.
   *

   * This method blocks until the byte is read, the end of the stream
   * is detected, or an exception is thrown.
   *
   * @return the next byte of this file, interpreted as an unsigned
   *         8-bit number.
   * @throws EOFException if this file has reached the end.
   * @throws IOException  if an I/O error occurs.
   */
  public final int readUnsignedByte() throws IOException {
    int ch = this.read();
    if (ch < 0) {
      throw new EOFException();
    }
    return ch;
  }

  /**
   * Reads a signed 16-bit number from this file. The method reads 2
   * bytes from this file. If the two bytes read, in order, are
   * b1 and b2, where each of the two values is
   * between 0 and 255, inclusive, then the
   * result is equal to:
   *

   * (short)((b1 << 8) | b2)
   *

   *

   * This method blocks until the two bytes are read, the end of the
   * stream is detected, or an exception is thrown.
   *
   * @return the next two bytes of this file, interpreted as a signed
   *         16-bit number.
   * @throws EOFException if this file reaches the end before reading
   *                      two bytes.
   * @throws IOException  if an I/O error occurs.
   */
  public final short readShort() throws IOException {
    int ch1 = this.read();
    int ch2 = this.read();
    if ((ch1 | ch2) < 0) {
      throw new EOFException();
    }
    if (bigEndian) {
      return (short) ((ch1 << 8) + (ch2));
    } else {
      return (short) ((ch2 << 8) + (ch1));
    }
  }

  /**
   * Read an array of shorts
   *
   * @param pa    read into this array
   * @param start starting at pa[start]
   * @param n     read this many elements
   * @throws IOException on read error
   */
  public final void readShort(short[] pa, int start, int n) throws IOException {
    for (int i = 0; i < n; i++) {
      pa[start + i] = readShort();
    }
  }

  /**
   * Reads an unsigned 16-bit number from this file. This method reads
   * two bytes from the file. If the bytes read, in order, are
   * b1 and b2, where
   * 0 <= b1, b2 <= 255,
   * then the result is equal to:
   *

   * (b1 << 8) | b2
   *

   *

   * This method blocks until the two bytes are read, the end of the
   * stream is detected, or an exception is thrown.
   *
   * @return the next two bytes of this file, interpreted as an unsigned
   *         16-bit integer.
   * @throws EOFException if this file reaches the end before reading
   *                      two bytes.
   * @throws IOException  if an I/O error occurs.
   */
  public final int readUnsignedShort() throws IOException {
    int ch1 = this.read();
    int ch2 = this.read();
    if ((ch1 | ch2) < 0) {
      throw new EOFException();
    }
    if (bigEndian) {
      return ((ch1 << 8) + (ch2));
    } else {
      return ((ch2 << 8) + (ch1));
    }
  }

  /*
  * Reads a signed 24-bit integer from this file. This method reads 3
  * bytes from the file. If the bytes read, in order, are b1,
  * b2, and b3, where
  * 0 <= b1, b2, b3 <= 255,
  * then the result is equal to:
  *

  * (b1 << 16) | (b2 << 8) + (b3 << 0)
  *

  *

  * This method blocks until the three bytes are read, the end of the
  * stream is detected, or an exception is thrown.
  */

  /**
   * Reads a Unicode character from this file. This method reads two
   * bytes from the file. If the bytes read, in order, are
   * b1 and b2, where
   * 0 <= b1, b2 <= 255,
   * then the result is equal to:
   *

   * (char)((b1 << 8) | b2)
   *

   *

   * This method blocks until the two bytes are read, the end of the
   * stream is detected, or an exception is thrown.
   *
   * @return the next two bytes of this file as a Unicode character.
   * @throws EOFException if this file reaches the end before reading
   *                      two bytes.
   * @throws IOException  if an I/O error occurs.
   */
  public final char readChar() throws IOException {
    int ch1 = this.read();
    int ch2 = this.read();
    if ((ch1 | ch2) < 0) {
      throw new EOFException();
    }
    if (bigEndian) {
      return (char) ((ch1 << 8) + (ch2));
    } else {
      return (char) ((ch2 << 8) + (ch1));
    }
  }

  /**
   * Reads a signed 32-bit integer from this file. This method reads 4
   * bytes from the file. If the bytes read, in order, are b1,
   * b2, b3, and b4, where
   * 0 <= b1, b2, b3, b4 <= 255,
   * then the result is equal to:
   *

   * (b1 << 24) | (b2 << 16) + (b3 << 8) + b4
   *

   *

   * This method blocks until the four bytes are read, the end of the
   * stream is detected, or an exception is thrown.
   *
   * @return the next four bytes of this file, interpreted as an
   *         int.
   * @throws EOFException if this file reaches the end before reading
   *                      four bytes.
   * @throws IOException  if an I/O error occurs.
   */
  public final int readInt() throws IOException {
    int ch1 = this.read();
    int ch2 = this.read();
    int ch3 = this.read();
    int ch4 = this.read();
    if ((ch1 | ch2 | ch3 | ch4) < 0) {
      throw new EOFException();
    }

    if (bigEndian) {
      return ((ch1 << 24) + (ch2 << 16) + (ch3 << 8) + (ch4));
    } else {
      return ((ch4 << 24) + (ch3 << 16) + (ch2 << 8) + (ch1));
    }
  }

  /**
   * Read an integer at the given position, bypassing all buffering.
   *
   * @param pos read a byte at this position
   * @return The int that was read
   * @throws IOException if an I/O error occurs.
   */
  public final int readIntUnbuffered(long pos) throws IOException {
    byte[] bb = new byte[4];
    read_(pos, bb, 0, 4);
    int ch1 = bb[0] & 0xff;
    int ch2 = bb[1] & 0xff;
    int ch3 = bb[2] & 0xff;
    int ch4 = bb[3] & 0xff;
    if ((ch1 | ch2 | ch3 | ch4) < 0) {
      throw new EOFException();
    }

    if (bigEndian) {
      return ((ch1 << 24) + (ch2 << 16) + (ch3 << 8) + (ch4));
    } else {
      return ((ch4 << 24) + (ch3 << 16) + (ch2 << 8) + (ch1));
    }
  }

  /**
   * Read an array of ints
   *
   * @param pa    read into this array
   * @param start starting at pa[start]
   * @param n     read this many elements
   * @throws IOException on read error
   */
  public final void readInt(int[] pa, int start, int n) throws IOException {
    for (int i = 0; i < n; i++) {
      pa[start + i] = readInt();
    }
  }

  /**
   * Reads a signed 64-bit integer from this file. This method reads eight
   * bytes from the file. If the bytes read, in order, are
   * b1, b2, b3,
   * b4, b5, b6,
   * b7, and b8, where:
   *

   * 0 <= b1, b2, b3, b4, b5, b6, b7, b8 <=255,
   *

   *

   * then the result is equal to:
   *

   *     ((long)b1 << 56) + ((long)b2 << 48)
   *     + ((long)b3 << 40) + ((long)b4 << 32)
   *     + ((long)b5 << 24) + ((long)b6 << 16)
   *     + ((long)b7 << 8) + b8
   *

   *

   * This method blocks until the eight bytes are read, the end of the
   * stream is detected, or an exception is thrown.
   *
   * @return the next eight bytes of this file, interpreted as a
   *         long.
   * @throws EOFException if this file reaches the end before reading
   *                      eight bytes.
   * @throws IOException  if an I/O error occurs.
   */
  public final long readLong() throws IOException {
    if (bigEndian) {
      return ((long) (readInt()) << 32) + (readInt() & 0xFFFFFFFFL);  // tested ok
    } else {
      return ((readInt() & 0xFFFFFFFFL) + ((long) readInt() << 32)); // not tested yet ??
    }

    /*     int ch1 = this.read();
      int ch2 = this.read();
      int ch3 = this.read();
      int ch4 = this.read();
      int ch5 = this.read();
      int ch6 = this.read();
      int ch7 = this.read();
      int ch8 = this.read();
      if ((ch1 | ch2 | ch3 | ch4 | ch5 | ch6 | ch7 | ch8) < 0)
         throw new EOFException();

      if (bigEndian)
        return ((long)(ch1 << 56)) + (ch2 << 48) + (ch3 << 40) + (ch4 << 32) + (ch5 << 24) + (ch6 << 16) + (ch7 << 8) + (ch8 << 0));
      else
        return ((long)(ch8 << 56) + (ch7 << 48) + (ch6 << 40) + (ch5 << 32) + (ch4 << 24) + (ch3 << 16) + (ch2 << 8) + (ch1 << 0));
    */
  }

  /**
   * Read an array of longs
   *
   * @param pa    read into this array
   * @param start starting at pa[start]
   * @param n     read this many elements
   * @throws IOException on read error
   */
  public final void readLong(long[] pa, int start, int n) throws IOException {
    for (int i = 0; i < n; i++) {
      pa[start + i] = readLong();
    }
  }

  /**
   * Reads a float from this file. This method reads an
   * int value as if by the readInt method
   * and then converts that int to a float
   * using the intBitsToFloat method in class
   * Float.
   *

   * This method blocks until the four bytes are read, the end of the
   * stream is detected, or an exception is thrown.
   *
   * @return the next four bytes of this file, interpreted as a
   *         float.
   * @throws EOFException if this file reaches the end before reading
   *                      four bytes.
   * @throws IOException  if an I/O error occurs.
   * @see java.io.RandomAccessFile#readInt()
   * @see java.lang.Float#intBitsToFloat(int)
   */
  public final float readFloat() throws IOException {
    return Float.intBitsToFloat(readInt());
  }

  /**
   * Read an array of floats
   *
   * @param pa    read into this array
   * @param start starting at pa[start]
   * @param n     read this many elements
   * @throws IOException on read error
   */
  public final void readFloat(float[] pa, int start, int n) throws IOException {
    for (int i = 0; i < n; i++) {
      pa[start + i] = Float.intBitsToFloat(readInt());
    }
  }

  /**
   * Reads a double from this file. This method reads a
   * long value as if by the readLong method
   * and then converts that long to a double
   * using the longBitsToDouble method in
   * class Double.
   *

   * This method blocks until the eight bytes are read, the end of the
   * stream is detected, or an exception is thrown.
   *
   * @return the next eight bytes of this file, interpreted as a
   *         double.
   * @throws EOFException if this file reaches the end before reading
   *                      eight bytes.
   * @throws IOException  if an I/O error occurs.
   * @see java.io.RandomAccessFile#readLong()
   * @see java.lang.Double#longBitsToDouble(long)
   */
  public final double readDouble() throws IOException {
    return Double.longBitsToDouble(readLong());
  }

  /**
   * Read an array of doubles
   *
   * @param pa    read into this array
   * @param start starting at pa[start]
   * @param n     read this many elements
   * @throws IOException on read error
   */
  public final void readDouble(double[] pa, int start, int n) throws IOException {
    for (int i = 0; i < n; i++) {
      pa[start + i] = Double.longBitsToDouble(readLong());
    }
  }

  /**
   * Reads the next line of text from this file.  This method successively
   * reads bytes from the file, starting at the current file pointer,
   * until it reaches a line terminator or the end
   * of the file.  Each byte is converted into a character by taking the
   * byte's value for the lower eight bits of the character and setting the
   * high eight bits of the character to zero.  This method does not,
   * therefore, support the full Unicode character set.
   *
   *
 A line of text is terminated by a carriage-return character
   * ('\r'), a newline character ('\n'), a
   * carriage-return character immediately followed by a newline character,
   * or the end of the file.  Line-terminating characters are discarded and
   * are not included as part of the string returned.
   *
   *
 This method blocks until a newline character is read, a carriage
   * return and the byte following it are read (to see if it is a newline),
   * the end of the file is reached, or an exception is thrown.
   *
   * @return the next line of text from this file, or null if end
   *             of file is encountered before even one byte is read.
   * @exception IOException  if an I/O error occurs.
   */
  public final String readLine() throws IOException {
    StringBuilder input = new StringBuilder();
    int c = -1;
    boolean eol = false;

    while (!eol) {
      switch (c = read()) {
        case -1:
        case '\n':
          eol = true;
          break;
        case '\r':
          eol = true;
          long cur = getFilePointer();
          if ((read()) != '\n') {
            seek(cur);
          }
          break;
        default:
          input.append((char) c);
          break;
      }
    }

    if ((c == -1) && (input.length() == 0)) {
      return null;
    }
    return input.toString();
  }

  /**
   * Reads in a string from this file. The string has been encoded
   * using a modified UTF-8 format.
   *

   * The first two bytes are read as if by
   * readUnsignedShort. This value gives the number of
   * following bytes that are in the encoded string, not
   * the length of the resulting string. The following bytes are then
   * interpreted as bytes encoding characters in the UTF-8 format
   * and are converted into characters.
   *

   * This method blocks until all the bytes are read, the end of the
   * stream is detected, or an exception is thrown.
   *
   * @return a Unicode string.
   * @throws EOFException           if this file reaches the end before
   *                                reading all the bytes.
   * @throws IOException            if an I/O error occurs.
   * @throws UTFDataFormatException if the bytes do not represent
   *                                valid UTF-8 encoding of a Unicode string.
   * @see java.io.RandomAccessFile#readUnsignedShort()
   */
  public final String readUTF() throws IOException {
    return DataInputStream.readUTF(this);
  }

  /**
   * Read a String of known length.
   *
   * @param nbytes number of bytes to read
   * @return String wrapping the bytes.
   * @throws IOException if an I/O error occurs.
   */
  public String readString(int nbytes) throws IOException {
    byte[] data = new byte[nbytes];
    readFully(data);
    return new String(data, CDM.utf8Charset);
  }

  /**
   * Read a String of max length, zero terminate.
   *
   * @param nbytes number of bytes to read
   * @return String wrapping the bytes.
   * @throws IOException if an I/O error occurs.
   */
  public String readStringMax(int nbytes) throws IOException {
    byte[] b = new byte[nbytes];
    readFully(b);
    int count;
    for (count = 0; count < nbytes; count++)
      if (b[count] == 0) break;
    return new String(b, 0, count, CDM.utf8Charset);
  }

  //
  // DataOutput methods.
  //

  /**
   * Writes a boolean to the file as a 1-byte value. The
   * value true is written out as the value
   * (byte)1; the value false is written out
   * as the value (byte)0.
   *
   * @param v a boolean value to be written.
   * @throws IOException if an I/O error occurs.
   */
  public final void writeBoolean(boolean v) throws IOException {
    write(v ? 1 : 0);
  }

  /**
   * Write an array of booleans
   *
   * @param pa    write from this array
   * @param start starting with this element in the array
   * @param n     write this number of elements
   * @throws IOException on read error
   */
  public final void writeBoolean(boolean[] pa, int start, int n) throws IOException {
    for (int i = 0; i < n; i++) {
      writeBoolean(pa[start + i]);
    }
  }

  /**
   * Writes a byte to the file as a 1-byte value.
   *
   * @param v a byte value to be written.
   * @throws IOException if an I/O error occurs.
   */
  public final void writeByte(int v) throws IOException {
    write(v);
  }

  /**
   * Writes a short to the file as two bytes, high byte first.
   *
   * @param v a short to be written.
   * @throws IOException if an I/O error occurs.
   */
  public final void writeShort(int v) throws IOException {
    write((v >>> 8) & 0xFF);
    write((v) & 0xFF);
  }

  /**
   * Write an array of shorts
   *
   * @param pa    write from this array
   * @param start starting with this element in the array
   * @param n     this number of elements
   * @throws IOException on read error
   */
  public final void writeShort(short[] pa, int start, int n) throws IOException {
    for (int i = 0; i < n; i++) {
      writeShort(pa[start + i]);
    }
  }

  /**
   * Writes a char to the file as a 2-byte value, high
   * byte first.
   *
   * @param v a char value to be written.
   * @throws IOException if an I/O error occurs.
   */
  public final void writeChar(int v) throws IOException {
    write((v >>> 8) & 0xFF);
    write((v) & 0xFF);
  }

  /**
   * Write an array of chars
   *
   * @param pa    write from this array
   * @param start starting with this element in the array
   * @param n     this number of elements
   * @throws IOException on read error
   */
  public final void writeChar(char[] pa, int start, int n) throws IOException {
    for (int i = 0; i < n; i++) {
      writeChar(pa[start + i]);
    }
  }

  /**
   * Writes an int to the file as four bytes, high byte first.
   *
   * @param v an int to be written.
   * @throws IOException if an I/O error occurs.
   */
  public final void writeInt(int v) throws IOException {
    write((v >>> 24) & 0xFF);
    write((v >>> 16) & 0xFF);
    write((v >>> 8) & 0xFF);
    write((v) & 0xFF);
  }

  /**
   * Write an array of ints
   *
   * @param pa    write from this array
   * @param start starting with this element in the array
   * @param n     write this number of elements
   * @throws IOException on read error
   */
  public final void writeInt(int[] pa, int start, int n) throws IOException {
    for (int i = 0; i < n; i++) {
      writeInt(pa[start + i]);
    }
  }

  /**
   * Writes a long to the file as eight bytes, high byte first.
   *
   * @param v a long to be written.
   * @throws IOException if an I/O error occurs.
   */
  public final void writeLong(long v) throws IOException {
    write((int) (v >>> 56) & 0xFF);
    write((int) (v >>> 48) & 0xFF);
    write((int) (v >>> 40) & 0xFF);
    write((int) (v >>> 32) & 0xFF);
    write((int) (v >>> 24) & 0xFF);
    write((int) (v >>> 16) & 0xFF);
    write((int) (v >>> 8) & 0xFF);
    write((int) (v) & 0xFF);
  }

  /**
   * Write an array of longs
   *
   * @param pa    write from this array
   * @param start starting with this element in the array
   * @param n     write this number of elements
   * @throws IOException on read error
   */
  public final void writeLong(long[] pa, int start, int n) throws IOException {
    for (int i = 0; i < n; i++) {
      writeLong(pa[start + i]);
    }
  }

  /**
   * Converts the float argument to an int using the
   * floatToIntBits method in class Float,
   * and then writes that int value to the file as a
   * 4-byte quantity, high byte first.
   *
   * @param v a float value to be written.
   * @throws IOException if an I/O error occurs.
   * @see java.lang.Float#floatToIntBits(float)
   */
  public final void writeFloat(float v) throws IOException {
    writeInt(Float.floatToIntBits(v));
  }

  /**
   * Write an array of floats
   *
   * @param pa    write from this array
   * @param start starting with this element in the array
   * @param n     write this number of elements
   * @throws IOException on read error
   */
  public final void writeFloat(float[] pa, int start, int n) throws IOException {
    for (int i = 0; i < n; i++) {
      writeFloat(pa[start + i]);
    }
  }

  /**
   * Converts the double argument to a long using the
   * doubleToLongBits method in class Double,
   * and then writes that long value to the file as an
   * 8-byte quantity, high byte first.
   *
   * @param v a double value to be written.
   * @throws IOException if an I/O error occurs.
   * @see java.lang.Double#doubleToLongBits(double)
   */
  public final void writeDouble(double v) throws IOException {
    writeLong(Double.doubleToLongBits(v));
  }

  /**
   * Write an array of doubles
   *
   * @param pa    write from this array
   * @param start starting with this element in the array
   * @param n     write this number of elements
   * @throws IOException on read error
   */
  public final void writeDouble(double[] pa, int start, int n) throws IOException {
    for (int i = 0; i < n; i++) {
      writeDouble(pa[start + i]);
    }
  }

  /**
   * Writes the string to the file as a sequence of bytes. Each
   * character in the string is written out, in sequence, by discarding
   * its high eight bits.
   *
   * @param s a string of bytes to be written.
   * @throws IOException if an I/O error occurs.
   */
  public final void writeBytes(String s) throws IOException {
    int len = s.length();
    for (int i = 0; i < len; i++) {
      write((byte) s.charAt(i));
    }
  }

  /**
   * Writes the character array to the file as a sequence of bytes. Each
   * character in the string is written out, in sequence, by discarding
   * its high eight bits.
   *
   * @param b   a character array of bytes to be written.
   * @param off the index of the first character to write.
   * @param len the number of characters to write.
   * @throws IOException if an I/O error occurs.
   */
  public final void writeBytes(char b[], int off, int len) throws IOException {
    for (int i = off; i < len; i++) {
      write((byte) b[i]);
    }
  }

  /**
   * Writes a string to the file as a sequence of characters. Each
   * character is written to the data output stream as if by the
   * writeChar method.
   *
   * @param s a String value to be written.
   * @throws IOException if an I/O error occurs.
   * @see java.io.RandomAccessFile#writeChar(int)
   */
  public final void writeChars(String s) throws IOException {
    int len = s.length();
    for (int i = 0; i < len; i++) {
      int v = s.charAt(i);
      write((v >>> 8) & 0xFF);
      write((v) & 0xFF);
    }
  }

  /**
   * Writes a string to the file using UTF-8 encoding in a
   * machine-independent manner.
   *

   * First, two bytes are written to the file as if by the
   * writeShort method giving the number of bytes to
   * follow. This value is the number of bytes actually written out,
   * not the length of the string. Following the length, each character
   * of the string is output, in sequence, using the UTF-8 encoding
   * for each character.
   *
   * @param str a string to be written.
   * @throws IOException if an I/O error occurs.
   */
  public final void writeUTF(String str) throws IOException {
    int strlen = str.length();
    int utflen = 0;

    for (int i = 0; i < strlen; i++) {
      int c = str.charAt(i);
      if ((c >= 0x0001) && (c <= 0x007F)) {
        utflen++;
      } else if (c > 0x07FF) {
        utflen += 3;
      } else {
        utflen += 2;
      }
    }
    if (utflen > 65535) {
      throw new UTFDataFormatException();
    }

    write((utflen >>> 8) & 0xFF);
    write((utflen) & 0xFF);
    for (int i = 0; i < strlen; i++) {
      int c = str.charAt(i);
      if ((c >= 0x0001) && (c <= 0x007F)) {
        write(c);
      } else if (c > 0x07FF) {
        write(0xE0 | ((c >> 12) & 0x0F));
        write(0x80 | ((c >> 6) & 0x3F));
        write(0x80 | ((c) & 0x3F));
      } else {
        write(0xC0 | ((c >> 6) & 0x1F));
        write(0x80 | ((c) & 0x3F));
      }
    }
  }

  /**
   * Create a string representation of this object.
   *
   * @return a string representation of the state of the object.
   */
  public String toString() {
    return location;
    /* return "fp=" + filePosition + ", bs=" + bufferStart + ", de="
            + dataEnd + ", ds=" + dataSize + ", bl=" + buffer.length
            + ", readonly=" + readonly + ", bm=" + bufferModified; */
  }

  /////////////////////////////////////////////////

  /**
   * Search forward from the current pos, looking for a match.
   *
   * @param match    the match to look for.
   * @param maxBytes maximum number of bytes to search. use -1 for all
   * @return true if found, file position will be at the start of the match.
   * @throws IOException on read error
   */
  public boolean searchForward(KMPMatch match, int maxBytes) throws IOException {
    long start = getFilePointer();
    long last = (maxBytes < 0) ? length() : Math.min(length(), start + maxBytes);
    long needToScan = last - start;

    // check what ever is now in the buffer
    int bytesAvailable = (int) (dataEnd - filePosition);
    if (bytesAvailable < 1) {
      seek(filePosition); // read a new buffer
      bytesAvailable = (int) (dataEnd - filePosition);
    }
    int bufStart = (int) (filePosition - bufferStart);
    int scanBytes = (int) Math.min(bytesAvailable, needToScan);
    int pos = match.indexOf(buffer, bufStart, scanBytes);
    if (pos >= 0) {
      seek(bufferStart + pos);
      return true;
    }

    int matchLen = match.getMatchLength();
    needToScan -= scanBytes - matchLen;

    while (needToScan > matchLen) {
      readBuffer(dataEnd - matchLen); // force new buffer

      scanBytes = (int) Math.min(buffer.length, needToScan);
      pos = match.indexOf(buffer, 0, scanBytes);
      if (pos > 0) {
        seek(bufferStart + pos);
        return true;
      }

      needToScan -= scanBytes - matchLen;
    }

    // failure
    seek(last);
    return false;
  }

}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

// $Id: Selector.java 48 2006-07-12 16:15:40Z caron $

package thredds.catalog.query;

import thredds.catalog.InvDocumentation;
import ucar.unidata.util.StringUtil2;

import java.util.*;

/**
 * Abstract class for DQC selector elements.
 *
 * @author john caron
 */

public abstract class Selector {
  protected List compound; // List<Selector> : only one is operable
  protected boolean isUsed; //

  protected String title, id, template;
  protected boolean required, multiple;
  protected InvDocumentation desc;

  protected Selector( ) { }

  /**
   * Construct from fields in XML catalog.
   * @param title : human displayable name
   * @param id : unique id
   * @param template : for the query string
   * @param required : true or false
   * @param multiple : true or false
   */
  protected Selector( String title, String id, String template, String required, String multiple )
  {
    this.title = title;
    this.id = id;
    this.template = template;
    this.required = (required == null) || !required.equals("false");
    this.multiple = (multiple != null) && multiple.equals("true");
  }

  public void setDescription( InvDocumentation desc) { this.desc = desc; }
  public InvDocumentation getDescription() { return desc; }

  public String getTitle() { return title; }
  public void setTitle(String title) { this.title = title; }

  public String getId() { return id; }
  public void setId( String id) { this.id = id; }

  public String getTemplate() { return template; }
  public void setTemplate (String template) { this.template = template; }

  public boolean isRequired() { return required; }
  public void setRequired( String required) {
     this.required = (required == null) || !required.equals("false");
  }

  public boolean isMultiple() { return multiple; }
  public void setMultiple( String multiple) {
    this.multiple = (multiple != null) && multiple.equals("true");
  }
  public String getSelectType() { return multiple ? "multiple" : "single"; }

  public void setCompoundSelectors(List compound) {
    this.compound = compound;
  }
  public boolean isUsed() { return isUsed; }
  public void setUsed (boolean isUsed) {
    this.isUsed = isUsed;
    if (isUsed && compound != null) {
      for (Object aCompound : compound) {
        Selector s = (Selector) aCompound;
        if (s != this)
          s.setUsed(false);
      }
    }
  }

  /////////////////////////////////////////////////////////////////////
  // forming the query string

  /**
   * Create the selector result string, and append.
   * @param sbuff append here
   * @param values list of selected values, each value is a pair (String, Object), where the
   *  String is name of the value, and the Object is the value itself. We use the toString()
   *  method on the object to get its String representation.
   */
  public void appendQuery( StringBuffer sbuff, ArrayList values) {
    if (template != null)
      appendQueryFromTemplate( sbuff, values);
    else
      appendQueryFromParamValue( sbuff, values);
  }

  private void appendQueryFromParamValue( StringBuffer sbuff, ArrayList choices) {
    for (int i=1; i<choices.size(); i+=2) {
      sbuff.append(getId());
      sbuff.append("=");
      sbuff.append(choices.get(i).toString());
      sbuff.append("&");
    }
  }

  private void appendQueryFromTemplate( StringBuffer sbuff, ArrayList choices) {
    StringBuilder templateBuff = new StringBuilder( template);

    for (int i=0; i<choices.size(); i+=2) {
      StringUtil2.substitute(templateBuff, choices.get(i).toString(), choices.get(i + 1).toString());
    }
    sbuff.append( templateBuff.toString());
  }

  /** Instances which have same id are equal.*/
  public boolean equals(Object o) {
     if (this == o) return true;
     if (!(o instanceof Selector)) return false;
     return o.hashCode() == this.hashCode();
  }

  /**
   * Override Object.hashCode() to be consistent with this equals.
   */
  public int hashCode() {
    if (null != getId())
      return getId().hashCode();
    return super.hashCode();
  }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package thredds.server.ncss.validation;

import thredds.server.ncss.params.NcssGridParamsBean;
import ucar.nc2.time.CalendarDate;

import javax.validation.ConstraintValidator;
import javax.validation.ConstraintValidatorContext;

/**
 * Validator for ncss GridRequest
 * 1) if has either latitude or longitude, has both
 * 2) if has any of latlon bb, has all. north > south, east > west
 * 3) if has any of projection bb, has all. min < max
 * <p>
 * Messages in WEB-INF/classes/ValidationMessages.properties
 *
 * @author caron
 * @since 10/9/13
 */
public class NcssGridRequestValidator implements ConstraintValidator<NcssGridRequestConstraint, NcssGridParamsBean> {

  /* (non-Javadoc)
   * @see javax.validation.ConstraintValidator#initialize(java.lang.annotation.Annotation)
   */
  @Override
  public void initialize(NcssGridRequestConstraint arg0) {
    // TODO Auto-generated method stub
  }

  /*
   * since none of these are required, can only do consistency checks
   * @see "http://www.unidata.ucar.edu/software/thredds/current/tds/reference/NetcdfSubsetServiceReference.html"
   */
  @Override
  public boolean isValid(NcssGridParamsBean params, ConstraintValidatorContext constraintValidatorContext) {

    constraintValidatorContext.disableDefaultConstraintViolation();
    boolean isValid = true;

    // lat/lon point
    if (params.getLatitude() != null || params.getLongitude() != null) {
      if (!params.hasLatLonPoint()) {
        isValid = false;
        constraintValidatorContext.buildConstraintViolationWithTemplate("{thredds.server.ncSubset.validation.lat_or_lon_missing}").addConstraintViolation();
      }
    }

    // lat/lon bb
    if (params.getNorth() != null || params.getSouth() != null || params.getEast() != null || params.getWest() != null) {
      if (!params.hasLatLonBB()) {
        isValid = false;
        constraintValidatorContext.buildConstraintViolationWithTemplate("{thredds.server.ncSubset.validation.wrong_bbox}").addConstraintViolation();
      }

      if (params.getNorth() < params.getSouth()) {
        isValid = false;
        constraintValidatorContext.buildConstraintViolationWithTemplate("{thredds.server.ncSubset.validation.north_south}").addConstraintViolation();
      }
      if (params.getEast() < params.getWest()) {
        isValid = false;
        constraintValidatorContext.buildConstraintViolationWithTemplate("{thredds.server.ncSubset.validation.east_west}").addConstraintViolation();
      }
    }

    // proj bb
    if (params.getMaxx() != null || params.getMinx() != null || params.getMaxy() != null || params.getMiny() != null) {
      if (!params.hasProjectionBB()) {
        isValid = false;
        constraintValidatorContext.buildConstraintViolationWithTemplate("{thredds.server.ncSubset.validation.wrong_pbox}").addConstraintViolation();
      }

      if (params.getMaxx() < params.getMinx()) {
        isValid = false;
        constraintValidatorContext.buildConstraintViolationWithTemplate("{thredds.server.ncSubset.validation.rangex}").addConstraintViolation();
      }
      if (params.getMaxy() < params.getMiny()) {
        isValid = false;
        constraintValidatorContext.buildConstraintViolationWithTemplate("{thredds.server.ncSubset.validation.rangey}").addConstraintViolation();
      }
    }

    // runtime: latest, all, or time
    if (params.getRuntime() != null) {
      if ("latest".equalsIgnoreCase(params.getRuntime())) {
        params.setLatestRuntime(true);

      } else if ("all".equalsIgnoreCase(params.getRuntime())) {
        params.setAllRuntime(true);

      } else {
        CalendarDate cd = TimeParamsValidator.validateISOString(params.getRuntime(), "{thredds.server.ncSubset.validation.param.runtime}", constraintValidatorContext);
        if (cd != null)
          params.setRuntimeDate(cd);
      }
    }

    // timeOffset: first or double
    if (params.getTimeOffset() != null) {
      if ("first".equalsIgnoreCase(params.getTimeOffset())) {
        params.setFirstTimeOffset(true);

      } else {
        try {
          double val = Double.parseDouble(params.getTimeOffset());
          params.setTimeOffsetVal(val);
        } catch (NumberFormatException e) {
          constraintValidatorContext.buildConstraintViolationWithTemplate("{thredds.server.ncSubset.validation.param.time_offset}").addConstraintViolation();
        }
      }
    }

    return isValid;
  }
}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package thredds.inventory.partition;

import thredds.featurecollection.FeatureCollectionConfig;
import thredds.inventory.CollectionUpdateType;
import thredds.inventory.MCollection;
import thredds.inventory.MFile;
import ucar.nc2.util.Indent;

import java.io.IOException;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.attribute.FileTime;
import java.util.ArrayList;
import java.util.Formatter;
import java.util.Iterator;
import java.util.List;

/**
 * A Builder of DirectoryPartitions and DirectoryCollections.
 * Each DirectoryBuilder is associated with one directory, and one ncx index.
 * This may contain collections of files (MFiles in a DirectoryCollection), or subdirectories (MCollections in a DirectoryPartition).
 *
 * @author caron
 * @since 11/10/13
 */
public class DirectoryBuilder {

  // returns a DirectoryPartition or DirectoryCollection
  static public MCollection factory(FeatureCollectionConfig config, Path topDir, boolean isTop, IndexReader indexReader, String suffix, org.slf4j.Logger logger) throws IOException {
    DirectoryBuilder builder = new DirectoryBuilder(config.collectionName, topDir.toString(), suffix);

    DirectoryPartition dpart = new DirectoryPartition(config, topDir, isTop, indexReader, suffix, logger);
    if (!builder.isLeaf(indexReader))  { // its a partition
      return dpart;
    }

    // its a collection
    boolean hasIndex = builder.findIndex();
    if (hasIndex) {
      return dpart.makeChildCollection(builder);
    } else {
      DirectoryCollection result = new DirectoryCollection(config.collectionName, topDir, isTop, config.olderThan, logger); // no index file
      return result;
    }
  }

  private enum PartitionStatus {unknown, isDirectoryPartition, isLeaf}

  //////////////////////////////////////////////////////////////////////////////////////////////

  private static final boolean debug = false;
  private final String suffix;
  private final String topCollectionName;  // collection name
  private final String partitionName;      // partition name
  private final Path dir;                  // the directory
  private final FileTime dirLastModified;  // directory last modified
  private Path index;                      // TimePartition index file (ncx2 with magic = TimePartition)
  private FileTime indexLastModified;      // index last modified
  private long indexSize;                  // index size

  private boolean childrenConstructed = false;
  private List<DirectoryBuilder> children = new ArrayList<>(25);
  private PartitionStatus partitionStatus = PartitionStatus.unknown;

  public DirectoryBuilder(String topCollectionName, String dirFilename, String suffix) throws IOException {
    this(topCollectionName, Paths.get(dirFilename), null, suffix);
  }

  /**
   * Create a DirectoryBuilder for the named directory
   * @param topCollectionName  from config, name of the collection
   * @param dir covers this directory
   * @param attr file attributes, may be null
   * @throws IOException
   */
  public DirectoryBuilder(String topCollectionName, Path dir, BasicFileAttributes attr, String suffix) throws IOException {
    this.topCollectionName = topCollectionName;
    this.dir = dir;
    this.partitionName = DirectoryCollection.makeCollectionName(topCollectionName, dir);
    this.suffix = suffix;

    if (attr == null)
      attr = Files.readAttributes(this.dir, BasicFileAttributes.class);
    if (!attr.isDirectory())
      throw new IllegalArgumentException("DirectoryPartitionBuilder needs a directory");
    dirLastModified = attr.lastModifiedTime();

    // see if we can find the index
    findIndex();
  }

  //public void setChildrenConstructed(boolean childrenConstructed) { this.childrenConstructed = childrenConstructed; }

  /**
   * Find the index file, using its canonical name
   * @return true if found
   * @throws IOException
   */
  public boolean findIndex() throws IOException {
    Path indexPath = Paths.get(dir.toString(), partitionName + suffix);
    if (Files.exists(indexPath)) {
      this.index = indexPath;
      BasicFileAttributes attr = Files.readAttributes(indexPath, BasicFileAttributes.class);
      this.indexLastModified = attr.lastModifiedTime();
      this.indexSize = attr.size();
      return true;
    }
    return false;
  }

  /**
   * Scans first 100 files to decide if its a leaf. If so, it becomes a DirectoryCollection, else a PartitionCollection.
   * @param indexReader reads the index
   * @return true if partition, false if file collection
   * @throws IOException on IO error
   */
  private boolean isLeaf(IndexReader indexReader) throws IOException {
    if (partitionStatus == PartitionStatus.unknown) {

        int countDir=0, countFile=0, count =0;
        try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(dir)) {
          Iterator<Path> iterator = dirStream.iterator();
          while (iterator.hasNext() && count++ < 100) {
            Path p = iterator.next();
            BasicFileAttributes attr = Files.readAttributes(p, BasicFileAttributes.class);
            if (attr.isDirectory()) countDir++;
            else countFile++;
          }
        }
      partitionStatus = (countFile > countDir) ? PartitionStatus.isLeaf : PartitionStatus.isDirectoryPartition;
    }

    return partitionStatus == PartitionStatus.isLeaf;
  }

  /**
   * Find all children directories. Does not recurse.
   * We separate this from the constructor so it can be done on demand
   * Public for debugging.
   *
   * Look for children by:
   *

   *
If index exists , use the children inside there./li>
   *
(or) scan the directory for children partitions

   *

   *
   * @param indexReader  this reads the index, and calls AddChild.addchild() for each child
   * @return children, may be empty but not null
   * @throws IOException
   */
  public List<DirectoryBuilder> constructChildren(IndexReader indexReader, CollectionUpdateType forceCollection) throws IOException {
    if (childrenConstructed) return children;

    if (index != null && forceCollection == CollectionUpdateType.nocheck) { // use index if it exists
      constructChildrenFromIndex(indexReader, false);

    } else {
      scanForChildren();
    }

    //once we have found children, we know that this is a time partition
    partitionStatus = (children.size() > 0) ?  PartitionStatus.isDirectoryPartition : PartitionStatus.isLeaf;
    childrenConstructed = true;  // otherwise we are good

    return children;
  }

  public List<DirectoryBuilder> constructChildrenFromIndex(IndexReader indexReader, boolean substituteParentDir) throws IOException {
    if (!indexReader.readChildren(index, new AddChildSub(substituteParentDir))) {
      partitionStatus =  PartitionStatus.isLeaf;
    }
    return children;
  }

  // add a child partition from the index file (callback from constructChildren)
  // we dont know at this point if its another partition or a gribCollection
  private class AddChild implements IndexReader.AddChildCallback {
    public void addChild(String dirName, String indexFilename, long lastModified) throws IOException {
      Path indexPath = Paths.get(indexFilename);
      DirectoryBuilder child = new DirectoryBuilder(topCollectionName, indexPath, lastModified, suffix);
      children.add(child);
    }
  }

  private class AddChildSub implements IndexReader.AddChildCallback {
    boolean substituteParentDir;
    AddChildSub(boolean substituteParentDir) {
      this.substituteParentDir = substituteParentDir;
    }
    public void addChild(String dirName, String indexFilename, long lastModified) throws IOException {
      Path indexPath = Paths.get(dirName, indexFilename);
      if (substituteParentDir) {
        Path parent = index.getParent();
        indexPath = parent.resolve( indexFilename);
      }
      DirectoryBuilder child = new DirectoryBuilder(topCollectionName, indexPath, lastModified, suffix);
      children.add(child);
    }
  }

  // coming in from the index reader
  private DirectoryBuilder(String topCollectionName, Path indexFile, long indexLastModified, String suffix) throws IOException {
    this.topCollectionName = topCollectionName;
    if (Files.exists(indexFile)) {
      this.index = indexFile;
      this.indexLastModified = FileTime.fromMillis(indexLastModified);
    }

    this.dir = indexFile.getParent();
    this.partitionName = DirectoryCollection.makeCollectionName(topCollectionName, dir);

    BasicFileAttributes attr = Files.readAttributes(this.dir, BasicFileAttributes.class);
    if (!attr.isDirectory())
      throw new IllegalArgumentException("DirectoryPartition needs a directory");
    dirLastModified = attr.lastModifiedTime();

    this.suffix = suffix;
  }

  /**
   * Scan for subdirectories, make each into a DirectoryBuilder and add as a child
   */
  private void scanForChildren() {
    if (debug) System.out.printf("DirectoryBuilder.scanForChildren on %s ", dir);

    int count = 0;
    try (DirectoryStream<Path> ds = Files.newDirectoryStream(dir)) {
      for (Path p : ds) {
        BasicFileAttributes attr = Files.readAttributes(p, BasicFileAttributes.class);
        if (attr.isDirectory()) {
          children.add(new DirectoryBuilder(topCollectionName, p, attr, suffix));
          if (debug && (++count % 10 == 0)) System.out.printf("%d ", count);
        }
      }
    } catch (IOException e) {
      e.printStackTrace();
    }
    if (debug) System.out.printf("done=%d%n", count);
    childrenConstructed = true;
  }

  //////////////////////////////////////////////////////////////////////////////////////
  // read the list of files from the index

  public List<MFile> readFilesFromIndex(IndexReader indexReader) throws IOException {
    List<MFile> result = new ArrayList<>(100);
    if (index == null) return result;

    indexReader.readMFiles(index, result);
    return result;
  }

  ////////////////////////////////////////////////////////

  /**
    * The directory that the partition covers
    * @return directory
    */
   public Path getDir() {
     return dir;
   }

   /**
    * The ncx2 file
    * @return  ncx2 file path
    */
   public Path getIndex() {
     return index;
   }

   /**
    * May be null if constructChildren() was not called
    * @return children directories
    */
   public List<DirectoryBuilder> getChildren() {
     return children;
   }

   public String getPartitionName() {
     return partitionName;
   }

  public void show(Formatter out) {
    out.format("Collection %s%n", partitionName);
    toString(out, new Indent(2));
    out.format("%n%n");
  }

  private void toString(Formatter out, Indent indent) {
    out.format("%sDir '%s' (%s) index '%s' (%s)%n", indent, dir, dirLastModified, index, indexLastModified);
    indent.incr();
    for (DirectoryBuilder c : children)
      c.toString(out, indent);
    indent.decr();
  }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.nc2.dt.grid;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.Formatter;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

import ucar.nc2.Attribute;
import ucar.nc2.NetcdfFile;
import ucar.nc2.Variable;
import ucar.nc2.VariableSimpleIF;
import ucar.nc2.constants.CDM;
import ucar.nc2.constants.FeatureType;
import ucar.nc2.dataset.*;
import ucar.nc2.dataset.NetcdfDataset.Enhance;
import ucar.nc2.dt.GridCoordSystem;
import ucar.nc2.dt.GridDatatype;
import ucar.nc2.ft.FeatureDataset;
import ucar.nc2.time.CalendarDate;
import ucar.nc2.time.CalendarDateRange;
import ucar.nc2.units.DateRange;
import ucar.nc2.util.cache.FileCacheIF;
import ucar.unidata.geoloc.LatLonRect;
import ucar.unidata.geoloc.ProjectionRect;

/**
 * Make a NetcdfDataset into a collection of GeoGrids with Georeferencing coordinate systems.
 *

 *

 * A variable will be made into a GeoGrid if it has a Georeferencing coordinate system,
 * using GridCoordSys.isGridCoordSys(), and it has no extra dimensions, ie
 * GridCoordSys.isComplete( var) is true.
 * If it has multiple Georeferencing coordinate systems, any one that is a product set will be given preference.
 *

 * Example:
 *

 *

 * GridDataset gridDs = GridDataset.open (uriString);
 * List grids = gridDs.getGrids();
 * for (int i=0; i<grids.size(); i++) {
 *   GeoGrid grid = (Geogrid) grids.get(i);
 * }
 *

 *
 * @author caron
 */

public class GridDataset implements ucar.nc2.dt.GridDataset, FeatureDataset {
  private NetcdfDataset ncd;
  private ArrayList<GeoGrid> grids = new ArrayList<>();
  private Map<String, Gridset> gridsetHash = new HashMap<>();

  /**
   * Open a netcdf dataset, using NetcdfDataset.defaultEnhanceMode plus CoordSystems
   * and turn into a GridDataset.
   *
   * @param location netcdf dataset to open, using NetcdfDataset.acquireDataset().
   * @return GridDataset
   * @throws java.io.IOException on read error
   * @see ucar.nc2.dataset.NetcdfDataset#acquireDataset
   */
  static public GridDataset open(String location) throws java.io.IOException {
    return open(location, NetcdfDataset.getDefaultEnhanceMode());
  }

  /**
   * Open a netcdf dataset, using NetcdfDataset.defaultEnhanceMode plus CoordSystems
   * and turn into a GridDataset.
   *
   * @param location netcdf dataset to open, using NetcdfDataset.acquireDataset().
   * @param enhanceMode open netcdf dataset with this enhanceMode
   * @return GridDataset
   * @throws java.io.IOException on read error
   * @see ucar.nc2.dataset.NetcdfDataset#acquireDataset
   */
  static public GridDataset open(String location, Set<NetcdfDataset.Enhance> enhanceMode) throws java.io.IOException {
    NetcdfDataset ds = ucar.nc2.dataset.NetcdfDataset.acquireDataset(null, DatasetUrl.findDatasetUrl(location), enhanceMode, -1, null, null);
    return new GridDataset(ds, null);
  }

  /**
   * Create a GridDataset from a NetcdfDataset.
   *
   * @param ncd underlying NetcdfDataset, will do Enhance.CoordSystems if not already done.
   * @throws java.io.IOException on read error
   */
  public GridDataset(NetcdfDataset ncd) throws IOException {
    this(ncd, null);
  }

  /**
   * Create a GridDataset from a NetcdfDataset.
   *
   * @param ncd underlying NetcdfDataset, will do Enhance.CoordSystems if not already done.
   * @param parseInfo put parse info here, may be null
   * @throws java.io.IOException on read error
   */
  public GridDataset(NetcdfDataset ncd, Formatter parseInfo) throws IOException {
    this.ncd = ncd;
    // ds.enhance(EnumSet.of(NetcdfDataset.Enhance.CoordSystems));
    Set<Enhance> enhance = ncd.getEnhanceMode();
    if(enhance == null || enhance.isEmpty()) enhance = NetcdfDataset.getDefaultEnhanceMode();
    ncd.enhance(enhance);

    // look for geoGrids
    if (parseInfo != null) parseInfo.format("GridDataset look for GeoGrids%n");
    List<Variable> vars = ncd.getVariables();
    for (Variable var : vars) {
      VariableEnhanced varDS = (VariableEnhanced) var;
      constructCoordinateSystems(ncd, varDS, parseInfo);
    }
  }

  private void constructCoordinateSystems(NetcdfDataset ds, VariableEnhanced v, Formatter parseInfo) {

    if (v instanceof StructureDS) {
      StructureDS s = (StructureDS) v;
      List<Variable> members = s.getVariables();
      for (Variable nested : members) {
        // LOOK flatten here ??
        constructCoordinateSystems(ds, (VariableEnhanced) nested, parseInfo);
      }
    } else {

      // see if it has a GridCS
      // LOOK: should add geogrid it multiple times if there are multiple geoCS ??
      GridCoordSys gcs = null;
      List<CoordinateSystem> csys = v.getCoordinateSystems();
      for (CoordinateSystem cs : csys) {
        GridCoordSys gcsTry = GridCoordSys.makeGridCoordSys(parseInfo, cs, v);
        if (gcsTry != null) {
          gcs = gcsTry;
          if (gcsTry.isProductSet()) break;
        }
      }

      if (gcs != null)
        addGeoGrid((VariableDS) v, gcs, parseInfo);
    }

  }

  private LatLonRect llbbMax = null;
  private CalendarDateRange dateRangeMax = null;
  private ProjectionRect projBB = null;

  private void makeRanges() {

    for (ucar.nc2.dt.GridDataset.Gridset gset : getGridsets()) {
      GridCoordSystem gcs = gset.getGeoCoordSystem();

      ProjectionRect bb = gcs.getBoundingBox();
      if (projBB == null)
        projBB = bb;
      else
        projBB.add(bb);

      LatLonRect llbb = gcs.getLatLonBoundingBox();
      if (llbbMax == null)
        llbbMax = llbb;
      else
        llbbMax.extend(llbb);

      CalendarDateRange dateRange = gcs.getCalendarDateRange();
      if (dateRange != null) {
        if (dateRangeMax == null)
          dateRangeMax = dateRange;
        else
          dateRangeMax = dateRangeMax.extend(dateRange);
      }
    }
  }

  // stuff to satisfy ucar.nc2.dt.TypedDataset
  public String getTitle() {
    String title = ncd.getTitle();
    if (title == null)
      title = ncd.findAttValueIgnoreCase(null, CDM.TITLE, null);
    if (title == null)
      title = getName();
    return title;
  }

  public String getDescription() {
    String desc = ncd.findAttValueIgnoreCase(null, "description", null);
    if (desc == null)
      desc = ncd.findAttValueIgnoreCase(null, CDM.HISTORY, null);
    return (desc == null) ? getName() : desc;
  }

  public String getLocation() {
    return (ncd != null) ?  ncd.getLocation() : "";
  }

  /**
   * @deprecated use getCalendarDateRange
   */
  public DateRange getDateRange() {
    CalendarDateRange cdr = getCalendarDateRange();
    return (cdr != null) ? cdr.toDateRange() : null;
  }

  /**
   * @deprecated use getStartCalendarDate
   */
  public Date getStartDate() {
    DateRange dr = getDateRange();
    return (dr != null) ? dr.getStart().getDate() : null;
  }

  /**
   * @deprecated use getEndCalendarDate
   */
  public Date getEndDate() {
    DateRange dr = getDateRange();
    return (dr != null) ? dr.getEnd().getDate() : null;
  }

  public CalendarDateRange getCalendarDateRange() {
    if (dateRangeMax == null) makeRanges();
    return dateRangeMax;
  }

  public CalendarDate getCalendarDateStart() {
    if (dateRangeMax == null) makeRanges();
    return (dateRangeMax == null) ? null : dateRangeMax.getStart();
  }

  public CalendarDate getCalendarDateEnd() {
    if (dateRangeMax == null) makeRanges();
    return (dateRangeMax == null) ? null : dateRangeMax.getEnd();
  }

  public LatLonRect getBoundingBox() {
    if (llbbMax == null) makeRanges();
    return llbbMax;
  }

  public ProjectionRect getProjBoundingBox() {
    if (llbbMax == null) makeRanges();
    return projBB;
  }

  public void calcBounds() throws java.io.IOException {
    // not needed
  }

  public List<Attribute> getGlobalAttributes() {
    return ncd.getGlobalAttributes();
  }

  public Attribute findGlobalAttributeIgnoreCase(String name) {
    return ncd.findGlobalAttributeIgnoreCase(name);
  }

  public List<VariableSimpleIF> getDataVariables() {
    List<VariableSimpleIF> result = new ArrayList<>( grids.size());
    for (GridDatatype grid : getGrids()) {
      if (grid.getVariable() != null) // LOOK could make Adaptor if no variable
        result.add( grid.getVariable());
    }
    return result;
  }

  public VariableSimpleIF getDataVariable(String shortName) {
    return ncd.getRootGroup().findVariable(shortName);
  }

  public NetcdfFile getNetcdfFile() {
    return ncd;
  }

  private void addGeoGrid(VariableDS varDS, GridCoordSys gcs, Formatter parseInfo) {
    Gridset gridset;
    if (null == (gridset = gridsetHash.get(gcs.getName()))) {
      gridset = new Gridset(gcs);
      gridsetHash.put(gcs.getName(), gridset);
      if (parseInfo != null) parseInfo.format(" -make new GridCoordSys= %s%n",gcs.getName());
      gcs.makeVerticalTransform(this, parseInfo); // delayed until now LOOK why for each grid ??
    }

    GeoGrid geogrid = new GeoGrid(this, varDS, gridset.gcc);
    grids.add(geogrid);
    gridset.add(geogrid);
  }

  /**
   * the name of the dataset is the last part of the location
   * @return the name of the dataset
   */
  public String getName() {
    String loc = ncd.getLocation();
    int pos = loc.lastIndexOf('/');
    if (pos < 0)
      pos = loc.lastIndexOf('\\');
    return (pos < 0) ? loc : loc.substring(pos+1);
  }

  /**
   * @return the underlying NetcdfDataset
   */
  public NetcdfDataset getNetcdfDataset() {
    return ncd;
  }

  /**
   * @return the list of GeoGrid objects contained in this dataset.
   */
  public List<GridDatatype> getGrids() {
    return new ArrayList<GridDatatype>(grids);
  }

  public GridDatatype findGridDatatype(String name) {
    return findGridByName(name);
  }

  /**
   * Return GridDatatype objects grouped by GridCoordSys. All GridDatatype in a Gridset
   * have the same GridCoordSystem.
   *
   * @return List of type ucar.nc2.dt.GridDataset.Gridset
   */
  public List<ucar.nc2.dt.GridDataset.Gridset> getGridsets() {
    return new ArrayList<ucar.nc2.dt.GridDataset.Gridset>(gridsetHash.values());
  }

  /**
   * find the named GeoGrid.
   *
   * @param fullName find this GeoGrid by full name
   * @return the named GeoGrid, or null if not found
   */
  public GeoGrid findGridByName(String fullName) {
    for (GeoGrid ggi : grids) {
      if (fullName.equals(ggi.getFullName()))
        return ggi;
    }
    return null;
  }

  /**
   * find the named GeoGrid.
   *
   * @param shortName find this GeoGrid by short name
   * @return the named GeoGrid, or null if not found
   */
  public GeoGrid findGridByShortName(String shortName) {
    for (GeoGrid ggi : grids) {
      if (shortName.equals(ggi.getShortName()))
        return ggi;
    }
    return null;
  }

  public GeoGrid findGridDatatypeByAttribute(String attName, String attValue) {
    for (GeoGrid ggi : grids) {
      for (Attribute att : ggi.getAttributes())
        if (attName.equals(att.getShortName()) && attValue.equals(att.getStringValue()))
          return ggi;
    }
    return null;
  }

  /**
   * Get Details about the dataset.
   */
  public String getDetailInfo() {
    Formatter buff = new Formatter();
    getDetailInfo(buff);
    return buff.toString();
  }

  public void getDetailInfo(Formatter buff) {
    getInfo(buff);
    buff.format("%n%n----------------------------------------------------%n");
    try (NetcdfDatasetInfo info = new NetcdfDatasetInfo(ncd)) {
      buff.format("%s%n", info.writeXML());
      buff.format("------------------------------------------");
      buff.format("%s", info.getParseInfo());
    } catch (IOException e) {
      buff.format("NetcdfDatasetInfo failed");
    }
    buff.format("%n%n----------------------------------------------------%n");
    buff.format("%s", ncd.toString());
    buff.format("%n%n----------------------------------------------------%n");
  }

  /**
   * Show Grids and coordinate systems.
   * @param buf put info here
   */
  private void getInfo(Formatter buf) {
    int countGridset = 0;

    for (Gridset gs : gridsetHash.values()) {
      GridCoordSystem gcs = gs.getGeoCoordSystem();
      buf.format("%nGridset %d  coordSys=%s", countGridset,  gcs);
      buf.format(" LLbb=%s ", gcs.getLatLonBoundingBox());
      if ((gcs.getProjection() != null)  && !gcs.getProjection().isLatLon())
        buf.format(" bb= %s", gcs.getBoundingBox());
      buf.format("%n");
      buf.format("Name__________________________Unit__________________________hasMissing_Description%n");
      for (GridDatatype grid : gs.getGrids()) {
        buf.format("%s%n", grid.getInfo());
      }
      countGridset++;
      buf.format("%n");
    }

    buf.format("%nGeoReferencing Coordinate Axes%n");
    buf.format("Name__________________________Units_______________Type______Description%n");
    for (CoordinateAxis axis : ncd.getCoordinateAxes()) {
      if (axis.getAxisType() == null) continue;
      axis.getInfo(buf);
      buf.format("%n");
    }
  }

  /**
   * This is a set of GeoGrids with the same GeoCoordSys.
   */
  public static class Gridset implements ucar.nc2.dt.GridDataset.Gridset {

    private GridCoordSys gcc;
    private List<GridDatatype> grids = new ArrayList<>();

    private Gridset(GridCoordSys gcc) {
      this.gcc = gcc;
    }

    private void add(GeoGrid grid) {
      grids.add(grid);
    }

    /**
     * Get list of GeoGrid objects
     */
    public List<GridDatatype> getGrids() {
      return grids;
    }

    /**
     * all GridDatatype point to this GridCoordSystem
     */
    public GridCoordSystem getGeoCoordSystem() {
      return gcc;
    }

    /**
     * all GeoGrids point to this GeoCoordSysImpl.
     *
     * @deprecated use getGeoCoordSystem() if possible.
     */
    public GridCoordSys getGeoCoordSys() {
      return gcc;
    }

  }

  ////////////////////////////
  // for ucar.nc2.ft.FeatureDataset

  public FeatureType getFeatureType() {
    return FeatureType.GRID;
  }

  public String getImplementationName() {
    return ncd.getConventionUsed();
  }

  //////////////////////////////////////////////////
  //  FileCacheable

  @Override
  public synchronized void close() throws java.io.IOException {
    if (fileCache != null) {
      if (fileCache.release(this)) return;
    }

    try {
      if (ncd != null) ncd.close();
    } finally {
      ncd = null;
      }
  }

        // release any resources like file handles
  public void release() throws IOException {
    if (ncd != null) ncd.release();
  }

  // reacquire any resources like file handles
  public void reacquire() throws IOException {
    if (ncd != null) ncd.reacquire();
  }

  @Override
  public long getLastModified() {
    return (ncd != null) ? ncd.getLastModified() : 0;
  }

  protected FileCacheIF fileCache;

  @Override
  public synchronized void setFileCache(FileCacheIF fileCache) {
    this.fileCache = fileCache;
  }

  /////////////////////////////
  // deprecated

  /**
   * Open a netcdf dataset, parse Conventions, find all the geoGrids, return a GridDataset.
   *
   * @deprecated : use GridDataset.open().
   */
  static public GridDataset factory(String netcdfFileURI) throws java.io.IOException {
    return open(netcdfFileURI);
  }

}