/*
 * Copyright (c) 1998-2017 University Corporation for Atmospheric Research/Unidata
 * See LICENSE.txt for license information.
 */

package thredds.inventory;

import thredds.featurecollection.FeatureCollectionConfig;
import thredds.filesystem.MFileOS;
import thredds.inventory.filter.*;
import ucar.nc2.time.CalendarDate;
import ucar.nc2.units.TimeDuration;

import javax.annotation.concurrent.GuardedBy;
import javax.annotation.concurrent.ThreadSafe;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.atomic.AtomicLong;

/**
 * Manage Collections of MFiles.
 * Used in:
 *

 *
 thredds.inventory
 *
 ucar.nc2.ft.fmrc.Fmrc
 *
 ucar.nc2.ncml.Aggregation
 *

 *

 * we need to be thread safe, for InvDatasetFeatureCollection
 * @author caron
 * @since Jul 8, 2009
 */
@ThreadSafe
public class MFileCollectionManager extends CollectionManagerAbstract {
  static private MController controller;

  /**
   * Set the MController used by scan. Defaults to thredds.filesystem.ControllerOS() if not set.
   *
   * @param _controller use this MController
   */
  static public void setController(MController _controller) {
    controller = _controller;
  }

  static public MController getController() {
    if (null == controller) controller = new thredds.filesystem.ControllerOS();  // default
    return controller;
  }

  // called from Aggregation, Fmrc, FeatureDatasetFactoryManager
  static public MFileCollectionManager open(String collectionName, String collectionSpec, String olderThan, Formatter errlog) throws IOException {
    return new MFileCollectionManager(collectionName, collectionSpec, olderThan, errlog);
  }

  // retrofit to Aggregation
  static public MFileCollectionManager openWithRecheck(String collectionName, String recheckS) {
    return new MFileCollectionManager(collectionName, recheckS);
  }

  ////////////////////////////////////////////////////////////////////

  // these are final
  private final List<CollectionConfig> scanList = new ArrayList<>(); // an MCollection is a collection of managed files
  private final long olderThanInMsecs;  // LOOK why not use LastModifiedLimit filter ?
  //protected String rootDir;
  protected FeatureCollectionConfig config;

  @GuardedBy("this")
  private Map<String, MFile> map; // current map of MFile in the collection. this can change = keep under lock

  @GuardedBy("this")
  private long lastScanned;       // last time scanned
  @GuardedBy("this")
  private AtomicLong lastChanged = new AtomicLong(); // last time the set of files changed

  @GuardedBy("this")
  private Map<String, String> filesRunDateMap = new HashMap<>();

  private MFileCollectionManager(String collectionName, String collectionSpec, String olderThan, Formatter errlog) {
    super(collectionName, null);
    CollectionSpecParser sp = new CollectionSpecParser(collectionSpec, errlog);
    this.recheck = null;
    this.protoChoice = FeatureCollectionConfig.ProtoChoice.Penultimate; // default
    this.root = sp.getRootDir();

    CompositeMFileFilter filters = new CompositeMFileFilter();
    if (null != sp.getFilter())
      filters.addIncludeFilter(new WildcardMatchOnName(sp.getFilter()));
    olderThanInMsecs = parseOlderThanFilter(olderThan);

    dateExtractor = (sp.getDateFormatMark() == null) ? new DateExtractorNone() : new DateExtractorFromName(sp.getDateFormatMark(), true);
    scanList.add(new CollectionConfig(sp.getRootDir(), sp.getRootDir(), sp.wantSubdirs(), filters, null));
  }

  // this is the full featured constructor, using FeatureCollectionConfig for config.
  public MFileCollectionManager(FeatureCollectionConfig config, Formatter errlog, org.slf4j.Logger logger) {
    super(config.collectionName != null ? config.collectionName : config.spec, logger);
    this.config = config;

    CollectionSpecParser sp = config.getCollectionSpecParser(errlog);
    this.root = sp.getRootDir();

    CompositeMFileFilter filters = new CompositeMFileFilter();
    if (null != sp.getFilter())
      filters.addIncludeFilter(new WildcardMatchOnName(sp.getFilter()));
    olderThanInMsecs = parseOlderThanFilter(config.olderThan);

    if (config.dateFormatMark != null)
      dateExtractor = new DateExtractorFromName(config.dateFormatMark, false);
    else if (sp.getDateFormatMark() != null)
      dateExtractor = new DateExtractorFromName(sp.getDateFormatMark(), true);
    else
      dateExtractor = new DateExtractorNone();

    scanList.add(new CollectionConfig(sp.getRootDir(), sp.getRootDir(), sp.wantSubdirs(), filters, null));

    if (config.protoConfig != null)
      protoChoice = config.protoConfig.choice;

    if (config.updateConfig != null) {
      this.recheck = makeRecheck(config.updateConfig.recheckAfter);

      // static means never rescan on checkState; let it be externally triggered.
      if ((config.updateConfig.recheckAfter == null) && (config.updateConfig.rescan == null) && (config.updateConfig.deleteAfter == null))
        setStatic(true);
    }

    if (this.auxInfo == null) this.auxInfo = new HashMap<>(10);
    this.auxInfo.put(FeatureCollectionConfig.AUX_CONFIG, config);
  }

  private long parseOlderThanFilter(String olderThan) {
    if (olderThan != null) {
      try {
        TimeDuration tu = new TimeDuration(olderThan);
        return (long) (1000 * tu.getValueInSeconds());
      } catch (Exception e) {
        logger.error(collectionName + ": Invalid time unit for olderThan = {}", olderThan);
      }
    }
    return -1;
  }

  private TimeDuration makeRecheck(String recheckS) {
    if (recheckS != null) {
      try {
        return new TimeDuration(recheckS);
      } catch (Exception e) {
        logger.error(collectionName + ": Invalid time unit for recheckEvery = {}", recheckS);
      }
    }
    return null;
  }

  // for subclasses
  protected MFileCollectionManager(String name, org.slf4j.Logger logger) {
    super(name, logger);
    this.recheck = null;
    this.olderThanInMsecs = -1;
    this.protoChoice = FeatureCollectionConfig.ProtoChoice.Penultimate; // default
  }

  ////////////////////////////////////////////////////////////////////////////////

  public MFileCollectionManager(String name, String spec, Formatter errlog, org.slf4j.Logger logger) {
    super(name, logger);
    CollectionSpecParser sp = new CollectionSpecParser(spec, errlog);
    this.root = sp.getRootDir();

    CompositeMFileFilter filters = new CompositeMFileFilter();
    if (null != sp.getFilter())
      filters.addIncludeFilter(new WildcardMatchOnName(sp.getFilter()));

    dateExtractor = (sp.getDateFormatMark() == null) ? new DateExtractorNone() : new DateExtractorFromName(sp.getDateFormatMark(), true);
    scanList.add(new CollectionConfig(sp.getRootDir(), sp.getRootDir(), sp.wantSubdirs(), filters, null));

    this.recheck = null;
    this.protoChoice = FeatureCollectionConfig.ProtoChoice.Penultimate; // default
    this.olderThanInMsecs = -1;
  }

  public MFileCollectionManager(String name, CollectionConfig mc, CalendarDate startPartition, org.slf4j.Logger logger) {
    super(name, logger);
    this.startCollection = startPartition;
    this.scanList.add(mc);

    this.root = mc.getDirectoryName();
    this.recheck = null;
    this.protoChoice = FeatureCollectionConfig.ProtoChoice.Penultimate; // default
    this.olderThanInMsecs = -1;
  }

  ////////////////////////////////////////////////////////////////////
  // Aggregation retrofit

  /**
   * For retrofitting to Aggregation
   * Must also call addDirectoryScan one or more times
   *
   * @param recheckS a undunit time unit, specifying how often to rscan
   */
  private MFileCollectionManager(String collectionName, String recheckS) {
    super(collectionName, null);
    this.recheck = makeRecheck(recheckS);
    this.olderThanInMsecs = -1;
    this.protoChoice = FeatureCollectionConfig.ProtoChoice.Penultimate;
  }

  /**
   * Add a directory scan to the collection
   *
   * @param dirName             scan this directory
   * @param suffix              require this suffix (overriddden by regexp), may be null
   * @param regexpPatternString if present, use this reqular expression to filter files , may be null
   * @param subdirsS            if "true", descend into subdirectories, may be null
   * @param olderS              udunit time unit - files must be older than this amount of time (now - lastModified > olderTime), may be null
   *                            // * @param dateFormatString dateFormatMark string, may be null
   * @param auxInfo             attach this object to any MFile found by this scan
   */
  public void addDirectoryScan(String dirName, String suffix, String regexpPatternString, String subdirsS, String olderS, Object auxInfo) {
    CompositeMFileFilter filters = new CompositeMFileFilter();
    if (null != regexpPatternString)
      filters.addIncludeFilter(new RegExpMatchOnName(regexpPatternString));
    else if (suffix != null)
      filters.addIncludeFilter(new WildcardMatchOnPath("*" + suffix + "$"));

    if (olderS != null) {
      try {
        TimeDuration tu = new TimeDuration(olderS);
        filters.addAndFilter(new LastModifiedLimit((long) (1000 * tu.getValueInSeconds())));
      } catch (Exception e) {
        logger.error(collectionName + ": Invalid time unit for olderThan = {}", olderS);
      }
    }

    boolean wantSubdirs = true;
    if ((subdirsS != null) && subdirsS.equalsIgnoreCase("false"))
      wantSubdirs = false;

    CollectionConfig mc = new CollectionConfig(dirName, dirName, wantSubdirs, filters, auxInfo);

    // create name
    StringBuilder sb = new StringBuilder(dirName);
    if (wantSubdirs)
      sb.append("**/");
    if (null != regexpPatternString)
      sb.append(regexpPatternString);
    else if (suffix != null)
      sb.append(suffix);
    else
      sb.append("noFilter");

    collectionName = sb.toString();
    scanList.add(mc);
  }

  ////////////////////////////////////////////////////////////////////

  public long getOlderThanFilterInMSecs() {
    return olderThanInMsecs;
  }

  @Override
  public synchronized long getLastScanned() {
    return lastScanned;
  }

  @Override
  public synchronized long getLastChanged() {
    return lastChanged.get();
  }

  protected boolean hasScans() {
    return !scanList.isEmpty();
  }

  /**
   * Compute if synchronous scan is needed.
   * True if recheck is true and enough time has elapsed.
   * @return true if rescan is needed
   */
  @Override
  public boolean isScanNeeded() {
    // see if we need to recheck
    if (recheck == null) {
      logger.debug("{}: scan not needed, recheck null", collectionName);
      return false;
    }

    if (!hasScans()) {
      logger.debug("{}: scan not needed, no scanners", collectionName);
      return false;
    }

    synchronized (this) {
      if (map == null && !isStatic()) {
        logger.debug("{}: scan needed, never scanned", collectionName);
        return true;
      }
    }

    Date now = new Date();
    Date lastCheckedDate = new Date(getLastScanned());
    Date need = recheck.add(lastCheckedDate);
    if (now.before(need)) {
      logger.debug("{}: scan not needed, last scanned={}, now={}", collectionName, lastCheckedDate, now);
      return false;
    }

    return true;
  }

    ////////////////////////
  // experimental
  protected ChangeChecker changeChecker = null;

  public synchronized void setChangeChecker(ChangeChecker strat) {
    this.changeChecker = strat;
  }

  @Override
  public synchronized boolean scan(boolean sendEvent) throws IOException {
    if (map == null) {
      boolean changed = scanFirstTime();
      if (changed && sendEvent)
        sendEvent(new TriggerEvent(this, CollectionUpdateType.always));  // watch out for infinite loop
      return changed;
    }

    long olderThan = (olderThanInMsecs <= 0) ? -1 : System.currentTimeMillis() - olderThanInMsecs; // new files must be older than this.

    // rescan
    Map<String, MFile> oldMap = map;
    Map<String, MFile> newMap = new HashMap<>();

    if ((!hasScans())) {
      // if no directory scans, the map of files should not change
      // but we should still make a new map to see if the files
      // have been updated since the last recheck
      for (String file : oldMap.keySet()) {
        newMap.put(file, MFileOS.getExistingFile(file));
      }
    } else {
      // we have a directory scan, so scan it
      reallyScan(newMap);
    }

    // replace with previous datasets if they exist
    int nnew = 0;
    int nchange = 0;
    Iterator<MFile> iter = newMap.values().iterator(); // need iterator so we can remove()
    while (iter.hasNext()) {
      MFile newFile = iter.next();
      String path = newFile.getPath();
      MFile oldFile = oldMap.get(path);
      if (oldFile != null) {
        if (newFile.getLastModified() > oldFile.getLastModified()) { // the file has changed since last time
          nchange++;
          logger.debug("{}: scan found Dataset changed= {}", collectionName, path);

        } else if (changeChecker != null && changeChecker.hasntChangedSince(newFile, oldFile.getLastModified())) { // the ancilliary file hasnt changed
          nchange++;
          logger.debug("{}: scan changeChecker found Dataset changed= {}", collectionName, path);
        }
      } else { // oldFile doesnt exist
        if (olderThan > 0 && newFile.getLastModified() > olderThan) { // the file is too new
          iter.remove();
          logger.debug("{}: scan found new Dataset but its too recently modified = {}", collectionName, path);
        } else {
          nnew++;
          logger.debug("{}: scan found new Dataset= {} ", collectionName, path);
        }
      }
    }

    // check for deletions
    int ndelete = 0;
    for (MFile oldDataset : oldMap.values()) {
      String path = oldDataset.getPath();
      MFile newDataset = newMap.get(path);
      if (newDataset == null) {
        ndelete++;
        logger.debug("{}: scan found deleted Dataset={}", collectionName, path);
      }
    }

    boolean changed = (nnew > 0) || (ndelete > 0) || (nchange > 0);
    if (changed) {
      if (logger.isInfoEnabled())
        logger.info("{}: scan found changes {}: nnew={}, nchange={}, ndelete={}", collectionName, new Date(), nnew, nchange, ndelete);

        map = newMap;
        this.lastScanned = System.currentTimeMillis();
        this.lastChanged.set(this.lastScanned);
    } else {
        this.lastScanned = System.currentTimeMillis();
    }

    if (changed && sendEvent) {  // event is processed on this thread
      sendEvent(new TriggerEvent(this, CollectionUpdateType.always));  // watch out for infinite loop
    }

    return changed;
  }

  public void setFiles(Iterable<MFile> files) {
    Map<String, MFile> newMap = new HashMap<>();
    for (MFile file : files)
      newMap.put(file.getPath(), file);

    synchronized (this) {
      map = newMap;
      this.lastScanned = System.currentTimeMillis();
      this.lastChanged.set(this.lastScanned);
    }
  }

  public void setFilesAndRunDate(Map<String, String> filesRunDateMap) {
    // a simple mapping between file names and coordValue dates for
    // aggregations in which files are explicitly defined.
    this.filesRunDateMap = filesRunDateMap;
    // update MFileCollection map of files
    List<MFile> files = new ArrayList<>(filesRunDateMap.size());
    for (String file : filesRunDateMap.keySet()) {
      files.add(MFileOS.getExistingFile(file));
    }
    setFiles(files);
  }

  @Override
  public synchronized Iterable<MFile> getFilesSorted() {
    if (map == null)
      try {
        scanFirstTime(); // never scanned
      } catch (IOException e) {
        e.printStackTrace();
        return Collections.emptyList();
      }

    List<MFile> result = new ArrayList<>(map.values());
    if (hasDateExtractor()) {
      Collections.sort(result, new DateSorter());
    } else {
      Collections.sort(result);
    }

    return result;
  }

  @Override
  public boolean hasDateExtractor() {
    return (dateExtractor != null) && !(dateExtractor instanceof DateExtractorNone);
  }

  public synchronized Map<String, String> getFilesRunDateMap() {
    return filesRunDateMap;
  }

  // only called from synch methods
  private boolean scanFirstTime() throws IOException {
    Map<String, MFile> newMap = new HashMap<>();
    if (!hasScans()) {
        map = newMap;
        return false;
    }

    reallyScan(newMap);
    // deleteOld(newMap); // ?? hmmmmm LOOK this seems wrong; maintainence in background ?? generally collection doesnt exist

    // implement olderThan
    if (olderThanInMsecs > 0) {
      long olderThan = System.currentTimeMillis() - olderThanInMsecs; // new files must be older than this.
      Iterator<MFile> iter = newMap.values().iterator(); // need iterator so we can remove()
      while (iter.hasNext()) {
        MFile newFile = iter.next();
        String path = newFile.getPath();
        if (newFile.getLastModified() > olderThan) { // the file is too new
          iter.remove();
          logger.debug("{}: scan found new Dataset but its too recently modified = {}", collectionName, path);
        }
      }
    }

    map = newMap;
    this.lastScanned = System.currentTimeMillis();
    this.lastChanged.set(this.lastScanned);
    logger.debug("{} : initial scan found n datasets = {} ", collectionName, map.keySet().size());
    return map.keySet().size() > 0;
  }

  @Override
  CalendarDate extractRunDateWithError(MFile mfile) {
    CalendarDate result = super.extractRunDateWithError(mfile);
    // if there isn't a DateExtractor, see if a mapping exists between
    // filenames and runtimes as defied by the coordValue attribute
    // in explicitly defined file aggregations (i.e. not a directory scan)
    if (result == null)
      if (!this.filesRunDateMap.isEmpty()) {
        String dateString = filesRunDateMap.get(mfile.getPath());
        result = CalendarDate.parseISOformat(null, dateString);
      }
    if (result == null)
      logger.error("Failed to find a run date associated with file {}", mfile.getPath());
    return result;
  }

  protected void reallyScan(java.util.Map<String, MFile> map) throws IOException {
    getController(); // make sure a controller is instantiated

    // run through all scanners and collect MFile instances into the Map
    int count = 0;
    for (CollectionConfig mc : scanList) {
      long start = System.currentTimeMillis();
      // System.out.printf("MFileCollectionManager reallyScan %s %s%n", mc.getDirectoryName(), CalendarDate.present());

      // lOOK: are there any circumstances where we dont need to recheck against OS, ie always use cached values?
      Iterator<MFile> iter = (mc.wantSubdirs()) ? controller.getInventoryAll(mc, true) : controller.getInventoryTop(mc, true);  /// NCDC wants subdir /global/nomads/nexus/gfsanl/**/gfsanl_3_.*\.grb$
      if (iter == null) {
        logger.error(collectionName + ": Invalid collection= " + mc);
        continue;
      }

      while (iter.hasNext()) {
        MFile mfile = iter.next();
        mfile.setAuxInfo(mc.getAuxInfo());
        map.put(mfile.getPath(), mfile);
        count++;
      }

      long took = (System.currentTimeMillis() - start) / 1000;
      // System.out.printf("MFileCollectionManager reallyScan %s took %d secs%n", collectionName, took);
      if (logger.isDebugEnabled()) {
        long took2 = (System.currentTimeMillis() - start) / 1000;
        logger.debug("{} : was scanned nfiles= {} took={} secs", collectionName, count, took2);
     }
    }

    if (map.size() == 0) {
      if (hasScans()) {
        // only warn if a directory scan comes up with no files found
        logger.warn("MFileCollectionManager: No files found for {}", collectionName);
      }
    }
  }

  @Override
  public String toString() {
    Formatter f = new Formatter();
    f.format("DatasetCollectionManager{ collectionName='%s' recheck=%s ", collectionName, recheck);
    for (CollectionConfig mc : scanList) {
      f.format("%n dir=%s filter=%s", mc.getDirectoryName(), mc.getFileFilter());
    }
    return f.toString();
  }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.nc2;

import ucar.ma2.*;
import ucar.nc2.util.CancelTask;
import ucar.nc2.util.CancelTaskImpl;
import ucar.nc2.write.Nc4Chunking;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * Utility class for copying a NetcdfFile object, or parts of one, to a netcdf-3 or netcdf-4 disk file.
 * Uses NetcdfFileWriter.
 * This handles the entire CDM model (groups, etc) if you are writing to netcdf-4.
 *

 * The fileIn may be an NcML file which has a referenced dataset in the location URL, the underlying data
 * (modified by the NcML) is written to the new file. If the NcML does not have a referenced dataset,
 * then the new file is filled with fill values, like ncgen.
 *

 *
 Use a NetcdfFileWriter object for a lower level API.
 *
 * @see ucar.nc2.dt.grid.CFGridWriter2
 * @see ucar.nc2.ft.point.writer.CFPointWriter
 *
 * @author caron
 */

public class FileWriter2 {
  static private final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(FileWriter2.class);
  static private final long maxSize = 50 * 1000 * 1000; // 50 Mbytes
  static private boolean debug = false, debugWrite = false, debugChunk = false;

  /**
   * Set debugging flags
   *
   * @param debugFlags debug flags
   */
  public static void setDebugFlags(ucar.nc2.util.DebugFlags debugFlags) {
    debug = debugFlags.isSet("ncfileWriter2/debug");
    debugWrite = debugFlags.isSet("ncfileWriter2/debugWrite");
    debugChunk = debugFlags.isSet("ncfileWriter2/debugChunk");
  }

  //////////////////////////////////////////////////////////////////////////////////////
  private final NetcdfFile fileIn;
  private final NetcdfFileWriter writer;
  private final NetcdfFileWriter.Version version;

  private final Map<Variable, Variable> varMap = new HashMap<>(100);  // oldVar, newVar
  private final List<Variable> varList = new ArrayList<>(100);        // old Vars
  private final Map<String, Dimension> gdimHash = new HashMap<>(33); // name, newDim : global dimensions (classic mode)

  /**
   * Use this constructor to copy entire file. Use this.write() to do actual copy.
   *
   * @param fileIn      copy this file
   * @param fileOutName to this output file
   * @param version     output file version
   * @param chunker     chunking strategy (netcdf4 only)
   * @throws IOException on read/write error
   */
  public FileWriter2(NetcdfFile fileIn, String fileOutName, NetcdfFileWriter.Version version, Nc4Chunking chunker) throws IOException {
    this.fileIn = fileIn;
    this.writer = NetcdfFileWriter.createNew(version, fileOutName, chunker);
    this.version = version;
  }

  public enum N3StructureStrategy {flatten, exclude}

  private N3StructureStrategy n3StructureStrategy;
  public void setN3StructureStrategy(N3StructureStrategy n3StructureStrategy) {
    this.n3StructureStrategy = n3StructureStrategy;
  }

  public NetcdfFileWriter getNetcdfFileWriter() {
    return writer;
  }

  /////////////////////////////////////////////////////////////////////////////////////////////
  // might be better to push these next up into NetcdfCFWriter, but we want to use copyVarData

  /**
   * Use this constructor to copy specific variables to new file.
   * Only supports classic mode
   *

   * Use addVariable() to load in variables, then this.write().
   *
   * @param fileWriter this encapsolates new file.
   * @throws IOException on read/write error
   */
  public FileWriter2(NetcdfFileWriter fileWriter) throws IOException {
    this.fileIn = null;
    this.writer = fileWriter;
    this.version = fileWriter.getVersion();
  }

  /**
   * Specify which variable will get written
   *
   * @param oldVar add this variable, and all parent groups
   * @return new Variable.
   */
  public Variable addVariable(Variable oldVar) {
    List<Dimension> newDims = getNewDimensions(oldVar);

    Variable newVar;
    if ((oldVar.getDataType().equals(DataType.STRING)) && (!version.isExtendedModel())) {
      newVar = writer.addStringVariable(null, oldVar, newDims);
    } else {
      newVar = writer.addVariable(null, oldVar.getShortName(), oldVar.getDataType(), newDims);
    }
    varMap.put(oldVar, newVar);
    varList.add(oldVar);

    for (Attribute orgAtt : oldVar.getAttributes())
      writer.addVariableAttribute(newVar, convertAttribute(orgAtt));

    return newVar;
  }

  private List<Dimension> getNewDimensions(Variable oldVar) {
    List<Dimension> result = new ArrayList<>(oldVar.getRank());

    // dimensions
    for (Dimension oldD : oldVar.getDimensions()) {
      Dimension newD = gdimHash.get(oldD.getShortName());
      if (newD == null) {
        newD = writer.addDimension(null, oldD.getShortName(), oldD.isUnlimited() ? 0 : oldD.getLength(),
                oldD.isUnlimited(), oldD.isVariableLength());
        gdimHash.put(oldD.getShortName(), newD);
        if (debug) System.out.println("add dim= " + newD);
      }
      result.add(newD);
    }
    return result;
  }

  ////////////////////////////////////////////////////////////////////////////////////////////////

  public NetcdfFile write() throws IOException {
    return write(null);
  }

  /**
   * Write the input file to the output file.
   * @param cancel  allow user to cancel; may be null.
   * @return the open output file.
   * @throws IOException
   */
  public NetcdfFile write(CancelTask cancel) throws IOException {

    try {
      if (version.isExtendedModel())
        addGroupExtended(null, fileIn.getRootGroup());
      else
        addGroupClassic();

      if (cancel != null && cancel.isCancel()) return null;

      // create the file
      writer.create();

      if (cancel != null && cancel.isCancel()) return null;
      double total = copyVarData(varList, null, cancel);
      if (cancel != null && cancel.isCancel()) return null;

      writer.flush();
      if (debug) System.out.println("FileWriter done total bytes = " + total);

    } catch (IOException ioe) {
      ioe.printStackTrace();
      writer.abort();  // clean up
      throw ioe;
    }

    return writer.getNetcdfFile();
  }

  private void addGroupClassic() throws IOException {

    if (fileIn.getRootGroup().getGroups().size() != 0) {
      throw new IllegalStateException("Input file has nested groups: cannot write to netcdf-3 format");
    }

    // attributes
    for (Attribute orgAtt : fileIn.getGlobalAttributes()) {
      writer.addGroupAttribute(null, convertAttribute(orgAtt));
    }

    // dimensions
    Map<String, Dimension> dimHash = new HashMap<>();
    for (Dimension oldD : fileIn.getDimensions()) {
      Dimension newD = writer.addDimension(null, oldD.getShortName(), oldD.isUnlimited() ? 0 : oldD.getLength(),
              oldD.isUnlimited(), oldD.isVariableLength());
      dimHash.put(oldD.getShortName(), newD);
      if (debug) System.out.println("add dim= " + newD);
    }

    // Variables
    int anonCount = 0;
    for (Variable oldVar : fileIn.getVariables()) {
      if (oldVar instanceof Structure) continue; // ignore for the moment

      List<Dimension> dims = new ArrayList<>();
      for (Dimension oldD : oldVar.getDimensions()) {
        if (!oldD.isShared()) { // netcdf3 dimensions must be shared
          String anonName = "anon" + anonCount;
          anonCount++;
          Dimension newD = writer.addDimension(null, anonName, oldD.getLength());
          dims.add(newD);

        } else {
          Dimension dim = dimHash.get(oldD.getShortName());
          if (dim != null)
            dims.add(dim);
          else
            throw new IllegalStateException("Unknown dimension= " + oldD.getShortName());
        }
      }

      DataType newType = oldVar.getDataType();

      // convert STRING to CHAR
      if (oldVar.getDataType() == DataType.STRING) {
        Array data = oldVar.read();
        IndexIterator ii = data.getIndexIterator();
        int max_len = 0;
        while (ii.hasNext()) {
          String s = (String) ii.getObjectNext();
          max_len = Math.max(max_len, s.length());
        }

        // add last dimension
        String useName = oldVar.getShortName() + "_strlen";
        Dimension newD = writer.addDimension(null, useName, max_len);
        dims.add(newD);

        newType = DataType.CHAR;
      }

      Variable v = writer.addVariable(null, oldVar.getShortName(), newType, dims);
      if (debug) System.out.println("add var= " + v.getNameAndDimensions());
      varMap.put(oldVar, v);
      varList.add(oldVar);

      // attributes
      for (Attribute orgAtt : oldVar.getAttributes()) {
        writer.addVariableAttribute(v, convertAttribute(orgAtt));
      }
    }
  }

  private void addGroupExtended(Group newParent, Group oldGroup) throws IOException {
    Group newGroup = writer.addGroup(newParent, oldGroup.getShortName());

    // attributes
    for (Attribute att : oldGroup.getAttributes()) {
      writer.addGroupAttribute(newGroup, att); // atts are immutable
      if (debug) System.out.println("add gatt= " + att);
    }

    // typedefs
    for (EnumTypedef td : oldGroup.getEnumTypedefs()) {
      writer.addTypedef(newGroup, td); // td are immutable
      if (debug) System.out.println("add td= " + td);
    }

    // dimensions
    Map<String, Dimension> dimHash = new HashMap<>();
    for (Dimension oldD : oldGroup.getDimensions()) {
      Dimension newD = writer.addDimension(newGroup, oldD.getShortName(), oldD.isUnlimited() ? 0 : oldD.getLength(),
              oldD.isUnlimited(), oldD.isVariableLength());
      dimHash.put(oldD.getShortName(), newD);
      if (debug) System.out.println("add dim= " + newD);
    }

    // Variables
    for (Variable oldVar : oldGroup.getVariables()) {
      List<Dimension> dims = new ArrayList<>();
      for (Dimension oldD : oldVar.getDimensions()) {
        // in case the name changed
        Dimension newD = oldD.isShared() ? dimHash.get(oldD.getShortName()) : oldD;
        if (newD == null)
          newD = newParent.findDimension(oldD.getShortName());
        if (newD == null)
          throw new IllegalStateException("Cant find dimension " + oldD.getShortName());
        dims.add(newD);
      }

      DataType newType = oldVar.getDataType();
      Variable v;
      if (newType == DataType.STRUCTURE) {
        v = writer.addCopyOfStructure(newGroup, (Structure) oldVar, oldVar.getShortName(), dims);
      } else if(newType.isEnum()) {
        EnumTypedef en = oldVar.getEnumTypedef();
        v = writer.addVariable(newGroup, oldVar.getShortName(), newType, dims);
        v.setEnumTypedef(en);
      } else {
        v = writer.addVariable(newGroup, oldVar.getShortName(), newType, dims);
      }
      varMap.put(oldVar, v);
      varList.add(oldVar);
      if (debug) System.out.println("add var= " + v);

      // attributes
      for (Attribute att : oldVar.getAttributes())
        writer.addVariableAttribute(v, att); // atts are immutable
    }

    // nested groups
    for (Group nested : oldGroup.getGroups())
      addGroupExtended(newGroup, nested);

  }

  // munge attribute if needed
  private Attribute convertAttribute(Attribute org) {
    if (version.isExtendedModel()) return org;
    if (!org.getDataType().isUnsigned()) return org;
    Array orgValues = org.getValues();
    Array nc3Values = Array.makeFromJavaArray(orgValues.getStorage(), false);
    return new Attribute(org.getShortName(), nc3Values);
  }

  /**
   * Write data from varList into new file. Read/Write a maximum of  maxSize bytes at a time.
   * When theres a record variable, its much more efficient to use it.
   *
   * @param oldVars   list of variables from the original file, with data in them
   * @param recordVar the record variable from the original file, or null means dont use record variables
   * @param cancel  allow user to cancel, may be null.
   * @return total number of bytes written
   * @throws IOException if I/O error
   */
  public double copyVarData(List<Variable> oldVars, Structure recordVar, CancelTask cancel) throws IOException {

    boolean useRecordDimension = (recordVar != null);

    // write non-record data
    double total = 0;
    int countVars = 0;
    for (Variable oldVar : oldVars) {
      if (useRecordDimension && oldVar.isUnlimited())
        continue; // skip record variables
      if (oldVar == recordVar)
        continue;

      if (debug)
        System.out.println("write var= " + oldVar.getShortName() + " size = " + oldVar.getSize() + " type=" + oldVar.getDataType());
      if (cancel != null)
        cancel.setProgress("writing " + oldVar.getShortName(), countVars++);

      long size = oldVar.getSize() * oldVar.getElementSize();
      total += size;

      if (size <= maxSize) {
        copyAll(oldVar, varMap.get(oldVar));
      } else {
        copySome(oldVar, varMap.get(oldVar), maxSize, cancel);
      }

      if (cancel != null && cancel.isCancel()) return total;
    }

    // write record data
    if (useRecordDimension) {
      int[] origin = new int[]{0};
      int[] size = new int[]{1};

      int nrecs = (int) recordVar.getSize();
      int sdataSize = recordVar.getElementSize();
      Variable recordVarNew = varMap.get(recordVar);

      double totalRecordBytes = 0;
      for (int count = 0; count < nrecs; count++) {
        origin[0] = count;
        try {
          Array recordData = recordVar.read(origin, size);
          writer.write(recordVarNew, origin, recordData);  // rather magic here - only writes the ones in ncfile !!
          if (debug && (count == 0)) System.out.println("write record size = " + sdataSize);
        } catch (InvalidRangeException e) {
          e.printStackTrace();
          break;
        }
        totalRecordBytes += sdataSize;
        if (cancel != null && cancel.isCancel()) return total;
      }
      total += totalRecordBytes;
      totalRecordBytes /= 1000 * 1000;
      if (debug) System.out.println("write record var; total = " + totalRecordBytes + " Mbytes # recs=" + nrecs);
    }
    return total;
  }

  // copy all the data in oldVar to the newVar
  void copyAll(Variable oldVar, Variable newVar) throws IOException {

    Array data = oldVar.read();
    try {
      if (!version.isNetdf4format() && oldVar.getDataType() == DataType.STRING) {
        data = convertToChar(newVar, data);
      }
      if (data.getSize() > 0)  // zero when record dimension = 0
        writer.write(newVar, data);

    } catch (InvalidRangeException e) {
      e.printStackTrace();
      throw new IOException(e.getMessage() + " for Variable " + oldVar.getFullName());
    }
  }

  /**
   * Copies data from {@code oldVar} to {@code newVar}. The writes are done in a series of chunks no larger than
   * {@code maxChunkSize} bytes.
   *
   * @param oldVar       a variable from the original file to copy data from.
   * @param newVar       a variable from the original file to copy data from.
   * @param maxChunkSize the size, in bytes, of the largest chunk to write.
   * @param cancel      allow user to cancel, may be null.
   * @throws IOException if an I/O error occurs.
   */
  private void copySome(Variable oldVar, Variable newVar, long maxChunkSize, CancelTask cancel) throws IOException {
    long maxChunkElems = maxChunkSize / oldVar.getElementSize();
    long byteWriteTotal = 0;

    ChunkingIndex index = new ChunkingIndex(oldVar.getShape());
    while (index.currentElement() < index.getSize()) {
      try {
        int[] chunkOrigin = index.getCurrentCounter();
        int[] chunkShape = index.computeChunkShape(maxChunkElems);

        if (cancel != null) cancel.setProgress("Reading chunk "+new Section(chunkOrigin, chunkShape)+" from variable: " + oldVar.getShortName(), -1);
        /* writeProgressEvent.setWriteStatus("Reading chunk from variable: " + oldVar.getShortName());
        if (progressListeners != null) {
          for (FileWriterProgressListener listener : progressListeners) {
            listener.writeProgress(writeProgressEvent);
          }
        } */

        Array data = oldVar.read(chunkOrigin, chunkShape);
        if (!version.isNetdf4format() && oldVar.getDataType() == DataType.STRING) {
          data = convertToChar(newVar, data);
        }

        if (data.getSize() > 0) {// zero when record dimension = 0
          if (cancel != null) cancel.setProgress("Writing chunk "+new Section(chunkOrigin, chunkShape)+" from variable: " + oldVar.getShortName(), -1);

          writer.write(newVar, chunkOrigin, data);
          if (debugWrite)
            System.out.println(" write " + data.getSize() + " bytes at " + new Section(chunkOrigin, chunkShape));

          byteWriteTotal += data.getSize();
        }

        index.setCurrentCounter(index.currentElement() + (int) Index.computeSize(chunkShape));
        if (cancel != null && cancel.isCancel()) return;

      } catch (InvalidRangeException e) {
        e.printStackTrace();
        throw new IOException(e.getMessage());
      }
    }
  }

  private Array convertToChar(Variable newVar, Array oldData) {
    ArrayChar newData = (ArrayChar) Array.factory(DataType.CHAR, newVar.getShape());
    Index ima = newData.getIndex();
    IndexIterator ii = oldData.getIndexIterator();
    while (ii.hasNext()) {
      String s = (String) ii.getObjectNext();
      int[] c = ii.getCurrentCounter();
      for (int i = 0; i < c.length; i++)
        ima.setDim(i, c[i]);
      newData.setString(ima, s);
    }
    return newData;
  }

  /* private boolean hasRecordStructure(NetcdfFile file) {
    Variable v = file.findVariable("record");
    return (v != null) && (v.getDataType() == DataType.STRUCTURE);
  } */

  ///////////////////////////////////////////////////////////////////////////////////////////////////////////
  // contributed by  cwardgar@usgs.gov 4/12/2010

  /**
   * An index that computes chunk shapes. It is intended to be used to compute the origins and shapes for a series
   * of contiguous writes to a multidimensional array.
   * It writes the first n elements (n < maxChunkElems), then the next, etc.
   */
  public static class ChunkingIndex extends Index {
    public ChunkingIndex(int[] shape) {
      super(shape);
    }

    /**
     * Computes the shape of the largest possible contiguous chunk, starting at {@link #getCurrentCounter()}
     * and with {@code numElems <= maxChunkElems}.
     *
     * @param maxChunkElems the maximum number of elements in the chunk shape. The actual element count of the shape
     *                      returned is likely to be different, and can be found with {@link Index#computeSize}.
     * @return the shape of the largest possible contiguous chunk.
     */
    public int[] computeChunkShape(long maxChunkElems) {
      int[] chunkShape = new int[rank];

      for (int iDim = 0; iDim < rank; ++iDim) {
        int size = (int) (maxChunkElems / stride[iDim]);
        size = (size == 0) ? 1 : size;
        size = Math.min(size, shape[iDim] - current[iDim]);
        chunkShape[iDim] = size;
      }

      return chunkShape;
    }
  }

  ////////////////////////////////////////////////////////////////////////////////////////////////////////

  private static void usage() {
    System.out.println("usage: ucar.nc2.FileWriter2 -in <fileIn> -out <fileOut> [-netcdf4]");
  }

  /**
   * Better to use ucar.nc.dataset.NetcdfDataset main program instead.
   *
ucar.nc2.FileWriter -in fileIn -out fileOut.
   *
where:

   *
 fileIn : path of any CDM readable file
   *
 fileOut: local pathname where netdf-3 file will be written
   *

   *
   * @param arg -in fileIn -out fileOut [-netcdf4]
   * @throws IOException on read or write error
   */
  public static void main(String arg[]) throws IOException {
    if (arg.length < 4) {
      usage();
      System.exit(0);
    }

    String datasetIn = null, datasetOut = null;
    NetcdfFileWriter.Version version = NetcdfFileWriter.Version.netcdf3;
    for (int i = 0; i < arg.length; i++) {
      String s = arg[i];
      if (s.equalsIgnoreCase("-in")) datasetIn = arg[i + 1];
      if (s.equalsIgnoreCase("-out")) datasetOut = arg[i + 1];
      if (s.equalsIgnoreCase("-netcdf4")) version = NetcdfFileWriter.Version.netcdf4;
    }
    if ((datasetIn == null) || (datasetOut == null)) {
      usage();
      System.exit(0);
    }

    System.out.printf("FileWriter2 copy %s to %s ", datasetIn, datasetOut);
    CancelTaskImpl cancel = new CancelTaskImpl();
    NetcdfFile ncfileIn = ucar.nc2.NetcdfFile.open(datasetIn, cancel);
    if (cancel.isCancel()) return;

    FileWriter2 writer2 = new FileWriter2(ncfileIn, datasetOut, version, null); // currently only the default chunker
    NetcdfFile ncfileOut = writer2.write(cancel);
    if (ncfileOut != null) ncfileOut.close();
    ncfileIn.close();
    System.out.printf("%s%n", cancel);
  }

  // Q:/cdmUnitTest/formats/netcdf4/tst/tst_groups.nc

}


/* Copyright 2012, UCAR/Unidata.
   See the LICENSE file for more information.
*/

package dap4.cdm.nc2;

import dap4.cdm.CDMTypeFcns;
import dap4.core.data.DSP;
import dap4.core.data.DataCursor;
import dap4.core.dmr.DapSequence;
import dap4.core.dmr.DapStructure;
import dap4.core.dmr.DapType;
import dap4.core.dmr.DapVariable;
import dap4.core.util.*;
import ucar.ma2.*;
import ucar.nc2.Group;

import java.io.IOException;
import java.util.List;

/**
 * CDM now has an ArraySequence type intended to
 * support VLEN (aka CDM (*) dimension).
 * So, sequence is simulated as a rank n+1 structure where
 * the last dimension is "*" (i.e. variable length).
 * That is, given the following DAP4:
 * Sequence S {f1,f2,...fm} [d1][d2]...[dn]
 * Represent it in CDM as this:
 * Structure S {f1,f2,...fm} [d1][d2]...[dn][*]
 * We cannot subclass CDMArrayStructure because we need to subclass
 * ArraySequence, so we are forced to duplicate a lot of the CDMArrayStructure
 * code.
 * The important point to note is that for CDM, we do not need to support
 * Dimensioned sequences; the dimensions are supported by the enclosing
 * ArrayStructure covering the non-vlen dimensions
 */

/*package*/ class CDMArraySequence extends ArraySequence implements CDMArray
{

    //////////////////////////////////////////////////
    // Type decls

    // Define an open wrapper around a field array in order
    // to make the code somewhat more clear

    static protected class FieldSet
    {
        public Array[] fields;

        FieldSet(int nfields)
        {
            fields = new Array[nfields];
        }
    }

    static public class SDI implements StructureDataIterator
    {
        protected StructureData[] list;
        protected int position;

        public SDI()
        {
            this.list = null;
            this.position = 0;
        }

        public SDI
        setList(StructureData[] list)
        {
            this.list = list;
            return this;
        }

        public boolean hasNext() throws IOException
        {
            return position < list.length;
        }

        public StructureData next() throws IOException
        {
            if(position >= list.length)
                throw new IOException("No next element");
            return list[position++];
        }

        public StructureDataIterator reset()
        {
            position = 0;
            return this;
        }

        public int getCurrentRecno()
        {
            return position;
        }

    }

    //////////////////////////////////////////////////
    // Instance variables

    protected Group cdmroot = null;
    protected DSP dsp;
    protected DapVariable template;
    protected DapType basetype;
    protected long bytesize = 0;
    protected long recordcount = 0;
    protected int nmembers = 0;

    protected DataCursor seqdata = null;

    /**
     * Since in CDM a sequence is the last dimension of
     * array, we do not need to keep dimensionality info, only
     * the variable length stuff, which we do using
     * StructureDataA instances: 1 per record.
     */

    // Track the records of this sequence as an array
    // Note: term records here means the elements of the array,
    // not record as in Sequence

    protected FieldSet[] records = null; // list of records

    //////////////////////////////////////////////////
    // Constructor(s)

    /**
     * Constructor
     *
     * @param data
     */
    CDMArraySequence(Group group, DataCursor data)
            throws DapException
    {
        super(CDMArrayStructure.computemembers((DapVariable) data.getTemplate()),
                new SDI(), 0);
        this.template = (DapVariable) data.getTemplate();
        this.basetype =  this.template.getBaseType();
        // Currently do not allow non-scalar sequences
        if(this.template.getRank() != 0)
            throw new DapException("Non-scalar sequences unsupported through CDM interface");
        assert data.getScheme() == DataCursor.Scheme.SEQARRAY;
        this.cdmroot = group;
        this.dsp = dsp;
        // Since this is a scalar, pull out the single instance
        this.seqdata = ((DataCursor[])data.read(dap4.core.util.Index.SCALAR))[0];
        this.recordcount = this.seqdata.getRecordCount();
        this.nmembers = ((DapStructure)this.basetype).getFields().size();

        // Fill in the structdata (in parent) and record vectors
        super.sdata = new StructureDataA[(int) this.recordcount];
        records = new FieldSet[(int) this.recordcount];
        for(int i = 0; i < this.recordcount; i++) {
            super.sdata[i] = new StructureDataA(this, i);
            records[i] = new FieldSet(this.nmembers);
        }

        ((SDI) super.iter).setList(super.sdata);
    }

    //////////////////////////////////////////////////
    // Compiler API

    /*package*/
    void
    add(long recno, int fieldno, Array field)
    {
        //Make sure all the space is allocated
        if(records.length <= recno) {
            FieldSet[] newrecs = new FieldSet[(int) recno + 1];
            System.arraycopy(records, 0, newrecs, 0, records.length);
            records = newrecs;
        }
        FieldSet fs = records[(int) recno];
        if(fs == null) {
            records[(int) recno] = (fs = new FieldSet(this.nmembers));
        }
        fs.fields[fieldno] = field;
    }

    //////////////////////////////////////////////////
    // CDMArray Interface

    @Override
    public DapType getBaseType()
    {
        return this.basetype;
    }

    @Override
    public DSP getDSP()
    {
        return this.dsp;
    }

    @Override
    public DapVariable getTemplate()
    {
        return template;
    }

    //////////////////////////////////////////////////

    public String toString()
    {
        StringBuilder buf = new StringBuilder();
        DapVariable var = this.template;
        DapSequence seq = (DapSequence)this.basetype;
        long dimsize = DapUtil.dimProduct(var.getDimensions());
        for(int i = 0; i < dimsize; i++) {
            List<DapVariable> fields = seq.getFields();
            if(i < (dimsize - 1))
                buf.append("\n");
            buf.append("Sequence {\n");
            buf.append(String.format("} [%d/%d]", i, dimsize));
        }
        return buf.toString();
    }

    //////////////////////////////////////////////////
    // ArraySequence/ArrayStructure overrides

    @Override
    public int getStructureDataCount()
    {
        return this.records.length;
    }

    @Override
    protected StructureData makeStructureData(ArrayStructure as, int index)
    {
        throw new UnsupportedOperationException("Cannot subset a Sequence");
    }

    /**
     * Get the index'th StructureData(StructureDataA) object
     * We need instances of StructureData to give to the user.
     * We use StructureDataA so we can centralize everything
     * in this class. The total number of StructureData objects
     * is dimsize.
     *
     * @param index
     * @return
     */
    @Override
    public StructureData getStructureData(int index)
    {
        assert (super.sdata != null);
        if(index < 0 || index >= this.records.length)
            throw new IllegalArgumentException(index + " >= " + super.sdata.length);
        assert (super.sdata[index] != null);
        return super.sdata[index];
    }

    public ArraySequence getArraySequence(StructureMembers.Member m)
    {
        return this;
    }

    /////////////////////////
    // Define API required by StructureDataA
    @Override
    public Array copy()
    {
        return this; // temporary
    }

    /**
     * Get member data of any type for a specific record as an Array.
     * This may avoid the overhead of creating the StructureData object,
     * but is equivalent to getStructure(recno).getArray( Member m).
     *
     * @param recno get data from the recnum-th StructureData of the ArrayStructure.
     *              Must be less than getSize();
     * @param m     get data from this StructureMembers.Member.
     * @return Array values.
     */
    public Array getArray(int recno, StructureMembers.Member m)
    {
        return (ucar.ma2.Array) memberArray(recno, CDMArrayStructure.memberIndex(m));
    }

    /////////////////////////

    protected CDMArrayAtomic
    getAtomicArray(int index, StructureMembers.Member m)
    {
        Array dd = memberArray(index, CDMArrayStructure.memberIndex(m));
        if(dd.getDataType() != DataType.STRUCTURE && dd.getDataType() != DataType.SEQUENCE)
            return (CDMArrayAtomic) dd;
        throw new ForbiddenConversionException("Cannot convert structure to AtomicArray");
    }

    protected Array
    memberArray(int recno, int memberindex)
    {

        Object[] values = new Object[(int) this.recordcount];
        for(int i = 0; i < this.recordcount; i++) {
            FieldSet fs = records[i];
            values[i] = fs.fields[memberindex];
        }
        DapVariable field = ((DapStructure)this.basetype).getField(memberindex);
        DapType base = field.getBaseType();
        if(base == null)
            throw new IllegalStateException("Unknown field type: "+field);
        DataType dt = CDMTypeFcns.daptype2cdmtype(base);
        Class elemtype = CDMTypeFcns.cdmElementClass(dt);
        int shape[] = new int[]{(int) this.recordcount};
        return new ArrayObject(dt, elemtype, false, shape, values);
    }
}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.dataset;

import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Formatter;
import java.util.List;
import java.util.StringTokenizer;

import org.jdom2.Document;
import org.jdom2.Element;
import org.jdom2.output.Format;
import org.jdom2.output.XMLOutputter;
import ucar.nc2.Variable;
import ucar.nc2.constants.CDM;
import ucar.nc2.dt.grid.GridCoordSys;
import ucar.nc2.units.DateUnit;
import ucar.nc2.units.SimpleUnit;
import ucar.unidata.util.Parameter;

/**
 * Helper class for obtaining information about a NetcdfDataset.
 * Creates a "netcdfDatasetInfo" XML document, used by the TDS "Common Data Model Coordinate System Validation".
 *
 * @author john caron
 */
public class NetcdfDatasetInfo implements Closeable {
  private NetcdfDataset ds;
  private CoordSysBuilderIF builder;

  public NetcdfDatasetInfo( String location) throws IOException {
    ds = NetcdfDataset.openDataset(location, false, null);
    builder = ds.enhance();
  }

  public NetcdfDatasetInfo( NetcdfDataset ds) throws IOException {
    File loc = new File(ds.getLocation());
    if (loc.exists()) {
      this.ds = NetcdfDataset.openDataset(ds.getLocation(), false, null); // fresh enhancement
      builder = this.ds.enhance();
    } else {  // Aggregation, fc may not exist
      this.ds = ds;  // LOOK what can we do thats better ?
      builder = ds.enhance();
    }
  }

  public void close() throws IOException {
    if (ds != null) ds.close();
  }

  /**
   * Detailed information when the coordinate systems were parsed
   * @return String containing parsing info
   */
  public String getParseInfo( ) {
    return (builder == null) ? "" : builder.getParseInfo();
  }
  /**
   * Specific advice to a user about problems with the coordinate information in the file.
   * @return String containing advice to a user about problems with the coordinate information in the file.
   */
  public String getUserAdvice( ) {
    return (builder == null) ? "" : builder.getUserAdvice();
  }

  /**
   * Get the name of the CoordSysBuilder that parses this file.
   * @return the name of the CoordSysBuilder that parses this file.
   */
  public String getConventionUsed( ) {
    return (builder == null) ? "None" : builder.getConventionUsed();
  }

  /** Write the information as an XML document
   * @return String containing netcdfDatasetInfo XML
   */
  public String writeXML( )  {
    XMLOutputter fmt = new XMLOutputter( Format.getPrettyFormat());
    return fmt.outputString( makeDocument());
  }

  public GridCoordSys getGridCoordSys(VariableEnhanced ve) {
    List<CoordinateSystem> csList = ve.getCoordinateSystems();
    for (CoordinateSystem cs : csList) {
      if (GridCoordSys.isGridCoordSys(null, cs, ve)) {
        return new GridCoordSys(cs, null);
      }
    }
    return null;
  }

  public void writeXML(OutputStream os) throws IOException {
    XMLOutputter fmt = new XMLOutputter( Format.getPrettyFormat());
    fmt.output( makeDocument(), os);
  }

  /** Create an XML document from this info
   * @return netcdfDatasetInfo XML document
   */
  public Document makeDocument() {
    Element rootElem = new Element("netcdfDatasetInfo");
    Document doc = new Document(rootElem);
    rootElem.setAttribute("location", ds.getLocation());
    rootElem.addContent( new Element("convention").setAttribute("name", getConventionUsed()));

    int nDataVariables = 0;
    int nOtherVariables = 0;

    List<CoordinateAxis> axes = ds.getCoordinateAxes();
    int nCoordAxes = axes.size();
    for (CoordinateAxis axis : axes) {
      Element axisElem = new Element("axis");
      rootElem.addContent(axisElem);

      axisElem.setAttribute("name", axis.getFullName());
      axisElem.setAttribute("decl", getDecl(axis));
      if (axis.getAxisType() != null)
        axisElem.setAttribute("type", axis.getAxisType().toString());
      if (axis.getUnitsString() != null) {
        axisElem.setAttribute(CDM.UNITS, axis.getUnitsString());
        axisElem.setAttribute("udunits", isUdunits(axis.getUnitsString()));
      }
      if (axis instanceof CoordinateAxis1D) {
        CoordinateAxis1D axis1D = (CoordinateAxis1D) axis;
        if (axis1D.isRegular())
          axisElem.setAttribute("regular", ucar.unidata.util.Format.d(axis1D.getIncrement(), 5));
      }
    }

    List<CoordinateSystem> csList = ds.getCoordinateSystems();
    for (CoordinateSystem cs : csList) {
      Element csElem;
      if (GridCoordSys.isGridCoordSys(null, cs, null)) {
        GridCoordSys gcs = new GridCoordSys(cs, null);
        csElem = new Element("gridCoordSystem");
        csElem.setAttribute("name", cs.getName());
        csElem.setAttribute("horizX", gcs.getXHorizAxis().getFullName());
        csElem.setAttribute("horizY", gcs.getYHorizAxis().getFullName());
        if (gcs.hasVerticalAxis())
          csElem.setAttribute("vertical", gcs.getVerticalAxis().getFullName());
        if (gcs.hasTimeAxis())
          csElem.setAttribute("time", cs.getTaxis().getFullName());
      } else {
        csElem = new Element("coordSystem");
        csElem.setAttribute("name", cs.getName());
      }

      List<CoordinateTransform> coordTransforms = cs.getCoordinateTransforms();
      for (CoordinateTransform ct : coordTransforms) {
        Element ctElem = new Element("coordTransform");
        csElem.addContent(ctElem);
        ctElem.setAttribute("name", ct.getName());
        ctElem.setAttribute("type", ct.getTransformType().toString());
      }

      rootElem.addContent(csElem);
    }

    List<CoordinateTransform> coordTransforms = ds.getCoordinateTransforms();
    for (CoordinateTransform ct : coordTransforms) {
      Element ctElem = new Element("coordTransform");
      rootElem.addContent(ctElem);

      ctElem.setAttribute("name", ct.getName());
      ctElem.setAttribute("type", ct.getTransformType().toString());
      List<Parameter> params = ct.getParameters();
      for (Parameter pp : params) {
        Element ppElem = new Element("param");
        ctElem.addContent(ppElem);
        ppElem.setAttribute("name", pp.getName());
        ppElem.setAttribute("value", pp.getStringValue());
      }
    }

    for (Variable var : ds.getVariables()) {
      VariableEnhanced ve = (VariableEnhanced) var;
      if (ve instanceof CoordinateAxis) continue;
      GridCoordSys gcs = getGridCoordSys(ve);
      if (null != gcs) {
        nDataVariables++;
        Element gridElem = new Element("grid");
        rootElem.addContent(gridElem);

        gridElem.setAttribute("name", ve.getFullName());
        gridElem.setAttribute("decl", getDecl(ve));
        if (ve.getUnitsString() != null) {
          gridElem.setAttribute(CDM.UNITS, ve.getUnitsString());
          gridElem.setAttribute("udunits", isUdunits(ve.getUnitsString()));
        }
        gridElem.setAttribute("coordSys", gcs.getName());
      }
    }

    for (Variable var : ds.getVariables()) {
      VariableEnhanced ve =  (VariableEnhanced) var;
      if (ve instanceof CoordinateAxis) continue;
      GridCoordSys gcs = getGridCoordSys(ve);
      if (null == gcs) {
        nOtherVariables++;
        Element elem = new Element("variable");
        rootElem.addContent(elem);

        elem.setAttribute("name", ve.getFullName());
        elem.setAttribute("decl", getDecl(ve));
        if (ve.getUnitsString() != null) {
          elem.setAttribute(CDM.UNITS, ve.getUnitsString());
          elem.setAttribute("udunits", isUdunits(ve.getUnitsString()));
        }
        elem.setAttribute("coordSys", getCoordSys(ve));
      }
    }

    if (nDataVariables > 0) {
      rootElem.addContent( new Element("userAdvice").addContent( "Dataset contains useable gridded data."));
      if (nOtherVariables > 0)
        rootElem.addContent( new Element("userAdvice").addContent( "Some variables are not gridded fields; check that is what you expect."));
    } else {
      if (nCoordAxes == 0)
         rootElem.addContent( new Element("userAdvice").addContent( "No Coordinate Axes were found."));
      else
        rootElem.addContent( new Element("userAdvice").addContent( "No gridded data variables were found."));
    }

    String userAdvice = getUserAdvice();
    if (userAdvice.length() > 0) {
      StringTokenizer toker = new StringTokenizer(userAdvice, "\n");
      while (toker.hasMoreTokens())
        rootElem.addContent( new Element("userAdvice").addContent( toker.nextToken()));
    }

    return doc;
  }

  private String getDecl( VariableEnhanced ve) {
    Formatter sb = new Formatter();
    sb.format("%s ", ve.getDataType().toString());
    ve.getNameAndDimensions(sb, true, true);
    return sb.toString();
  }

  private String getCoordSys( VariableEnhanced ve) {
    List csList = ve.getCoordinateSystems();
    if (csList.size() == 1) {
      CoordinateSystem cs = (CoordinateSystem) csList.get(0);
      return cs.getName();
    } else if (csList.size() > 1) {
      StringBuilder sb = new StringBuilder();
      for (int i = 0; i < csList.size(); i++) {
        CoordinateSystem cs = (CoordinateSystem) csList.get(i);
        if (i > 0) sb.append(";");
        sb.append(cs.getName());
      }
      return sb.toString();
    }
    return " ";
  }

 private String isUdunits(String unit) {
    try {
      new DateUnit(unit);
      return "date";
    } catch (Exception e) {
      // ok
    }

    SimpleUnit su = SimpleUnit.factory(unit);
    if (null == su) return "false";
    return su.getCanonicalString();
  }

  /** debug */
  public static void main(String args[]) throws IOException {
    String url = "C:/data/badmodels/RUC_CONUS_80km_20051211_1900.nc";

    try (NetcdfDatasetInfo info = new NetcdfDatasetInfo(url)) {
      String infoString = info.writeXML();
      System.out.println(infoString);
    }
  }

}


package ucar.nc2.ogc;

import net.opengis.waterml.x20.CollectionDocument;
import org.apache.xmlbeans.*;
import org.joda.time.DateTime;
import ucar.nc2.VariableSimpleIF;
import ucar.nc2.ft.FeatureDatasetPoint;
import ucar.nc2.ogc.waterml.NcCollectionType;

import javax.xml.namespace.QName;
import java.io.IOException;
import java.io.OutputStream;
import java.util.*;
import java.util.function.Predicate;

/**
 * Created by cwardgar on 2014/04/29.
 */
public class MarshallingUtil {
    /////////////////////////////////////////////// Testing Util ///////////////////////////////////////////////

    public static DateTime fixedGenerationDate = null;
    public static DateTime fixedResultTime     = null;

    /////////////////////////////////////////////// ID Creation ///////////////////////////////////////////////

    private static Map<Class<?>, Integer> idsMap = new HashMap<>();

    public static String createIdForType(Class<?> clazz) {
        Integer numIds = idsMap.get(clazz);
        if (numIds == null) {
            numIds = 0;
        }

        String id = clazz.getSimpleName() + "." + ++numIds;
        idsMap.put(clazz, numIds);
        return id;
    }

    public static void resetIds() {
        idsMap.clear();
    }

    /////////////////////////////////////////////// Marshalling ///////////////////////////////////////////////

    public static void marshalPointDataset(FeatureDatasetPoint fdPoint, OutputStream outputStream)
            throws IOException, XmlException {
        marshalPointDataset(fdPoint, fdPoint.getDataVariables(), outputStream);
    }

    public static void marshalPointDataset(FeatureDatasetPoint fdPoint, List<VariableSimpleIF> dataVars,
            OutputStream outputStream) throws IOException, XmlException {
        resetIds();

        CollectionDocument collectionDoc = CollectionDocument.Factory.newInstance();
        NcCollectionType.initCollection(collectionDoc.addNewCollection(), fdPoint, dataVars);
        writeObject(collectionDoc, outputStream, true);
    }

    public static void writeObject(XmlObject doc, OutputStream out, boolean validate) throws IOException, XmlException {
        // Add xsi:schemaLocation
        XmlCursor cursor = doc.newCursor();
        if (cursor.toFirstChild()) {
            String location = "http://www.opengis.net/waterml/2.0 http://schemas.opengis.net/waterml/2.0/waterml2.xsd";
            cursor.setAttributeText(new QName("http://www.w3.org/2001/XMLSchema-instance", "schemaLocation"), location);
        }

        if (validate) {
            validate(doc, false);
        }

        doc.save(out, makeOptions());
    }

    private static XmlOptions makeOptions() {
        XmlOptions options = new XmlOptions();

        options.setSaveNamespacesFirst();
        options.setSavePrettyPrint();
        options.setSavePrettyPrintIndent(4);
        options.setSaveAggressiveNamespaces();
        options.setSaveSuggestedPrefixes(getSuggestedPrefixes());

        return options;
    }

    private static Map<String, String> getSuggestedPrefixes() {
        Map<String, String> prefixMap = new HashMap<>();

        prefixMap.put("http://www.w3.org/2001/XMLSchema-instance", "xsi");
        prefixMap.put("http://www.w3.org/1999/xlink", "xlink");
        prefixMap.put("http://www.opengis.net/gml/3.2", "gml");
        prefixMap.put("http://www.opengis.net/om/2.0", "om");
        prefixMap.put("http://www.opengis.net/sampling/2.0", "sam");
        prefixMap.put("http://www.opengis.net/samplingSpatial/2.0", "sams");
        prefixMap.put("http://www.opengis.net/swe/2.0", "swe");
        prefixMap.put("http://www.opengis.net/waterml/2.0", "wml2");
        prefixMap.put("http://www.opengis.net/waterml-dr/2.0", "wml2dr");
        prefixMap.put("http://www.opengis.net/gmlcov/1.0", "gmlcov");

        return prefixMap;
    }

    /////////////////////////////////////////////// Validation ///////////////////////////////////////////////

    /**
     * Validates an xml doc. If the validation fails, the exception contains a detailed list of errors.
     *
     * @param doc the document to validate
     * @throws XmlException thrown if the XML is incorrect
     */
    public static void validate(XmlObject doc, boolean strict) throws XmlException {
        // Create an XmlOptions instance and set the error listener.
        Set<XmlError> validationErrors = new HashSet<>();
        XmlOptions validationOptions = new XmlOptions();
        validationOptions.setErrorListener(validationErrors);

        // Validate the XML document
        final boolean isValid = doc.validate(validationOptions);

        // Create Exception with error message if the xml document is invalid
        if (!isValid && !strict) {
            // check if we have special validation cases which could let the message pass anyhow
            validationErrors = filterToOnlySerious(validationErrors);
        }

        if (!validationErrors.isEmpty()) {
            throw new XmlException(createErrorMessage(validationErrors));
        }
    }

    private static Set<XmlError> filterToOnlySerious(Iterable<XmlError> allValidationErrors) {
        Set<XmlError> filteredErrors = new HashSet<>();
        GML32AbstractFeatureLaxCase pred = new GML32AbstractFeatureLaxCase();

        for (final XmlError validationError : allValidationErrors) {
            if (!pred.test(validationError)) {
                filteredErrors.add(validationError);
            }
        }

        return filteredErrors;
    }

    private static String createErrorMessage(Collection<XmlError> errors) {
        StringBuilder errorBuilder = new StringBuilder();
        for (XmlError xmlError : errors) {
            errorBuilder.append(xmlError.getMessage()).append(";");
        }

        if (!errors.isEmpty()) {
            errorBuilder.deleteCharAt(errorBuilder.length() - 1);
        }
        return errorBuilder.toString();
    }

    /**
     * Allow substitutions of gml:AbstractFeature. This lax validation lets pass every child, hence
     * it checks not _if_ this is a valid substitution.
     * </p>
     * This is similar to org.n52.oxf.xmlbeans.parser.GMLAbstractFeatureCase, updated for GML 3.2.
     */
    private static class GML32AbstractFeatureLaxCase implements Predicate<XmlError> {
        private static final Object FEATURE_QN = new QName("http://www.opengis.net/gml/3.2", "AbstractFeature");

        // Returns 'true' if the error should be ignored because we're doing lax validation.
        @Override public boolean test(XmlError validationError) {
            if (!(validationError instanceof XmlValidationError)) return false;

            XmlValidationError xve = (XmlValidationError) validationError;
            return xve.getExpectedQNames() != null && xve.getExpectedQNames().contains(FEATURE_QN);
        }
    }

    private MarshallingUtil() { }
}


/* Copyright 2012, UCAR/Unidata.
   See the LICENSE file for more information.
*/

package dap4.core.util;

import java.util.ArrayList;
import java.util.List;

/**
 * A MultiSlice is a list of slices to support e.g. [x:y,a:b]
 * It subclasses Slice so it can appear in same places slices do;
 * code is expected to differentiate.
 */

public class MultiSlice extends Slice
{
    //////////////////////////////////////////////////
    // Instance Variables

    protected List<Slice> subslices;

    // cache some values
    protected long count = -1;

    //////////////////////////////////////////////////
    // Constructor(s)

    public MultiSlice(List<Slice> subslices)
            throws dap4.core.util.DapException
    {
        super();
        this.sort = Sort.Multi;
        this.subslices = subslices;
        finish();
        // provide values for first, last, etc
        this.first = UNDEFINED;
        this.stop = UNDEFINED;
        this.stride = UNDEFINED;
        this.maxsize = UNDEFINED;
        for(int i = 0; i < this.subslices.size(); i++) {
            Slice s = this.subslices.get(i);
            this.first = (this.first < 0 ? s.getFirst() : Math.min(this.first, s.getFirst()));
            this.stop = Math.max(this.stop, s.getStop());
            this.stride = Math.max(this.stride, s.getStride());
            this.maxsize = Math.max(this.maxsize, s.getMax());
        }
        this.whole = false;
        this.constrained = true; // by definition
    }

    //////////////////////////////////////////////////

    /**
     * Compare two slices for equality
     *
     * @param o the other slice to compare with
     * @return true if this and other are the same, false otherwise
     */
    @Override
    public boolean equals(Object o)
    {
        if(o == this) return true;
        if(o == null) return false;
        if(o instanceof Slice) try {
            List<Slice> tmp = new ArrayList<Slice>();
            tmp.add((Slice) o);
            o = new MultiSlice(tmp);
        } catch (dap4.core.util.DapException de) {
            throw new IllegalArgumentException();
        }
        if(!(o instanceof MultiSlice)) return false;
        Slice other = (Slice) o;
        return other.getFirst() == getFirst()
                && other.getLast() == getLast()
                && other.getStride() == getStride();
    }

    @Override
    public int hashCode()
    {
        long accum = 0;
        for(int i = 0; i < this.subslices.size(); i++) {
            Slice s = this.subslices.get(i);
            accum += s.hashCode() * i;
        }
        return (int) accum;
    }

    public String
    toString()
    {
        StringBuilder buf = new StringBuilder();
        buf.append("[");
        for(int i = 0; i < this.subslices.size(); i++) {
            if(i > 0)
                buf.append(",");
            buf.append(this.subslices.get(i).toString(false));
        }
        buf.append("]");
        return buf.toString();
    }

    /**
     * Convert this multislice to a string
     * suitable for use in a constraint
     *
     * @return constraint usable string
     * @throws DapException
     */
    @Override
    public String toConstraintString()
            throws DapException
    {
        assert this.first != UNDEFINED && this.stride != UNDEFINED && this.stop != UNDEFINED;
        StringBuilder buf = new StringBuilder();
        buf.append("[");
        boolean first = true;
        for(Slice sub: this.subslices) {
            if(!first) buf.append(",");
            first = false;
            if((sub.stop - sub.first) == 0) {
                buf.append("0");
            } else if(sub.stride == 1) {
                if((sub.stop - sub.first) == 1)
                    buf.append(sub.first);
                else
                    buf.append(String.format("%d:%d", sub.first, sub.stop - 1));
            } else
                buf.append(String.format("%d:%d:%d", sub.first, sub.stride, sub.stop - 1));
        }
        buf.append("]");
        return buf.toString();
    }

    @Override
    public Slice
    finish()
            throws dap4.core.util.DapException
    {
        this.maxsize = UNDEFINED;
        for(int i = 0; i < this.subslices.size(); i++) {// size is max size
            Slice sl = this.subslices.get(i);
            assert (sl.getSort() == Slice.Sort.Single);
            sl.finish();
            if(this.maxsize < sl.getMax())
                this.maxsize = sl.getMax();
        }
        if(this.maxsize < 0)
            throw new dap4.core.util.DapException("Cannot compute multislice size");
        for(int i = 0; i < this.subslices.size(); i++) {
            this.subslices.get(i).setMaxSize(this.maxsize);
        }
        this.count = 0;
        for(int i = 0; i < this.subslices.size(); i++) {
            count += this.subslices.get(i).getCount();
        }
        return this; // fluent interface
    }

    //////////////////////////////////////////////////
    // Accessors

    @Override
    public List<Slice>
    getSubSlices()
    {
        return this.subslices;
    }

    public Slice
    getSubSlice(int i)
    {
        return this.subslices.get(i);
    }

    @Override
    public long
    getCount()
    {
        return this.count;
    }

    @Override
    public Slice
    setMaxSize(long size)
            throws DapException
    {
        for(int i = 0; i < this.subslices.size(); i++) {
            this.subslices.get(i).setMaxSize(size);
        }
        return this;
    }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.dataset;

import ucar.nc2.*;
import ucar.nc2.constants.CDM;
import ucar.nc2.util.CancelTask;
import ucar.ma2.*;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;

/**
 * An "enhanced" Structure.
 *
 * @author john caron
 * @see NetcdfDataset
 */

public class StructureDS extends ucar.nc2.Structure implements VariableEnhanced {
  // static private org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(StructureDS.class);

  private EnhancementsImpl proxy; // API relies that this cant be null

  protected Structure orgVar; // wrap this Variable
  private String orgName; // in case Variable wwas renamed, and we need the original name for aggregation

  protected StructureDS(NetcdfFile ncfile, Group group, String shortName) {
    super(ncfile, group, null, shortName);
    this.proxy = new EnhancementsImpl(this);
  }

  /**
   * Constructor when theres no underlying variable. You better set the values too!
   *
   * @param ds              the containing NetcdfDataset.
   * @param group           the containing group; if null, use rootGroup
   * @param parentStructure parent Structure, may be null
   * @param shortName       variable shortName, must be unique within the Group
   * @param dims            list of dimension names, space delimited
   * @param units           unit string (may be null)
   * @param desc            description (may be null)
   */
  public StructureDS(NetcdfDataset ds, Group group, Structure parentStructure, String shortName,
                     String dims, String units, String desc) {

    super(ds, group, parentStructure, shortName);
    setDimensions(dims);
    this.proxy = new EnhancementsImpl(this, units, desc);

    if (units != null)
      addAttribute(new Attribute(CDM.UNITS, units));
    if (desc != null)
      addAttribute(new Attribute(CDM.LONG_NAME, desc));
  }

  /**
   * Create a StructureDS thats wraps a Structure
   *
   * @param g      parent group
   * @param orgVar original Structure
   */
  public StructureDS(Group g, ucar.nc2.Structure orgVar) { // , boolean reparent) {
    super(orgVar);
    setParentGroup(g);
    this.orgVar = orgVar;
    this.proxy = new EnhancementsImpl(this);

    // dont share cache, iosp : all IO is delegated
    this.ncfile = null;
    this.spiObject = null;
    createNewCache();

    if (orgVar instanceof StructureDS)
      return;

    // all member variables must be wrapped, reparented
    List<Variable> newList = new ArrayList<>(members.size());
    for (Variable v : members) {
      Variable newVar = convertVariable(g, v);
      newVar.setParentStructure(this);
      newList.add(newVar);
    }
    setMemberVariables(newList);
  }

  private Variable convertVariable(Group g, Variable v) {
    Variable newVar;
    if (v instanceof Sequence) {
      newVar = new SequenceDS(g, (Sequence) v);
    } else if (v instanceof Structure) {
      newVar = new StructureDS(g, (Structure) v);
    } else {
      newVar = new VariableDS(g, v, false); // enhancement done later
    }
    return newVar;
  }

  /**
   * Wrap the given Structure, making it into a StructureDS.
   * Delegate data reading to the original variable.
   * Does not share cache, iosp.
   * This is for NcML explicit mode
   *
   * @param ds        the containing NetcdfDataset.
   * @param group     the containing group; may not be null
   * @param parent    parent Structure, may be null
   * @param shortName variable shortName, must be unique within the Group
   * @param orgVar    the original Structure to wrap.
   */
  public StructureDS(NetcdfDataset ds, Group group, Structure parent, String shortName, Structure orgVar) {
    super(ds, group, parent, shortName);

    // dont share cache, iosp : all IO is delegated
    // this.ncfile = null;
    this.spiObject = null;
    createNewCache();

    this.orgVar = orgVar;
    this.proxy = new EnhancementsImpl(this);
  }

  // for section and slice and select

  @Override
  protected Variable copy() {
    return new StructureDS(getParentGroup(), this);
  }

  // copy() doesnt work because convert gets called twice

  @Override
  public Structure select(List<String> memberNames) {
    StructureDS result = new StructureDS(getParentGroup(), orgVar);
    List<Variable> members = new ArrayList<>();
    for (String name : memberNames) {
      Variable m = findVariable(name);
      if (null != m) members.add(m);
    }
    result.setMemberVariables(members);
    result.isSubset = true;
    return result;
  }

  /**
   * A StructureDS may wrap another Structure.
   *
   * @return original Structure or null
   */
  public ucar.nc2.Variable getOriginalVariable() {
    return orgVar;
  }

  /**
   * Set the Structure to wrap.
   *
   * @param orgVar original Variable, must be a Structure
   */
  public void setOriginalVariable(ucar.nc2.Variable orgVar) {
    if (!(orgVar instanceof Structure))
      throw new IllegalArgumentException("StructureDS must wrap a Structure; name=" + orgVar.getFullName());
    this.orgVar = (Structure) orgVar;
  }

  /**
   * When this wraps another Variable, get the original Variable's DataType.
   *
   * @return original Variable's DataType
   */
  public DataType getOriginalDataType() {
    return DataType.STRUCTURE;
  }

  /**
   * When this wraps another Variable, get the original Variable's DataType.
   *
   * @return original Variable's DataType
   */
  public String getOriginalName() {
    return orgName;
  }

  @Override
  public String setName(String newName) {
    this.orgName = getShortName();
    setShortName(newName);
    return newName;
  }

  // regular Variables.

  @Override
  public Array reallyRead(Variable client, CancelTask cancelTask) throws IOException {
    Array result;

    if (hasCachedData())
      result = super.reallyRead(client, cancelTask);
    else if (orgVar != null)
      result = orgVar.read();
    else {
      throw new IllegalStateException("StructureDS has no way to get data");
      //Object data = smProxy.getFillValue(getDataType());
      //return Array.factoryConstant(dataType.getPrimitiveClassType(), getShape(), data);
    }

    return convert(result, null);
  }

  // section of regular Variable

  @Override
  public Array reallyRead(Variable client, Section section, CancelTask cancelTask) throws IOException, InvalidRangeException {
    if (section.computeSize() == getSize())
      return _read();

    Array result;
    if (hasCachedData())
      result = super.reallyRead(client, section, cancelTask);
    else if (orgVar != null)
      result = orgVar.read(section);
    else {
      throw new IllegalStateException("StructureDS has no way to get data");
      //Object data = smProxy.getFillValue(getDataType());
      //return Array.factoryConstant(dataType.getPrimitiveClassType(), section.getShape(), data);
    }

    // do any needed conversions (enum/scale/offset/missing/unsigned, etc)
    return convert(result, section);
  }

  ///////////////////////////////////////

  // is conversion needed?

  private boolean convertNeeded(StructureMembers smData) {

    for (Variable v : getVariables()) {

      if (v instanceof VariableDS) {
        VariableDS vds = (VariableDS) v;
        if (vds.needConvert())
          return true;
      } else if (v instanceof StructureDS) {
        StructureDS nested = (StructureDS) v;
        if (nested.convertNeeded(null))
          return true;
      }

      // a variable with no data in the underlying smData
      if ((smData != null) && !varHasData(v, smData))
        return true;
    }

    return false;
  }

  // possible things needed:
  //   1) enum/scale/offset/missing/unsigned conversion
  //   2) name, info change
  //   3) variable with cached data added to StructureDS through NcML

  protected ArrayStructure convert(Array data, Section section) throws IOException {
    ArrayStructure orgAS = (ArrayStructure) data;
    if (!convertNeeded(orgAS.getStructureMembers())) {
      // name, info change only
      convertMemberInfo(orgAS.getStructureMembers());
      return orgAS;
    }

    // LOOK! converting to ArrayStructureMA
    // do any enum/scale/offset/missing/unsigned conversions
    ArrayStructure newAS = ArrayStructureMA.factoryMA(orgAS);
    for (StructureMembers.Member m : newAS.getMembers()) {
      VariableEnhanced v2 = (VariableEnhanced) findVariable(m.getName());
      if ((v2 == null) && (orgVar != null)) // these are from orgVar - may have been renamed
        v2 = findVariableFromOrgName(m.getName());
      if (v2 == null) continue;

      if (v2 instanceof VariableDS) {
        VariableDS vds = (VariableDS) v2;
        if (vds.needConvert()) {
          Array mdata = newAS.extractMemberArray(m);
          // mdata has not yet been enhanced, but vds would *think* that it has been if we used the 1-arg version of
          // VariableDS.convert(). So, we use the 2-arg version to explicitly request enhancement.
          mdata = vds.convert(mdata, vds.getEnhanceMode());
          newAS.setMemberArray(m, mdata);
        }

      } else if (v2 instanceof StructureDS) {
        StructureDS innerStruct = (StructureDS) v2;
        if (innerStruct.convertNeeded(null)) {

          if (innerStruct.getDataType() == DataType.SEQUENCE) {
            ArrayObject.D1 seqArray = (ArrayObject.D1) newAS.extractMemberArray(m);
            ArrayObject.D1 newSeq = (ArrayObject.D1) Array.factory(DataType.SEQUENCE, new int[] {(int) seqArray.getSize()});
            m.setDataArray(newSeq); // put back into member array

            // wrap each Sequence
            for (int i = 0; i < seqArray.getSize(); i++) {
              ArraySequence innerSeq = (ArraySequence) seqArray.get(i); // get old ArraySequence
              newSeq.set(i, new SequenceConverter(innerStruct, innerSeq)); // wrap in converter
            }

            // non-Sequence Structures
          } else {
            Array mdata = newAS.extractMemberArray(m);
            mdata = innerStruct.convert(mdata, null);
            newAS.setMemberArray(m, mdata);
          }

        }

        // always convert the inner StructureMembers
        innerStruct.convertMemberInfo(m.getStructureMembers());
      }
    }

    StructureMembers sm = newAS.getStructureMembers();
    convertMemberInfo(sm);

    // check for variables that have been added by NcML
    for (Variable v : getVariables()) {
      if (!varHasData(v, sm)) {
        try {
          Variable completeVar = getParentGroup().findVariable(v.getShortName()); // LOOK BAD
          Array mdata = completeVar.read(section);
          StructureMembers.Member m = sm.addMember(v.getShortName(), v.getDescription(), v.getUnitsString(), v.getDataType(), v.getShape());
          newAS.setMemberArray(m, mdata);
        } catch (InvalidRangeException e) {
          throw new IOException(e.getMessage());
        }
      }
    }

    return newAS;
  }

  /* convert original structureData to one that conforms to this Structure */

  protected StructureData convert(StructureData orgData, int recno) throws IOException {
    if (!convertNeeded(orgData.getStructureMembers())) {
      // name, info change only
      convertMemberInfo(orgData.getStructureMembers());
      return orgData;
    }

    // otherwise we create a new StructureData and convert to it. expensive
    StructureMembers smResult = new StructureMembers(orgData.getStructureMembers());
    StructureDataW result = new StructureDataW(smResult);

    for (StructureMembers.Member m : orgData.getMembers()) {
      VariableEnhanced v2 = (VariableEnhanced) findVariable(m.getName());
      if ((v2 == null) && (orgVar != null))     // why ?
        v2 = findVariableFromOrgName(m.getName());
      if (v2 == null) {
        findVariableFromOrgName(m.getName());  // debug
        // log.warn("StructureDataDS.convert Cant find member " + m.getName());
        continue;
      }
      StructureMembers.Member mResult = smResult.findMember(m.getName());

      if (v2 instanceof VariableDS) {
        VariableDS vds = (VariableDS) v2;
        Array mdata = orgData.getArray(m);

        if (vds.needConvert())
          // mdata has not yet been enhanced, but vds would *think* that it has been if we used the 1-arg version of
          // VariableDS.convert(). So, we use the 2-arg version to explicitly request enhancement.
          mdata = vds.convert(mdata, vds.getEnhanceMode());

        result.setMemberData(mResult, mdata);
      }

      // recurse into sub-structures
      if (v2 instanceof StructureDS) {
        StructureDS innerStruct = (StructureDS) v2;
        // if (innerStruct.convertNeeded(null)) {

        if (innerStruct.getDataType() == DataType.SEQUENCE) {
          Array a = orgData.getArray(m);

          if (a instanceof ArrayObject.D1) { // LOOK when does this happen vs ArraySequence?
            ArrayObject.D1 seqArray = (ArrayObject.D1) a;
            ArrayObject.D1 newSeq = (ArrayObject.D1) Array.factory(DataType.SEQUENCE, new int[] {(int) seqArray.getSize()});
            mResult.setDataArray(newSeq); // put into result member array

            for (int i = 0; i < seqArray.getSize(); i++) {
              ArraySequence innerSeq = (ArraySequence) seqArray.get(i); // get old ArraySequence
              newSeq.set(i, new SequenceConverter(innerStruct, innerSeq)); // wrap in converter
            }

          } else {
            ArraySequence seqArray = (ArraySequence) a;
            result.setMemberData(mResult, new SequenceConverter(innerStruct, seqArray)); // wrap in converter
          }

          // non-Sequence Structures
        } else {
          Array mdata = orgData.getArray(m);
          mdata = innerStruct.convert(mdata, null);
          result.setMemberData(mResult, mdata);
        }
        //}

        // always convert the inner StructureMembers
        innerStruct.convertMemberInfo(mResult.getStructureMembers());
      }
    }

    StructureMembers sm = result.getStructureMembers();
    convertMemberInfo(sm);

    // check for variables that have been added by NcML
    for (Variable v : getVariables()) {
      if (!varHasData(v, sm)) {
        try {
          Variable completeVar = getParentGroup().findVariable(v.getShortName()); // LOOK BAD
          Array mdata = completeVar.read(new Section().appendRange(recno, recno));
          StructureMembers.Member m = sm.addMember(v.getShortName(), v.getDescription(), v.getUnitsString(), v.getDataType(), v.getShape());
          result.setMemberData(m, mdata);
        } catch (InvalidRangeException e) {
          throw new IOException(e.getMessage());
        }
      }
    }

    return result;
  }

  // the wrapper StructureMembers must be converted to correspond to the wrapper Structure

  private void convertMemberInfo(StructureMembers wrapperSm) {
    for (StructureMembers.Member m : wrapperSm.getMembers()) {
      Variable v = findVariable(m.getName());
      if ((v == null) && (orgVar != null)) // may have been renamed
        v = (Variable) findVariableFromOrgName(m.getName());

      if (v != null) { // a section will have missing variables LOOK wrapperSm probably wrong in that case
        //  log.error("Cant find " + m.getName());
        //else
        m.setVariableInfo(v.getShortName(), v.getDescription(), v.getUnitsString(), v.getDataType());
      }

      // nested structures
      if (v instanceof StructureDS) {
        StructureDS innerStruct = (StructureDS) v;
        innerStruct.convertMemberInfo(m.getStructureMembers());
      }

    }
  }

  // look for the top variable that has an orgVar with the wanted orgName

  private VariableEnhanced findVariableFromOrgName(String orgName) {
    for (Variable vTop : getVariables()) {
      Variable v = vTop;
      while (v instanceof VariableEnhanced) {
        VariableEnhanced ve = (VariableEnhanced) v;
        if ((ve.getOriginalName() != null) && (ve.getOriginalName().equals(orgName)))
          return (VariableEnhanced) vTop;
        v = ve.getOriginalVariable();
      }
    }
    return null;
  }

  // verify that the variable has data in the data array

  private boolean varHasData(Variable v, StructureMembers sm) {
    if (sm.findMember(v.getShortName()) != null) return true;
    while (v instanceof VariableEnhanced) {
      VariableEnhanced ve = (VariableEnhanced) v;
      if (sm.findMember(ve.getOriginalName()) != null) return true;
      v = ve.getOriginalVariable();
    }
    return false;
  }

  /////////////////////////////////////////////////////////////////////////////////////////////////////////////

  private static class SequenceConverter extends ArraySequence {
    StructureDS orgStruct;
    ArraySequence orgSeq;

    SequenceConverter(StructureDS orgStruct, ArraySequence orgSeq) {
      super(orgSeq.getStructureMembers(), orgSeq.getShape());
      this.orgStruct = orgStruct;
      this.orgSeq = orgSeq;
      this.nelems = orgSeq.getStructureDataCount();

      // copy and convert the members
      members = new StructureMembers(orgSeq.getStructureMembers());
      orgStruct.convertMemberInfo(members);
    }

    @Override
    public StructureDataIterator getStructureDataIterator() { // throws java.io.IOException {
      return new StructureDataConverter(orgStruct, orgSeq.getStructureDataIterator());
    }
  }

  private static class StructureDataConverter implements StructureDataIterator {
    private StructureDataIterator orgIter;
    private StructureDS newStruct;
    private int count = 0;

    StructureDataConverter(StructureDS newStruct, StructureDataIterator orgIter) {
      this.newStruct = newStruct;
      this.orgIter = orgIter;
    }

    @Override
    public boolean hasNext() throws IOException {
      return orgIter.hasNext();
    }

    @Override
    public StructureData next() throws IOException {
      StructureData sdata = orgIter.next();
      return newStruct.convert(sdata, count++);
    }

    @Override
    public void setBufferSize(int bytes) {
      orgIter.setBufferSize(bytes);
    }

    @Override
    public StructureDataIterator reset() {
      orgIter = orgIter.reset();
      return (orgIter == null) ? null : this;
    }

    @Override
    public int getCurrentRecno() {
      return orgIter.getCurrentRecno();
    }

    @Override
    public void close() {
      orgIter.close();
    }
  }

  /* public StructureDataIterator getStructureIterator(int bufferSize) throws java.io.IOException {
    StructureDataIterator iter = orgVar.getStructureIterator(bufferSize);
    return new StructureDataConverter(this, iter);
  } */

  /* public StructureDataIterator getStructureIterator(int bufferSize) throws java.io.IOException {
     //StructureDataIterator iter = ncfile.getStructureIterator(this, bufferSize);
     //return (iter != null) ? iter : new Structure.Iterator(bufferSize);
     return new Structure.Iterator(bufferSize);
   } */

  /* Experimental - convert entire original array at once, to avoid one-by-one StructureData conversion.
private class Iterator implements StructureDataIterator {
  private int count = 0;
  private int recnum = (int) getSize();
  private int readStart = 0;
  private int readCount = 0;
  private int readAtaTime;
  private ArrayStructure as = null;
  private Index ii = null;

  protected Iterator(int bufferSize) {
    setBufferSize( bufferSize);
    reset();
  }

  public boolean hasNext() { return count < recnum; }

  public StructureDataIterator reset() {
    count = 0;
    readStart = 0;
    readCount = 0;
    ii = Index.factory(shape); // convert to nD index

    return this;
  }

  public StructureData next() throws IOException {
    if (count >= readStart) {
      if (getRank() == 1) readNextRank1();
      else readNextGeneralRank();
    }

    count++;
    return as.getStructureData( readCount++);
  }

  private void readNextRank1() throws IOException {
    int left = Math.min(recnum, readStart+readAtaTime); // dont go over recnum
    int need = left - readStart; // how many to read this time
    try {
      // System.out.println(" read start= "+readStart+" count= "+need);
      as = readStructure( readStart, need);

    } catch (InvalidRangeException e) {
      log.error("Structure.Iterator.readNext() ",e);
      throw new IllegalStateException("Structure.Iterator.readNext() ",e);
    } // cant happen

    readStart += need;
    readCount = 0;
  }

  private void readNextGeneralRank() throws IOException {
   ii.setCurrentCounter(index);
   int[] origin = ii.getCurrentCounter();
   section = new Section();
   for (int i=0;i<origin.length;i++)
     section.appendRange(origin[i], origin[i]);
  }

  public void setBufferSize(int bytes) {
    if (count > 0) return; // too late
    int structureSize = calcStructureSize();
    if (bytes <= 0)
      bytes = defaultBufferSize;
    readAtaTime = Math.max( 10, bytes / structureSize);
  }

}  */

  ///////////////////////////////////////////////////////////

  /**
   * DO NOT USE DIRECTLY. public by accident.
   * recalc any enhancement info
   */
  public void enhance(Set<NetcdfDataset.Enhance> mode) {
    for (Variable v : getVariables()) {
      VariableEnhanced ve = (VariableEnhanced) v;
      ve.enhance(mode);
    }
  }

  public void clearCoordinateSystems() {
    this.proxy = new EnhancementsImpl(this, getUnitsString(), getDescription());
  }

  public void addCoordinateSystem(ucar.nc2.dataset.CoordinateSystem p0) {
    proxy.addCoordinateSystem(p0);
  }

  public void removeCoordinateSystem(ucar.nc2.dataset.CoordinateSystem p0) {
    proxy.removeCoordinateSystem(p0);
  }

  public java.util.List<CoordinateSystem> getCoordinateSystems() {
    return proxy.getCoordinateSystems();
  }

  public java.lang.String getDescription() {
    return proxy.getDescription();
  }

  public java.lang.String getUnitsString() {
    return proxy.getUnitsString();
  }

  public void setUnitsString(String units) {
    proxy.setUnitsString(units);
  }

}