/*
 * Copyright (c) 1998-2019 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.ui.op;

import ucar.nc2.constants.CDM;
import ucar.nc2.iosp.bufr.*;
import ucar.nc2.iosp.bufr.tables.BufrTables;
import ucar.nc2.iosp.bufr.tables.TableB;
import ucar.nc2.ui.dialog.BufrBCompare;
import ucar.nc2.ui.widget.BAMutil;
import ucar.nc2.ui.widget.FileManager;
import ucar.nc2.ui.widget.IndependentWindow;
import ucar.nc2.ui.widget.PopupMenu;
import ucar.nc2.ui.widget.TextHistoryPane;
import ucar.nc2.units.SimpleUnit;
import ucar.unidata.util.StringUtil2;
import ucar.util.prefs.PreferencesExt;
import ucar.util.prefs.ui.BeanTable;
import ucar.unidata.io.RandomAccessFile;

import java.awt.*;
import java.awt.event.ActionEvent;
import java.beans.PropertyChangeEvent;
import java.beans.PropertyChangeListener;
import java.util.*;
import java.util.List;
import java.io.*;
import javax.swing.*;

/**
 * View BUFR Table B
 *
 * @author caron
 * @since Dec 1, 2009
 */

public class BufrTableBViewer extends JPanel {
  private PreferencesExt prefs;

  private BeanTable ddsTable, variantTable;
  private JSplitPane split;

  private TextHistoryPane compareTA;
  private IndependentWindow infoWindow;

  private TableB currTable, refTable;

  private FileManager fileChooser;

  public BufrTableBViewer(final PreferencesExt prefs, JPanel buttPanel) {
    this.prefs = prefs;

    ddsTable = new BeanTable(DdsBean.class, (PreferencesExt) prefs.node("DdsBean"), false);
    ddsTable.addListSelectionListener(e -> {
        DdsBean csb = (DdsBean) ddsTable.getSelectedBean();
        showVariants(csb);
    });

    variantTable = new BeanTable(DdsBean.class, (PreferencesExt) prefs.node("VariantBean"), false);
    /* variantTable.addListSelectionListener(e -> {
        variantTable.getSelectedBean();
    }); */

    PopupMenu varPopup = new PopupMenu(ddsTable.getJTable(), "Options");
    varPopup.addAction("Show uses", new AbstractAction() {
      public void actionPerformed(ActionEvent e) {
        Formatter out = new Formatter();
        DdsBean csb = (DdsBean) ddsTable.getSelectedBean();
        if (usedDds != null) {
          List<Message> list = usedDds.get(csb.getId());
          if (list != null) {
            for (Message use : list)
              use.dumpHeaderShort(out);
          }
        }
        compareTA.setText(out.toString());
        compareTA.gotoTop();
        infoWindow.setVisible(true);
      }
    });

    AbstractButton standardButton = BAMutil.makeButtcon("FontIncr", "Show union of tables", false);
    standardButton.addActionListener(e -> {
        showAll();
    });
    buttPanel.add(standardButton);

    AbstractButton diffButton = BAMutil.makeButtcon("Select", "Diff all variants", false);
    diffButton.addActionListener(e -> {
        diffVariants();
    });
    buttPanel.add(diffButton);

    AbstractButton compareButton = BAMutil.makeButtcon("Select", "Compare to standard table", false);
    compareButton.addActionListener(e -> {
        compareToStandard();
    });
    buttPanel.add(compareButton);

    AbstractAction refAction = new AbstractAction() {
      public void actionPerformed(ActionEvent e) {
        refTable = currTable;
        loadVariant(refTable.getName(), refTable);
      }
    };
    BAMutil.setActionProperties(refAction, "Dataset", "useAsRef", false, 'C', -1);
    BAMutil.addActionToContainer(buttPanel, refAction);

    AbstractAction usedAction = new AbstractAction() {
      public void actionPerformed(ActionEvent e) {
        try {
          if (fileChooser == null)
            fileChooser = new FileManager(null, null, null, (PreferencesExt) prefs.node("FileManager"));
          String filename = fileChooser.chooseFilename();
          if (filename == null) return;
          showUsed(filename);
        } catch (IOException e1) {
          e1.printStackTrace();
        }
      }
    };
    BAMutil.setActionProperties(usedAction, "dd", "checkUsed", false, 'C', -1);
    BAMutil.addActionToContainer(buttPanel, usedAction);

    AbstractAction diffAction = new AbstractAction() {
      public void actionPerformed(ActionEvent e) {
        try {
          String defloc = "C:/dev/tds/thredds/bufrTables/src/main/resources/resources/bufrTables/local";
          if (fileChooser == null)
            fileChooser = new FileManager(null, null, null, (PreferencesExt) prefs.node("FileManager"));

          String filename = fileChooser.chooseFilenameToSave(defloc + ".csv");
          if (filename == null) return;
          try (FileOutputStream out = new FileOutputStream(filename)) {
            OutputStreamWriter fout = new OutputStreamWriter(out, CDM.utf8Charset);
            BufferedWriter bw = new BufferedWriter(fout);
            Formatter f = new Formatter(bw);
            writeDiff(BufrTables.getWmoTableB(14), currTable, f);
          }
          JOptionPane.showMessageDialog(BufrTableBViewer.this, filename + " successfully written");

        } catch (Exception ex) {
          JOptionPane.showMessageDialog(BufrTableBViewer.this, "ERROR: " + ex.getMessage());
          ex.printStackTrace();
        }
      }
    };
    BAMutil.setActionProperties(diffAction, "dd", "write diff", false, 'C', -1);
    BAMutil.addActionToContainer(buttPanel, diffAction);

    AbstractAction localAction = new AbstractAction() {
      public void actionPerformed(ActionEvent e) {
        try {
          String defloc = "C:/dev/tds/thredds/bufrTables/src/main/resources/resources/bufrTables/local";
          if (fileChooser == null)
            fileChooser = new FileManager(null, null, null, (PreferencesExt) prefs.node("FileManager"));

          String filename = fileChooser.chooseFilenameToSave(defloc + ".csv");
          if (filename == null) return;
          try (FileOutputStream out = new FileOutputStream(filename)) {
            OutputStreamWriter fout = new OutputStreamWriter(out, CDM.utf8Charset);
            BufferedWriter bw = new BufferedWriter(fout);
            Formatter f = new Formatter(bw);
            writeLocal(currTable, f);
          }
          JOptionPane.showMessageDialog(BufrTableBViewer.this, filename + " successfully written");

        } catch (Exception ex) {
          JOptionPane.showMessageDialog(BufrTableBViewer.this, "ERROR: " + ex.getMessage());
          ex.printStackTrace();
        }
      }
    };
    BAMutil.setActionProperties(localAction, "dd", "write local", false, 'C', -1);
    BAMutil.addActionToContainer(buttPanel, localAction);

    // the info window
    compareTA = new TextHistoryPane();
    infoWindow = new IndependentWindow("Extra Information", BAMutil.getImage("netcdfUI"), compareTA);
    infoWindow.setBounds((Rectangle) prefs.getBean("InfoWindowBounds", new Rectangle(300, 300, 800, 600)));

    /* the info window 2
    infoTA2 = new TextHistoryPane();
    infoWindow2 = new IndependentWindow("Extra Information-2", BAMutil.getImage("netcdfUI"), infoTA2);
    infoWindow2.setBounds((Rectangle) prefs.getBean("InfoWindowBounds2", new Rectangle(300, 300, 500, 300)));

    split2 = new JSplitPane(JSplitPane.VERTICAL_SPLIT, false, ddsTable, obsTable);
    split2.setDividerLocation(prefs.getInt("splitPos2", 800)); */

    split = new JSplitPane(JSplitPane.VERTICAL_SPLIT, false, ddsTable, variantTable);
    split.setDividerLocation(prefs.getInt("splitPos", 500));

    setLayout(new BorderLayout());
    add(split, BorderLayout.CENTER);
  }

  public void save() {
    ddsTable.saveState(false);
    variantTable.saveState(false);
    prefs.putBeanObject("InfoWindowBounds", infoWindow.getBounds());
    //prefs.putBeanObject("InfoWindowBounds2", infoWindow2.getBounds());
    prefs.putInt("splitPos", split.getDividerLocation());
    //prefs.putInt("splitPos2", split2.getDividerLocation());
    if (fileChooser != null) fileChooser.save();
  }

  public void setBufrTableB(String filename, BufrTables.Format format) throws IOException {
    TableB tableB = BufrTables.readTableB(filename, format, true); // force read
    setBufrTableB(tableB);
  }

  private void setBufrTableB(TableB tb) throws IOException {
    String location = tb.getLocation();
    int pos = location.lastIndexOf("/");
    String src = (pos > 0) ? location.substring(pos + 1) : location;

    List<TableB.Descriptor> listDesc = new ArrayList<>(tb.getDescriptors());
    Collections.sort(listDesc);
    List<DdsBean> dds = new ArrayList<>(listDesc.size());
    for (TableB.Descriptor d : listDesc) {
      dds.add(new DdsBean(src, d));
    }
    ddsTable.setBeans(dds);
    currTable = tb;
  }

  private String checkDiff(TableB.Descriptor want) {
    if (refTable == null) return "n/a";
    for (TableB.Descriptor d1 : refTable.getDescriptors()) {
      if (d1.getId() != want.getId()) continue;
      if (d1.getScale() != want.getScale()) return "diff";
      if (d1.getRefVal() != want.getRefVal()) return "diff";
      if (d1.getDataWidth() != want.getDataWidth()) return "diff";
      return "";
    }
    return want.isLocal() ? "local" : "new";
  }

  /*
Class,FXY,enElementName,BUFR_Unit,BUFR_Scale,BUFR_ReferenceValue,BUFR_DataWidth_Bits,CREX_Unit,CREX_Scale,CREX_DataWidth
20,20009,General weather indicator (TAF/METAR),Code table,0,0,4,Code table,0,2
   */
  private void writeDiff(TableB wmo, TableB t, Formatter out) {
    out.format("#%n# BUFR diff written from %s against %s %n#%n", t.getName(), wmo.getName());
    out.format("Class,FXY,enElementName,BUFR_Unit,BUFR_Scale,BUFR_ReferenceValue,BUFR_DataWidth_Bits%n");
    List<TableB.Descriptor> listDesc = new ArrayList<>(t.getDescriptors());
    Collections.sort(listDesc);
    for (TableB.Descriptor d1 : listDesc) {
      TableB.Descriptor d2 = wmo.getDescriptor(d1.getId());
      if ((d2 == null) || (d1.getScale() != d2.getScale()) || (d1.getRefVal() != d2.getRefVal()) || (d1.getDataWidth() != d2.getDataWidth())) {
        short fxy = d1.getId();
//        int f = (fxy & 0xC000) >> 14;
        int x  = (fxy & 0x3F00) >> 8;
        int y  = fxy & 0xFF;
        out.format("%d,%2d%03d,\"%s\",%s,%d,%d,%d%n",x,x,y, d1.getName(), d1.getUnits(), d1.getScale(), d1.getRefVal(), d1.getDataWidth());
      }
    }
    out.flush();
  }

  private void writeLocal(TableB t, Formatter out) {
    out.format("#%n# BUFR local only from %s %n#%n", t.getName());
    out.format("Class,FXY,enElementName,BUFR_Unit,BUFR_Scale,BUFR_ReferenceValue,BUFR_DataWidth_Bits%n");
    List<TableB.Descriptor> listDesc = new ArrayList<>(t.getDescriptors());
    Collections.sort(listDesc);
    for (TableB.Descriptor d1 : listDesc) {
      if (d1.isLocal()) {
        short fxy = d1.getId();
//        int f = (fxy & 0xC000) >> 14;
        int x  = (fxy & 0x3F00) >> 8;
        int y  = fxy & 0xFF;
        out.format("%d,%d%03d,\"%s\",%s,%d,%d,%d%n",x,x,y,d1.getName(), d1.getUnits(), d1.getScale(), d1.getRefVal(), d1.getDataWidth());
      }
    }
    out.flush();
  }

  private char[] remove = new char[]{'(', ')', ' ', '"', ',', '*', '-'};
  private String[] replace = new String[]{"", "", "", "", "", "", ""};

  private boolean equiv(String org1, String org2) {
    String s1 = StringUtil2.replace(org1, remove, replace).toLowerCase();
    String s2 = StringUtil2.replace(org2, remove, replace).toLowerCase();
    return s1.equals(s2);
  }

  private boolean equivUnits(String unitS1, String unitS2) {
    String lower1 = unitS1.toLowerCase();
    String lower2 = unitS2.toLowerCase();
    if (lower1.equals(lower2)) return true;
    if (lower1.startsWith("code") && lower2.startsWith("code")) return true;
    if (lower1.startsWith("flag") && lower2.startsWith("flag")) return true;
    if (unitS1.startsWith("CCITT") && unitS2.startsWith("CCITT")) return true;

    try {
      return SimpleUnit.isCompatibleWithExceptions(unitS1, unitS2);

    } catch (Exception e) {
      return equiv(unitS1, unitS2);
    }
  }

  //////////////////////////////////////////////////////////

  private BufrBCompare dialog = null;
  public void compareToStandard() {
    if (dialog == null) {
      dialog = new BufrBCompare(null);
      dialog.pack();
      dialog.addPropertyChangeListener("OK", new PropertyChangeListener() {
        public void propertyChange(PropertyChangeEvent evt) {
          compareToStandard((BufrBCompare.Data) evt.getNewValue());
        }
      });
    }
    dialog.setVisible(true);
  }

  private void compareToStandard(BufrBCompare.Data data) {

    try {
      Formatter out = new Formatter();
      if (refTable == null) {
        refTable = BufrTables.getWmoTableBlatest();
      }
      compare(refTable, !data.compareNames, !data.compareUnits, out);

      compareTA.setText(out.toString());
      compareTA.gotoTop();
      infoWindow.setVisible(true);

    } catch (Throwable ioe) {
      StringWriter sw = new StringWriter(10000);
      ioe.printStackTrace(new PrintWriter(sw));
      compareTA.setText(sw.toString());
      compareTA.gotoTop();
      infoWindow.setVisible(true);
    }
  }

  private void compare(TableB t2, boolean skipNames, boolean skipUnits, Formatter out) {
    Map<Short, DdsBean> mapBeans = new HashMap<>(3000);

    out.format("Compare Current Table (%s) to %s %n", currTable.getName(), t2.getName());
    for (Object beano : ddsTable.getBeans()) {
      DdsBean bean = (DdsBean) beano;
      TableB.Descriptor d1 = bean.dds;
      TableB.Descriptor d2 = t2.getDescriptor(d1.getId());
      if (d2 == null)
        out.format("**No key %s in second table; local=%s%n", d1.getFxy(), d1.isLocal());
      else {
        if (!skipNames) {
          if (!equiv(d1.getName(), d2.getName()))
            out.format(" %s name%n   %s%n   %s%n", d1.getFxy(), d1.getName(), d2.getName());
        }
        if (!skipUnits) {
          if (!equivUnits(d1.getUnits(), d2.getUnits()))
            out.format(" %s units%n   %s%n   %s%n", d1.getFxy(), d1.getUnits(), d2.getUnits());
        }
        if (d1.getScale() != d2.getScale())
          out.format(" %s scale %d != %d %n", d1.getFxy(), d1.getScale(), d2.getScale());
        if (d1.getRefVal() != d2.getRefVal())
          out.format(" %s refVal %d != %d %n", d1.getFxy(), d1.getRefVal(), d2.getRefVal());
        if (d1.getDataWidth() != d2.getDataWidth())
          out.format(" %s bitWidth %d != %d %n", d1.getFxy(), d1.getDataWidth(), d2.getDataWidth());
      }
      mapBeans.put(d1.getId(), bean);
    }

    // see whats missing
    List<TableB.Descriptor> listDesc2 = new ArrayList<>(t2.getDescriptors());
    out.format("%n Missing in first table %n");
    Collections.sort(listDesc2);
    for (TableB.Descriptor d2 : listDesc2) {
      DdsBean bean = mapBeans.get(d2.getId());
      if (bean == null) {
        out.format("   %s%n", d2.getFxy());
      }
    }
  }

  ///////////////////////

  private HashMap<Short, List<Message>> usedDds = null;

  private void showUsed(String filename) throws IOException {
    usedDds = new HashMap<>(3000);
    scanFileForDds(filename);
  }

  public void scanFileForDds(String filename) throws IOException {
    RandomAccessFile raf = new RandomAccessFile(filename, "r");

    MessageScanner scan = new MessageScanner(raf);
    while (scan.hasNext()) {
      Message m = scan.next();
      if (m == null) continue;
      // String src = m.getHeader().trim() +"("+Integer.toHexString(m.hashCode())+")";
      setDataDescriptors(m, m.getRootDataDescriptor());
    }
  }

  private void setDataDescriptors(Message src, DataDescriptor dds) {
    for (DataDescriptor key : dds.getSubKeys()) {
      List<Message> list = usedDds.get(key.getFxy());
      if (list == null) {
        list = new ArrayList<>();
        usedDds.put(key.getFxy(), list);
      }
      if (!list.contains(src))
        list.add(src);

      if (key.getSubKeys() != null)
        setDataDescriptors(src, key);
    }
  }
  ////////////////////////////////////////////////////////

  /*private HashMap<String, TableB> standardTables = null;
  private void initStandardTables() throws IOException {
    standardTables = new HashMap<String, TableB>();
    standardTables.put("WMO.07", BufrTables.getWmoTableB(7));
    standardTables.put("WMO.08", BufrTables.getWmoTableB(8));
    standardTables.put("WMO.09", BufrTables.getWmoTableB(9));
    standardTables.put("WMO.10", BufrTables.getWmoTableB(10));
    standardTables.put("WMO.11", BufrTables.getWmoTableB(11));
    standardTables.put("WMO.12", BufrTables.getWmoTableB(12));
    standardTables.put("WMO.13", BufrTables.getWmoTableB(13));
    standardTables.put("WMO.14", BufrTables.getWmoTableB(14));
    standardTables.put("WMO.15", BufrTables.getWmoTableB(15));
    standardTables.put("WMO.16", BufrTables.getWmoTableB(16));
  }

  private TableB getStandardTable(String name) throws IOException {
    if (standardTables == null) initStandardTables();
    return standardTables.get(name);
  }

  private void loadStandardVariants() throws IOException {
    if (standardTables == null) initStandardTables();
    for (String key : standardTables.keySet()) {
      loadVariant(key, standardTables.get(key));
    }
    standardVariantsLoaded = true;
  }
  */

  //////////////////////////////////////////////////////
  private HashMap<Short, List<DdsBean>> allVariants = null;
  private Set<String> varKeys = null;
  private boolean standardVariantsLoaded = false;
  private void loadStandardVariants() {
    for (BufrTables.TableConfig tc : BufrTables.getTables()) {
      try {
        if (!tc.getName().startsWith("FNMOC"))  // exclude crap
          loadVariant(tc.getName(), BufrTables.readTableB(tc.getTableBname(), tc.getTableBformat(), false));
      } catch (IOException e) {
        e.printStackTrace();
      }
    }
    standardVariantsLoaded = true;
  }

  private void loadVariant(String key, TableB tableB) {
    if (allVariants == null) allVariants = new HashMap<>();
    if (varKeys == null) varKeys = new HashSet<>();
    if (varKeys.contains(key)) return; // dont add again

    List<TableB.Descriptor> listDesc = new ArrayList<>(tableB.getDescriptors());
    for (TableB.Descriptor d : listDesc) {
      List<DdsBean> list = allVariants.get(d.getId());
      if (list == null) {
        list = new ArrayList<>(10);
        allVariants.put(d.getId(), list);
      }
      list.add(new DdsBean(key, d));
    }
  }

  private void showVariants(DdsBean bean) {
    if (!standardVariantsLoaded) loadStandardVariants();

    List<DdsBean> all = allVariants.get(bean.getId());
    List<DdsBean> ddsBean = new ArrayList<>(10);
    ddsBean.add(bean);
    if (all != null) ddsBean.addAll(all);
    variantTable.setBeans(ddsBean);
  }

  private void showAll() {
    if (!standardVariantsLoaded) loadStandardVariants();

    Map<Short, DdsBean> allDesc = new HashMap<>(3000);
    Set<String> keys = new HashSet<>();

    for (BufrTables.TableConfig tc : BufrTables.getTables()) {
      String filename = tc.getTableBname();
      if (keys.contains(filename)) continue;
      keys.add(filename);

      try {
        TableB tb = BufrTables.readTableB(filename, tc.getTableBformat(), false);
        addVariant(tc.getName(), tb, allDesc);
      } catch (IOException e) {
        e.printStackTrace();
      }
    }

    List<DdsBean> beans = new ArrayList<>(allDesc.values());
    Collections.sort(beans);
    ddsTable.setBeans(beans);
    currTable = null;
  }

  private void addVariant(String key, TableB tableB, Map<Short, DdsBean> allDesc) {
    List<TableB.Descriptor> listDesc = new ArrayList<>(tableB.getDescriptors());
    for (TableB.Descriptor d : listDesc) {
      if (!allDesc.containsKey(d.getId()))
        allDesc.put(d.getId(), new DdsBean(key, d));
    }
  }

  private void diffVariants() {
    if (!standardVariantsLoaded) loadStandardVariants();

    Set<String> keys = new HashSet<>();

    for (Object beano : ddsTable.getBeans()) {
      DdsBean bean = (DdsBean) beano;
      bean.setDiff(null);
    }

    for (BufrTables.TableConfig tc : BufrTables.getTables()) {
      if (tc.getName().startsWith("FNMOC")) continue; // exclude crap

      String filename = tc.getTableBname();
      if (keys.contains(filename)) continue;
      keys.add(filename);

      try {
        TableB tb = BufrTables.readTableB(filename, tc.getTableBformat(), false);
        diffVariant(tb, true, true);
      } catch (IOException e) {
        e.printStackTrace();
      }
    }

    int count = 0;
    int total = 0;
    for (Object beano : ddsTable.getBeans()) {
      DdsBean bean = (DdsBean) beano;
      if (bean.getDiff() != null)
        count++;
      if (!bean.dds.isLocal())
        total++;
    }
    System.out.printf("Diff non-local = %d out of %d%n", count, total);
  }

  private void diffVariant(TableB t2, boolean skipNames, boolean skipUnits) {

    for (Object beano : ddsTable.getBeans()) {
      DdsBean bean = (DdsBean) beano;
      if (bean.getDiff() != null) continue;
      TableB.Descriptor d1 = bean.dds;
      if (d1.isLocal()) continue;

      TableB.Descriptor d2 = t2.getDescriptor(d1.getId());
      if (d2 != null) {
        if (!skipNames) {
          if (!equiv(d1.getName(), d2.getName()))
            bean.setDiff("diff");
        }
        if (!skipUnits) {
          if (!equivUnits(d1.getUnits(), d2.getUnits()))
            bean.setDiff("diff");
        }
        if (d1.getScale() != d2.getScale())
          bean.setDiff("diff");
        if (d1.getRefVal() != d2.getRefVal())
          bean.setDiff("diff");
        if (d1.getDataWidth() != d2.getDataWidth())
          bean.setDiff("diff");
      }
    }

  }

  ///////////////////////////////////////////////////////////////////////////////////////
  public class DdsBean implements Comparable<DdsBean> {
    TableB.Descriptor dds;
    String source;
    String udunits;
    boolean checkDiff;
    String isDiff;

    // create from a dataset
    public DdsBean(String source, TableB.Descriptor dds) {
      this.source = source;
      this.dds = dds;
    }

    public String getFxy() {
      return dds.getFxy();
    }

    public short getId() {
      return dds.getId();
    }

    public String getSource() {
      return source;
    }

    public String getName() {
      return dds.getName();
    }

    public String getUnits() {
      return dds.getUnits();
    }

    public String getDesc() {
      return dds.getDesc();
    }

    public String getUdunits() {
      if (udunits == null) {
        try {
          SimpleUnit su = SimpleUnit.factoryWithExceptions(dds.getUnits());
          udunits = su.isUnknownUnit() ? "" : su.toString();

        } catch (Exception ioe) {
          udunits = " unit convert failed ";
        }
      }
      return udunits;
    }

    public int getWidth() {
      return dds.getDataWidth();
    }

    public int getScale() {
      return dds.getScale();
    }

    public int getReference() {
      return dds.getRefVal();
    }

    public boolean isNumeric() {
      return dds.isNumeric();
    }

    public boolean isLocal() {
      return dds.isLocal();
    }

    public String getDiff() {
      if (!checkDiff && refTable != null) {
        isDiff = checkDiff(dds);
        checkDiff = true;
      }
      return isDiff;
    }

    public void setDiff(String diff) {
      checkDiff = (diff != null);
      isDiff = diff;
    }

    public int getUsed() {
      if (usedDds == null) return 0;
      List<Message> list = usedDds.get(dds.getId());
      if (list == null) return 0;
      return list.size();
    }

    @Override
    public int compareTo(DdsBean o1) {
      return dds.compareTo(o1.dds);
    }
  }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.ft.fmrc;

import thredds.featurecollection.FeatureCollectionConfig;
import ucar.ma2.*;
import ucar.nc2.*;
import ucar.nc2.constants.*;
import ucar.nc2.dataset.*;
import ucar.nc2.dt.GridCoordSystem;
import ucar.nc2.dt.GridDatatype;
import ucar.nc2.dt.grid.GridDataset;
import ucar.nc2.ncml.NcMLReader;
import ucar.nc2.time.CalendarDate;
import ucar.nc2.time.CalendarDateRange;
import ucar.nc2.util.CancelTask;

import javax.annotation.concurrent.ThreadSafe;
import java.io.IOException;
import java.util.*;

/**
 * Helper class for Fmrc.
 * The various GridDatasets must be thread-safe.
 *

 * Time coordinate values come from the FmrcInv, so there is little I/O here.
 * Non-aggregation variables are either cached or have DatasetProxyReaders set so the file is opened when the variable needs to be read.
 *

 * The prototype dataset is kept separate, since the common case is that just the time coordinates have changed.
 *

 * This replaces ucar.nc2.dt.fmrc.FmrcImpl
 *
 *
 * @author caron
 * @since Jan 19, 2010
 */
@ThreadSafe
class FmrcDataset {
  static private final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FmrcDataset.class);
  static private final boolean debugEnhance = false, debugRead = false;

  private final FeatureCollectionConfig config;
  // private final Element ncmlOuter, ncmlInner;

  //private List<String> protoList; // the list of datasets in the proto that have proxy reader, so these need to exist. not implemented yet

  // allow to build a new state while old state can still be safely used
  private static class State {
    NetcdfDataset proto; // once built, the proto doesnt change until setInventory is called with forceProto = true
    FmrcInvLite lite; // lightweight version of the FmrcInv

    private State(NetcdfDataset proto, FmrcInvLite lite) {
      this.proto = proto;
      this.lite = lite;
    }
  }
  private State state;
  private final Object lock= new Object();

  FmrcDataset(FeatureCollectionConfig config) { // }, Element ncmlInner, Element ncmlOuter) {
    this.config = config;
    //this.ncmlInner = ncmlInner;
    //this.ncmlOuter = ncmlOuter;
  }

  List<CalendarDate> getRunDates() {
    State localState;
    synchronized (lock) {
      localState = state;
    }
    return localState.lite.getRunDates();
  }

  List<CalendarDate> getForecastDates() {
    State localState;
    synchronized (lock) {
      localState = state;
    }
    return localState.lite.getForecastDates();
  }

  // for making offset datasets
  double[] getForecastOffsets() {
    State localState;
    synchronized (lock) {
      localState = state;
    }
    return localState.lite.getForecastOffsets();
  }

  public CalendarDateRange getDateRangeForRun(CalendarDate run) {
    State localState;
    synchronized (lock) {
      localState = state;
    }
    int runidx = localState.lite.findRunIndex(run);
    if (runidx < 0) return null;

    double min = Double.MAX_VALUE;
    double max = Double.MIN_VALUE;
    for (FmrcInvLite.Gridset gs : localState.lite.gridSets) {
      for (int i=0; i<gs.noffsets; i++) {
        double time = gs.getTimeCoord(runidx, i);
        if (Double.isNaN(time)) continue;
        min = Math.min(min, time);
        max = Math.max(max, time);
      }
    }
    return CalendarDateRange.of(FmrcInv.makeOffsetDate(localState.lite.base, min), FmrcInv.makeOffsetDate(localState.lite.base, max));
  }

  public CalendarDateRange getDateRangeForOffset(double offset) {
    State localState;
    synchronized (lock) {
      localState = state;
    }
    List<CalendarDate> runs = localState.lite.getRunDates();
    int n = runs.size();
    return CalendarDateRange.of(FmrcInv.makeOffsetDate(runs.get(0), offset), FmrcInv.makeOffsetDate(runs.get(n-1), offset));
  }

  /**
   * Make the 2D dataset
   * @param result     use this empty NetcdfDataset, may be null (used by NcML)
   * @return 2D dataset
   * @throws IOException on read error
   */
  GridDataset getNetcdfDataset2D(NetcdfDataset result) throws IOException {
    State localState;
    synchronized (lock) {
      localState = state;
    }
    return buildDataset2D(result,  localState.proto, localState.lite);
  }

  GridDataset getBest() throws IOException {
    State localState;
    synchronized (lock) {
      localState = state;
    }
    return buildDataset1D( localState.proto, localState.lite, localState.lite.makeBestDatasetInventory());
  }

  GridDataset getBest(FeatureCollectionConfig.BestDataset bd) throws IOException {
    State localState;
    synchronized (lock) {
      localState = state;
    }
    return buildDataset1D( localState.proto, localState.lite, localState.lite.makeBestDatasetInventory(bd));
  }

  GridDataset getRunTimeDataset(CalendarDate run) throws IOException {
    State localState;
    synchronized (lock) {
      localState = state;
    }
    return buildDataset1D(  localState.proto, localState.lite, localState.lite.makeRunTimeDatasetInventory( run));
  }

  GridDataset getConstantForecastDataset(CalendarDate run) throws IOException {
    State localState;
    synchronized (lock) {
      localState = state;
    }
    return buildDataset1D(  localState.proto, localState.lite, localState.lite.getConstantForecastDataset( run));
  }

  GridDataset getConstantOffsetDataset(double offset) throws IOException {
    State localState;
    synchronized (lock) {
      localState = state;
    }
    return buildDataset1D(  localState.proto, localState.lite, localState.lite.getConstantOffsetDataset( offset));
  }

  /**
   * Set a new FmrcInv and optionally a proto dataset
   *
   * @param fmrcInv    based on this inventory
   * @param forceProto create a new proto, else use existing one
   * @throws IOException on read error
   */
  void setInventory(FmrcInv fmrcInv, boolean forceProto) throws IOException {
    NetcdfDataset protoLocal = null;

    // make new proto if needed
    if (state == null || forceProto) {
      protoLocal = buildProto(fmrcInv, config.protoConfig);
    }

    // switch to FmrcInvLite to reduce memory usage
    FmrcInvLite liteLocal = new FmrcInvLite(fmrcInv);
    synchronized (lock) {
      if (protoLocal == null && state != null) protoLocal = state.proto;
      state = new State(protoLocal, liteLocal);
    }
  }

   /////////////////////////////////////////////////////////////////////////////
  // the prototypical dataset

  private NetcdfDataset buildProto(FmrcInv fmrcInv, FeatureCollectionConfig.ProtoConfig protoConfig) throws IOException {
    NetcdfDataset result = new NetcdfDataset(); // empty

    // choose some run in the list
    List<FmrInv> list = fmrcInv.getFmrInv();
    if (list.size() == 0) {
      logger.error("Fmrc collection is empty ="+fmrcInv.getName());
      throw new IllegalStateException("Fmrc collection is empty ="+fmrcInv.getName());
    }

    int protoIdx = 0;
    switch (protoConfig.choice) {
      case First:
        protoIdx = 0;
        break;
      case Random:
        Random r = new Random(System.currentTimeMillis());
        protoIdx = r.nextInt(list.size() - 1);
        break;
      case Penultimate:
        protoIdx = Math.max(list.size() - 2, 0);
        break;
      case Latest:
        protoIdx = Math.max(list.size() - 1, 0);
        break;

      case Run:
        int runOffset = 0;
        if (protoConfig.param != null)
          runOffset = Integer.parseInt(protoConfig.param);
        for (int i=0; i<list.size(); i++) {
          FmrInv fmr = list.get(i);
          CalendarDate cd = fmr.getRunDate();
          int hour = cd.getHourOfDay();
          if (hour == runOffset)
            protoIdx = i; // later ones override
        }
        break;
    }
    FmrInv proto = list.get(protoIdx); // use this one

    Map<String, NetcdfDataset> openFilesProto = new HashMap<>();

    try {
      // create the union of all objects in that run
      // this covers the case where the variables are split across files
      Set<GridDatasetInv> files = proto.getFiles();
      if (logger.isDebugEnabled())
        logger.debug("FmrcDataset: proto= " + proto.getName() + " " + proto.getRunDate() + " collection= " + fmrcInv.getName());
      for (GridDatasetInv file : files) {
        NetcdfDataset ncfile = open(file.getLocation(), openFilesProto);
        if (ncfile != null)
          transferGroup(ncfile.getRootGroup(), result.getRootGroup(), result);
        else
          logger.warn("Failed to open "+file.getLocation());
        if (logger.isDebugEnabled()) logger.debug("FmrcDataset: proto dataset= " + file.getLocation());
      }

      // some additional global attributes
      Group root = result.getRootGroup();
      Attribute orgConv =  root.findAttributeIgnoreCase(CDM.CONVENTIONS);
      String convAtt = CoordSysBuilder.buildConventionAttribute("CF-1.4", (orgConv == null ? null : orgConv.getStringValue()));
      root.addAttribute(new Attribute(CDM.CONVENTIONS, convAtt));
      root.addAttribute(new Attribute("cdm_data_type", FeatureType.GRID.toString()));
      root.addAttribute(new Attribute(CF.FEATURE_TYPE, FeatureType.GRID.toString()));
      root.addAttribute(new Attribute("location", "Proto "+fmrcInv.getName()));

      // remove some attributes that can cause trouble
      root.remove(root.findAttribute(_Coordinate.ModelRunDate));

      // protoList = new ArrayList<String>();
      // these are the non-agg variables - store data or ProxyReader in proto
      List<Variable> copyList = new ArrayList<>(root.getVariables()); // use copy since we may be removing some variables
      for (Variable v : copyList) {
        // see if its a non-agg variable
        FmrcInv.UberGrid grid = fmrcInv.findUberGrid(v.getFullName());
        if (grid == null) { // only non-agg vars need to be cached
          Variable orgV = (Variable) v.getSPobject();
          if (orgV.getSize() > 10 * 1000 * 1000) {
            logger.info("FMRCDataset build Proto cache >10M var= "+orgV.getNameAndDimensions());
          } else {
            v.setCachedData(orgV.read()); // read from original - store in proto
          }
        }

        v.setSPobject(null); // clear the reference to orgV for all of proto
      }

      result.finish();

      // enhance the proto. becomes the template for coordSystems in the derived datasets
      CoordSysBuilderIF builder = result.enhance();
      if (debugEnhance) System.out.printf("proto.enhance() parseInfo = %s%n", builder.getParseInfo());

      // turn it into a GridDataset, so we can add standard metadata to result, not dependent on CoordSysBuilder
      // also see  ucar.nc2.dt.grid.NetcdfCFWriter - common code could be extracted
      Formatter parseInfo = new Formatter();
      GridDataset gds = new ucar.nc2.dt.grid.GridDataset(result, parseInfo); // LOOK not sure coord axes will read ??
      if (debugEnhance) System.out.printf("proto GridDataset parseInfo = %s%n", parseInfo);

      // now make standard CF metadata for gridded data
      for (GridDatatype grid : gds.getGrids()) {
        Variable newV = result.findVariable(grid.getFullName());
        if (newV == null) {
          logger.warn("FmrcDataset cant find "+grid.getFullName()+" in proto gds ");
          continue;
        }

        // annotate Variable for CF
        StringBuilder sbuff = new StringBuilder();
        GridCoordSystem gcs = grid.getCoordinateSystem();
        for (CoordinateAxis axis : gcs.getCoordinateAxes()) {
          if ((axis.getAxisType() != AxisType.Time) && (axis.getAxisType() != AxisType.RunTime)) // these are added later
            sbuff.append(axis.getFullName()).append(" ");
        }
        newV.addAttribute(new Attribute(CF.COORDINATES, sbuff.toString())); // LOOK what about adding lat/lon variable

        // looking for coordinate transform variables
        for (CoordinateTransform ct : gcs.getCoordinateTransforms()) {
          Variable ctv = result.findVariable(ct.getName());
          if ((ctv != null) && (ct.getTransformType() == TransformType.Projection))
            newV.addAttribute(new Attribute(CF.GRID_MAPPING, ctv.getFullName()));
        }

        // LOOK is this needed ?
        for (CoordinateAxis axis : gcs.getCoordinateAxes()) {
          Variable coordV = result.findVariable(axis.getFullNameEscaped());
          if ((axis.getAxisType() == AxisType.Height) || (axis.getAxisType() == AxisType.Pressure) || (axis.getAxisType() == AxisType.GeoZ)) {
            if (null != axis.getPositive())
              coordV.addAttribute(new Attribute(CF.POSITIVE, axis.getPositive()));
          }
          if (axis.getAxisType() == AxisType.Lat) {
            coordV.addAttribute(new Attribute(CDM.UNITS, CDM.LAT_UNITS));
            coordV.addAttribute(new Attribute(CF.STANDARD_NAME, "latitude"));
          }
          if (axis.getAxisType() == AxisType.Lon) {
            coordV.addAttribute(new Attribute(CDM.UNITS, CDM.LON_UNITS));
            coordV.addAttribute(new Attribute(CF.STANDARD_NAME, "longitude"));
          }
          if (axis.getAxisType() == AxisType.GeoX) {
            coordV.addAttribute(new Attribute(CF.STANDARD_NAME, "projection_x_coordinate"));
          }
          if (axis.getAxisType() == AxisType.GeoY) {
            coordV.addAttribute(new Attribute(CF.STANDARD_NAME, "projection_y_coordinate"));
          }
          if (axis.getAxisType() == AxisType.Time) {
            Attribute att = axis.findAttribute("bounds");  // LOOK nasty : remove time bounds from proto
            if ((att != null) && att.isString())
              result.removeVariable(null, att.getStringValue());
          }
        }
      }

      // more troublesome attributes, use pure CF
      for (Variable v : result.getVariables()) {
        Attribute att;
        if (null != (att = v.findAttribute(_Coordinate.Axes)))
           v.remove(att);
        if (null != (att = v.findAttribute(_Coordinate.Systems)))
           v.remove(att);
        if (null != (att = v.findAttribute(_Coordinate.SystemFor)))
          v.remove(att);
        if (null != (att = v.findAttribute(_Coordinate.Transforms)))
          v.remove(att);
      }

      // apply ncml if it exists
      if (protoConfig.outerNcml != null)
        NcMLReader.mergeNcMLdirect(result, protoConfig.outerNcml);

      return result;

    } finally {
      // data is read and cached, can close files now
      closeAll(openFilesProto);
    }
  }

  // transfer the objects in src group to the target group, unless that name already exists
  // Dimensions, Variables, Groups are not transferred, but an equivalent object is created with same metadata
  // Attributes and EnumTypedef are transferred, these are immutable with no references to container

  private void transferGroup(Group srcGroup, Group targetGroup, NetcdfDataset target) throws IOException {
    // group attributes
    DatasetConstructor.transferGroupAttributes(srcGroup, targetGroup);

    // dimensions
    for (Dimension d : srcGroup.getDimensions()) {
      if (null == targetGroup.findDimensionLocal(d.getShortName())) {
        Dimension newd = new Dimension(d.getShortName(), d.getLength(), d.isShared(), d.isUnlimited(), d.isVariableLength());
        targetGroup.addDimension(newd);
      }
    }

    // transfer variables - eliminate any references to component files
    for (Variable v : srcGroup.getVariables()) {
      Variable targetV = targetGroup.findVariable(v.getShortName());

      if (null == targetV) { // add it
        if (v instanceof Structure) {
          targetV = new StructureDS(target, targetGroup, null, v.getShortName(), v.getDimensionsString(), v.getUnitsString(), v.getDescription());
          //LOOK - not adding the members here - what to do ??

        } else {
          targetV = new VariableDS(target, targetGroup, null, v.getShortName(), v.getDataType(), v.getDimensionsString(), v.getUnitsString(), v.getDescription());
        }

        DatasetConstructor.transferVariableAttributes(v, targetV);
        VariableDS vds = (VariableDS) v;
        targetV.setSPobject(vds); //temporary, for non-agg variables when proto is made
        if (vds.hasCachedDataRecurse()) {
          if (vds.getSize() > 1000 * 1000) {
            boolean wtf = vds.hasCachedDataRecurse();
          }
          targetV.setCachedData(vds.read()); //
        }
        targetGroup.addVariable(targetV);
      }
    }

    // nested groups - check if target already has it
    for (Group srcNested : srcGroup.getGroups()) {
      Group nested = targetGroup.findGroup(srcNested.getShortName());
      if (null == nested) {
        nested = new Group(target, targetGroup, srcNested.getShortName());
        targetGroup.addGroup(nested);
        for (EnumTypedef et : srcNested.getEnumTypedefs()) {
          targetGroup.addEnumeration(et);
        }
      }
      transferGroup(srcNested, nested, target);
    }
  }

  /////////////////////////////////////////////////////////
  // constructing the dataset

  // private static final String FTIME_PREFIX = "";

  private String getRunDimensionName() {
    return "run";
  }

  private String makeCoordinateList(VariableDS aggVar, String timeCoordName, boolean is2D) {
    String coords = "";
    Attribute att = aggVar.findAttribute(CF.COORDINATES);
    if (att == null)
      att = aggVar.findAttribute(_Coordinate.Axes);
    if (att != null)
      coords = att.getStringValue();

    if (is2D)
      return getRunDimensionName() + " " + timeCoordName + " " + coords;
    else
      return timeCoordName + "_" +  getRunDimensionName() + " " + timeCoordName + " " + coords;
  }

  private void addAttributeInfo(NetcdfDataset result, String attName, String info) {
    Attribute att = result.findGlobalAttribute(attName);
    if (att == null)
      result.addAttribute(null, new Attribute(attName, info));
    else {
      String oldValue = att.getStringValue();
      result.addAttribute(null, new Attribute(attName, oldValue +" ;\n"+ info));
    }
  }

  /**
   * Build the 2D time dataset, make it immutable so it can be shared across threads
   *
   * @param result     place results in here, if null create a new one. must be threadsafe (immutable)
   * @param proto      current proto dataset
   * @param lite       current inventory
   * @return resulting GridDataset
   * @throws IOException on read error
   */
  private GridDataset buildDataset2D(NetcdfDataset result, NetcdfDataset proto, FmrcInvLite lite) throws IOException {
    if (lite == null) return null;
    // make a copy, so that this object can coexist with previous incarnations
    if (result == null) result = new NetcdfDataset();
    result.setLocation(lite.collectionName);
    transferGroup(proto.getRootGroup(), result.getRootGroup(), result);
    result.finish();
    //CoordSysBuilderIF builder = result.enhance();
    //if (debugEnhance) System.out.printf("buildDataset2D.enhance() parseInfo = %s%n", builder.getParseInfo());

    addAttributeInfo(result, CDM.HISTORY, "FMRC 2D Dataset");

    // create runtime aggregation dimension
    double[] runOffset = lite.runOffset;
    String runtimeDimName = getRunDimensionName();
    int nruns = runOffset.length;
    Dimension runDim = new Dimension(runtimeDimName, nruns);
    result.removeDimension(null, runtimeDimName); // remove previous declaration, if any
    result.addDimension(null, runDim);

    // deal with promoteGlobalAttribute
    // promoteGlobalAttributes((AggregationOuterDimension.DatasetOuterDimension) typicalDataset);

    ProxyReader2D proxyReader2D = new ProxyReader2D();

    // extract a copy of the runtimes for thread safety
    // List<Date> runTimes = new ArrayList<Date>(fmrcInv.getRunTimes());

    // create runtime aggregation coordinate variable
    DataType coordType = DataType.DOUBLE; // LOOK getCoordinateType();
    VariableDS runtimeCoordVar = new VariableDS(result, null, null, runtimeDimName, coordType, runtimeDimName, null, null);
    runtimeCoordVar.addAttribute(new Attribute(CDM.LONG_NAME, "Run time for ForecastModelRunCollection"));
    runtimeCoordVar.addAttribute(new ucar.nc2.Attribute("standard_name", "forecast_reference_time"));
    runtimeCoordVar.addAttribute(new ucar.nc2.Attribute(CDM.UNITS, "hours since " + lite.base));
    runtimeCoordVar.addAttribute(new ucar.nc2.Attribute(_Coordinate.AxisType, AxisType.RunTime.toString()));
    result.removeVariable(null, runtimeCoordVar.getShortName());
    result.addVariable(null, runtimeCoordVar);
    if (logger.isDebugEnabled()) logger.debug("FmrcDataset: added runtimeCoordVar " + runtimeCoordVar.getFullName());

    // make the runtime coordinates
    Array runCoordVals = ArrayDouble.factory(DataType.DOUBLE, new int[] {nruns}, runOffset);
    runtimeCoordVar.setCachedData(runCoordVals);

    // make the time coordinate(s) as 2D
    List<Variable> nonAggVars = result.getVariables();
    for (FmrcInvLite.Gridset gridset : lite.gridSets) {
      Group newGroup = result.getRootGroup(); // can it be different ??

      //int noffsets = runSeq.getNTimeOffsets();
      Dimension timeDim = new Dimension(gridset.gridsetName, gridset.noffsets);
      result.removeDimension(null, gridset.gridsetName); // remove previous declaration, if any
      result.addDimension(null, timeDim);

      DataType dtype = DataType.DOUBLE;
      String dims = getRunDimensionName() + " " + gridset.gridsetName;
      VariableDS timeVar = new VariableDS(result, newGroup, null, gridset.gridsetName, dtype, dims, null, null); // LOOK could just make a CoordinateAxis1D
      timeVar.addAttribute(new Attribute(CDM.LONG_NAME, "Forecast time for ForecastModelRunCollection"));
      timeVar.addAttribute(new ucar.nc2.Attribute("standard_name", "time"));
      timeVar.addAttribute(new ucar.nc2.Attribute(CDM.UNITS, "hours since " + lite.base));
      timeVar.addAttribute(new ucar.nc2.Attribute(CDM.MISSING_VALUE, Double.NaN));
      timeVar.addAttribute(new ucar.nc2.Attribute(_Coordinate.AxisType, AxisType.Time.toString()));

      // remove the old one if any
      newGroup.removeVariable(gridset.gridsetName);
      newGroup.addVariable(timeVar);

      Array timeCoordVals = Array.factory(DataType.DOUBLE, timeVar.getShape(), gridset.timeOffset);
      timeVar.setCachedData(timeCoordVals);

      if (gridset.timeBounds != null) {
        String bname = timeVar.getShortName() + "_bounds";
        timeVar.addAttribute(new ucar.nc2.Attribute("bounds", bname));
        Dimension bd = ucar.nc2.dataset.DatasetConstructor.getBoundsDimension( result);
        VariableDS boundsVar = new VariableDS(result, newGroup, null, bname, dtype, dims+" "+bd.getShortName(), null, null);
        boundsVar.addAttribute(new Attribute(CDM.LONG_NAME, "bounds for "+ timeVar.getShortName()));
        boundsVar.setCachedData(Array.factory( DataType.DOUBLE, new int[] {nruns, gridset.noffsets, 2}, gridset.timeBounds));
        newGroup.addVariable(boundsVar);
      }

      // promote all grid variables to agg variables
      for (FmrcInvLite.Gridset.Grid ugrid : gridset.grids) {
        VariableDS aggVar = (VariableDS) result.findVariable(ugrid.name);
        if (aggVar == null) { // a ugrid is not in the proto
          logger.error("buildDataset2D: cant find ugrid variable "+ugrid.name+" in collection "+lite.collectionName+debugMissingVar(proto, result));
          continue; // skip
        }

        // create dimension list
        List<Dimension> dimList = aggVar.getDimensions();
        dimList = dimList.subList(1, dimList.size());  // LOOK assumes time is outer dimension
        dimList.add(0, timeDim);
        dimList.add(0, runDim);

        aggVar.setDimensions(dimList);
        aggVar.setProxyReader(proxyReader2D);
        aggVar.setSPobject(ugrid);
        nonAggVars.remove(aggVar);

        // we need to explicitly list the coordinate axes, because time coord is now 2D
        String coords = makeCoordinateList(aggVar, gridset.gridsetName, true);
        aggVar.removeAttribute(_Coordinate.Axes);
        aggVar.addAttribute(new Attribute(CF.COORDINATES, coords));

        /* transfer Coordinate Systems
        VariableDS protoV = (VariableDS) proto.findVariable(aggVar.getName());
        for (CoordinateSystem protoCs : protoV.getCoordinateSystems()) {
          CoordinateSystem cs = findReplacementCs(protoCs, gridset.gridsetName, result);
          aggVar.addCoordinateSystem(cs);
        } */
      }
    }

    result.finish();  // this puts the new dimensions into the global structures

    //CoordSysBuilderIF builder = result.enhance();
    //if (debugEnhance) System.out.printf("parseInfo = %s%n", builder.getParseInfo());

    // LOOK better not to do this when you only want the NetcdfDataset
    Formatter parseInfo = new Formatter();
    GridDataset gds = new ucar.nc2.dt.grid.GridDataset(result, parseInfo);
    if (debugEnhance) System.out.printf("GridDataset2D parseInfo = %s%n", parseInfo);
    return gds;
  }

  private CoordinateSystem findReplacementCs(CoordinateSystem protoCs, String timeDim, NetcdfDataset result) {
    CoordinateSystem replace = result.findCoordinateSystem(protoCs.getName());
    if (replace != null) return replace;

    List<CoordinateAxis> axes = new ArrayList<>();
    for (CoordinateAxis axis : protoCs.getCoordinateAxes()) {
      CoordinateAxis ra = result.findCoordinateAxis(axis.getFullNameEscaped());
      axes.add(ra);
    }

    // coord transforms are immutable and can be shared
    CoordinateSystem cs = new CoordinateSystem(result, axes, protoCs.getCoordinateTransforms());
    result.addCoordinateSystem(cs);
    return cs;
  }

  /*
  private ArrayDouble.D2 makeTimeCoordinateData2D(Date baseDate, FmrcInv.RunSeq runSeq, VariableDS timeVar) { // throws IOException {
    int[] shape = timeVar.getShape();
    int nruntimes = shape[0];
    int noffsets = shape[1];

    ArrayDouble.D2 timeCoordVals = new ArrayDouble.D2(nruntimes, noffsets);
    Index ima = timeCoordVals.getIndex();
    MAMath.setDouble(timeCoordVals, Double.NaN);  // all default to missing

    int runIdx = 0;
    for (TimeCoord tc : runSeq.getTimes()) {
      double runOffset = FmrcInv.getOffsetInHours(baseDate, tc.getRunDate());
      double[] offset = tc.getOffsetHours();
      for (int j = 0; j < offset.length; j++)
        timeCoordVals.setDouble(ima.set(runIdx, j), offset[j] + runOffset); // offset reletive to base date
      runIdx++;
    }

    return timeCoordVals;
  }

  // keep track of the ugrid and 2D time coords for each variable, put in SPobject

  private class Vstate2D {
    String varName;
    ArrayObject.D2 location;
    ArrayInt.D2 invIndex;
    Date baseDate;

    private Vstate2D(FmrcInv.UberGrid ugrid, Date baseDate, ArrayDouble.D2 timeCoordVals, List<Date> runDates) {
      this.varName = ugrid.getName();
      this.baseDate = baseDate;

      int[] shape = timeCoordVals.getShape();
      int nruns = shape[0]; // this will always equal the complete set of runs
      int ntimes = shape[1];

      this.location = (ArrayObject.D2) Array.factory(DataType.STRING, shape);
      this.invIndex = (ArrayInt.D2) Array.factory(DataType.INT, shape);

      // loop over runDates
      int gridIdx = 0;
      List<FmrInv.GridVariable> grids = ugrid.getRuns(); // must be sorted by rundate

      for (int runIdx=0; runIdx<nruns; runIdx++ ) {
        Date runDate =  runDates.get(runIdx);

        // do we have a grid for this runDate?
        FmrInv.GridVariable grid = grids.get(gridIdx);
        if (!grid.getRunDate().equals(runDate)) continue;
        gridIdx++; // for next loop

        // loop over actual inventory
        for (GridDatasetInv.Grid inv : grid.getInventory()) {
          double invOffset = FmrcInv.getOffsetInHours(baseDate, inv.tc.getRunDate()); // offset reletive to inv
          double[] offsets = inv.tc.getOffsetHours();
          for (int i = 0; i < offsets.length; i++) {
            int timeIdx = findIndex(timeCoordVals, runIdx, ntimes, invOffset + offsets[i]);
            if (timeIdx >= 0) {
              location.set(runIdx, timeIdx, inv.getLocation());
              invIndex.set(runIdx, timeIdx, i);
            }
          }
        }
      }

    }

    // look in the runIdx row of coords to see if a value matches want, return index else -1
    private int findIndex(ArrayDouble.D2 coords, int runIdx, int ntimes, double want) {
      for (int j=0; j<ntimes; j++)
        if (Misc.nearlyEquals(coords.get(runIdx, j), want)) return j;
      return -1;
    }

    private TimeInstance findInventory(int runIdx, int timeIdx) {
      String loc = (String) location.get(runIdx, timeIdx);
      if (loc == null) return null;
      return new TimeInstance(loc, invIndex.get(runIdx, timeIdx));
    }

  }  */

  private class ProxyReader2D implements ProxyReader {

    @Override
    public Array reallyRead(Variable mainv, CancelTask cancelTask) throws IOException {
      try {
        return reallyRead(mainv, mainv.getShapeAsSection(), cancelTask);
      } catch (InvalidRangeException e) {
        throw new IOException(e);
      }
    }

    // here is where agg variables get read

    @Override
    public Array reallyRead(Variable mainv, Section section, CancelTask cancelTask) throws IOException, InvalidRangeException {
      FmrcInvLite.Gridset.Grid gridLite = (FmrcInvLite.Gridset.Grid) mainv.getSPobject();

      // read the original type - if its been promoted to a new type, the conversion happens after this read
      DataType dtype = (mainv instanceof VariableDS) ? ((VariableDS) mainv).getOriginalDataType() : mainv.getDataType();

      Array allData = Array.factory(dtype, section.getShape());
      int destPos = 0;

      // assumes the first two dimensions are runtime and time: LOOK: ensemble ??
      List<Range> ranges = section.getRanges();
      Range runRange = ranges.get(0);
      Range timeRange = ranges.get(1);
      List<Range> innerSection = ranges.subList(2, ranges.size());

      // keep track of open file - must be local variable for thread safety
      HashMap<String, NetcdfDataset> openFilesRead = new HashMap<>();
      try {

        // iterate over the desired runs
        for (int runIdx : runRange) {
          //Date runDate = vstate.runTimes.get(runIdx);

          // iterate over the desired forecast times
          for (int timeIdx : timeRange) {
            Array result = null;

            // find the inventory for this grid, runtime, and hour
            TimeInventory.Instance timeInv = gridLite.getInstance(runIdx, timeIdx);
            if (timeInv != null) {
              if (debugRead) System.out.printf("HIT %d %d ", runIdx, timeIdx);
              result = read(timeInv, gridLite.name, innerSection, openFilesRead); // may return null
              result = MAMath.convert(result, dtype); // just in case it need to be converted
            }

            // missing data
            if (result == null) {
              int[] shape = new Section(innerSection).getShape();
              result = ((VariableDS) mainv).getMissingDataArray(shape); // fill with missing values
              if (debugRead) System.out.printf("MISS %d %d ", runIdx, timeIdx);
            }

            if (debugRead)
              System.out.printf("%d %d reallyRead %s %d bytes start at %d total size is %d%n",
                      runIdx, timeIdx, mainv.getFullName(), result.getSize(), destPos, allData.getSize());

            Array.arraycopy(result, 0, allData, destPos, (int) result.getSize());
            destPos += result.getSize();
          }
        }
        return allData;

      } finally {
        // close any files used during this operation
        closeAll(openFilesRead);
      }
    }

  }

 /////////////////////////////////////////////////////////////////////////
  // 1D

  /**
   * Build a dataset with a 1D time coordinate.
   *
   * @param proto      current proto dataset
   * @param lite       current inventory
   * @param timeInv         use this to generate the time coordinates
   * @return resulting GridDataset
   * @throws IOException on read error
   */
  private GridDataset buildDataset1D(NetcdfDataset proto, FmrcInvLite lite, TimeInventory timeInv) throws IOException {
    if (timeInv == null) return null;
    NetcdfDataset result = new NetcdfDataset(); // make a copy, so that this object can coexist with previous incarnations
    result.setLocation(lite.collectionName);
    transferGroup(proto.getRootGroup(), result.getRootGroup(), result);
    result.finish();
    addAttributeInfo(result, CDM.HISTORY, "FMRC "+timeInv.getName()+" Dataset");

    //DateFormatter dateFormatter = new DateFormatter();
    ProxyReader1D proxyReader1D = new ProxyReader1D();

    // make the time coordinate(s) for each runSeq
    List<Variable> nonAggVars = result.getVariables();
    for (FmrcInvLite.Gridset gridset : lite.gridSets) {
      Group group = result.getRootGroup(); // can it be different ??
      String timeDimName = gridset.gridsetName;

      // construct the dimension
      int ntimes = timeInv.getTimeLength(gridset);
      if (ntimes == 0) {   // eg a constant offset dataset for variables that dont have that offset
        // remove all variables that are in this gridset
        for (FmrcInvLite.Gridset.Grid ugrid : gridset.grids) {
          result.removeVariable(group, ugrid.name);
          logger.warn("buildDataset1D "+timeInv.getName()+" remove "+ugrid.name);
        }
        continue; // skip the rest
      }

      Dimension timeDim = new Dimension(timeDimName, ntimes);
      result.removeDimension(group, timeDimName); // remove previous declaration, if any
      result.addDimension(group, timeDim);

      // optional time coordinate
      group.removeVariable(timeDimName);
      FmrcInvLite.ValueB timeCoordValues = timeInv.getTimeCoords(gridset);
      if (timeCoordValues != null) {
        makeTimeCoordinate(result, group, timeDimName, lite.base, timeCoordValues);
      }

      // optional runtime coordinate
      group.removeVariable(timeDimName+"_run");
      double[] runtimeCoordValues = timeInv.getRunTimeCoords(gridset);
      if (runtimeCoordValues != null) {
        makeRunTimeCoordinate(result, group, timeDimName, lite.base, runtimeCoordValues);
      }

      // optional offset coordinate
      group.removeVariable(timeDimName+"_offset");
      double[] offsetCoordValues = timeInv.getOffsetCoords(gridset);
      if (offsetCoordValues != null) {
        makeOffsetCoordinate(result, group, timeDimName, lite.base, offsetCoordValues);
      }

      // promote all grid variables to agg variables
      for (FmrcInvLite.Gridset.Grid ugrid : gridset.grids) {
        //BestInventory bestInv = makeBestInventory(bestTimeCoord, ugrid);

        VariableDS aggVar = (VariableDS) result.findVariable(ugrid.name);
        if (aggVar == null) { // a ugrid is not in the proto
          logger.error("buildDataset1D "+lite.collectionName+": cant find ugrid variable "+ugrid.name+" in collection "+lite.collectionName+debugMissingVar(proto, result));
          continue; // skip
        }

        // create dimension list
        List<Dimension> dimList = aggVar.getDimensions();
        dimList = dimList.subList(1, dimList.size());  // LOOK assumes time is outer dimension
        dimList.add(0, timeDim);

        aggVar.setDimensions(dimList);
        aggVar.setProxyReader(proxyReader1D);
        aggVar.setSPobject( new Vstate1D(ugrid, timeInv));
        nonAggVars.remove(aggVar);

        // we need to explicitly list the coordinate axes
        String coords = makeCoordinateList(aggVar, timeDimName, false);
        aggVar.removeAttribute(_Coordinate.Axes);
        aggVar.addAttribute(new Attribute(CF.COORDINATES, coords)); // CF

       // if (logger.isDebugEnabled()) logger.debug("FmrcDataset: added grid " + aggVar.getName());
      }
    }

    result.finish();  // this puts the new dimensions into the global structures

    // these are the non-agg variables - get data or ProxyReader from proto
    for (Variable v : nonAggVars) {
      VariableDS protoV = (VariableDS) proto.findVariable(v.getFullNameEscaped());
      if (protoV.hasCachedDataRecurse()) {
        v.setCachedData(protoV.read()); // read from original
      } else {
        v.setProxyReader(protoV.getProxyReader());
      }
    }

    if (debugEnhance) {
      CoordSysBuilderIF builder = result.enhance();
      System.out.printf("GridDataset1D parseInfo = %s%n", builder.getParseInfo());
    }

    return new ucar.nc2.dt.grid.GridDataset(result);
  }

  private String debugMissingVar(NetcdfFile proto, NetcdfFile result) {
    Formatter f = new Formatter();
    f.format("%nresult dataset %s%n", result.getLocation());
    for (Variable v: result.getVariables())
      f.format(" %s%n", v.getNameAndDimensions());
    f.format("%n");
    f.format("proto dataset %s%n", proto.getLocation());
    for (Variable v: proto.getVariables())
      f.format(" %s%n", v.getNameAndDimensions());

    return f.toString();
  }

  private VariableDS makeTimeCoordinate(NetcdfDataset result, Group group, String dimName, CalendarDate base, FmrcInvLite.ValueB valueb) {
    DataType dtype = DataType.DOUBLE;
    VariableDS timeVar = new VariableDS(result, group, null, dimName, dtype, dimName, null, null); // LOOK could just make a CoordinateAxis1D
    timeVar.addAttribute(new Attribute(CDM.LONG_NAME, "Forecast time for ForecastModelRunCollection"));
    timeVar.addAttribute(new ucar.nc2.Attribute("standard_name", "time"));
    timeVar.addAttribute(new ucar.nc2.Attribute(CF.CALENDAR, base.getCalendar().name() ));
    //timeVar.addAttribute(new ucar.nc2.Attribute(CDM.UNITS, "hours since " + base));

    //Ensure a valid udunit
    timeVar.addAttribute(new ucar.nc2.Attribute(CDM.UNITS, "hours since " + base.getTimeUnits()));

    timeVar.addAttribute(new ucar.nc2.Attribute(CDM.MISSING_VALUE, Double.NaN));
    timeVar.addAttribute(new ucar.nc2.Attribute(_Coordinate.AxisType, AxisType.Time.toString()));

    // construct the values
    int ntimes = valueb.offset.length;
    timeVar.setCachedData(Array.factory( DataType.DOUBLE, new int[] {ntimes}, valueb.offset));
    group.addVariable(timeVar);

    if (valueb.bounds != null) {
      String bname = timeVar.getShortName() + "_bounds";
      timeVar.addAttribute(new ucar.nc2.Attribute("bounds", bname));
      Dimension bd = ucar.nc2.dataset.DatasetConstructor.getBoundsDimension( result);
      VariableDS boundsVar = new VariableDS(result, group, null, bname, dtype, dimName+" " + bd.getShortName(), null, null);
      boundsVar.addAttribute(new Attribute(CDM.LONG_NAME, "bounds for "+ timeVar.getShortName()));
      boundsVar.setCachedData(Array.factory( DataType.DOUBLE, new int[] {ntimes, 2}, valueb.bounds));
      group.addVariable(boundsVar);
    }

    return timeVar;
  }

  private VariableDS makeRunTimeCoordinate(NetcdfDataset result, Group group, String dimName, CalendarDate base, double[] values) {
    DataType dtype = DataType.DOUBLE;
    VariableDS timeVar = new VariableDS(result, group, null, dimName+"_run", dtype, dimName, null, null); // LOOK could just make a CoordinateAxis1D
    timeVar.addAttribute(new Attribute(CDM.LONG_NAME, "run times for coordinate = " + dimName));
    timeVar.addAttribute(new ucar.nc2.Attribute("standard_name", "forecast_reference_time"));
    timeVar.addAttribute(new ucar.nc2.Attribute(CF.CALENDAR, base.getCalendar().name() ));
    //timeVar.addAttribute(new ucar.nc2.Attribute(CDM.UNITS, "hours since " + base));
    timeVar.addAttribute(new ucar.nc2.Attribute(CDM.UNITS, "hours since " + base.getTimeUnits()  ));

    timeVar.addAttribute(new ucar.nc2.Attribute(CDM.MISSING_VALUE, Double.NaN));
    timeVar.addAttribute(new ucar.nc2.Attribute(_Coordinate.AxisType, AxisType.RunTime.toString())); // if theres already a time coord, dont put in coordSys - too complicated

    // construct the values
    int ntimes = values.length;
    ArrayDouble.D1 timeCoordVals = (ArrayDouble.D1) Array.factory( DataType.DOUBLE, new int[] {ntimes}, values);
    timeVar.setCachedData(timeCoordVals);
    group.addVariable(timeVar);

    return timeVar;
  }

  private VariableDS makeOffsetCoordinate(NetcdfDataset result, Group group, String dimName, CalendarDate base, double[] values) {
    DataType dtype = DataType.DOUBLE;
    VariableDS timeVar = new VariableDS(result, group, null, dimName+"_offset", dtype, dimName, null, null); // LOOK could just make a CoordinateAxis1D
    timeVar.addAttribute(new Attribute(CDM.LONG_NAME, "offset hour from start of run for coordinate = " + dimName));
    timeVar.addAttribute(new ucar.nc2.Attribute("standard_name", "forecast_period"));
    timeVar.addAttribute(new ucar.nc2.Attribute(CF.CALENDAR, base.getCalendar().name() ));
    timeVar.addAttribute(new ucar.nc2.Attribute(CDM.UNITS, "hours since " + base));
    timeVar.addAttribute(new ucar.nc2.Attribute(CDM.MISSING_VALUE, Double.NaN));

    // construct the values
    int ntimes = values.length;
    ArrayDouble.D1 timeCoordVals = (ArrayDouble.D1) Array.factory( DataType.DOUBLE, new int[] {ntimes}, values);
    timeVar.setCachedData(timeCoordVals);
    group.addVariable(timeVar);

    return timeVar;
  }

  private static class Vstate1D {
    FmrcInvLite.Gridset.Grid gridLite;
    TimeInventory timeInv;

    private Vstate1D(FmrcInvLite.Gridset.Grid gridLite, TimeInventory timeInv) {
      this.gridLite = gridLite;
      this.timeInv = timeInv;
    }
  }

  private class ProxyReader1D implements ProxyReader {

    @Override
    public Array reallyRead(Variable mainv, CancelTask cancelTask) throws IOException {
      try {
        return reallyRead(mainv, mainv.getShapeAsSection(), cancelTask);

      } catch (InvalidRangeException e) {
        throw new IOException(e);
      }
    }

    // here is where 1D agg variables get read
    @Override
    public Array reallyRead(Variable mainv, Section section, CancelTask cancelTask) throws IOException, InvalidRangeException {
      Vstate1D vstate = (Vstate1D) mainv.getSPobject();

     // read the original type - if its been promoted to a new type, the conversion happens after this read
     DataType dtype = (mainv instanceof VariableDS) ? ((VariableDS) mainv).getOriginalDataType() : mainv.getDataType();

     Array allData = Array.factory(dtype, section.getShape());
     int destPos = 0;

     // assumes the first dimension is time: LOOK: what about ensemble ??
     List<Range> ranges = section.getRanges();
     Range timeRange = ranges.get(0);
     List<Range> innerSection = ranges.subList(1, ranges.size());

       // keep track of open files - must be local variable for thread safety
      HashMap<String, NetcdfDataset> openFilesRead = new HashMap<>();

      try {

        // iterate over the desired forecast times
        for (int timeIdx : timeRange) {
          Array result = null;

          // find the inventory for this grid, runtime, and hour
          TimeInventory.Instance timeInv =  vstate.timeInv.getInstance(vstate.gridLite, timeIdx);
          if (timeInv == null) {
            if (logger.isDebugEnabled())
              logger.debug("Missing Inventory timeInx="+timeIdx+ " for "+ mainv.getFullName()+" in "+state.lite.collectionName);
            // vstate.timeInv.getInstance(vstate.gridLite, timeIdx); // allow debugger
          }

          else if (timeInv.getDatasetLocation() != null) {
            if (debugRead) System.out.printf("HIT %s%n", timeInv);
            result = read(timeInv, mainv.getFullNameEscaped(), innerSection, openFilesRead); // may return null
            result = MAMath.convert(result, dtype); // just in case it need to be converted
          }

          // may have missing data
          if (result == null) {
            int[] shape = new Section(innerSection).getShape();
            result = ((VariableDS) mainv).getMissingDataArray(shape); // fill with missing values
            if (debugRead) System.out.printf("MISS %d ", timeIdx);
          }

          if (debugRead)
            System.out.printf("%d reallyRead %s %d bytes start at %d total size is %d%n", timeIdx, mainv.getFullName(), result.getSize(), destPos, allData.getSize());

          Array.arraycopy(result, 0, allData, destPos, (int) result.getSize());
          destPos += result.getSize();
        }
        return allData;

      } finally {
        // close any files used during this operation
        closeAll(openFilesRead);
      }
    }

  }

  ////////////////////////////////////////////////////////////////////////////////////////////////////

  /* private BestInventory makeBestInventory(TimeCoord.TimeResult bestTimeCoord, FmrcInv.UberGrid ugrid) {
    BestInventory bestInv = new BestInventory(bestTimeCoord, ugrid);

    // each run
    for (FmrInv.GridVariable grid : ugrid.getRuns()) {

      // each file in the run
      for (GridDatasetInv.Grid inv : grid.getInventory()) {
        double invOffset = FmrcInv.getOffsetInHours(bestTimeCoord.base, inv.tc.getRunDate()); // offset reletive to inv

        // set of offsets are relative to inv.tc.getRunDate()
        double[] offsets = inv.tc.getOffsetHours();
        for (int i = 0; i < offsets.length; i++) {
          bestInv.setOffset(offsets[i] + invOffset, inv.getLocation(), i); // later ones override
        }
      }
    }

    return bestInv;
  }

  public static class BestInventory {
    String varName;
    double[] offsets;
    String[] location; // file location
    int[] invIndex; // the index in the file

    BestInventory(TimeCoord.TimeResult tc, FmrcInv.UberGrid ugrid) {
      this.varName = ugrid.getName();
      this.offsets = tc.offsets; // all the TimeCoords possible
      this.location = new String[tc.offsets.length];
      this.invIndex = new int[tc.offsets.length];
    }

    void setOffset(double offsetHour, String location, int invIndex) {
      int offsetIndex = findIndex(offsetHour);
      if (offsetIndex < 0)
        throw new IllegalStateException("BestInventory cant find hour " + offsetHour + " in " + varName);
      this.location[offsetIndex] = location;
      this.invIndex[offsetIndex] = invIndex;
    }

    // linear search - barf
     private int findIndex(double offsetHour) {
      for (int i = 0; i < offsets.length; i++)
        if (Misc.nearlyEquals(offsets[i], offsetHour))
          return i;
      return -1;
    }

  }  */

  ///////////////////////////////////////////////////////////////////////////////////////////////////

  // the general case is to get only one time per read - probably not too inefficient, eg GRIB, except maybe for remote reads

  private Array read(TimeInventory.Instance timeInstance, String fullNameEsc, List<Range> innerSection, HashMap<String, NetcdfDataset> openFilesRead) throws IOException, InvalidRangeException {
    NetcdfFile ncfile = open(timeInstance.getDatasetLocation(), openFilesRead);
    if (ncfile == null) return null; // file might be deleted ??

    Variable v = ncfile.findVariable(fullNameEsc);
    if (v == null) return null; // v could be missing, return missing data i think

    // assume time is first dimension LOOK: out of-order; ensemble; section different ??
    Range timeRange = new Range(timeInstance.getDatasetIndex(), timeInstance.getDatasetIndex());
    Section s = new Section(innerSection);
    s.insertRange(0, timeRange);
    return v.read(s);
  }

  /**
   * Open a file, keep track of open files
   * @param location open this location
   * @param openFiles keep track of open files
   * @return file or null if not found
   */
  private NetcdfDataset open(String location, Map<String, NetcdfDataset> openFiles)  throws IOException {
    NetcdfDataset ncd;

    if (openFiles != null) {
      ncd = openFiles.get(location);
      if (ncd != null) return ncd;
    }

    if (config.innerNcml == null) {
      ncd = NetcdfDataset.acquireDataset(new DatasetUrl(null, location), true, null);  // default enhance

    } else {
      NetcdfFile nc = NetcdfDataset.acquireFile(new DatasetUrl(null, location), null);
      ncd = NcMLReader.mergeNcML(nc, config.innerNcml); // create new dataset
      ncd.enhance(); // now that the ncml is added, enhance "in place", ie modify the NetcdfDataset
    }

    if (openFiles != null && ncd != null) {
      openFiles.put(location, ncd);
    }

    return ncd;
  }

  /*  from Aggregation.Dataset
  public NetcdfFile acquireFile(CancelTask cancelTask) throws IOException {
    if (debugOpenFile) System.out.println(" try to acquire " + cacheLocation);
    long start = System.currentTimeMillis();

    NetcdfFile ncfile = NetcdfDataset.acquireFile(reader, null, cacheLocation, -1, cancelTask, spiObject);

    // must merge NcML before enhancing
    if (mergeNcml != null)
      ncfile = NcMLReader.mergeNcML(ncfile, mergeNcml); // create new dataset
    if (enhance == null || enhance.isEmpty()) {
      if (debugOpenFile) System.out.println(" acquire (no enhance) " + cacheLocation + " took " + (System.currentTimeMillis() - start));
      return ncfile;
    }

    // must enhance
    NetcdfDataset ds;
    if (ncfile instanceof NetcdfDataset) {
      ds = (NetcdfDataset) ncfile;
      ds.enhance(enhance); // enhance "in place", ie modify the NetcdfDataset
    } else {
      ds = new NetcdfDataset(ncfile, enhance); // enhance when wrapping
    }

    if (debugOpenFile) System.out.println(" acquire (enhance) " + cacheLocation + " took " + (System.currentTimeMillis() - start));
    return ds;
  } */

  private void closeAll(Map<String, NetcdfDataset> openFiles) throws IOException {
    for (NetcdfDataset ncfile : openFiles.values())
      ncfile.close();
    openFiles.clear();
  }

  ////////////////////////////////////////////////////////////////////////////////////
  // all normal (non agg, non cache) variables must use a proxy to acquire the file
  // the openFiles map is null - should be ok since its a non-agg, so file gets opened (and closed) for each read.

  protected class DatasetProxyReader implements ProxyReader {
    String location;

    DatasetProxyReader(String location) {
      this.location = location;
    }

    public Array reallyRead(Variable client, CancelTask cancelTask) throws IOException {
      try (NetcdfFile ncfile= open(location, null)) {
        if ((cancelTask != null) && cancelTask.isCancel()) return null;
        Variable proxyV = findVariable(ncfile, client);
        return proxyV.read();
      }
    }

    public Array reallyRead(Variable client, Section section, CancelTask cancelTask) throws IOException, InvalidRangeException {
      try (NetcdfFile ncfile = open(location, null)) {
        Variable proxyV = findVariable(ncfile, client);
        if ((cancelTask != null) && cancelTask.isCancel()) return null;
        return proxyV.read(section);
      }
    }
  }

  protected Variable findVariable(NetcdfFile ncfile, Variable client) {
    Variable v = ncfile.findVariable(client.getFullNameEscaped());
    if (v == null) {  // might be renamed
      VariableEnhanced ve = (VariableEnhanced) client;
      v = ncfile.findVariable(ve.getOriginalName());
    }
    return v;
  }

  public void showDetails(Formatter out) {
    out.format("==========================%nproto=%n%s%n", state.proto);
  }

}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.nc2.ui.grid;

import ucar.ma2.IsMissingEvaluator;
import ucar.nc2.ui.widget.FontUtil;

import ucar.nc2.util.ListenerManager;
import ucar.unidata.util.Format;

import java.awt.*;
import java.awt.event.*;
import java.beans.PropertyChangeListener;
import java.beans.PropertyChangeEvent;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.IOException;
import javax.swing.*;

/**
 * A ColorScale is used for false-color data mapping. It contains an array of java.awt.Color,
 * along with an assignment of a data interval to each Color.
 *

 * ColorScale.Panel handles the displaying of a ColorScale. It also works with ColorScaleManager
 * to allow editing and defining new ColorScales.
 *
 * @author caron
 */

public class ColorScale implements Cloneable, java.io.Serializable {
  public static enum MinMaxType {horiz, log, hold};

  public static final int VERTICAL = 0;
  public static final int HORIZONTAL = 1;

  private static final long serialVersionUID = -1L;       // disconnect java version checking
  private static final int objectVersion = 1;             // our version control
  private static final boolean debugColors = false;
  private static int sigfig = 4;

  // this is all that needs to be serialized
  private String name;
  private int ncolors;
  private Color[] colors;

  // reset after deserializing
  private Color[] useColors;
  private ListenerManager lm;

  // this is set for each grid
  private IsMissingEvaluator gg;
  private double[] edge;
  private int[] hist;
  private double min, max, interval;
  private boolean hasMissingData = false;
  private Color missingDataColor = Color.white;

  // kludge to make life easier
  static final public Color[] redHot = {
          new java.awt.Color(0x5cacee),    //cyan
          new java.awt.Color(0xfff5eb),
          new java.awt.Color(0xffe6cc),
          new java.awt.Color(0xffd9b0),
          new java.awt.Color(0xffc485),
          new java.awt.Color(0xffb86b),
          new java.awt.Color(0xffad56),
          new java.awt.Color(0xff992b),
          new java.awt.Color(0xff7e00),
          new java.awt.Color(0xff6a00),
          new java.awt.Color(0xf15800),
          new java.awt.Color(0xe24400),
          new java.awt.Color(0xd22f00),
          new java.awt.Color(0xc11900),
          new java.awt.Color(0xa00300),
          new java.awt.Color(0xff00ff)      // magenta
  };

  static final public Color[] redBlue = {
          new java.awt.Color(1, 57, 255),
          new java.awt.Color(0, 140, 255),
          new java.awt.Color(1, 209, 255),
          new java.awt.Color(1, 255, 232),
          new java.awt.Color(1, 255, 171),
          new java.awt.Color(1, 255, 79),
          new java.awt.Color(43, 255, 0),
          new java.awt.Color(166, 255, 2),
          new java.awt.Color(227, 255, 1),
          new java.awt.Color(255, 198, 0),
          new java.awt.Color(255, 168, 1),
          new java.awt.Color(255, 145, 1),
          new java.awt.Color(255, 130, 1),
          new java.awt.Color(255, 107, 0),
          new java.awt.Color(255, 84, 0),
          new java.awt.Color(255, 7, 0)
  };
  static final public Color[] hueBands = {
          new java.awt.Color(204, 0, 0),
          new java.awt.Color(255, 0, 0),
          new java.awt.Color(255, 51, 51),
          new java.awt.Color(255, 102, 102),
          new java.awt.Color(255, 153, 153),
          new java.awt.Color(255, 102, 0),
          new java.awt.Color(255, 153, 0),
          new java.awt.Color(255, 204, 102),
          new java.awt.Color(255, 255, 0),
          new java.awt.Color(255, 255, 51),
          new java.awt.Color(255, 255, 102),
          new java.awt.Color(255, 255, 153),
          new java.awt.Color(0, 255, 0),
          new java.awt.Color(51, 255, 51),
          new java.awt.Color(102, 255, 102),
          new java.awt.Color(153, 255, 153),
          new java.awt.Color(204, 255, 204),
          new java.awt.Color(0, 255, 255),
          new java.awt.Color(51, 255, 255),
          new java.awt.Color(102, 255, 255),
          new java.awt.Color(153, 255, 255),
          new java.awt.Color(204, 255, 255),
          new java.awt.Color(0, 0, 255),
          new java.awt.Color(51, 51, 255),
          new java.awt.Color(102, 102, 255),
          new java.awt.Color(153, 153, 255),
          new java.awt.Color(255, 0, 255),
          new java.awt.Color(255, 51, 255),
          new java.awt.Color(255, 102, 255),
          new java.awt.Color(255, 153, 255),
          new java.awt.Color(255, 204, 255)
  };

  static final public Color[] spectrum2 = {
          new java.awt.Color(204, 0, 0),
          new java.awt.Color(255, 31, 0),
          new java.awt.Color(255, 69, 0),
          new java.awt.Color(255, 107, 0),
          new java.awt.Color(255, 138, 0),
          new java.awt.Color(255, 168, 0),
          new java.awt.Color(255, 199, 0),
          new java.awt.Color(255, 244, 0),
          new java.awt.Color(227, 255, 0),
          new java.awt.Color(196, 255, 0),
          new java.awt.Color(165, 255, 0),
          new java.awt.Color(127, 255, 0),
          new java.awt.Color(89, 255, 0),
          new java.awt.Color(43, 255, 0),
          new java.awt.Color(0, 255, 40),
          new java.awt.Color(0, 255, 80),
          new java.awt.Color(0, 255, 110),
          new java.awt.Color(0, 255, 140),
          new java.awt.Color(0, 255, 170),
          new java.awt.Color(0, 255, 201),
          new java.awt.Color(0, 255, 232),
          new java.awt.Color(0, 232, 255),
          new java.awt.Color(0, 209, 255),
          new java.awt.Color(0, 170, 255),
          new java.awt.Color(0, 140, 255),
          new java.awt.Color(0, 79, 255),
          new java.awt.Color(0, 57, 255),
          new java.awt.Color(0, 34, 255),
          new java.awt.Color(0, 3, 255),
          new java.awt.Color(29, 1, 255),
          new java.awt.Color(59, 1, 255),
          new java.awt.Color(82, 1, 255),
          new java.awt.Color(112, 1, 255),
          new java.awt.Color(125, 1, 255),
          new java.awt.Color(166, 1, 255),
          new java.awt.Color(189, 1, 255),
          new java.awt.Color(219, 1, 255)
  };
  static final private Color[] defaultColors = redBlue;

  /* Constructor.
   * @param name of this colorscale.
   * @param c array of colors.
   */
  public ColorScale(String name, Color [] c) {
    this.name = name;
    this.ncolors = c.length;
    colors = new Color[ ncolors];
    System.arraycopy(c, 0, colors, 0, ncolors);

    constructTransient();
  }

  /* Constructor. Use default colors.
   * @param name of this colorscale.
   */
  public ColorScale(String name) {
    this(name, defaultColors);
/*    this.name = new String(name);
    this.ncolors = ncolors;
    colors = new Color[ ncolors];

      // set default colors
    int n = Math.min(ncolors, defaultColors.length);
    for (int i=0; i<n; i++)
      colors[i] = defaultColors[i];
    for (int i=n; i<ncolors; i++)
      colors[i] = defaultColors[n-1];

    constructTransient(); */
  }

  /* Constructor. Use default colors, no name.
   */
  public ColorScale() {
    this("", defaultColors);
  }

  // rest of stuff for construction/deserialization
  private void constructTransient() {
    useColors = colors;

    edge = new double[ ncolors];
    hist = new int[ ncolors + 1];
    lm = new ListenerManager(
            "java.beans.PropertyChangeListener",
            "java.beans.PropertyChangeEvent",
            "propertyChange");
    missingDataColor = Color.white;
  }

  /**
   * add action event listener
   */
  public void addPropertyChangeListener(PropertyChangeListener l) {
    lm.addListener(l);
  }

  /**
   * remove action event listener
   */
  public void removePropertyChangeListener(PropertyChangeListener l) {
    lm.removeListener(l);
  }

  /**
   * Get the colorscale name.
   */
  public String getName() {
    return name;
  }

  /**
   * Set the colorscale name.
   */
  public void setName(String name) {
    this.name = name;
  }

  /**
   * Get the number of colors in the colorscale.
   */
  public int getNumColors() {
    return ncolors;
  }

  /**
   * Set the number of colors in the colorscale.
   */
  public void setNumColors(int n) {
    if (n != ncolors) {
      colors = new Color[n];
      int prevn = Math.min(ncolors, n);
      System.arraycopy(useColors, 0, colors, 0, prevn);
      for (int i = ncolors; i < n; i++)
        colors[i] = Color.white;

      useColors = colors;
      ncolors = n;
      edge = new double[ ncolors];
      hist = new int[ ncolors + 1];
    }
  }

  /* Get the color at the given index, or the missing data color.
   * @param i index into color array, or ncolors to get missingDataColor.
   * @exception IllegalArgumentException if (i<0) || (i>ncolors)
   */
  public Color getColor(int i) {
    if (i >= 0 && i < ncolors)
      return useColors[i];
    else if (i == ncolors && hasMissingData)
      return missingDataColor;
    else
      throw new IllegalArgumentException("Color Scale getColor " + i);
  }

  /**
   * Get the edge of the ith interval. see @link setMinMax
   */
  public double getEdge(int index) {
    return edge[index];
  }

  /* Set whether there is missing data, what it is, and what color to use.
   * @param i: index into color array, or ncolors for missingDataColor.
   * @exception IllegalArgumentException if (i<0) || (i>ncolors)
   *
  public void setMissingDataColor( Color missingDataColor) {
    this.missingDataColor = missingDataColor;
  } */

  public void setGeoGrid(IsMissingEvaluator gg) {
    this.gg = gg;
    hasMissingData = gg.hasMissing();
  }

  public Color getMissingDataColor() {
    return missingDataColor;
  }

  /*public String getLabel(int i) {
    if (i >=0 && i < ncolors)
      return label[i];
    else
      throw new IllegalArgumentException("Color Scale getLabel "+i);
  } */

  /**
   * Get which color interval this value lies in.
   *
   * @param value minimum data value.
   * @return the color index.
   */
  public int getIndexFromValue(double value) {
    int index;
    if (hasMissingData && gg.isMissing(value))
      index = ncolors;  // missing data
    else if (value <= min)
      index = 0;
    else if (value >= max)
      index = ncolors - 1;
    else
      index = (int) ((value - min) / interval) + 1;

    hist[index]++;
    return index;
  }

  public int getIndexFromValueLog(double value) {
    int index;
    if (hasMissingData && gg.isMissing(value))
      index = ncolors;  // missing data
    else if (value <= min)
      index = 0;
    else if (value >= max)
      index = ncolors - 1;
    else
      index = (int) ((value - min) / interval) + 1;

    hist[index]++;
    return index;
  }

  /**
   * Set the data min/max interval. The color intervals are set based on this.
   * A PropertyChangeEvent is sent when this is called. Currently the intervals are
   * calculated in the following way (where incr = (max-min)/(n-2)) :
   *

   *

   *      edge           data interval
   *  0    min             value <= min
   *  1    min+incr        min <= value < min + incr
   *  2    min+2*incr      min+incr <= value < min+2*incr
   *  ith  min+i*incr      min+(i-1)*incr <= value < min+i*incr
   *  n-2  max             min+(n-3)*incr <= value < max
   *  n-1  max             max < value
   *  n                    value = missingDataValue
   *

   *
   * @param min minimum data value
   * @param max maximum data value
   */
  public void setMinMax(double min, double max) {
    this.min = min;
    this.max = max;
    interval = (max - min) / (ncolors - 2);

    // set edges
    for (int i = 0; i < ncolors; i++)
      edge[i] = min + i * interval;

    lm.sendEvent(new PropertyChangeEvent(this, "ColorScaleLimits", null, this));
  }

  /**
   * This is an optimization for counting the number of colors in each interval.
   * the histpogram is populated by calls to getIndexFromValue().
   *
   * @return the index with the maximum histogram count.
   */
  public int getHistMax() {
    int max = 0, maxi = 0;
    for (int i = 0; i <= ncolors; i++)
      if (hist[i] > max) {
        max = hist[i];
        maxi = i;
      }
    return maxi;
  }

  /**
   * reset the histogram.
   */
  public void resetHist() {
    for (int i = 0; i <= ncolors; i++)
      hist[i] = 0;
  }

  public String toString() {
    return name;
  }

  public Object clone() {
    ColorScale cl = new ColorScale(name, colors);
    /*try {
      cl = (ColorScale) super.clone();
    } catch(CloneNotSupportedException e) {
      return null;
    } // ignore

    // non primitive fields must be cloned separately
    cl.name = new String(name);
    cl.set(this);
    cl.construct();  */
    return (Object) cl;
  }

  /////////// private ////////////////////

  // this is for editing a colorscale
  private void editModeBegin() {
    Color [] editColors = new Color[ ncolors];
    System.arraycopy(colors, 0, editColors, 0, ncolors);
    useColors = editColors;
  }

  private void editModeEnd(boolean accept) {
    if (accept) {
      System.arraycopy(useColors, 0, colors, 0, ncolors);
    }
    useColors = colors;
  }

  private void setColor(int i, Color c) {
    if (i >= 0 && i < ncolors)
      useColors[i] = c;
  }

  /* private void set( ColorScale cs) {
   set(cs.getColors());
 } */
  private void setColors(Color[] c) {
    ncolors = c.length;
    colors = new Color[ ncolors];
    System.arraycopy(c, 0, colors, 0, ncolors);
    edge = new double[ ncolors];
    hist = new int[ ncolors + 1];
    useColors = colors;        // ??
  }

  // serialization
  private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundException {
    s.readInt();
    this.name = s.readUTF();
    this.colors = (Color[]) s.readObject();
    this.ncolors = colors.length;
    constructTransient();
  }

  private void writeObject(ObjectOutputStream s) throws IOException {
    s.writeInt(objectVersion);
    s.writeUTF(this.name);
    s.writeObject(this.colors);
  }

  // heres the swing component part of a ColorScale: static so its not part of serialization
  // originally designed to allow popup editor to change colors; this has been disabled; all
  // changes made by COlorManager. design could be cleaned up, but maybe I want that back later?
  public static class Panel extends JPanel {
    private int type;

    private int size = 50;
    private ColorScale cs;
    private JLabel unitLabel = new JLabel("unit", SwingConstants.CENTER);
    private JPanel lpanel;

    private boolean editable = false;
    private int selected = -1;

    private int nColorInterval;
    private String[] label;
    private boolean useLabel = true;
    private FontUtil.StandardFont sf = FontUtil.getStandardFont(10);

    public Panel(Component parent) {
      this(parent, ColorScale.VERTICAL, null);
    }

    public Panel(Component parent, ColorScale cscale) {
      this(parent, ColorScale.VERTICAL, cscale);
    }

    public Panel(Component parent, int type, ColorScale cscale) {
      this.cs = (cscale == null) ? new ColorScale("default") : cscale;
      this.type = type;

      if (type == ColorScale.VERTICAL) {
        setPreferredSize(new Dimension(size, 400));
        setLayout(new BoxLayout(this, BoxLayout.Y_AXIS));
      } else {
        setPreferredSize(new Dimension(400, size));
        setLayout(new BoxLayout(this, BoxLayout.X_AXIS));
      }

      setListener();

      nColorInterval = cs.getNumColors();
      for (int i = 0; i < nColorInterval; i++) {
        ColorInterval intv = new ColorInterval(nColorInterval - i - 1);
        add(intv);
      }
      lpanel = new JPanel();
      lpanel.add(unitLabel);
      //unitLabel.setBorder( new javax.swing.border.EtchedBorder());
      if (type == ColorScale.VERTICAL)
        lpanel.setPreferredSize(new Dimension(size, 0));
      else
        lpanel.setPreferredSize(new Dimension(0, size));
      add(lpanel);

      label = new String[ nColorInterval];
      calcLabels();

      /* creates a popup men, attaches it to this Panel
     PopupMenu popupMenu = new PopupMenu(this, "options");
     popupMenu.add("Edit", new AbstractAction() {
       public void actionPerformed(ActionEvent e) {
         System.out.println("popup edit action");
         dialog.show();
       }
     }); */

    }

    private void calcLabels() {
      label[0] = "<" + Format.d(cs.getEdge(0), sigfig);
      for (int i = 1; i < nColorInterval - 1; i++) {
        label[i] = Format.d(cs.getEdge(i), sigfig);
      }
      label[nColorInterval - 1] = ">" + Format.d(cs.getEdge(nColorInterval - 2), sigfig);
    }

    private void setListener() {
      // listen for changes so we can redraw
      cs.addPropertyChangeListener(new java.beans.PropertyChangeListener() {
        public void propertyChange(java.beans.PropertyChangeEvent e) {
          if (e.getPropertyName().equals("ColorScaleLimits"))
            calcLabels();
          repaint();
        }
      });
    }

    public ColorScale getColorScale() {
      return cs;
    }

    // change the current cscale
    public void setColorScale(ColorScale cscale) {
      if (nColorInterval != cscale.getNumColors()) {
        removeAll();
        nColorInterval = cscale.getNumColors();
        label = new String[ nColorInterval];
        for (int i = 0; i < nColorInterval; i++) {
          ColorInterval intv = new ColorInterval(nColorInterval - i - 1);
          add(intv);
          label[i] = "none";
        }
        add(lpanel);
        revalidate();
      }

      this.cs = cscale;
      setListener();
      calcLabels();
      repaint();
    }

    // set existing colorscale to have new colors
    public void setColors(Color[] c) {
      if (nColorInterval != c.length) {
        removeAll();
        nColorInterval = c.length;
        label = new String[ nColorInterval];
        for (int i = 0; i < nColorInterval; i++) {
          ColorInterval intv = new ColorInterval(nColorInterval - i - 1);
          add(intv);
          label[i] = "none";
        }
        add(lpanel);
        revalidate();
      }

      cs.setColors(c);
      cs.editModeBegin(); // again

      if (debugColors) {
        for (int i = 0; i < cs.getNumColors(); i++)
          System.out.println(cs.getColor(i));
      }
      calcLabels();
      repaint();
    }

    public void setColor(Color c) {
      cs.setColor(selected, c);
    }

    //public void setEditable( boolean b) { editable = b; }
    public void setEditMode(boolean on, boolean accept) {
      if (on) {
        editable = true;
        cs.editModeBegin();
      } else {
        cs.editModeEnd(accept);
        if (accept)
          setColorScale(cs);
        selected = -1;
        editable = false;
      }
      repaint();
    }

    public void setSelected(int i) {
      selected = i;
    }

    public void setShowText(boolean b) {
      useLabel = b;
    }

    /*private void edit(int which) {
      if (!editable)
        return;
      selected = which;
      repaint();
      cs.setEditMode(true);
      //dialog.show();
    }*/
    public void setUnitString(String s) {
      unitLabel.setText(s);
      //System.out.println("new text = "+s);
      //unitLabel.repaint(s);
    }

    public void print(Graphics2D g, double x, double y, double width, double height) {
      int n = cs.getNumColors();
      double size = (type == ColorScale.VERTICAL) ? height / n : width / n;
      int count = 0;
      for (int i = 0; i < getComponentCount(); i++) {
        Component c = getComponent(i);
        if (c instanceof ColorInterval) {
          ColorInterval intv = (ColorInterval) c;
          if (type == ColorScale.VERTICAL)
            intv.printV(g, (int) x, (int) (y + count * size), (int) width, (int) size);
          else {
            double xpos = x + width - (count + 1) * size;
            intv.printH(g, (int) xpos, (int) y, (int) size, (int) height);
          }
          count++;
        }
      }
    }

    /* private class OkListener implements ActionListener {
      public void actionPerformed(ActionEvent e) {
        selected = -1;
        cs.accept();
        repaint();
      }
    }  // end inner class OkListener

    private class CancelListener implements ActionListener {
      public void actionPerformed(ActionEvent e) {
        selected = -1;
        cs.cancel();
        repaint();
      }
    } // end inner class CancelListener  */

    private class ColorInterval extends JComponent {
      private int rank;

      ColorInterval(int r) {
        this.rank = r;
        //setPreferredSize( new Dimension(40,40));
        addMouseListener(new MouseAdapter() {
          public void mousePressed(MouseEvent e) {
            if (editable) {
              selected = rank;
              Panel.this.repaint();
            }
          }
        });
      }

      public void printV(Graphics2D g, int x, int y, int width, int height) {
        int textSize = 15; // LOOK : neeed to calculate this

        g.setColor(cs.getColor(rank));
        g.fillRect(x, y + textSize, width, height - textSize);

        //g.setColor( Color.black);
        //g.drawRect( x, y+textSize, width, height-textSize);

        if (useLabel) {
          g.setColor(Color.black);
          g.setFont(sf.getFont());
          g.drawString(label[rank], x + 3, y + 10);
        }
      }

      public void printH(Graphics2D g, int x, int y, int width, int height) {
        int textSize = 15; // LOOK : neeed to calculate this

        g.setColor(cs.getColor(rank));
        g.fillRect(x, y + textSize, width, height - 2 * textSize);

        g.setColor(Color.white);
        g.drawRect(x, y + textSize, width, height - 2 * textSize);

        if (useLabel) {
          g.setColor(Color.black);
          g.setFont(sf.getFont());
          if (rank % 2 == 0) // even
            g.drawString(label[rank], x, y + textSize);
          else
            g.drawString(label[rank], x, y + height);
        }
      }

      public void paintComponent(Graphics g) {
        Rectangle b = getBounds();

        g.setColor(cs.getColor(rank));
        g.fillRect(0, 0, b.width - 1, b.height - 1);

        g.setColor((selected == rank) ? Color.magenta : Color.black);
        g.drawRect(0, 0, b.width - 1, b.height - 1);

        if (selected == rank) {
          g.drawLine(0, 0, b.width, b.height);
          g.drawLine(0, b.height, b.width, 0);
        }

        if (useLabel) {
          g.setColor(Color.black);
          g.setFont(sf.getFont());
          g.drawString(label[rank], 3, 10);
        }
      }
    } // end inner class ColorInterval

  } // end inner class ColorScale.Panel
}