/*
 * Copyright (c) 1998-2019 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.ui.op;

import java.awt.*;
import java.awt.event.ActionEvent;
import java.awt.event.WindowAdapter;
import java.awt.event.WindowEvent;
import java.beans.PropertyChangeEvent;
import java.beans.PropertyChangeListener;
import java.io.*;
import java.nio.channels.WritableByteChannel;
import java.util.ArrayList;
import java.util.Formatter;
import java.util.List;
import javax.swing.*;

import ucar.nc2.ui.StructureTable;

import org.jdom2.Element;
import ucar.ma2.Array;
import ucar.ma2.InvalidRangeException;
import ucar.nc2.Attribute;
import ucar.nc2.FileWriter2;
import ucar.nc2.NCdumpW;
import ucar.nc2.NetcdfFile;
import ucar.nc2.NetcdfFileWriter;
import ucar.nc2.ParsedSectionSpec;
import ucar.nc2.Structure;
import ucar.nc2.Variable;
import ucar.nc2.dataset.NetcdfDataset;
import ucar.nc2.ncml.NcMLWriter;
import ucar.nc2.stream.NcStreamWriter;
import ucar.nc2.ui.dialog.CompareDialog;
import ucar.nc2.ui.dialog.NetcdfOutputChooser;
import ucar.nc2.ui.widget.BAMutil;
import ucar.nc2.ui.widget.FileManager;
import ucar.nc2.ui.widget.IndependentWindow;
import ucar.nc2.ui.widget.PopupMenu;
import ucar.nc2.ui.widget.TextHistoryPane;
import ucar.nc2.util.CompareNetcdf2;
import ucar.nc2.write.Nc4ChunkingStrategy;
import ucar.util.prefs.PreferencesExt;
import ucar.util.prefs.ui.BeanTable;
import ucar.util.prefs.ui.Debug;

/**
 * A Swing widget to view the content of a netcdf dataset.
 * It uses a DatasetTree widget and nested BeanTable widget, by
 * wrapping the Variables in a VariableBean.
 * A pop-up menu allows to view a Structure in a StructureTable.
 *
 * @author caron
 */

public class DatasetViewer extends JPanel {
  private FileManager fileChooser;

  private PreferencesExt prefs;
  private NetcdfFile ds;

  private List<NestedTable> nestedTableList = new ArrayList<>();
  private BeanTable attTable;

  private JPanel tablePanel;
  private JSplitPane mainSplit;

  private JComponent currentComponent;
  private DatasetTreeView datasetTree;
  private NCdumpPane dumpPane;
  private VariablePlot dataPlot;
  private VariableTable variableTable;

  private TextHistoryPane infoTA;
  private StructureTable dataTable;
  private IndependentWindow infoWindow, dataWindow, plotWindow, dumpWindow, attWindow;

  private boolean eventsOK = true;

  public DatasetViewer(PreferencesExt prefs, FileManager fileChooser) {
    this.prefs = prefs;
    this.fileChooser = fileChooser;

    // create the variable table(s)
    tablePanel = new JPanel( new BorderLayout());
    setNestedTable(0, null);

    // the tree view
    datasetTree = new DatasetTreeView();
    datasetTree.addPropertyChangeListener(new PropertyChangeListener() {
      public void propertyChange(PropertyChangeEvent e) {
        if (e.getPropertyName().equals("Selection")) {
          setSelected((Variable) e.getNewValue());
        }
      }
    });

    // layout
    mainSplit = new JSplitPane(JSplitPane.HORIZONTAL_SPLIT, false, datasetTree, tablePanel);
    mainSplit.setDividerLocation(prefs.getInt("mainSplit", 100));

    setLayout(new BorderLayout());
    add(mainSplit, BorderLayout.CENTER);

    // the info window
    infoTA = new TextHistoryPane();
    infoWindow = new IndependentWindow("Variable Information", BAMutil.getImage( "netcdfUI"), infoTA);
    infoWindow.setBounds( (Rectangle) prefs.getBean("InfoWindowBounds", new Rectangle( 300, 300, 500, 300)));

    // the data Table
    dataTable = new StructureTable( (PreferencesExt) prefs.node("structTable"));
    variableTable = new VariableTable( (PreferencesExt) prefs.node("variableTable"));
    dataWindow = new IndependentWindow("Data Table", BAMutil.getImage( "netcdfUI"), dataTable);
    dataWindow.setBounds( (Rectangle) prefs.getBean("dataWindowBounds", new Rectangle( 50, 300, 1000, 1200)));

    // the ncdump Pane
    dumpPane = new NCdumpPane((PreferencesExt) prefs.node("dumpPane"));
    dumpWindow = new IndependentWindow("NCDump Variable Data", BAMutil.getImage( "netcdfUI"), dumpPane);
    dumpWindow.setBounds( (Rectangle) prefs.getBean("DumpWindowBounds", new Rectangle( 300, 300, 300, 200)));

    // the plot Pane
    dataPlot = new VariablePlot((PreferencesExt) prefs.node("plotPane"));
    plotWindow = new IndependentWindow("Plot Variable Data", BAMutil.getImage( "netcdfUI"), dataPlot);
    plotWindow.setBounds( (Rectangle) prefs.getBean("PlotWindowBounds", new Rectangle( 300, 300, 300, 200)));
    plotWindow.addWindowListener(new WindowAdapter() {
        @Override
        public void windowClosing(WindowEvent e) {
            dataPlot.clear();
        }
    });
  }

  NetcdfOutputChooser outChooser;
  public void addActions(JPanel buttPanel) {

    AbstractAction netcdfAction = new AbstractAction() {
      public void actionPerformed(ActionEvent e) {
      if (ds == null) return;
      if (outChooser == null) {
        outChooser = new NetcdfOutputChooser((Frame) null);
        outChooser.addPropertyChangeListener("OK", new PropertyChangeListener() {
           public void propertyChange(PropertyChangeEvent evt) {
             writeNetcdf((NetcdfOutputChooser.Data) evt.getNewValue());
           }
         });
      }
      outChooser.setOutputFilename(ds.getLocation());
      outChooser.setVisible(true);
      }
    };
    BAMutil.setActionProperties(netcdfAction, "netcdf", "Write netCDF file", false, 'S', -1);
    BAMutil.addActionToContainer(buttPanel, netcdfAction);

    AbstractButton compareButton = BAMutil.makeButtcon("Select", "Compare to another file", false);
    compareButton.addActionListener(e -> {
        compareDataset();
    });
    buttPanel.add(compareButton);

    AbstractAction attAction = new AbstractAction() {
      public void actionPerformed(ActionEvent e) {
        showAtts();
      }
    };
    BAMutil.setActionProperties(attAction, "FontDecr", "global attributes", false, 'A', -1);
    BAMutil.addActionToContainer(buttPanel, attAction);
  }

  ///////////////////////////////////////

  void writeNetcdf(NetcdfOutputChooser.Data data) {
    if (data.version == NetcdfFileWriter.Version.ncstream) {
      writeNcstream(data.outputFilename);
      return;
    }

    try {
      FileWriter2 writer = new FileWriter2(ds, data.outputFilename, data.version,
              Nc4ChunkingStrategy.factory(data.chunkerType, data.deflate, data.shuffle));
      NetcdfFile result = writer.write();
      result.close();
      JOptionPane.showMessageDialog(this, "File successfully written");
    } catch (Exception ioe) {
      JOptionPane.showMessageDialog(this, "ERROR: " + ioe.getMessage());
      ioe.printStackTrace();
    }
  }

  void writeNcstream(String filename) {
    try {
      NcStreamWriter writer = new NcStreamWriter(ds, null);
      try (OutputStream fos = new BufferedOutputStream( new FileOutputStream(filename), 50 * 1000)) {
        writer.streamAll(fos);
      }
      JOptionPane.showMessageDialog(this, "File successfully written");
    } catch (Exception ioe) {
      JOptionPane.showMessageDialog(this, "ERROR: " + ioe.getMessage());
      ioe.printStackTrace();
    }
  }

  void writeNcstreamHeader(String filename) {
    try {
      NcStreamWriter writer = new NcStreamWriter(ds, null);
      try (FileOutputStream fos = new FileOutputStream(filename)) {
        writer.sendHeader(fos);
      }
      JOptionPane.showMessageDialog(this, "File successfully written");
    } catch (Exception ioe) {
      JOptionPane.showMessageDialog(this, "ERROR: " + ioe.getMessage());
      ioe.printStackTrace();
    }
  }

  private CompareDialog dialog = null;
  public void compareDataset() {
    if (ds == null) return;
    if (dialog == null) {
      dialog = new CompareDialog(null, fileChooser);
      dialog.pack();
      dialog.addPropertyChangeListener("OK", new PropertyChangeListener() {
        public void propertyChange(PropertyChangeEvent evt) {
          CompareDialog.Data data = (CompareDialog.Data) evt.getNewValue();
          //System.out.printf("name=%s %s%n", evt.getPropertyName(), data);
          compareDataset(data);
        }
      });
    }
    dialog.setVisible(true);
  }

  private void compareDataset(CompareDialog.Data data) {
    if (data.name == null) return;

    NetcdfFile compareFile = null;
    try {
      compareFile = NetcdfDataset.openFile(data.name, null);
      Formatter f = new Formatter();

      CompareNetcdf2 cn = new CompareNetcdf2(f, data.showCompare, data.showDetails, data.readData);
      if (data.howMuch == CompareDialog.HowMuch.All)
        cn.compare(ds, compareFile);

      else if (data.howMuch == CompareDialog.HowMuch.varNameOnly) {
        f.format(" First file = %s%n", ds.getLocation());
        f.format(" Second file= %s%n", compareFile.getLocation());
        CompareNetcdf2.compareLists(getVariableNames(ds), getVariableNames(compareFile), f);

      } else {
        NestedTable nested = nestedTableList.get(0);
        Variable org = getCurrentVariable(nested.table);
        if (org == null) return;
        Variable ov = compareFile.findVariable(org.getFullNameEscaped());
        if (ov != null)
          cn.compareVariable(org, ov);
      }

      infoTA.setText(f.toString());
      infoTA.gotoTop();
      infoWindow.setTitle("Compare");
      infoWindow.show();

    } catch (Throwable ioe) {
      StringWriter sw = new StringWriter(10000);
      ioe.printStackTrace(new PrintWriter(sw));
      infoTA.setText(sw.toString());
      infoTA.gotoTop();
      infoWindow.show();

    } finally {
      if (compareFile != null)
        try {
          compareFile.close();
        }
        catch (Exception eek) {
            eek.printStackTrace();
        }
    }
  }

  private List<String> getVariableNames(NetcdfFile nc) {
    List<String> result = new ArrayList<>();
    for (Variable v : nc.getVariables())
      result.add(v.getFullName());
    return result;
  }

  ////////////////////////////////////////////////

  public void showAtts() {
    if (ds == null) return;
    if (attTable == null) {
      // global attributes
      attTable = new BeanTable(AttributeBean.class, (PreferencesExt) prefs.node("AttributeBeans"), false);
      PopupMenu varPopup = new PopupMenu(attTable.getJTable(), "Options");
      varPopup.addAction("Show Attribute", new AbstractAction() {
        public void actionPerformed(ActionEvent e) {
          AttributeBean bean = (AttributeBean) attTable.getSelectedBean();
          if (bean != null) {
            infoTA.setText(bean.att.toString());
            infoTA.gotoTop();
            infoWindow.show();
          }
        }
      });
      attWindow = new IndependentWindow("Global Attributes", BAMutil.getImage( "netcdfUI"), attTable);
      attWindow.setBounds( (Rectangle) prefs.getBean("AttWindowBounds", new Rectangle( 300, 100, 500, 800)));
    }

    List<AttributeBean> attlist = new ArrayList<>();
    for (Attribute att : ds.getGlobalAttributes()) {
      attlist.add(new AttributeBean(att));
    }
    attTable.setBeans(attlist);
    attWindow.show();
  }

  public NetcdfFile getDataset() {
    return this.ds;
  }

  public void clear() {
    this.ds = null;
    if (attTable != null) attTable.clearBeans();
    for (NestedTable nt : nestedTableList) nt.table.clearBeans();
    datasetTree.clear();
  }

  public void setDataset(NetcdfFile ds) {
    this.ds = ds;
    NestedTable nt = nestedTableList.get(0);
    nt.table.setBeans( getVariableBeans(ds));
    hideNestedTable( 1);

    datasetTree.setFile( ds);
  }

  private void setSelected( Variable v ) {
    eventsOK = false;

    List<Variable> vchain = new ArrayList<>();
    vchain.add( v);

    Variable vp = v;
    while (vp.isMemberOfStructure()) {
      vp = vp.getParentStructure();
      vchain.add( 0, vp); // reverse
    }

    for (int i=0; i<vchain.size(); i++) {
      vp = vchain.get(i);
      NestedTable ntable = setNestedTable(i, vp.getParentStructure());
      ntable.setSelected( vp);
    }

    eventsOK = true;
  }

  private NestedTable setNestedTable(int level, Structure s) {
    NestedTable ntable;
    if (nestedTableList.size() < level+1) {
      ntable = new NestedTable(level);
      nestedTableList.add( ntable);

    } else {
      ntable = nestedTableList.get(level);
    }

    if (s != null) // variables inside of records
      ntable.table.setBeans( getStructureVariables( s));

    ntable.show();
    return ntable;
  }

  private void hideNestedTable(int level) {
    int n = nestedTableList.size();
    for (int i=n-1; i>=level; i--) {
      NestedTable ntable = nestedTableList.get(i);
      ntable.hide();
    }
  }

  private class NestedTable {
    int level;
    PreferencesExt myPrefs;

    BeanTable table; // always the left component
    JSplitPane split = null; // right component (if exists) is the nested dataset.
    int splitPos = 100;
    boolean isShowing = false;

    NestedTable(int level) {
      this.level = level;
      myPrefs = (PreferencesExt) prefs.node("NestedTable"+level);

      table = new BeanTable(VariableBean.class, myPrefs, false);

      JTable jtable = table.getJTable();
      PopupMenu csPopup = new PopupMenu(jtable, "Options");
      csPopup.addAction("Show Declaration", new AbstractAction() {
        public void actionPerformed(ActionEvent e) {
          showDeclaration(table, false);
        }
      });
      csPopup.addAction("Show NcML", new AbstractAction() {
        public void actionPerformed(ActionEvent e) {
          showDeclaration(table, true);
        }
      });
      csPopup.addAction("Show Data", new AbstractAction() {
        public void actionPerformed(ActionEvent e) {
          showData(table);
        }
      });
      csPopup.addAction("NCdump Data", "Dump", new AbstractAction() {
        public void actionPerformed(ActionEvent e) {
          dumpData(table);
        }
      });
      csPopup.addAction("Write binary Data to file", new AbstractAction() {
        public void actionPerformed(ActionEvent e) {
          String binaryFilePath = fileChooser.chooseFilenameToSave("data.bin");
          if (binaryFilePath != null) {
            writeData(table, new File(binaryFilePath));
          }
        }
      });
      if (level == 0) {
        csPopup.addAction("Data Table", new AbstractAction() {
          public void actionPerformed(ActionEvent e) {
            dataTable(table);
          }
        });
        csPopup.addAction("Data Plot", new AbstractAction() {
            public void actionPerformed(ActionEvent e) {
              dataPlot(table);
            }
          });
      }

      // get selected variable, see if its a structure
      table.addListSelectionListener(e -> {
          Variable v = getCurrentVariable(table);
          if ((v != null) && (v instanceof Structure)) {
            hideNestedTable(NestedTable.this.level+2);
            setNestedTable(NestedTable.this.level+1, (Structure) v);

          } else {
            hideNestedTable(NestedTable.this.level+1);
          }
          if (eventsOK) {
            datasetTree.setSelected( v);
          }
      });

      // layout
      if (currentComponent == null) {
        currentComponent = table;
        tablePanel.add(currentComponent, BorderLayout.CENTER);
        isShowing = true;

      } else {
        split = new JSplitPane(JSplitPane.VERTICAL_SPLIT, false, currentComponent, table);
        splitPos = myPrefs.getInt("splitPos"+level, 500);
        if (splitPos > 0)
          split.setDividerLocation(splitPos);

        show();
      }
    }

    void show() {
      if (isShowing) return;

      tablePanel.remove( currentComponent);
      split.setLeftComponent( currentComponent);
      split.setDividerLocation( splitPos);
      currentComponent = split;
      tablePanel.add(currentComponent, BorderLayout.CENTER);
      tablePanel.revalidate();
      isShowing = true;
    }

    void hide() {
      if (!isShowing) return;
      tablePanel.remove( currentComponent);

      if (split != null) {
        splitPos = split.getDividerLocation();
        currentComponent = (JComponent) split.getLeftComponent();
        tablePanel.add(currentComponent, BorderLayout.CENTER);
      }

      tablePanel.revalidate();
      isShowing = false;
    }

    void setSelected( Variable vs) {

      List beans = table.getBeans();
      for (Object bean1 : beans) {
        VariableBean bean = (VariableBean) bean1;
        if (bean.vs == vs) {
          table.setSelectedBean(bean);
          return;
        }
      }
    }

    void saveState() {
      table.saveState( false);
      if (split != null) myPrefs.putInt("splitPos"+level, split.getDividerLocation());
    }
  }

  /* public void showTreeViewWindow() {
    if (treeWindow == null) {
      datasetTree = new DatasetTreeView();
      treeWindow = new IndependentWindow("TreeView", datasetTree);
      treeWindow.setIconImage(thredds.ui.BAMutil.getImage("netcdfUI"));
      treeWindow.setBounds( (Rectangle) prefs.getBean("treeWindow", new Rectangle( 150, 100, 400, 700)));
    }

    datasetTree.setDataset( ds);
    treeWindow.show();
  } */

  private void showDeclaration(BeanTable from, boolean isNcml) {
    Variable v = getCurrentVariable(from);
    if (v == null) return;
    infoTA.clear();

    if (isNcml) {
      NcMLWriter ncmlWriter = new NcMLWriter();
      ncmlWriter.setNamespace(null);
      ncmlWriter.getXmlFormat().setOmitDeclaration(true);

      Element varElement = ncmlWriter.makeVariableElement(v, false);
      infoTA.appendLine(ncmlWriter.writeToString(varElement));
    } else {
      infoTA.appendLine(v.toString());
    }

    if (Debug.isSet( "Xdeveloper")) {
      infoTA.appendLine("\n");
      infoTA.appendLine( "FULL NAME = "+ v.getFullName());
      infoTA.appendLine("\n");
      infoTA.appendLine(v.toStringDebug());
    }

    infoTA.gotoTop();
    infoWindow.setTitle("Variable Info");
    infoWindow.show();
  }

  private void showData(BeanTable from) {
    Variable v = getCurrentVariable(from);
    if (v == null) return;
    infoTA.clear();

    try {
      Array data = v.read();
      infoTA.setText( NCdumpW.toString(data, v.getFullName(), null));

    } catch (Exception ex) {
      StringWriter s = new StringWriter();
      ex.printStackTrace( new PrintWriter(s));
      infoTA.setText( s.toString());
    }

    infoTA.gotoTop();
    infoWindow.setTitle("Variable Info");
    infoWindow.show();

  }

  private void dumpData(BeanTable from) {
    Variable v = getCurrentVariable(from);
    if (v == null) return;

    dumpPane.clear();
    String spec;

    try {
      spec = ParsedSectionSpec.makeSectionSpecString(v, null);
      dumpPane.setContext(ds, spec);

    } catch (Exception ex) {
      StringWriter s = new StringWriter();
      ex.printStackTrace( new PrintWriter(s));
      dumpPane.setText( s.toString());
    }

    dumpWindow.show();
  }

  private void writeData(BeanTable from, File name) {
    Variable v = getCurrentVariable(from);
    if (v == null || name == null) return;

    try (FileOutputStream stream = new FileOutputStream(name)) {
      WritableByteChannel channel = stream.getChannel();
      v.readToByteChannel(v.getShapeAsSection(), channel);
      System.out.printf("Write ok to %s%n", name);

    } catch (InvalidRangeException | IOException e) {
      e.printStackTrace();
    }

  }

  /* private void showMissingData(BeanTable from) {
    VariableBean vb = (VariableBean) from.getSelectedBean();
    if (vb == null) return;
    Variable v = vb.vs;
    if ((v != null) && (v.getDataType() == ucar.nc2.DataType.STRUCTURE)) {
      showMissingStructureData( (Structure) v);
    }
    if (!vb.vs.hasMissing()) return;

    int count = 0, total = 0;
    infoTA.clear();
    infoTA.appendLine( v.toString());
    try {

      Array data = null;
      if (v.isMemberOfStructure())
        data = v.readAllStructures((List)null, true);
      else
        data = v.read();

      IndexIterator iter = data.getIndexIterator();
      while (iter.hasNext()) {
        if (vb.vs.isMissing( iter.getDoubleNext()))
          count++;
        total++;
      }

      double p = ((100.0 * count) / total);
      infoTA.appendLine( " missing values = "+count);
      infoTA.appendLine( " total values = "+total);
      infoTA.appendLine( " percent missing values = "+ Format.d(p, 2) +" %");

    } catch( InvalidRangeException e ) {
      infoTA.appendLine( "ERROR= "+e.getMessage());
    } catch( IOException ioe ) {
      infoTA.appendLine( "ERROR= "+ioe.getMessage());
    }
    infoTA.gotoTop();
    infoWindow.showIfNotIconified();
  }

  private void showMissingStructureData(Structure s) {
    ArrayList members = new ArrayList();
    List allMembers = s.getVariables();
    for (int i=0; i<allMembers.size(); i++) {
      Variable vs = (Variable) allMembers.get(i);
      if (vs.hasMissing())
        members.add( vs);
    }

    if (members.size() == 0) return;
    int[] count = new int[ members.size()];
    int[] total = new int[ members.size()];

    infoTA.clear();
    try {

     Structure.Iterator iter = s.getStructureIterator();
     while (iter.hasNext()) {
       StructureData sdata = iter.next();

       for (int i=0; i<members.size(); i++) {
         Variable vs = (Variable) members.get(i);

         Array data = sdata.findMemberArray( vs.getShortName());
         IndexIterator dataIter = data.getIndexIterator();
         while (dataIter.hasNext()) {
           if (vs.isMissing(dataIter.getDoubleNext()))
             count[i]++;
           total[i]++;
         }
       }
     }
     int countAll = 0, totalAll = 0;
     infoTA.appendLine("      name                missing   total     percent missing");
     for (int i=0; i<members.size(); i++) {
       Variable vs = (Variable) members.get(i);
       double p = ( (100.0 * count[i]) / total[i]);
       infoTA.appendLine(Format.s(vs.getShortName(), 25) +
                         " "+ Format.i(count[i], 7) +
                         "   "+ Format.i(total[i], 7) +
                         "   "+ Format.d(p, 2) + "%");
       countAll += count[i];
       totalAll += total[i];
     }

     infoTA.appendLine("");
     double p = ( (100.0 * countAll) / totalAll);
     infoTA.appendLine(Format.s("TOTAL ALL", 25) +
                       " "+ Format.i(countAll, 7) +
                       "   "+ Format.i(totalAll, 7) +
                       "   "+ Format.d(p, 2) + "%");

    } catch( IOException ioe ) {
      infoTA.appendLine( "ERROR= "+ioe.getMessage());
    }
    infoTA.gotoTop();
    infoWindow.showIfNotIconified();
  } */

  private void dataTable(BeanTable from) {
    VariableBean vb = (VariableBean) from.getSelectedBean();
    if (vb == null) return;
    Variable v = vb.vs;
    if (v instanceof Structure) {
      try {
        dataTable.setStructure((Structure)v);
      }
      catch (Exception ex) {
        ex.printStackTrace();
      }
      dataWindow.setComponent(dataTable);
  	}
  	else {
	  List<VariableBean> l = from.getSelectedBeans();
	  List<Variable> vl = new ArrayList<>();

	  for(VariableBean vb1  : l) {
		  if (vb1 == null) return;
		  v = vb1.vs;
		  if (v != null) {
			  vl.add(v);
		  }
		  else return;
	  }

	  variableTable.setDataset(ds);
	  variableTable.setVariableList(vl);
	  variableTable.createTable();

	  dataWindow.setComponent(variableTable);
    }
    Rectangle r = (Rectangle) prefs.getBean("dataWindowBounds", new Rectangle( 50, 300, 1000, 1200));
    dataWindow.setBounds(r);
  	dataWindow.show();
  }

  private void dataPlot(BeanTable from) {
    List<VariableBean> l = from.getSelectedBeans();

    for (VariableBean vb : l) {
      if (vb == null) return;
      Variable v = vb.vs;
      if (v != null) {
        try {
          dataPlot.setDataset(ds);
          dataPlot.setVariable(v);
        } catch (Exception ex) {
          ex.printStackTrace();
        }
      } else return;
    }
    plotWindow.show();
  }

  private Variable getCurrentVariable(BeanTable from) {
    VariableBean vb = (VariableBean) from.getSelectedBean();
    if (vb == null) return null;
    return vb.vs;
  }

  public PreferencesExt getPrefs() { return prefs; }

  public void save() {
    dumpPane.save();
    fileChooser.save();

    for (NestedTable nt : nestedTableList) {
      nt.saveState();
    }
    prefs.putBeanObject("InfoWindowBounds", infoWindow.getBounds());
    prefs.putBeanObject("dataWindowBounds", dataWindow.getBounds());
    prefs.putBeanObject("DumpWindowBounds", dumpWindow.getBounds());
    prefs.putBeanObject("PlotWindowBounds", plotWindow.getBounds());
    if (attWindow != null) prefs.putBeanObject("AttWindowBounds", attWindow.getBounds());

    prefs.putInt("mainSplit", mainSplit.getDividerLocation());
  }

  public List<VariableBean> getVariableBeans(NetcdfFile ds) {
    List<VariableBean> vlist = new ArrayList<>();
    for (Variable v : ds.getVariables()) {
      vlist.add(new VariableBean(v));
    }
    return vlist;
  }

  public List<VariableBean> getStructureVariables(Structure s) {
    List<VariableBean> vlist = new ArrayList<>();
    for (Variable v : s.getVariables()) {
      vlist.add( new VariableBean( v));
    }
    return vlist;
  }

  public class VariableBean {
    // static public String editableProperties() { return "title include logging freq"; }
    private Variable vs;
    private String name, dimensions, desc, units, dataType, shape;

    // create from a dataset
    public VariableBean( Variable vs) {
      this.vs = vs;
      //vs = (v instanceof VariableEnhanced) ? (VariableEnhanced) v : new VariableStandardized( v);

      setName( vs.getShortName());
      setDescription( vs.getDescription());
      setUnits( vs.getUnitsString());
      setDataType( vs.getDataType().toString());

      //Attribute csAtt = vs.findAttribute("_coordSystems");
      //if (csAtt != null)
      //  setCoordSys( csAtt.getStringValue());

      // collect dimensions
      StringBuilder lens = new StringBuilder();
      StringBuilder names = new StringBuilder();
      List dims = vs.getDimensions();
      for (int j=0; j<dims.size(); j++) {
        ucar.nc2.Dimension dim = (ucar.nc2.Dimension) dims.get(j);
        if (j>0) {
          lens.append(",");
          names.append(",");
        }
        String name = dim.isShared() ? dim.getShortName() : "anon";
        names.append(name);
        lens.append(dim.getLength());
      }
      setDimensions( names.toString());
      setShape( lens.toString());
    }

    public String getName() { return name; }
    public void setName(String name) { this.name = name; }

    public String getGroup() {
      return vs.getParentGroup().getFullName();
    }

    public String getDimensions() { return dimensions; }
    public void setDimensions(String dimensions) { this.dimensions = dimensions; }

    /* public boolean isCoordVar() { return isCoordVar; }
    public void setCoordVar(boolean isCoordVar) { this.isCoordVar = isCoordVar; }

    /* public boolean isAxis() { return axis; }
    public void setAxis(boolean axis) { this.axis = axis; }

    public boolean isGeoGrid() { return isGrid; }
    public void setGeoGrid(boolean isGrid) { this.isGrid = isGrid; }

    public String getAxisType() { return axisType; }
    public void setAxisType(String axisType) { this.axisType = axisType; } */

    //public String getCoordSys() { return coordSys; }
    //public void setCoordSys(String coordSys) { this.coordSys = coordSys; }

    /* public String getPositive() { return positive; }
    public void setPositive(String positive) { this.positive = positive; }

    */

    public String getDescription() { return desc; }
    public void setDescription(String desc) { this.desc = desc; }

    public String getUnits() { return units; }
    public void setUnits(String units) { this.units = units; }

    public String getDataType() { return dataType; }
    public void setDataType( String dataType) { this.dataType = dataType; }

    public String getShape() { return shape; }
    public void setShape( String shape) { this.shape = shape; }

    /** Get hasMissing */
   // public boolean getHasMissing() { return hasMissing; }
    /** Set hasMissing */
   // public void setHasMissing( boolean hasMissing) { this.hasMissing = hasMissing; }

  }

  public class AttributeBean {
    private Attribute att;

    // create from a dataset
    public AttributeBean( Attribute att) {
      this.att = att;
    }

    public String getName() { return att.getShortName(); }
    public String getValue() {
      Array value = att.getValues();
      return NCdumpW.toString(value, null, null);
    }

  }

  /* public class StructureBean {
    // static public String editableProperties() { return "title include logging freq"; }

    private String name;
    private int domainRank, rangeRank;
    private boolean isGeoXY, isLatLon, isProductSet, isZPositive;

    // no-arg constructor
    public StructureBean() {}

    // create from a dataset
    public StructureBean( Structure s) {

      setName( cs.getName());
      setGeoXY( cs.isGeoXY());
      setLatLon( cs.isLatLon());
      setProductSet( cs.isProductSet());
      setDomainRank( cs.getDomain().size());
      setRangeRank( cs.getCoordinateAxes().size());
      //setZPositive( cs.isZPositive());
    }

    public String getName() { return name; }
    public void setName(String name) { this.name = name; }

    public boolean isGeoXY() { return isGeoXY; }
    public void setGeoXY(boolean isGeoXY) { this.isGeoXY = isGeoXY; }

    public boolean getLatLon() { return isLatLon; }
    public void setLatLon(boolean isLatLon) { this.isLatLon = isLatLon; }

    public boolean isProductSet() { return isProductSet; }
    public void setProductSet(boolean isProductSet) { this.isProductSet = isProductSet; }

    public int getDomainRank() { return domainRank; }
    public void setDomainRank(int domainRank) { this.domainRank = domainRank; }

    public int getRangeRank() { return rangeRank; }
    public void setRangeRank(int rangeRank) { this.rangeRank = rangeRank; }

    //public boolean isZPositive() { return isZPositive; }
    //public void setZPositive(boolean isZPositive) { this.isZPositive = isZPositive; }
  }  */

}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.ui.grib;

import org.jdom2.Document;
import org.jdom2.Element;
import org.jdom2.output.Format;
import org.jdom2.output.XMLOutputter;
import thredds.inventory.*;
import thredds.inventory.MCollection;
import ucar.nc2.NetcdfFile;
import ucar.nc2.dataset.CoordinateAxis1D;
import ucar.nc2.dataset.NetcdfDataset;
import ucar.nc2.dt.GridCoordSystem;
import ucar.nc2.grib.GribData;
import ucar.nc2.grib.collection.Grib;
import ucar.nc2.grib.grib1.*;
import ucar.nc2.Attribute;
import ucar.nc2.dt.GridDatatype;
import ucar.nc2.dt.grid.GridDataset;
import ucar.nc2.grib.grib1.tables.Grib1Customizer;
import ucar.nc2.grib.grib1.tables.Grib1ParamTableReader;
import ucar.nc2.grib.grib1.tables.Grib1ParamTables;
import ucar.nc2.ui.ReportPanel;
import ucar.nc2.util.Counters;
import ucar.unidata.io.RandomAccessFile;
import ucar.util.prefs.PreferencesExt;

import java.io.*;
import java.util.*;

/**
 * Run through collections of Grib 1 files and make reports
 *
 * @author John
 * @since 8/28/11
 */
public class Grib1ReportPanel extends ReportPanel {

  public enum Report {
    checkTables, showLocalParams, summary, rename, showEncoding, gribIndex, gds
  }

  private Grib1Customizer cust = null;

  public Grib1ReportPanel(PreferencesExt prefs) {
    super(prefs);
  }

  @Override
  public Object[] getOptions() {
    return Grib1ReportPanel.Report.values();
  }

  @Override
  protected void doReport(Formatter f, Object option, MCollection dcm, boolean useIndex, boolean eachFile, boolean extra) throws IOException {
    cust = null;

    switch ((Report) option) {
      case checkTables:
        doCheckTables(f, dcm, useIndex);
        break;
      case gds:
        doUniqueGds(f, dcm, useIndex);
        break;
      case showLocalParams:
        doCheckLocalParams(f, dcm, useIndex);
        break;
      case summary:
        doScanIssues(f, dcm, useIndex, eachFile, extra);
        break;
      case rename:
        doRename(f, dcm, useIndex);
        break;
      case showEncoding:
        doShowEncoding(f, dcm);
        break;
      case gribIndex:
        doGribIndex(f, dcm, eachFile);
        break;
    }
  }

  private Grib1Index createIndex(MFile mf) throws IOException {
    String path = mf.getPath();
    Grib1Index index = new Grib1Index();
    if (!index.readIndex(path, mf.getLastModified())) {
      // make sure its a grib1 file
      try (RandomAccessFile raf = new RandomAccessFile(path, "r")) {
        if (!Grib1RecordScanner.isValidFile(raf)) return null;
        index.makeIndex(path, raf);
      }
    }
    return index;
  }

  /////////////////////////////////////////////////////////////////

  private void doGribIndex(Formatter f, MCollection dcm, boolean eachFile) throws IOException {
    Counters counters = new Counters();

    // must open collection again without gbx filtering
    try (MCollection dcm2 = getCollectionUnfiltered(spec, f)) {

      for (MFile mfile : dcm2.getFilesSorted()) {
        String path = mfile.getPath();
        f.format(" %s%n", path);
        doGribIndex(f, mfile, counters, eachFile);
      }
    }

    counters.show(f);
  }

  private void doGribIndex(Formatter fm, MFile ff, Counters counters, boolean eachFile) throws IOException {
    String path = ff.getPath();
    Grib1Index g1idx = new Grib1Index();
    g1idx.readIndex(path, 0, thredds.inventory.CollectionUpdateType.nocheck);
    counters.count("GDS", g1idx.getGds().size());

    // count unique hash
    Set<Integer> gdsHash = new HashSet<>();
    for (Grib1SectionGridDefinition gdss : g1idx.getGds()) {
      gdsHash.add(gdss.getGDS().hashCode());
    }
    counters.count("GDShashes", gdsHash.size());
    if (eachFile) {
      fm.format("   count=%d countHash=%d%n", g1idx.getGds().size(), gdsHash.size());
    }
  }

  ///////////////////////////////////////////////

  private void doCheckLocalParams(Formatter f, MCollection dcm, boolean useIndex) throws IOException {
    f.format("Check Grib-1 Parameter Tables for local entries%n");
    int[] accum = new int[4];

    for (MFile mfile : dcm.getFilesSorted()) {
      String path = mfile.getPath();
      f.format("%n %s%n", path);
      try {
        doCheckLocalParams(mfile, f, accum);
      } catch (Throwable t) {
        System.out.printf("FAIL on %s%n", mfile.getPath());
        t.printStackTrace();
      }
    }

    f.format("%nGrand total=%d local = %d missing = %d%n", accum[0], accum[2], accum[3]);
  }

  private void doCheckLocalParams(MFile ff, Formatter fm, int[] accum) throws IOException {
    int local = 0;
    int miss = 0;
    int nonop = 0;
    int total = 0;

    try (GridDataset ncfile = GridDataset.open(ff.getPath())) {
      Attribute gatt = ncfile.findGlobalAttributeIgnoreCase("GRIB table");
      if (gatt != null) {
        String[] s = gatt.getStringValue().split("-");
        Grib1ParamTableReader gtable = new Grib1ParamTables().getParameterTable(Integer.parseInt(s[0]), Integer.parseInt(s[1]), Integer.parseInt(s[2]));
        fm.format("  %s == %s%n", gatt, gtable.getPath());
      }
      for (GridDatatype dt : ncfile.getGrids()) {
        String currName = dt.getName();
        total++;

        Attribute att = dt.findAttributeIgnoreCase("Grib_Parameter");
        int number = (att == null) ? 0 : att.getNumericValue().intValue();
        if (number >= 128) {
          fm.format("  local parameter = %s (%d) units=%s %n", currName, number, dt.getUnitsString());
          local++;
          if (currName.startsWith("VAR")) miss++;
        }

      }
    }

    fm.format("total=%d local = %d miss=%d %n", total, local, miss);
    accum[0] += total;
    accum[1] += nonop;
    accum[2] += local;
    accum[3] += miss;
  }

  /////////////////////////////////////////////////////////////////

  // Look through the collection and find what GDS templates are used.
  private void doCheckTables(Formatter f, MCollection dcm, boolean useIndex) throws IOException {
    Counters counters = new Counters();

    for (MFile mfile : dcm.getFilesSorted()) {
      String path = mfile.getPath();
      f.format(" %s%n", path);
      if (useIndex)
        doCheckTablesWithIndex(f, mfile, counters);
      else
        doCheckTablesNoIndex(f, mfile, counters);
    }

    f.format("Check Parameter Tables%n");
    counters.show(f);
  }

  private void doCheckTablesWithIndex(Formatter mf, MFile ff,Counters counters) throws IOException {
    Grib1Index index = createIndex(ff);
    if (index == null) return;
    for (ucar.nc2.grib.grib1.Grib1Record gr : index.getRecords()) {
      doCheckTables(gr, counters);
    }
  }

  private void doCheckTablesNoIndex(Formatter fm, MFile ff, Counters counters) throws IOException {
    String path = ff.getPath();
    try (RandomAccessFile raf = new ucar.unidata.io.RandomAccessFile(path, "r")) {

      raf.order(ucar.unidata.io.RandomAccessFile.BIG_ENDIAN);
      raf.seek(0);

      Grib1RecordScanner reader = new Grib1RecordScanner(raf);
      while (reader.hasNext()) {
        ucar.nc2.grib.grib1.Grib1Record gr = reader.next();
        if (gr == null) break;
        doCheckTables(gr, counters);
      }

    } catch (Throwable ioe) {
      fm.format("Failed on %s == %s%n", path, ioe.getMessage());
      System.out.printf("Failed on %s%n", path);
      ioe.printStackTrace();
    }
  }

  private void doCheckTables(ucar.nc2.grib.grib1.Grib1Record gr, Counters counters) throws IOException {
    Grib1SectionProductDefinition pds = gr.getPDSsection();
    String key = Grib1Utils.extractParameterCode(gr);
    counters.count("center", pds.getCenter());
    counters.count("tableVersion", key);

    if (pds.getParameterNumber() > 127)
      counters.count("local", key);

    Grib1ParamTableReader table = new Grib1ParamTables().getParameterTable(pds.getCenter(), pds.getSubCenter(), pds.getTableVersion());
    if (table == null || null == table.getParameter(pds.getParameterNumber()))
      counters.count("missing", key);
  }

  /////////////////////////////////////////////////////////////////

  private void doScanIssues(Formatter f, MCollection dcm, boolean useIndex, boolean eachFile, boolean extraInfo) throws IOException {
    Counters countersAll = new Counters();

    for (MFile mfile : dcm.getFilesSorted()) {
      Counters countersOneFile = countersAll.makeSubCounters();

      String path = mfile.getPath();
      f.format(" %s%n", path);
      if (useIndex)
        doScanIssuesWithIndex(f, mfile, extraInfo, countersOneFile);
      else
        doScanIssuesNoIndex(f, mfile, extraInfo, countersOneFile);

      if (eachFile) {
        countersOneFile.show(f);
      }

      countersAll.addTo(countersOneFile);
    }

    f.format("ScanIssues - all files%n");
    countersAll.show(f);
  }

  private void doScanIssuesWithIndex(Formatter fm, MFile ff, boolean extraInfo, Counters counters) throws IOException {
    Grib1Index index = createIndex(ff);
    if (index == null) return;
    for (ucar.nc2.grib.grib1.Grib1Record gr : index.getRecords()) {
      doScanIssues(gr, fm, ff.getPath(), extraInfo, counters);
    }
  }

  private void doScanIssuesNoIndex(Formatter fm, MFile ff, boolean extraInfo, Counters counters) throws IOException {

    String path = ff.getPath();
    try (RandomAccessFile raf = new ucar.unidata.io.RandomAccessFile(path, "r")) {
      raf.order(ucar.unidata.io.RandomAccessFile.BIG_ENDIAN);
      raf.seek(0);

      Grib1RecordScanner reader = new Grib1RecordScanner(raf);
      while (reader.hasNext()) {
        ucar.nc2.grib.grib1.Grib1Record gr = reader.next();
        if (gr == null) break;
        doScanIssues(gr, fm, path, extraInfo, counters);
      }

    } catch (Throwable ioe) {
      fm.format("Failed on %s == %s%n", path, ioe.getMessage());
      System.out.printf("Failed on %s%n", path);
      ioe.printStackTrace();
    }
  }

  private void doScanIssues(ucar.nc2.grib.grib1.Grib1Record gr, Formatter fm, String path, boolean extraInfo, Counters counters) throws IOException {

    Grib1SectionGridDefinition gdss = gr.getGDSsection();
    Grib1Gds gds = gdss.getGDS();
    Grib1SectionProductDefinition pds = gr.getPDSsection();
    String table = pds.getCenter() + "-" + pds.getSubCenter() + "-" + pds.getTableVersion();
    counters.count("table version", table);
    counters.count("param", pds.getParameterNumber());
    counters.count("timeRangeIndicator", pds.getTimeRangeIndicator());
    counters.count("vertCoord", pds.getLevelType());
    counters.count("referenceDate", pds.getReferenceDate().toString());

    if (cust == null) cust = Grib1Customizer.factory(gr, null);

    Grib1ParamTime ptime = cust.getParamTime(pds);
    counters.count("timeCoord", ptime.getTimeCoord());
    counters.count("earthShape", gds.getEarthShape());
    counters.count("UVisReletiveToEastNorth", gds.getUVisReletiveToEastNorth() ? "true" : "false");

    if (gdss.isThin()) {
      if (extraInfo) fm.format("  THIN= (gds=%d) %s%n", gdss.getGridTemplate(), path);
      counters.count("thin", gdss.getGridTemplate());
    }

    if (!pds.gdsExists()) {
      if (extraInfo) fm.format("   PREDEFINED GDS= %s%n", path);
      counters.count("predefined", gdss.getPredefinedGridDefinition());
    }

    if (gdss.hasVerticalCoordinateParameters()) {
      if (extraInfo) fm.format("   Has vertical coordinates in GDS= %s%n", path);
      counters.count("vertCoordInGDS", pds.getLevelType());
    }

  }

  /////////////////////////////////////////////////////////////////

  private void doShowEncoding(Formatter f, MCollection dcm) throws IOException {
    Counters countersAll = new Counters();
    for (MFile mfile : dcm.getFilesSorted()) {
      f.format(" %s%n", mfile.getPath());
      // need dataRaf, so cant useIndex
      doShowEncodingNoIndex(f, mfile, countersAll);
    }

    countersAll.show(f);
  }

  private void doShowEncodingNoIndex(Formatter fm, MFile ff, Counters counters) throws IOException {
    String path = ff.getPath();
    try (RandomAccessFile raf = new ucar.unidata.io.RandomAccessFile(path, "r")) {
      raf.order(ucar.unidata.io.RandomAccessFile.BIG_ENDIAN);
      raf.seek(0);

      Grib1RecordScanner reader = new Grib1RecordScanner(raf);
      while (reader.hasNext()) {
        ucar.nc2.grib.grib1.Grib1Record gr = reader.next();
        if (gr == null) break;
        GribData.Info info = gr.getBinaryDataInfo(raf);
        counters.count("decimalScale", info.decimalScaleFactor);
        counters.count("binScale", info.binaryScaleFactor);
        counters.count("nbits", info.numberOfBits);
        counters.count("gridType", info.getGridPointS());
        counters.count("packing", info.getPackingS());
        counters.count("dataType", info.getDataTypeS());
        counters.count("hasOctet14", info.hasOctet14() ? 1 : 0);

        if (info.binaryScaleFactor != 0 && info.decimalScaleFactor != 0) {
          counters.count("scale", 1);
        } else {
          counters.count("scale", 0);
        }
      }

    } catch (Throwable ioe) {
      fm.format("Failed on %s == %s%n", path, ioe.getMessage());
      System.out.printf("Failed on %s%n", path);
      ioe.printStackTrace();
    }
  }

  ///////////////////////////////////////////////////////////////////////////////////

  private void doRename(Formatter f, MCollection dcm, boolean useIndex) throws IOException {
    f.format("CHECK Grib-1 Names: Old vs New for collection %s%n", dcm.getCollectionName());

    List<VarName> varNames = new ArrayList<>(3000);
    Map<String, List<String>> gridsAll = new HashMap<>(1000); // old -> list<new>

    for (MFile mfile : dcm.getFilesSorted()) {
      f.format("%n%s%n", mfile.getPath());
      Map<Integer, GridMatch> gridsNew = getGridsNew(mfile, f);
      Map<Integer, GridMatch> gridsOld = getGridsOld(mfile, f);

      // look for exact match
      for (GridMatch gm : gridsNew.values()) {
        GridMatch match = gridsOld.get(gm.hashCode());
        if (match != null) {
          gm.match = match;
          match.match = gm;
        }
      }

      // look for alternative match
      for (GridMatch gm : gridsNew.values()) {
        if (gm.match == null) {
          GridMatch match = altMatch(gm, gridsOld.values());
          if (match != null) {
            gm.match = match;
            match.match = gm;
          }
        }
      }

      f.format("%n");
      List<GridMatch> listNew = new ArrayList<>(gridsNew.values());
      Collections.sort(listNew);
      for (GridMatch gm : listNew) {
        f.format(" %s%n", gm.grid.getFullName());
        if (gm.match != null)
          f.format(" %s%n", gm.match.grid.getFullName());
        f.format("%n");
      }

      f.format("%nMISSING MATCHES IN NEW%n");
      List<GridMatch> list = new ArrayList<>(gridsNew.values());
      Collections.sort(list);
      for (GridMatch gm : list) {
        if (gm.match == null)
          f.format(" %s (%s) == %s%n", gm.grid.getFullName(), gm.show(), gm.grid.getDescription());
      }

      f.format("%nMISSING MATCHES IN OLD%n");
      List<GridMatch> listOld = new ArrayList<>(gridsOld.values());
      Collections.sort(listOld);
      for (GridMatch gm : listOld) {
        if (gm.match == null)
          f.format(" %s (%s)%n", gm.grid.getFullName(), gm.show());
      }

      // add to gridsAll
      for (GridMatch gmOld : listOld) {
        String key = gmOld.grid.getFullName();
        List<String> newGrids = gridsAll.computeIfAbsent(key, k -> new ArrayList<>());
        if (gmOld.match != null) {
          String keyNew = gmOld.match.grid.getFullName() + " == " + gmOld.match.grid.getDescription();
          if (!newGrids.contains(keyNew)) newGrids.add(keyNew);
        }
      }

      // add matches to VarNames
      for (GridMatch gmOld : listOld) {
        if (gmOld.match == null) {
          f.format("MISSING %s (%s)%n", gmOld.grid.getFullName(), gmOld.show());
          continue;
        }
        Attribute att = gmOld.match.grid.findAttributeIgnoreCase(Grib.VARIABLE_ID_ATTNAME);
        String varId = att == null ? "" : att.getStringValue();
        varNames.add(new VarName(mfile.getName(), gmOld.grid.getShortName(), gmOld.match.grid.getShortName(), varId));
      }

    }

    f.format("%nOLD -> NEW MAPPINGS%n");
    List<String> keys = new ArrayList<>(gridsAll.keySet());
    int total = keys.size();
    int dups = 0;
    Collections.sort(keys);
    for (String key : keys) {
      f.format(" OLD %s%n", key);
      List<String> newGrids = gridsAll.get(key);
      Collections.sort(newGrids);
      if (newGrids.size() > 1) dups++;
      for (String newKey : newGrids)
        f.format(" NEW %s%n", newKey);
      f.format("%n");
    }
    f.format("Number with more than one map=%d total=%d%n", dups, total);

    // old -> new mapping xml table
    if (!useIndex) {
      Element rootElem = new Element("gribVarMap");
      Document doc = new Document(rootElem);
      rootElem.setAttribute("collection", dcm.getCollectionName());

      String currentDs = null;
      Element dsElem = null;
      for (VarName vn : varNames) {
        if (!vn.dataset.equals(currentDs)) {
          dsElem = new Element("dataset");
          rootElem.addContent(dsElem);
          dsElem.setAttribute("name", vn.dataset);
          currentDs = vn.dataset;
        }
        Element param = new Element("param");
        dsElem.addContent(param);
        param.setAttribute("oldName", vn.oldVar);
        param.setAttribute("newName", vn.newVar);
        param.setAttribute("varId", vn.varId);
      }

      FileOutputStream fout = new FileOutputStream("C:/tmp/grib1VarMap.xml");
      XMLOutputter fmt = new XMLOutputter(Format.getPrettyFormat());
      fmt.output(doc, fout);
      fout.close();
    }

  }

  private static class VarName {
    String dataset;
    String oldVar;
    String newVar;
    String varId;

    private VarName(String dataset, String oldVar, String newVar, String varId) {
      this.dataset = dataset;
      this.oldVar = oldVar;
      this.newVar = newVar;
      this.varId = varId;
    }
  }

  private GridMatch altMatch(GridMatch want, Collection<GridMatch> test) {
    // look for scale factor errors in prob
    for (GridMatch gm : test) {
      if (gm.match != null) continue; // already matched
      if (gm.altMatch(want)) return gm;
    }

    // give up matching the prob
    for (GridMatch gm : test) {
      if (gm.match != null) continue; // already matched
      if (gm.altMatchNoProb(want)) return gm;
    }

    return null;
  }

  private static class GridMatch implements Comparable<GridMatch> {
    GridDatatype grid;
    GridMatch match;
    boolean isNew;
    int[] param = new int[3];
    int level;
    boolean isLayer, isError;
    int interval = -1;
    int prob = -1;
    int ens = -1;
    int probLimit;

    private GridMatch(GridDataset gds, GridDatatype grid, boolean aNew) {
      this.grid = grid;
      isNew = aNew;

      GridCoordSystem gcs = grid.getCoordinateSystem();
      CoordinateAxis1D zaxis = gcs.getVerticalAxis();
      if (zaxis != null) isLayer = zaxis.isInterval();

      if (isNew) {
      /* :Grib1_Center = 7; // int
       :Grib1_Subcenter = 0; // int
       :Grib1_TableVersion = 2; // int
       :Grib1_Parameter = 33; */
        Attribute att = grid.findAttributeIgnoreCase("Grib1_Center");
        param[0] = att.getNumericValue().intValue();
        att = grid.findAttributeIgnoreCase("Grib1_Subcenter");
        param[1] = att.getNumericValue().intValue();
        att = grid.findAttributeIgnoreCase("Grib1_Parameter");
        param[2] = att.getNumericValue().intValue();

        att = grid.findAttributeIgnoreCase("Grib1_Level_Type");
        level = att.getNumericValue().intValue();
        isError = grid.getName().contains("error");

        att = grid.findAttributeIgnoreCase("Grib1_Statistical_Interval_Type");
        if (att != null) {
          int intv = att.getNumericValue().intValue();
          if (intv != 255) interval = intv;
        }

        att = grid.findAttributeIgnoreCase("Grib1_Probability_Type"); // ??
        if (att != null) prob = att.getNumericValue().intValue();

        att = grid.findAttributeIgnoreCase("Grib1_Probability_Name"); // ??
        if (att != null) {
          String pname = att.getStringValue();
          int pos = pname.indexOf('_');
          pname = pname.substring(pos + 1);
          probLimit = (int) (1000.0 * Double.parseDouble(pname));
        }

        att = grid.findAttributeIgnoreCase("Grib1_Ensemble_Derived_Type");
        if (att != null) ens = att.getNumericValue().intValue();

      } else { // OLD
        Attribute att = grid.findAttributeIgnoreCase("GRIB_center_id");
        param[0] = att.getNumericValue().intValue();
        att = gds.findGlobalAttributeIgnoreCase("Originating_subcenter_id");
        param[1] = att.getNumericValue().intValue();
        att = grid.findAttributeIgnoreCase("GRIB_param_number");
        param[2] = att.getNumericValue().intValue();

        att = grid.findAttributeIgnoreCase("GRIB_level_type");
        level = att.getNumericValue().intValue();
        isError = grid.getName().contains("error");

        String desc = grid.getDescription();
        if (desc.contains("Accumulation"))
          interval = 4;
        else if (desc.contains("Accumulation"))
          interval = 4;

        att = grid.findAttributeIgnoreCase("GRIB_probability_type");
        if (att != null) prob = att.getNumericValue().intValue();
        if (prob == 0) {
          att = grid.findAttributeIgnoreCase("GRIB_probability_lower_limit");
          if (att != null) probLimit = (int) (1000 * att.getNumericValue().doubleValue());
          //if (Math.abs(probLimit) > 100000) probLimit /= 1000; // wierd bug in 4.2
        } else if (prob == 1) {
          att = grid.findAttributeIgnoreCase("GRIB_probability_upper_limit"); // GRIB_probability_upper_limit = 12.89; // double
          if (att != null) probLimit = (int) (1000 * att.getNumericValue().doubleValue());
          //if (Math.abs(probLimit) > 100000) probLimit /= 1000; // wierd bug in 4.2
        }

        att = grid.findAttributeIgnoreCase("GRIB_ensemble_derived_type");
        if (att != null) ens = att.getNumericValue().intValue();
      }
    }

    @Override
    public boolean equals(Object o) {
      if (this == o) return true;
      if (o == null || getClass() != o.getClass()) return false;

      GridMatch gridMatch = (GridMatch) o;

      if (param != gridMatch.param) return false;
      if (ens != gridMatch.ens) return false;
      if (interval != gridMatch.interval) return false;
      if (isError != gridMatch.isError) return false;
      if (isLayer != gridMatch.isLayer) return false;
      if (level != gridMatch.level) return false;
      if (prob != gridMatch.prob) return false;
      if (probLimit != gridMatch.probLimit) return false;

      return true;
    }

    public boolean altMatch(GridMatch gridMatch) {
      if (!altMatchNoProb(gridMatch)) return false;

      if (probLimit / 1000 == gridMatch.probLimit) return true;
      if (probLimit == gridMatch.probLimit / 1000) return true;

      return false;
    }

    public boolean altMatchNoProb(Object o) {
      if (this == o) return true;
      if (o == null || getClass() != o.getClass()) return false;

      GridMatch gridMatch = (GridMatch) o;

      if (ens != gridMatch.ens) return false;
      if (interval != gridMatch.interval) return false;
      if (isError != gridMatch.isError) return false;
      if (isLayer != gridMatch.isLayer) return false;
      if (level != gridMatch.level) return false;
      if (prob != gridMatch.prob) return false;

      return true;
    }

    @Override
    public int hashCode() {
      int result = 1;
      result = 31 * result + level;
      result = 31 * result + param[0];
      result = 31 * result + (isLayer ? 1 : 0);
      result = 31 * result + (isError ? 1 : 0);
      result = 31 * result + param[1];
      result = 31 * result + interval;
      result = 31 * result + prob;
      result = 31 * result + param[2];
      result = 31 * result + ens;
      result = 31 * result + probLimit;
      return result;
    }

    @Override
    public int compareTo(GridMatch o) {
      return grid.compareTo(o.grid);
    }

    String show() {
      Formatter f = new Formatter();
      f.format("%d-%d-%d-", param[0], param[1], param[2]);
      f.format("%d", level);
      if (isLayer) f.format("_layer");
      if (interval >= 0) f.format("_intv%d", interval);
      if (prob >= 0) f.format("_prob%d_%d", prob, probLimit);
      if (ens >= 0) f.format("_ens%d", ens);
      if (isError) f.format("_error");
      return f.toString();
    }
  }

  private Map<Integer, GridMatch> getGridsNew(MFile ff, Formatter f) throws IOException {
    Map<Integer, GridMatch> grids = new HashMap<>(100);
    try (GridDataset ncfile = GridDataset.open(ff.getPath())) {
      for (GridDatatype dt : ncfile.getGrids()) {
        GridMatch gm = new GridMatch(ncfile, dt, true);
        GridMatch dup = grids.get(gm.hashCode());
        if (dup != null)
          f.format(" DUP NEW (%d == %d) = %s (%s) and DUP %s (%s)%n", gm.hashCode(), dup.hashCode(), gm.grid.getFullName(), gm.show(), dup.grid.getFullName(), dup.show());
        else
          grids.put(gm.hashCode(), gm);
      }
    }
    return grids;
  }

  private Map<Integer, GridMatch> getGridsOld(MFile ff, Formatter f) throws IOException {
    Map<Integer, GridMatch> grids = new HashMap<>(100);
    try (NetcdfFile ncfile = NetcdfFile.open(ff.getPath(), "ucar.nc2.iosp.grib.GribServiceProvider", -1, null, null)) {
      NetcdfDataset ncd = new NetcdfDataset(ncfile);
      GridDataset grid = new GridDataset(ncd);
      for (GridDatatype dt : grid.getGrids()) {
        GridMatch gm = new GridMatch(grid, dt, false);
        GridMatch dup = grids.get(gm.hashCode());
        if (dup != null)
          f.format(" DUP OLD (%d == %d) = %s (%s) and DUP %s (%s)%n", gm.hashCode(), dup.hashCode(),
              gm.grid.getFullName(), gm.show(), dup.grid.getFullName(), dup.show());
        else
          grids.put(gm.hashCode(), gm);
      }
    } catch (Throwable t) {
      t.printStackTrace();
    }
    return grids;
  }

  ///////////////////////////////////////////////

  /*
    private void doLocalUseSection(Formatter f, CollectionManager dcm, boolean useIndex) throws IOException {
      f.format("Show Local Use Section%n");

      for (MFile mfile : dcm.getFiles()) {
        f.format(" %s%n", mfile.getPath());
        doLocalUseSection(mfile, f, useIndex);
      }
    }

    private void doLocalUseSection(MFile mf, Formatter f, boolean useIndex) throws IOException {
      f.format("File = %s%n", mf);

      String path = mf.getPath();
      GribIndex index = new GribIndex();
      if (!index.readIndex(path, mf.getLastModified()))
        index.makeIndex(path, f);

      for (Grib2Record gr : index.getRecords()) {
        Grib2SectionLocalUse lus = gr.getLocalUseSection();
        if (lus == null || lus.getRawBytes() == null)
          f.format(" %10d == none%n", gr.getDataSection().getStartingPosition());
        else
          f.format(" %10d == %s%n", gr.getDataSection().getStartingPosition(), Misc.showBytes(lus.getRawBytes()));
      }
    } */

    ///////////////////////////////////////////////

    // Look through the collection and find what GDS templates are used.
    private void doUniqueGds(Formatter f, MCollection dcm, boolean useIndex) throws IOException {
      f.format("Show Unique GDS%n");

      Map<Integer, GdsList> gdsSet = new HashMap<>();
      for (MFile mfile : dcm.getFilesSorted()) {
        f.format(" %s%n", mfile.getPath());
        doUniqueGds(mfile, gdsSet, f);
      }

      List<GdsList> sorted = new ArrayList<>(gdsSet.values());
      Collections.sort(sorted);

      for (GdsList gdsl : sorted) {
        f.format("%nGDS %s template= %d %n", gdsl.gds.getNameShort(), gdsl.gds.template);
        for (FileCount fc : gdsl.fileList) {
          f.format("  %5d %s %n", fc.countRecords, fc.f.getPath());
        }
      }
    }

    private void doUniqueGds(MFile mf, Map<Integer, GdsList> gdsSet, Formatter f) throws IOException {
      String path = mf.getPath();
      Grib1Index g1idx = new Grib1Index();
      boolean ok = g1idx.readIndex(path, 0, thredds.inventory.CollectionUpdateType.nocheck);
      if (!ok) {
        f.format("**Cant open %s%n", path);
        return;
      }

      for (Grib1Record gr : g1idx.getRecords()) {
        int template = gr.getGDSsection().getGDS().template;
        gdsSet.computeIfAbsent(template, k -> new GdsList(gr.getGDSsection().getGDS()));
        GdsList gdsList = gdsSet.get(template);
        FileCount fc = gdsList.findOrAdd(mf);
        fc.countRecords++;
      }
    }

    private class GdsList implements Comparable<GdsList> {
      Grib1Gds gds;
      java.util.List<FileCount> fileList = new ArrayList<>();

      private GdsList(Grib1Gds gds) {
        this.gds = gds;
      }

      FileCount findOrAdd(MFile f) {
        for (FileCount fc : fileList) {
          if (fc.f.getPath().equals(f.getPath()))
            return fc;
        }

        FileCount fc = new FileCount(f);
        fileList.add(fc);
        return fc;
      }

      @Override
      public int compareTo(GdsList o) {
        return gds.template - o.gds.template;
      }
    }

    private class FileCount {
      private FileCount(MFile f) {
        this.f = f;
      }

      MFile f;
      int countRecords = 0;
    }

    ///////////////////////////////////////////////
  /*

    private int countPDS, countPDSdup;

    private void doDuplicatePds(Formatter f, CollectionManager dcm, boolean useIndex) throws IOException {
      countPDS = 0;
      countPDSdup = 0;
      f.format("Show Duplicate PDS%n");
      for (MFile mfile : dcm.getFiles()) {
        doDuplicatePds(f, mfile);
      }
      f.format("Total PDS duplicates = %d / %d%n%n", countPDSdup, countPDS);
    }

    private void doDuplicatePds(Formatter f, MFile mfile) throws IOException {
      Set<Long> pdsMap = new HashSet<Long>();
      int dups = 0;
      int count = 0;

      RandomAccessFile raf = new RandomAccessFile(mfile.getPath(), "r");
      Grib2RecordScanner scan = new Grib2RecordScanner(raf);
      while (scan.hasNext()) {
        ucar.nc2.grib.grib2.Grib2Record gr = scan.next();
        Grib2SectionProductDefinition pds = gr.getPDSsection();
        long crc = pds.calcCRC();
        if (pdsMap.contains(crc))
          dups++;
        else
          pdsMap.add(crc);
        count++;
      }
      raf.close();

      f.format("PDS duplicates = %d / %d for %s%n%n", dups, count, mfile.getPath());
      countPDS += count;
      countPDSdup += dups;
    }

    ///////////////////////////////////////////////

    private class Counter {
      Map<Integer, Integer> set = new HashMap<Integer, Integer>();
      String name;

      private Counter(String name) {
        this.name = name;
      }

      void count(int value) {
        Integer count = set.get(value);
        if (count == null)
          set.put(value, 1);
        else
          set.put(value, count + 1);
      }

      void show(Formatter f) {
        f.format("%n%s%n", name);
        java.util.List<Integer> list = new ArrayList<Integer>(set.keySet());
        Collections.sort(list);
        for (int template : list) {
          int count = set.get(template);
          f.format("   %3d: count = %d%n", template, count);
        }
      }

    }

    int total = 0;
    int prob = 0;

    private void doPdsSummary(Formatter f, CollectionManager dcm, boolean useIndex) throws IOException {
      if (useIndex) {
        f.format("Check Grib-2 PDS probability and statistical variables%n");
        total = 0;
        prob = 0;
        for (MFile mfile : dcm.getFiles()) {
          f.format("%n %s%n", mfile.getPath());
          doPdsSummaryIndexed(f, mfile);
        }
        f.format("problems = %d/%d%n", prob, total);

      } else {
        Counter templateSet = new Counter("template");
        Counter timeUnitSet = new Counter("timeUnit");
        Counter statTypeSet = new Counter("statType");
        Counter NTimeIntervals = new Counter("NTimeIntervals");
        Counter processId = new Counter("processId");

        for (MFile mfile : dcm.getFiles()) {
          f.format(" %s%n", mfile.getPath());
          doPdsSummary(f, mfile, templateSet, timeUnitSet, statTypeSet, NTimeIntervals, processId);
        }

        templateSet.show(f);
        timeUnitSet.show(f);
        statTypeSet.show(f);
        NTimeIntervals.show(f);
        processId.show(f);
      }
    }

    private void doPdsSummaryIndexed(Formatter fm, MFile ff) throws IOException {
      String path = ff.getPath();

      //make sure indexes exist
      GribIndex index = new GribIndex();
      if (!index.readIndex(path, ff.getLastModified()))
        index.makeIndex(path, fm);
      GribCollection gc = GribCollectionBuilder.createFromSingleFile(new File(path), fm);
      gc.close();

      GridDataset ncfile = null;
      try {
        ncfile = GridDataset.open(path + GribCollection.IDX_EXT);
        for (GridDatatype dt : ncfile.getGrids()) {
          String currName = dt.getName();
          total++;

          Attribute att = dt.findAttributeIgnoreCase("Grib_Probability_Type");
          if (att != null) {
            fm.format("  %s (PROB) desc=%s %n", currName, dt.getDescription());
            prob++;
          }

          att = dt.findAttributeIgnoreCase(GribIosp.GRIB_STAT_TYPE);
          if (att != null) {
            int statType = att.getNumericValue().intValue();
            if ((statType == 7) || (statType == 9)) {
              fm.format("  %s (STAT type %s) desc=%s %n", currName, statType, dt.getDescription());
              prob++;
            }
          }

          att = dt.findAttributeIgnoreCase("Grib_Ensemble_Derived_Type");
          if (att != null) {
            int type = att.getNumericValue().intValue();
            if ((type > 9)) {
              fm.format("  %s (DERIVED type %s) desc=%s %n", currName, type, dt.getDescription());
              prob++;
            }
          }

        }
      } catch (Throwable ioe) {
        fm.format("Failed on %s == %s%n", path, ioe.getMessage());
        System.out.printf("Failed on %s%n", path);
        ioe.printStackTrace();

      } finally {
        if (ncfile != null) ncfile.close();
      }
    }

    private void doPdsSummary(Formatter f, MFile mf, Counter templateSet, Counter timeUnitSet, Counter statTypeSet, Counter NTimeIntervals,
                              Counter processId) throws IOException {
      String path = mf.getPath();
      GribIndex index = new GribIndex();
      if (!index.readIndex(path, mf.getLastModified()))
        index.makeIndex(path, f);

      for (ucar.nc2.grib.grib2.Grib2Record gr : index.getRecords()) {
        Grib2Pds pds = gr.getPDS();
        templateSet.count(pds.getTemplateNumber());
        timeUnitSet.count(pds.getTimeUnit());
        processId.count(pds.getGenProcessId());

        if (pds instanceof Grib2Pds.PdsInterval) {
          Grib2Pds.PdsInterval pdsi = (Grib2Pds.PdsInterval) pds;
          for (Grib2Pds.TimeInterval ti : pdsi.getTimeIntervals())
            statTypeSet.count(ti.statProcessType);
          NTimeIntervals.count(pdsi.getTimeIntervals().length);
        }
      }
    }

    ///////////////////////////////////////////////

    private void doIdProblems(Formatter f, CollectionManager dcm, boolean useIndex) throws IOException {
      f.format("Look for ID Problems%n");

      Counter disciplineSet = new Counter("discipline");
      Counter masterTable = new Counter("masterTable");
      Counter localTable = new Counter("localTable");
      Counter centerId = new Counter("centerId");
      Counter subcenterId = new Counter("subcenterId");
      Counter genProcess = new Counter("genProcess");
      Counter backProcess = new Counter("backProcess");

      for (MFile mfile : dcm.getFiles()) {
        f.format(" %s%n", mfile.getPath());
        doIdProblems(f, mfile, useIndex,
                disciplineSet, masterTable, localTable, centerId, subcenterId, genProcess, backProcess);
      }

      disciplineSet.show(f);
      masterTable.show(f);
      localTable.show(f);
      centerId.show(f);
      subcenterId.show(f);
      genProcess.show(f);
      backProcess.show(f);
    }

    private void doIdProblems(Formatter f, MFile mf, boolean showProblems,
                              Counter disciplineSet, Counter masterTable, Counter localTable, Counter centerId,
                              Counter subcenterId, Counter genProcessC, Counter backProcessC) throws IOException {
      String path = mf.getPath();
      GribIndex index = new GribIndex();
      if (!index.readIndex(path, mf.getLastModified()))
        index.makeIndex(path, f);

      // these should be the same for the entire file
      int center = -1;
      int subcenter = -1;
      int master = -1;
      int local = -1;
      int genProcess = -1;
      int backProcess = -1;

      for (ucar.nc2.grib.grib2.Grib2Record gr : index.getRecords()) {
        disciplineSet.count(gr.getDiscipline());
        masterTable.count(gr.getId().getMaster_table_version());
        localTable.count(gr.getId().getLocal_table_version());
        centerId.count(gr.getId().getCenter_id());
        subcenterId.count(gr.getId().getSubcenter_id());
        genProcessC.count(gr.getPDS().getGenProcessId());
        backProcessC.count(gr.getPDS().getBackProcessId());

        if (!showProblems) continue;

        if (gr.getDiscipline() == 255) {
          f.format("  bad discipline= ");
          gr.show(f);
          f.format("%n");
        }

        int val = gr.getId().getCenter_id();
        if (center < 0) center = val;
        else if (center != val) {
          f.format("  center %d != %d ", center, val);
          gr.show(f);
          f.format(" %s%n", gr.getId());
        }

        val = gr.getId().getSubcenter_id();
        if (subcenter < 0) subcenter = val;
        else if (subcenter != val) {
          f.format("  subcenter %d != %d ", subcenter, val);
          gr.show(f);
          f.format(" %s%n", gr.getId());
        }

        val = gr.getId().getMaster_table_version();
        if (master < 0) master = val;
        else if (master != val) {
          f.format("  master %d != %d ", master, val);
          gr.show(f);
          f.format(" %s%n", gr.getId());
        }

        val = gr.getId().getLocal_table_version();
        if (local < 0) local = val;
        else if (local != val) {
          f.format("  local %d != %d ", local, val);
          gr.show(f);
          f.format(" %s%n", gr.getId());
        }

        val = gr.getPDS().getGenProcessId();
        if (genProcess < 0) genProcess = val;
        else if (genProcess != val) {
          f.format("  genProcess %d != %d ", genProcess, val);
          gr.show(f);
          f.format(" %s%n", gr.getId());
        }

        val = gr.getPDS().getBackProcessId();
        if (backProcess < 0) backProcess = val;
        else if (backProcess != val) {
          f.format("  backProcess %d != %d ", backProcess, val);
          gr.show(f);
          f.format(" %s%n", gr.getId());
        }

      }
    }

    ///////////////////////////////////////////////

    private void doDrsSummary(Formatter f, CollectionManager dcm, boolean useIndex) throws IOException {
      f.format("Show Unique DRS Templates%n");
      Counter template = new Counter("DRS template");
      Counter prob = new Counter("DRS template 40 signed problem");

      for (MFile mfile : dcm.getFiles()) {
        f.format(" %s%n", mfile.getPath());
        doDrsSummary(f, mfile, useIndex, template, prob);
      }

      template.show(f);
      prob.show(f);
    }

    private void doDrsSummary(Formatter f, MFile mf, boolean useIndex, Counter templateC, Counter probC) throws IOException {
      String path = mf.getPath();
      GribIndex index = new GribIndex();
      if (!index.readIndex(path, mf.getLastModified()))
        index.makeIndex(path, f);
      RandomAccessFile raf = new RandomAccessFile(path, "r");

      for (ucar.nc2.grib.grib2.Grib2Record gr : index.getRecords()) {
        Grib2SectionDataRepresentation drss = gr.getDataRepresentationSection();
        int template = drss.getDataTemplate();
        templateC.count(template);

        if (useIndex && template == 40) {  // expensive
          Grib2Drs.Type40 drs40 = gr.readDataTest(raf);
          if (drs40 != null) {
            if (drs40.hasSignedProblem())
              probC.count(1);
            else
              probC.count(0);
          }
        }
      }
      raf.close();
    }

    ///////////////////////////////////////////////

    private void doGdsTemplate(Formatter f, CollectionManager dcm, boolean useIndex) throws IOException {
      f.format("Show Unique GDS Templates%n");

      Map<Integer, Integer> drsSet = new HashMap<Integer, Integer>();
      for (MFile mfile : dcm.getFiles()) {
        f.format(" %s%n", mfile.getPath());
        doGdsTemplate(f, mfile, drsSet);
      }

      for (int template : drsSet.keySet()) {
        int count = drsSet.get(template);
        f.format("%nGDS template = %d count = %d%n", template, count);
      }
    }

    private void doGdsTemplate(Formatter f, MFile mf, Map<Integer, Integer> gdsSet) throws IOException {
      String path = mf.getPath();
      GribIndex index = new GribIndex();
      if (!index.readIndex(path, mf.getLastModified()))
        index.makeIndex(path, f);

      for (Grib2SectionGridDefinition gds : index.getGds()) {
        int template = gds.getGDSTemplateNumber();
        Integer count = gdsSet.get(template);
        if (count == null)
          gdsSet.put(template, 1);
        else
          gdsSet.put(template, count + 1);
      }
    }   */

}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package thredds.client.catalog.tools;

import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Formatter;
import java.util.List;

import thredds.client.catalog.Access;
import thredds.client.catalog.Catalog;
import thredds.client.catalog.Dataset;
import thredds.client.catalog.Property;
import thredds.client.catalog.ServiceType;
import thredds.client.catalog.builder.CatalogBuilder;
import ucar.nc2.Attribute;
import ucar.nc2.constants.DataFormatType;
import ucar.nc2.constants.FeatureType;
import ucar.nc2.dataset.DatasetUrl;
import ucar.nc2.dataset.NetcdfDataset;
import ucar.nc2.ft.FeatureDataset;
import ucar.nc2.ft.FeatureDatasetFactoryManager;
import ucar.nc2.ft.remote.CdmrFeatureDataset;
import ucar.nc2.stream.CdmRemote;
import ucar.nc2.util.Optional;
import ucar.unidata.util.StringUtil2;

import javax.annotation.Nonnull;

/**
 * DataFactory for THREDDS client catalogs
 *
 * @author caron
 * @since 1/8/2015
 */
public class DataFactory {
  static public final String PROTOCOL = "thredds";
  static public final String SCHEME = PROTOCOL + ":";

  static private ServiceType[] preferAccess;

  static public void setPreferCdm(boolean prefer) {
    preferAccess = prefer ? new ServiceType[] {ServiceType.CdmRemote} : null;
  }
  static public void setPreferAccess(ServiceType... prefer) {
    preferAccess = prefer;
  }

  static public void setDebugFlags(ucar.nc2.util.DebugFlags debugFlag) {
    debugOpen = debugFlag.isSet("thredds/debugOpen");
    debugTypeOpen = debugFlag.isSet("thredds/openDatatype");
  }

  static private boolean debugOpen = false;
  static private boolean debugTypeOpen = false;

  /**
   * The result of trying to open a THREDDS dataset.
   * If fatalError is true, the operation failed, errLog should indicate why.
   * Otherwise, the FeatureType and FeatureDataset is valid.
   * There may still be warning or diagnostic errors in errLog.
   */
  public static class Result implements Closeable {
    public boolean fatalError;
    public Formatter errLog = new Formatter();

    public FeatureType featureType;
    public FeatureDataset featureDataset;
    public String imageURL;

    public String location;
    public Access accessUsed;

    @Override
    public String toString() {
      final StringBuilder sb = new StringBuilder();
      sb.append("Result");
      sb.append("{fatalError=").append(fatalError);
      sb.append(", errLog=").append(errLog);
      sb.append(", featureType=").append(featureType);
      sb.append(", featureDataset=").append(featureDataset);
      sb.append(", imageURL='").append(imageURL).append('\'');
      sb.append(", location='").append(location).append('\'');
      sb.append(", accessUsed=").append(accessUsed);
      sb.append('}');
      return sb.toString();
    }

    @Override
    public void close() throws IOException {
      if (featureDataset != null) featureDataset.close();
    }
  }

  /**
   * Open a FeatureDataset from a URL location string. Example URLS:

   *
http://localhost:8080/test/addeStationDataset.xml#surfaceHourly
   *
thredds:http://localhost:8080/test/addeStationDataset.xml#surfaceHourly
   *
thredds://localhost:8080/test/addeStationDataset.xml#surfaceHourly
   *
thredds:file:c:/test/data/catalog/addeStationDataset.xml#AddeSurfaceData (absolute file)
   *
thredds:resolve:resolveURL
   *

   *
   * @param urlString [thredds:]catalog.xml#datasetId
   * @param task      may be null
   * @return ThreddsDataFactory.Result check fatalError for validity
   * @throws java.io.IOException on read error
   */
  public DataFactory.Result openFeatureDataset(String urlString, ucar.nc2.util.CancelTask task) throws IOException {

    DataFactory.Result result = new DataFactory.Result();
    Dataset dataset = openCatalogFromLocation(urlString, task, result);
    if (result.fatalError || dataset == null)
      return result;

    return openFeatureDataset(null, dataset, task, result);
  }

  /**
   * Open a FeatureDataset from a URL location string, and a desired type (may by NONE or null).
   *
   * @param wantFeatureType desired feature type, may be NONE or null
   * @param urlString       [thredds:]catalog.xml#datasetId
   * @param task            may be null
   * @return ThreddsDataFactory.Result check fatalError for validity
   * @throws java.io.IOException on read error
   */
  public DataFactory.Result openFeatureDataset(FeatureType wantFeatureType, String urlString, ucar.nc2.util.CancelTask task)
          throws IOException {
    DataFactory.Result result = new DataFactory.Result();
    Dataset ds = openCatalogFromLocation(urlString, task, result);
    if (result.fatalError || ds == null)
      return result;

    return openFeatureDataset(wantFeatureType, ds, task, result);
  }

  private Dataset openCatalogFromLocation(String location, ucar.nc2.util.CancelTask task, Result result) throws IOException {
    location = location.trim();
    location = StringUtil2.replace(location, '\\', "/");

    if (location.startsWith(SCHEME))
      location = location.substring(8);

    if (location.startsWith("resolve:")) {
      location = location.substring(8);
      return openResolver(location, task, result);
    }

    if (!location.startsWith("http:") && !location.startsWith("file:"))   // LOOK whats this for??
      location = "http:" + location;

    Catalog catalog;
    String datasetId;

    int pos = location.indexOf('#');
    if (pos < 0) {
      result.fatalError = true;
      result.errLog.format("Must have the form catalog.xml#datasetId%n");
      return null;
    }

    CatalogBuilder catFactory = new CatalogBuilder();
    String catalogLocation = location.substring(0, pos);
    catalog = catFactory.buildFromLocation(catalogLocation, null);
    if (catalog == null) {
      result.errLog.format("Invalid catalog from Resolver <%s>%n%s%n", catalogLocation, catFactory.getErrorMessage());
      result.fatalError = true;
      return null;
    }

    datasetId = location.substring(pos + 1);
    Dataset ds = catalog.findDatasetByID(datasetId);
    if (ds == null) {
      result.fatalError = true;
      result.errLog.format("Could not find dataset %s in %s %n", datasetId, catalogLocation);
      return null;
    }

    return ds;
  }

  /**
   * Open a FeatureDataset from an Dataset object, deciding on which Access to use.
   *
   * @param Dataset use this to figure out what type, how to open, etc
   * @param task       allow user to cancel; may be null
   * @return ThreddsDataFactory.Result check fatalError for validity
   * @throws IOException on read error
   */
  @Nonnull
  public DataFactory.Result openFeatureDataset(Dataset Dataset, ucar.nc2.util.CancelTask task) throws IOException {
    return openFeatureDataset(null, Dataset, task, new Result());
  }

  @Nonnull
  public DataFactory.Result openFeatureDataset(FeatureType wantFeatureType, Dataset ds, ucar.nc2.util.CancelTask task, Result result)
          throws IOException {

    // deal with RESOLVER type
    Access resolverAccess = findAccessByServiceType(ds.getAccess(), ServiceType.Resolver);
    if (resolverAccess != null) {
      Dataset rds = openResolver(resolverAccess.getStandardUrlName(), task, result);
      if (rds != null) ds = rds;
    }

    try {
      result.featureType = ds.getFeatureType();
      if (result.featureType == null)
        result.featureType = wantFeatureType;

      // first choice would be remote FeatureDataset
      Access access = findAccessByServiceType(ds.getAccess(), ServiceType.CdmrFeature);
      if (access != null)
        return openFeatureDataset(result.featureType, access, task, result);

      // special handling for images
      if (result.featureType == FeatureType.IMAGE) {
        access = getImageAccess(ds, task, result);
        if (access != null) {
          return openFeatureDataset(result.featureType, access, task, result);
        } else
          result.fatalError = true;
        return result;
      }

      // otherwise, open as a remote NetcdfDataset (cdmremote or opendap)
      NetcdfDataset ncd = openDataset(ds, true, task, result);
      if (null != ncd)
        result.featureDataset = FeatureDatasetFactoryManager.wrap(result.featureType, ncd, task, result.errLog);

      if (null == result.featureDataset)
        result.fatalError = true;
      else {
        result.location = result.featureDataset.getLocation();
        if (result.featureType == null)
          result.featureType = result.featureDataset.getFeatureType();
      }

      return result;

    } catch (Throwable t) {
      result.close();
      if (t instanceof IOException) throw (IOException) t;
      throw new RuntimeException(t);
    }
  }

  /**
   * Open a FeatureDataset from an Access object.
   *
   * @param access use this Access.
   * @param task   may be null
   * @return ThreddsDataFactory.Result check fatalError for validity
   * @throws IOException on read error
   */
  public DataFactory.Result openFeatureDataset(Access access, ucar.nc2.util.CancelTask task) throws IOException {
    Dataset ds = access.getDataset();
    DataFactory.Result result = new Result();
    if (ds.getFeatureType() == null) {
      result.errLog.format("InvDatasert must specify a FeatureType%n");
      result.fatalError = true;
      return result;
    }

    return openFeatureDataset(ds.getFeatureType(), access, task, result);
  }

  private DataFactory.Result openFeatureDataset(FeatureType wantFeatureType, Access access, ucar.nc2.util.CancelTask task, Result result)
          throws IOException {
    result.featureType = wantFeatureType;
    result.accessUsed = access;

    // special handling for IMAGE
    if (result.featureType == FeatureType.IMAGE) {
      result.imageURL = access.getStandardUrlName();
      result.location = result.imageURL;
      return result;
    }

    if (access.getService().getType() == ServiceType.CdmrFeature) {
      Optional<FeatureDataset> opt = CdmrFeatureDataset.factory(wantFeatureType, access.getStandardUrlName());
      if (opt.isPresent())
        result.featureDataset = opt.get();
      else
        result.errLog.format("%s", opt.getErrorMessage());

    } else {

      // all other datatypes
      NetcdfDataset ncd = openDataset(access, true, task, result);
      if (null != ncd) {
        result.featureDataset = FeatureDatasetFactoryManager.wrap(result.featureType, ncd, task, result.errLog);
      }
    }

    if (null == result.featureDataset)
      result.fatalError = true;
    else {
      result.location = result.featureDataset.getLocation();
      result.featureType = result.featureDataset.getFeatureType();
    }

    return result;
  }

//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

  /**
   * Open a NetcdfDataset from a URL location string. Example URLS:

   *
http://localhost:8080/test/addeStationDataset.xml#surfaceHourly
   *
thredds:http://localhost:8080/test/addeStationDataset.xml#surfaceHourly
   *
thredds://localhost:8080/test/addeStationDataset.xml#surfaceHourly
   *
thredds:file:c:/dev/netcdf-java-2.2/test/data/catalog/addeStationDataset.xml#AddeSurfaceData (absolute file)
   *
thredds:resolve:resolveURL
   *

   *
   * @param location catalog.xml#datasetId, may optionally start with "thredds:"
   * @param task     may be null
   * @param log      error messages gp here, may be null
   * @param acquire  if true, aquire the dataset, else open it
   * @return NetcdfDataset
   * @throws java.io.IOException on read error
   */
  public NetcdfDataset openDataset(String location, boolean acquire, ucar.nc2.util.CancelTask task, Formatter log) throws IOException {
    Result result = new Result();
    Dataset dataset = openCatalogFromLocation(location, task, result);
    if (result.fatalError || dataset == null) {
      if (log != null) log.format("%s", result.errLog);
      result.close();
      return null;
    }

    return openDataset(dataset, acquire, task, result);
  }

  /**
   * Try to open as a NetcdfDataset.
   *
   * @param Dataset open this
   * @param acquire    if true, aquire the dataset, else open it
   * @param task       may be null
   * @param log        error message, may be null
   * @return NetcdfDataset or null if failure
   * @throws IOException on read error
   */
  public NetcdfDataset openDataset(Dataset Dataset, boolean acquire, ucar.nc2.util.CancelTask task, Formatter log) throws IOException {
    Result result = new Result();
    NetcdfDataset ncd = openDataset(Dataset, acquire, task, result);
    if (log != null) log.format("%s", result.errLog);
    if (result.fatalError) {
      result.close();
      if (ncd != null) ncd.close();
    }
    return (result.fatalError) ? null : ncd;
  }

  private NetcdfDataset openDataset(Dataset dataset, boolean acquire, ucar.nc2.util.CancelTask task, Result result) throws IOException {

    IOException saveException = null;

    List<Access> accessList = new ArrayList<>(dataset.getAccess()); // a list of all the accesses
    while (accessList.size() > 0) {
      Access access = chooseDatasetAccess(accessList);

      // no valid access
      if (access == null) {
        result.errLog.format("No access that could be used in dataset %s %n", dataset);
        if (saveException != null)
          throw saveException;
        return null;
      }

      String datasetLocation = access.getStandardUrlName();
      ServiceType serviceType = access.getService().getType();
      if (debugOpen)
        System.out.println("ThreddsDataset.openDataset try " + datasetLocation + " " + serviceType);

      // deal with RESOLVER type
      if (serviceType == ServiceType.Resolver) {
        Dataset rds = openResolver(datasetLocation, task, result);
        if (rds == null) return null;
        accessList = new ArrayList<>(rds.getAccess());
        continue;
      }

      // ready to open it through netcdf API
      NetcdfDataset ds;

      // try to open
      try {
        ds = openDataset(access, acquire, task, result);

      } catch (IOException e) {
        result.errLog.format("Cant open %s %n err=%s%n", datasetLocation, e.getMessage());
        if (debugOpen) {
          System.out.println("Cant open= " + datasetLocation + " " + serviceType);
          e.printStackTrace();
        }
        String mess = e.getMessage();
        if (mess != null && mess.contains("Unauthorized"))
          break; // bail out

        accessList.remove(access);
        saveException = e;
        continue;
      }

      result.accessUsed = access;
      return ds;
    } // loop over accesses

    // if (saveException != null) throw saveException;
    return null;
  }

  /**
   * Try to open Access as a NetcdfDataset.
   *
   * @param access  open this Access
   * @param acquire if true, aquire the dataset, else open it
   * @param task    may be null
   * @param log     error message, may be null
   * @return NetcdfDataset or null if failure
   * @throws IOException on read error
   */
  public NetcdfDataset openDataset(Access access, boolean acquire, ucar.nc2.util.CancelTask task, Formatter log)
          throws IOException {
    try (Result result = new Result()) {
      NetcdfDataset ncd = openDataset(access, acquire, task, result);
      if (log != null) log.format("%s", result.errLog);
      if (result.fatalError && ncd != null)
        ncd.close();
      return (result.fatalError) ? null : ncd;

      // result.close() will be called at the end of this block, which will close the Result's FeatureDataset, if it
      // exists. That data member won't have been set by any of the code above, so the close() will essentially be a
      // no-op.
    }
  }

  private NetcdfDataset openDataset(Access access, boolean acquire, ucar.nc2.util.CancelTask task, Result result) throws IOException {
    Dataset ds = access.getDataset();
    String datasetId = ds.getId();
    String title = ds.getName();

    String datasetLocation = access.getStandardUrlName();
    ServiceType serviceType = access.getService().getType();
    if (debugOpen) System.out.println("ThreddsDataset.openDataset= " + datasetLocation);

    // deal with RESOLVER type
    if (serviceType == ServiceType.Resolver) {
      Dataset rds = openResolver(datasetLocation, task, result);
      if (rds == null) return null;
      return openDataset(rds, acquire, task, result);
    }

    // ready to open it through netcdf API
    DatasetUrl durl = new DatasetUrl(serviceType, datasetLocation);
    NetcdfDataset ncd = acquire ? NetcdfDataset.acquireDataset(durl, true, task) : NetcdfDataset.openDataset(durl, null, -1, task, null);

    /* String prefix = null;
    if ((serviceType == ServiceType.OPENDAP)|| (serviceType == ServiceType.DODS))
        prefix = "dods:";
    else if(serviceType == ServiceType.DAP4)
        prefix = "dap4:";

    boolean enhanceMode = true;
    if (prefix != null) {
      String curl = datasetLocation;
      if (curl.startsWith("http:")) {
          curl = prefix + datasetLocation.substring(5);
      } else if (curl.startsWith("https:")) {
          curl =  prefix + curl.substring(6);
      }
      ncd = acquire ? NetcdfDataset.acquireDataset(curl, enhanceMode, task) : NetcdfDataset.openDataset(curl, enhanceMode, task);
    }

    // open CdmRemote
    else if (serviceType == ServiceType.CdmRemote) {
      String curl = CdmRemote.canonicalURL(datasetLocation);
      ncd = acquire ? NetcdfDataset.acquireDataset(curl, enhanceMode, task) : NetcdfDataset.openDataset(curl, enhanceMode, task);
    }

    // open HTTPServer
    else if (serviceType == ServiceType.HTTPServer) {
      String curl =  (datasetLocation.startsWith("http:")) ? "httpserver:" + datasetLocation.substring(5) : datasetLocation;
      ncd = acquire ? NetcdfDataset.acquireDataset(curl, enhanceMode, task) : NetcdfDataset.openDataset(curl, enhanceMode, task);
    }

    else {
      // open through NetcdfDataset API
      ncd = acquire ? NetcdfDataset.acquireDataset(datasetLocation, enhanceMode, task) : NetcdfDataset.openDataset(datasetLocation, enhanceMode, task);
    } */

    result.accessUsed = access;
    ncd.setId(datasetId);
    ncd.setTitle(title);
    annotate(ds, ncd);

    /* see if there's NcML metadata LOOK whats this
    List<Metadata> list = ds.getMetadata(Metadata.Type.NcML);
    if (list != null && list.size() > 0) {
      Metadata ncmlMetadata = list.get(0);
      NcMLReader.wrapNcML(ds, ncmlMetadata.getXlinkHref(), null);
    }  */

    return ncd;
  }

  /**
   * Find the "best" access in case theres more than one, based on what the CDM knows
   * how to open and use.
   *
   * @param accessList choose from this list.
   * @return best access method.
   */
  public Access chooseDatasetAccess(List<Access> accessList) {
    if (accessList.size() == 0)
      return null;

    Access access = null;
    if (preferAccess != null) {
      for (ServiceType type : preferAccess) {
        access = findAccessByServiceType(accessList, type);
        if (access != null) break;
      }
    }

    // the order indicates preference
    if (access == null)
      access = findAccessByServiceType(accessList, ServiceType.CdmRemote);
    if (access == null)
      access = findAccessByServiceType(accessList, ServiceType.DODS);
    if (access == null)
      access = findAccessByServiceType(accessList, ServiceType.OPENDAP);
    if (access == null)
       access = findAccessByServiceType(accessList, ServiceType.DAP4);
    if (access == null)
      access = findAccessByServiceType(accessList, ServiceType.File); // should mean that it can be opened through netcdf API
    if (access == null)
      access = findAccessByServiceType(accessList, ServiceType.HTTPServer); // should mean that it can be opened through netcdf API

    /* look for HTTP with format we can read
    if (access == null) {
      Access tryAccess = findAccessByServiceType(accessList, ServiceType.HTTPServer);

      if (tryAccess != null) {
        DataFormatType format = tryAccess.getDataFormatType();

        // these are the file types we can read
        if ((DataFormatType.NCML == format) || (DataFormatType.NETCDF == format)) {   // removed 4/4/2015 jc
        //if ((DataFormatType.BUFR == format) || (DataFormatType.GINI == format) || (DataFormatType.GRIB1 == format)
        //        || (DataFormatType.GRIB2 == format) || (DataFormatType.HDF5 == format) || (DataFormatType.NCML == format)
       //         || (DataFormatType.NETCDF == format) || (DataFormatType.NEXRAD2 == format) || (DataFormatType.NIDS == format)) {
          access = tryAccess;
        }
      }
    } */

    // ADDE
    if (access == null)
      access = findAccessByServiceType(accessList, ServiceType.ADDE);

    // RESOLVER
    if (access == null) {
      access = findAccessByServiceType(accessList, ServiceType.Resolver);
    }

    return access;
  }

  private Dataset openResolver(String urlString, ucar.nc2.util.CancelTask task, Result result) throws IOException {
    CatalogBuilder catFactory = new CatalogBuilder();
    Catalog catalog = catFactory.buildFromLocation(urlString, null);
    if (catalog == null) {
      result.errLog.format("Couldnt open Resolver %s err=%s%n ", urlString, catFactory.getErrorMessage());
      return null;
    }

    for (Dataset ds : catalog.getDatasetsLocal()) {
      if (ds.hasAccess()) return ds;
      for (Dataset nested : ds.getDatasetsLocal())   // cant be more than one deep
        if (nested.hasAccess()) return nested;
    }

    return null;
  }

  /**
   * Add information from the Dataset to the NetcdfDataset.
   *
   * @param ds        get info from here
   * @param ncDataset add to here
   */
  public static void annotate(Dataset ds, NetcdfDataset ncDataset) {
    ncDataset.setTitle(ds.getName());
    ncDataset.setId(ds.getId());

    // add properties as global attributes
    for (Property p : ds.getProperties()) {
      String name = p.getName();
      if (null == ncDataset.findGlobalAttribute(name)) {
        ncDataset.addAttribute(null, new Attribute(name, p.getValue()));
      }
    }

    /* ThreddsMetadata.GeospatialCoverage geoCoverage = ds.getGeospatialCoverage();
   if (geoCoverage != null) {
     if ( null != geoCoverage.getNorthSouthRange()) {
       ncDataset.addAttribute(null, new Attribute("geospatial_lat_min", new Double(geoCoverage.getLatSouth())));
       ncDataset.addAttribute(null, new Attribute("geospatial_lat_max", new Double(geoCoverage.getLatNorth())));
     }
     if ( null != geoCoverage.getEastWestRange()) {
       ncDataset.addAttribute(null, new Attribute("geospatial_lon_min", new Double(geoCoverage.getLonWest())));
       ncDataset.addAttribute(null, new Attribute("geospatial_lon_max", new Double(geoCoverage.getLonEast())));
     }
     if ( null != geoCoverage.getUpDownRange()) {
       ncDataset.addAttribute(null, new Attribute("geospatial_vertical_min", new Double(geoCoverage.getHeightStart())));
       ncDataset.addAttribute(null, new Attribute("geospatial_vertical_max", new Double(geoCoverage.getHeightStart() + geoCoverage.getHeightExtent())));
     }
   }

   DateRange timeCoverage = ds.getTimeCoverage();
   if (timeCoverage != null) {
     ncDataset.addAttribute(null, new Attribute("time_coverage_start", timeCoverage.getStart().toDateTimeStringISO()));
     ncDataset.addAttribute(null, new Attribute("time_coverage_end", timeCoverage.getEnd().toDateTimeStringISO()));
   } */

    ncDataset.finish();
  }

//////////////////////////////////////////////////////////////////////////////////
// image

// look for an access method for an image datatype

  private Access getImageAccess(Dataset ds, ucar.nc2.util.CancelTask task, Result result) throws IOException {

    List<Access> accessList = new ArrayList<>(ds.getAccess()); // a list of all the accesses
    while (accessList.size() > 0) {
      Access access = chooseImageAccess(accessList);
      if (access == null) {
        result.errLog.format("No access that could be used for Image Type %s %n", ds);
        return null;
      }

      // deal with RESOLVER type
      String datasetLocation = access.getStandardUrlName();
      Dataset rds = openResolver(datasetLocation, task, result);
      if (rds == null)
        return null;

      // use the access list from the resolved dataset
      accessList = new ArrayList<>(ds.getAccess());
    }

    return null;
  }

  private Access chooseImageAccess(List<Access> accessList) {
    Access access;

    access = findAccessByDataFormatType(accessList, DataFormatType.JPEG);
    if (access != null) return access;

    access = findAccessByDataFormatType(accessList, DataFormatType.GIF);
    if (access != null) return access;

    access = findAccessByDataFormatType(accessList, DataFormatType.TIFF);
    if (access != null) return access;

    access = findAccessByServiceType(accessList, ServiceType.ADDE);
    if (access != null) {
      String datasetLocation = access.getStandardUrlName();
      if (datasetLocation.indexOf("image") > 0)
        return access;
    }

    return access;
  }

///////////////////////////////////////////////////////////////////

// works against the accessList instead of the dataset list, so we can remove and try again

  private Access findAccessByServiceType(List<Access> accessList, ServiceType type) {
    for (Access a : accessList) {
      ServiceType stype = a.getService().getType();
      if (stype != null && stype.toString().equalsIgnoreCase(type.toString()))
        return a;
    }
    return null;
  }

// works against the accessList instead of the dataset list, so we can remove and try again

  private Access findAccessByDataFormatType(List<Access> accessList, DataFormatType type) {
    for (Access a : accessList) {
      DataFormatType has = a.getDataFormatType();
      if (has != null && type.toString().equalsIgnoreCase(has.toString()))
        return a;
    }
    return null;
  }
}