package thredds.tdm;

import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.ParameterException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import thredds.featurecollection.FeatureCollectionConfig;
import thredds.featurecollection.FeatureCollectionType;
import thredds.inventory.*;
import thredds.inventory.filter.StreamFilter;
import thredds.inventory.partition.*;
import ucar.nc2.grib.GribIndex;
import ucar.nc2.grib.GribIndexCache;
import ucar.nc2.grib.GribUtils;
import ucar.nc2.grib.collection.Grib1Iosp;
import ucar.nc2.grib.collection.GribCdmIndex;
import ucar.nc2.grib.grib1.*;
import ucar.nc2.grib.grib1.tables.Grib1Customizer;
import ucar.nc2.grib.grib2.*;
import ucar.nc2.grib.grib2.table.Grib2Tables;
import ucar.nc2.time.CalendarDate;
import ucar.nc2.util.CloseableIterator;
import ucar.nc2.util.Counters;
import ucar.nc2.util.DiskCache2;
import ucar.nc2.util.Indent;
import ucar.unidata.io.RandomAccessFile;
import ucar.unidata.util.StringUtil2;

import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;

/**
 * GribCollection Building - pass1 : gather information, optionally make gbx9 files
 *
 * @author caron
 * @since 11/15/2014
 */
public class GCpass1 {
  static private final Logger logger = LoggerFactory.getLogger(GCpass1.class);

  /*
String usage = "usage: thredds.tdm.GCpass1 -spec <collectionSpec> [-isGrib2] -partition [none|file|directory] -useTableVersion [true|false] "+
      "-useCacheDir <dir>"; */
  private static class CommandLine {
    @Parameter(names = {"-spec"}, description = "Collection specification string, exactly as in the <featureCollection>.", required = false)
    public String spec;

    @Parameter(names = {"-rootDir"}, description = "Collection rootDir, exactly as in the <featureCollection>.", required = false)
    public String rootDir;

    @Parameter(names = {"-regexp"}, description = "Collection regexp string, exactly as in the <featureCollection>.", required = false)
    public String regexp;

    @Parameter(names = {"-isGrib2"}, description = "Is Grib2 collection.", required = false)
    public boolean isGrib2 = false;

    @Parameter(names = {"-partition"}, description = "Partition type: none, directory, file", required = false)
    public FeatureCollectionConfig.PartitionType partitionType = FeatureCollectionConfig.PartitionType.directory;

    @Parameter(names = {"-useTableVersion"}, description = "Use Table version to make seperate variables.", required = false)
    public boolean useTableVersion = false;

    @Parameter(names = {"-useCacheDir"}, description = "Set the Grib index cache directory.", required = false)
    public String cacheDir;

    @Parameter(names = {"-h", "--help"}, description = "Display this help and exit", help = true)
    public boolean help = false;

    private final JCommander jc;

    public CommandLine(String progName, String[] args) throws ParameterException {
      this.jc = new JCommander(this, args);  // Parses args and uses them to initialize *this*.
      jc.setProgramName(progName);           // Displayed in the usage information.
    }

    public void printUsage() {
      jc.usage();
    }

    FeatureCollectionType getFeatureCollectionType() {
      return isGrib2 ? FeatureCollectionType.GRIB2 : FeatureCollectionType.GRIB1;
    }
  }

  public static void main(String[] args) throws IOException {
    CommandLine cmdLine = new CommandLine("thredds.tdm.GCpass1", args);

    if (cmdLine.help) {
      cmdLine.printUsage();
      return;
    }

    if (cmdLine.cacheDir != null) {
      DiskCache2 gribCache = DiskCache2.getDefault();
      gribCache.setRootDirectory(cmdLine.cacheDir);
      gribCache.setAlwaysUseCache(true);
      GribIndexCache.setDiskCache2(gribCache);
    }

    /*
      public FeatureCollectionConfig(String name, String path, FeatureCollectionType fcType, String spec, String collectionName,
                                 String dateFormatMark, String olderThan, String timePartition, Element innerNcml)
     */
    FeatureCollectionConfig config = new FeatureCollectionConfig("GCpass1", "GCpass1", cmdLine.getFeatureCollectionType(),
            cmdLine.spec, null, null, null, cmdLine.partitionType.toString(), null);
    FeatureCollectionConfig.GribConfig gribConfig = config.gribConfig;
    gribConfig.useTableVersion = cmdLine.useTableVersion;

    if (cmdLine.rootDir != null && cmdLine.regexp != null)
      config.setFilter(cmdLine.rootDir,cmdLine.regexp);

    Formatter fm = new Formatter(System.out);
    GCpass1 pass1 = new GCpass1(config, fm);
    pass1.scanAndReport();
  }

  public static class Accum {
    CalendarDate last = null;
    int nfiles;
    int nrecords;
    float fileSize;
    float indexSize;

    void add(Accum a) {
      this.nfiles += a.nfiles;
      this.nrecords += a.nrecords;
      this.fileSize += a.fileSize;
      this.indexSize += a.indexSize;
    }
  }

  static class Variable implements Comparable<Variable> {
    int cdmHash;
    String name;

    Variable(int cdmHash, String name) {
      this.cdmHash = cdmHash;
      this.name = name;
    }

    @Override
    public boolean equals(Object o) {
      if (this == o) return true;
      if (o == null || getClass() != o.getClass()) return false;
      Variable variable = (Variable) o;
      return cdmHash == variable.cdmHash;
    }

    @Override
    public int hashCode() {
      return cdmHash;
    }

    @Override
    public int compareTo(Variable o) {
      return name.compareTo(o.name);
    }

    @Override
    public String toString() {
      return name;
    }
  }

  ///////////////////////////////////
  FeatureCollectionConfig config;
  FeatureCollectionConfig.GribConfig gribConfig;
  Formatter fm;
  Counters countersAll;
  Accum accumAll = new Accum();

  public GCpass1(FeatureCollectionConfig config, Formatter fm) {
    this.config = config;
    this.gribConfig = config.gribConfig;
    this.fm = fm;

    this.config.show(fm);
    fm.format("%n");

    countersAll = new Counters();
    countersAll.add("referenceDate").setShowRange(true);
    countersAll.add("table version");
    countersAll.add("variable");
    countersAll.add("gds");

    countersAll.add("gdsTemplate");
    countersAll.add("vertCoordInGDS");
    countersAll.add("predefined");
    countersAll.add("thin");
  }

  public void reportAll(Indent indent, Formatter fm) {
    fm.format("%n");
    reportOneHeader(indent, fm);
    fm.format("%s%60s", indent, "grand total");
    fm.format("%8d ", accumAll.nfiles);
    fm.format("%8d ", accumAll.nrecords);
    fm.format("%8.0f ", accumAll.indexSize);
    fm.format("%8.0f ", accumAll.fileSize);
    fm.format("%8d ", countersAll.get("variable").getUnique());
    fm.format("%8d ", countersAll.get("referenceDate").getUnique());
    fm.format("%8d ", countersAll.get("gds").getUnique());
    fm.format("%n");

    countersAll.show(fm);

    if (gds1set.size() > 1) {
      fm.format("gds1%n");
      for (Grib1Record gr1 : gds1set.values()) {
        Grib1Gds gds = gr1.getGDS();
        fm.format(" hash %s == %s%n", gds.hashCode(), gds);
      }
    }

    if (gds2set.size() > 1) {
      fm.format("gds2%n");
      for (Integer key : gds2set.keySet()) {
        Grib2Record gr2 = gds2set.get(key);
        Grib2Gds gds = gr2.getGDS();
        fm.format(" key = %d hash = %s%n", key, gds.hashCode());
        Grib2Show.showGdsTemplate(gr2.getGDSsection(), fm, cust2);
        fm.format("%n");
      }
    }
  }

  public void reportOneHeader(Indent indent, Formatter fm) {
    fm.format("%s%60s #files   #records #idxSize #dataSize #vars  #runtimes    #gds%n", indent, "");
  }

  public CalendarDate reportOneDir(String dir, Accum accum, Counters countersOne, Indent indent, CalendarDate last) {
    fm.format("%s%60s", indent, dir + " total");
    fm.format("%8d ", accum.nfiles);
    fm.format("%8d ", accum.nrecords);
    fm.format("%8.3f ", accum.indexSize);
    fm.format("%8.3f ", accum.fileSize);
    fm.format("%8d ", countersOne.get("variable").getUnique());
    fm.format("%8d ", countersOne.get("referenceDate").getUnique());
    fm.format("%8d ", countersOne.get("gds").getUnique());
    fm.format("%s ", countersOne.get("referenceDate").showRange());

    CalendarDate first = (CalendarDate) countersOne.get("referenceDate").getFirst();
    if (last != null && first.isBefore(last))
      fm.format(" ***");
    fm.format("%n");

    return (CalendarDate) countersOne.get("referenceDate").getLast();
  }

  public void reportOneFileHeader(Indent indent, Formatter fm) {
    fm.format("%s%40s #records   #vars  #runtimes #gds%n", indent, "");
  }

  public void reportOneFile(MFile mfile, int nrecords, Counters countersOne, Indent indent, Formatter fm) {
    fm.format("%s%40s", indent, mfile.getName());
    fm.format("%8d ", nrecords);
    fm.format("%8d ", countersOne.get("variable").getUnique());
    fm.format("%8d ", countersOne.get("referenceDate").getUnique());
    fm.format("%8d ", countersOne.get("gds").getUnique());

    int vertCoordInGDS = countersOne.get("vertCoordInGDS").getUnique();
    int predefined = countersOne.get("predefined").getUnique();
    int thin = countersOne.get("thin").getUnique();
    if (vertCoordInGDS != 0) fm.format("vertCoordInGDS=%d ", vertCoordInGDS);
    if (predefined != 0) fm.format("predefined=%d ", vertCoordInGDS);
    if (thin != 0) fm.format("thin=%d ", vertCoordInGDS);
    fm.format("%n");
  }

  public void scanAndReport() throws IOException {
    Indent indent = new Indent(2);
    if (config.ptype != FeatureCollectionConfig.PartitionType.file)
      reportOneHeader(indent, fm);

    Formatter errlog = new Formatter();
    CollectionSpecParser specp = config.getCollectionSpecParser(errlog);
    Path rootPath = Paths.get(specp.getRootDir());
    boolean isGrib1 = config.type == FeatureCollectionType.GRIB1;

    try (MCollection topCollection = DirectoryBuilder.factory(config, rootPath, true, null, GribCdmIndex.NCX_SUFFIX, logger)) {

      if (topCollection instanceof DirectoryPartition) {
        DirectoryPartition dpart = (DirectoryPartition) topCollection;
        dpart.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config);
        accumAll.add(scanDirectoryPartitionRecurse(isGrib1, dpart, config, countersAll, logger, indent, fm));

      } else if (topCollection instanceof DirectoryCollection) {
        // otherwise its a leaf directory
        accumAll.add(scanLeafDirectoryCollection(isGrib1, config, countersAll, logger, rootPath, true, indent, fm));
      }

      reportAll(indent, fm);
    }
  }

  private Accum scanDirectoryPartitionRecurse(boolean isGrib1, DirectoryPartition dpart,
                                              FeatureCollectionConfig config,
                                              Counters countersParent,
                                              Logger logger, Indent indent, Formatter fm) throws IOException {

    fm.format("%n%sDirectory %s%n", indent, dpart.getRoot());
    indent.incr();
    Counters countersPart = countersParent.makeSubCounters();
    Accum accum = new Accum();

    for (MCollection part : dpart.makePartitions(CollectionUpdateType.always)) {
      part.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config);
      try {
        if (part instanceof DirectoryPartition) {
          accum.add(scanDirectoryPartitionRecurse(isGrib1, (DirectoryPartition) part, config, countersPart, logger, indent, fm));
        } else {
          Path partPath = Paths.get(part.getRoot());
          accum.add(scanLeafDirectoryCollection(isGrib1, config, countersPart, logger, partPath, false, indent, fm));
        }
      } catch (Throwable t) {
        logger.warn("Error making partition " + part.getRoot(), t);
        dpart.removePartition(part);
      }
    }   // loop over partitions

    countersParent.addTo(countersPart);
    accum.last = reportOneDir(dpart.getRoot(), accum, countersPart, indent, accum.last);
    indent.decr();
    return accum;
  }

  /**
   * Update all the grib indices in one directory, and the collection index for that directory
   *
   * @param config  FeatureCollectionConfig
   * @param dirPath directory path
   * @throws IOException
   */
  private Accum scanLeafDirectoryCollection(boolean isGrib1, FeatureCollectionConfig config,
                                            Counters parentCounters,
                                            Logger logger, Path dirPath, boolean isTop,
                                            Indent indent, Formatter fm) throws IOException {

    if (config.ptype == FeatureCollectionConfig.PartitionType.file) {
      reportOneFileHeader(indent, fm);
      fm.format("%sDirectory %s%n", indent, dirPath);
    }
    Accum accum = new Accum();
    int nfiles = 0;

    Counters countersThisDir = parentCounters.makeSubCounters();

    Formatter errlog = new Formatter();
    CollectionSpecParser specp = config.getCollectionSpecParser(errlog);

    DirectoryCollection dcm = new DirectoryCollection(config.collectionName, dirPath, isTop, config.olderThan, logger);
    // dcm.setUseGribFilter(false);
    dcm.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config);
    if (specp.getFilter() != null)
      dcm.setStreamFilter(new StreamFilter(specp.getFilter(), specp.getFilterOnName()));

    try (CloseableIterator<MFile> iter = dcm.getFileIterator()) {
      while (iter.hasNext()) {
        MFile mfile = iter.next();
        Counters countersOneFile = countersThisDir.makeSubCounters();
        int nrecords = 0;

        if (isGrib1) {
          Grib1Index grib1Index = readGrib1Index(mfile, false);
          if (grib1Index == null) {
            System.out.printf("%s%s: read or create failed%n", indent, mfile.getPath());
            continue;
          }
          for (ucar.nc2.grib.grib1.Grib1Record gr : grib1Index.getRecords()) {
            accumGrib1Record(gr, countersOneFile);
            nrecords++;
          }
        } else {
          Grib2Index grib2Index = readGrib2Index(mfile, false);
          if (grib2Index == null) {
            System.out.printf("%s%s: read or create failed%n", indent, mfile.getPath());
            continue;
          }
          for (ucar.nc2.grib.grib2.Grib2Record gr : grib2Index.getRecords()) {
            accumGrib2Record(gr, countersOneFile);
            nrecords++;
          }
        }

        accum.nrecords += nrecords;
        countersThisDir.addTo(countersOneFile);
        if (config.ptype == FeatureCollectionConfig.PartitionType.file)
          reportOneFile(mfile, nrecords, countersOneFile, indent, fm);
        nfiles++;

        // get file sizes
        String path = mfile.getPath();
        if (path.endsWith(GribIndex.GBX9_IDX)) {
          accum.indexSize += ((float) mfile.getLength() / (1000 * 1000)); // mb
        } else {
          accum.fileSize += ((float) mfile.getLength() / (1000 * 1000)); // mb
          File idxFile = GribIndexCache.getExistingFileOrCache(path + GribIndex.GBX9_IDX);
          if (idxFile.exists())
            accum.indexSize += ((float) idxFile.length() / (1000 * 1000)); // mb
        }
      }
    }

    parentCounters.addTo(countersThisDir);
    accum.nfiles += nfiles;
    accum.last = reportOneDir(dirPath.toString(), accum, countersThisDir, indent, accum.last);
    return accum;
  }

  Grib1Customizer cust1 = null;

  private void accumGrib1Record(ucar.nc2.grib.grib1.Grib1Record gr, Counters counters) throws IOException {

    if (cust1 == null) { // first record
      cust1 = Grib1Customizer.factory(gr, null);
    }

    Grib1SectionGridDefinition gdss = gr.getGDSsection();
    Grib1SectionProductDefinition pds = gr.getPDSsection();
    String table = pds.getCenter() + "-" + pds.getSubCenter() + "-" + pds.getTableVersion();
    counters.count("table version", table);
    counters.count("referenceDate", pds.getReferenceDate());

    int gdsHash = gr.getGDS().hashCode();
    int cdmHash = Grib1Variable.cdmVariableHash(cust1, gr, gdsHash, gribConfig.useTableVersion, gribConfig.intvMerge, gribConfig.useCenter);
    String name = Grib1Iosp.makeVariableName(cust1, gribConfig, pds);
    counters.count("variable", new Variable(cdmHash, name));
    if (counters.count("gds", gdsHash)) storeGrib1Record(gdsHash, gr);
    counters.count("gdsTemplate", gdss.getGridTemplate());

    if (gdss.isThin()) {
      //fm.format("  THIN= (gds=%d)%n", gdss.getGridTemplate());
      counters.count("thin", gdss.getGridTemplate());
    }

    if (!pds.gdsExists()) {
      //fm.format("   PREDEFINED GDS%n");
      counters.count("predefined", gdss.getPredefinedGridDefinition());
    }

    if (gdss.hasVerticalCoordinateParameters()) {
      //fm.format("   Has vertical coordinates in GDS%n");
      counters.count("vertCoordInGDS", pds.getLevelType());
    }

  }

  Grib2Tables cust2 = null;

  private void accumGrib2Record(ucar.nc2.grib.grib2.Grib2Record gr, Counters counters) throws IOException {
    if (cust2 == null) {                              // first record LOOK test if assumption is valid
      cust2 = Grib2Tables.factory(gr);
    }

    Grib2SectionIdentification id = gr.getId();
    //Grib2SectionProductDefinition pds = gr.getPDSsection();
    //Grib2Pds pdss = gr.getPDSsection().getPDS();

    String table = id.getCenter_id() + "-" + id.getSubcenter_id() + "-" + id.getMaster_table_version() + "-" + id.getLocal_table_version();
    counters.count("table version", table);
    counters.count("referenceDate", gr.getReferenceDate());

    //counters.countS("param", gr.getDiscipline() + "-" + pdss.getParameterCategory() + "-" + pdss.getParameterNumber());
    int cdmHash = Grib2Variable.cdmVariableHash(cust2, gr, 0, gribConfig.intvMerge, gribConfig.useGenType); // LOOK needs gdsHashOverride
    String name = GribUtils.makeNameFromDescription(cust2.getVariableName(gr));
    counters.count("variable", new Variable(cdmHash, name));
    int gdsHash = gr.getGDS().hashCode();
    if (counters.count("gds", gdsHash)) storeGrib2Record(gdsHash, gr);
    counters.count("gdsTemplate", gr.getGDSsection().getGDSTemplateNumber());
  }

  private Grib1Index readGrib1Index(MFile mf, boolean readOnly) throws IOException {
    String path = mf.getPath();
    if (path.endsWith(Grib1Index.GBX9_IDX))
      path = StringUtil2.removeFromEnd(path, Grib1Index.GBX9_IDX);

    Grib1Index index = new Grib1Index();
    if (!index.readIndex(path, mf.getLastModified(), CollectionUpdateType.test)) {
      if (readOnly) return null;
      try (RandomAccessFile raf = new RandomAccessFile(path, "r")) {
        if (!Grib1RecordScanner.isValidFile(raf)) return null;
        index.makeIndex(path, raf);
      }
    }
    return index;
  }

  private Grib2Index readGrib2Index(MFile mf, boolean readOnly) throws IOException {
    String path = mf.getPath();
    if (path.endsWith(Grib1Index.GBX9_IDX))
      path = StringUtil2.removeFromEnd(path, Grib1Index.GBX9_IDX);

    Grib2Index index = new Grib2Index();
    if (!index.readIndex(path, mf.getLastModified(), CollectionUpdateType.test)) {
      if (readOnly) return null;
      try (RandomAccessFile raf = new RandomAccessFile(path, "r")) {
        if (!Grib2RecordScanner.isValidFile(raf)) return null;
        index.makeIndex(path, raf);
      }
    }
    return index;
  }

  Map<Integer, Grib1Record> gds1set = new HashMap<>();
  private void storeGrib1Record( int hash, Grib1Record gr1) {
    gds1set.put(hash, gr1);
  }

  Map<Integer, Grib2Record> gds2set = new HashMap<>();
  private void storeGrib2Record( int hash, Grib2Record gr2) {
    gds2set.put(hash, gr2);
  }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.grib.collection;

import com.google.protobuf.ByteString;
import javax.annotation.Nullable;
import thredds.featurecollection.FeatureCollectionConfig;
import thredds.inventory.CollectionUpdateType;
import thredds.inventory.MCollection;
import thredds.inventory.MFile;
import thredds.inventory.partition.PartitionManager;
import ucar.nc2.constants.CDM;
import ucar.nc2.grib.GribIndexCache;
import ucar.nc2.grib.coord.CalendarDateFactory;
import ucar.nc2.grib.coord.Coordinate;
import ucar.nc2.grib.coord.CoordinateBuilder;
import ucar.nc2.grib.coord.CoordinateEns;
import ucar.nc2.grib.coord.CoordinatePartitionUnionizer;
import ucar.nc2.grib.coord.CoordinateRuntime;
import ucar.nc2.grib.coord.CoordinateSharer;
import ucar.nc2.grib.coord.CoordinateSharerBest;
import ucar.nc2.grib.coord.CoordinateTime;
import ucar.nc2.grib.coord.CoordinateTime2D;
import ucar.nc2.grib.coord.CoordinateTimeAbstract;
import ucar.nc2.grib.coord.CoordinateTimeIntv;
import ucar.nc2.grib.coord.CoordinateVert;
import ucar.nc2.stream.NcStream;
import ucar.nc2.time.CalendarDateRange;
import ucar.unidata.io.RandomAccessFile;
import ucar.unidata.util.StringUtil2;

import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.util.*;

/**
 * superclass to build Grib1/2 PartitionCollections
 *
 * @author caron
 * @since 2/21/14
 */
abstract class GribPartitionBuilder {

  private final PartitionManager partitionManager; // defines the partition
  protected final String name;            // collection name
  protected final org.slf4j.Logger logger;
  protected PartitionCollectionMutable result;  // build this object

  GribPartitionBuilder(String name, PartitionManager tpc, org.slf4j.Logger logger) {
    this.name = name;
    this.partitionManager = tpc;
    this.logger = logger;
  }

  boolean updateNeeded(CollectionUpdateType ff) throws IOException {
    if (ff == CollectionUpdateType.never) return false;
    if (ff == CollectionUpdateType.always) return true;

    File collectionIndexFile = GribIndexCache.getExistingFileOrCache(partitionManager.getIndexFilename(GribCdmIndex.NCX_SUFFIX));
    if (collectionIndexFile == null) return true;

    if (ff == CollectionUpdateType.nocheck) return false;

    // now check children
    return needsUpdate(ff, collectionIndexFile);
  }

  // LOOK need an option to only scan latest last partition or something
  private boolean needsUpdate(CollectionUpdateType ff, File collectionIndexFile) throws IOException {
    long collectionLastModified = collectionIndexFile.lastModified();
    Set<String> newFileSet = new HashSet<>();
    for (MCollection dcm : partitionManager.makePartitions(CollectionUpdateType.test)) {
      String partitionIndexFilename = StringUtil2.replace(dcm.getIndexFilename(GribCdmIndex.NCX_SUFFIX), '\\', "/");
      File partitionIndexFile = GribIndexCache.getExistingFileOrCache(partitionIndexFilename);
      if (partitionIndexFile == null)                                 // make sure each partition has an index
        return true;
      if (collectionLastModified < partitionIndexFile.lastModified())  // and the partition index is earlier than the collection index
        return true;
      newFileSet.add(partitionIndexFilename);
    }

    if (ff == CollectionUpdateType.testIndexOnly) return false;

    // now see if any files were deleted
    GribCdmIndex reader = new GribCdmIndex(logger);
    List<MFile> oldFiles = new ArrayList<>();
    reader.readMFiles(collectionIndexFile.toPath(), oldFiles);
    Set<String> oldFileSet = new HashSet<>();
    for (MFile oldFile : oldFiles) {
      if (!newFileSet.contains(oldFile.getPath()))
        return true;  // got deleted - must recreate the index
      oldFileSet.add(oldFile.getPath());
    }

    // now see if any files were added
    for (String newFilename : newFileSet) {
      if (!oldFileSet.contains(newFilename))
        return true;  // got added - must recreate the index
    }

    return false;
  }

  ///////////////////////////////////////////////////
  // build the index

  // return true if changed, exception on failure
  boolean createPartitionedIndex(CollectionUpdateType forcePartition, Formatter errlog) throws IOException {
    if (errlog == null) errlog = new Formatter(); // info will be discarded

    // create partitions from the partitionManager
    for (MCollection dcmp : partitionManager.makePartitions(forcePartition)) {
      dcmp.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, partitionManager.getAuxInfo(FeatureCollectionConfig.AUX_CONFIG));
      result.addPartition(dcmp);
    }
    result.sortPartitions(); // after this the partition list is immutable

    // choose the "canonical" partition, aka prototype
    // only used in copyInfo
    int n = result.getPartitionSize();
    if (n == 0) {
      errlog.format("ERR Nothing in this partition = %s%n", result.showLocation());
      throw new IllegalStateException("Nothing in this partition =" + result.showLocation());
    }
    int idx = partitionManager.getProtoIndex(n);
    PartitionCollectionMutable.Partition canon = result.getPartition(idx);
    logger.debug("     Using canonical partition {}", canon.getDcm().getCollectionName());

    try (GribCollectionMutable gc = canon.makeGribCollection()) {  // LOOK open/close canonical partition
      if (gc == null)
        throw new IllegalStateException("canon.makeGribCollection failed on =" + result.showLocation() + " " + canon.getName() + "; errs=" + errlog);

      // copy info from canon gribCollection to result partitionCollection
      result.copyInfo(gc);
      result.isPartitionOfPartitions = (gc instanceof PartitionCollectionMutable);

      result.dateRange = gc.dateRange;
    }

    // check consistency across vert and ens coords
    // create partitioned variables
    // partition index is used - do not resort partitions
    GribCollectionMutable.Dataset ds2D = makeDataset2D(errlog);
    if (ds2D == null) {
      errlog.format(" ERR makeDataset2D failed, index not written on %s%n", result.showLocation());
      throw new IllegalStateException("makeDataset2D failed, index not written on =" + result.showLocation() + "; errs=" + errlog);
    }

    // Make Best for a TwoD
    if (ds2D.gctype == GribCollectionImmutable.Type.TwoD)
      makeDatasetBest(ds2D, false);
    //else if (ds2D.gctype == GribCollectionImmutable.Type.MRUTC)
    //  makeTime2runtime(ds2D, false);

    // ready to write the index file
    return writeIndex(result, errlog);
  }

  // each dataset / group has one of these, across all partitions
  private class GroupPartitions {
    final GribCollectionMutable.GroupGC resultGroup;
    final GribCollectionMutable.GroupGC[] componentGroups; // one for each partition; may be null if group is not in the partition
    final int[] componentGroupIndex;                 // one for each partition; the index into the partition.ds2d.groups() array
    final int npart;

    GroupPartitions(GribCollectionMutable.GroupGC resultGroup, int npart) {
      this.resultGroup = resultGroup;
      this.npart = npart;
      this.componentGroups = new GribCollectionMutable.GroupGC[npart];
      this.componentGroupIndex = new int[npart];
    }

    void makeVariableIndexPartitioned() {
      // find unique variables across all partitions
      Map<GribCollectionMutable.VariableIndex, GribCollectionMutable.VariableIndex> varMap = new HashMap<>(2 * resultGroup.variList.size());
      for (GribCollectionMutable.GroupGC group : componentGroups) {
        if (group == null) continue;
        for (GribCollectionMutable.VariableIndex vi : group.variList)
          varMap.put(vi, vi); // this will use the last one found
      }
      for (GribCollectionMutable.VariableIndex vi : varMap.values()) {
        // convert each VariableIndex to VariableIndexPartitioned in result. note not using canon vi, but last one found
        result.makeVariableIndexPartitioned(resultGroup, vi, npart); // this adds to resultGroup
      }
    }
  }

  @Nullable
  private GribCollectionMutable.Dataset makeDataset2D(Formatter f) throws IOException {
    FeatureCollectionConfig config = (FeatureCollectionConfig) partitionManager.getAuxInfo(FeatureCollectionConfig.AUX_CONFIG);
    FeatureCollectionConfig.GribIntvFilter intvMap = (config != null) ? config.gribConfig.intvFilter : null;
    GribCollectionMutable.Dataset ds2D = result.makeDataset(GribCollectionImmutable.Type.TwoD);
    int npart = result.getPartitionSize();

    // make a list of unique groups across all partitions as well as component groups for each group
    List<CoordinateRuntime> masterRuntimes = new ArrayList<>();
    Map<Object, GroupPartitions> groupMap = new HashMap<>(40);  // gdsHashObject, GroupPartition
    CoordinateBuilder runtimeAllBuilder = new CoordinateRuntime.Builder2(null); // ok to use Builder2 for both grib1 and grib2 because not extracting

    int countPartition = 0;
    CalendarDateRange dateRangeAll = null;
    boolean rangeOverlaps = false;
    for (PartitionCollectionMutable.Partition tpp : result.getPartitions()) {
      try (GribCollectionMutable gc = tpp.makeGribCollection()) {  // LOOK open/close each child partition. could leave open ? they are NOT in cache
        if (gc == null) continue; // skip if they dont exist

        // note its not recursive, maybe leave open, or cache; actually we keep a pointer to the partition's group in the GroupPartitions
        CoordinateRuntime partRuntime = gc.masterRuntime;
        runtimeAllBuilder.addAll(partRuntime);  // make a complete set of runtime Coordinates
        masterRuntimes.add(partRuntime);        // make master runtimes

        GribCollectionMutable.Dataset ds2dp = gc.getDatasetCanonical(); // the twoD or GC dataset

        // date ranges must not overlap in order to use MRUTP
        if (dateRangeAll == null)
          dateRangeAll = gc.dateRange;
        else if (!rangeOverlaps) {
          rangeOverlaps = dateRangeAll.intersects(gc.dateRange);
          dateRangeAll = dateRangeAll.extend(gc.dateRange);
        }

        /* see if its only got one time coord
        if (ds2dp.gctype == GribCollectionImmutable.Type.SRC) {
          for (GribCollectionMutable.GroupGC group : ds2dp.getGroups()) {
            for (Coordinate coord : group.getCoordinates()) { // all time coords must have only one time
              if (coord instanceof CoordinateTime2D) {
                CoordinateTime2D coord2D = (CoordinateTime2D) coord;
                if (coord2D.getNtimes() > 1)
                  allAre1D = false;

            } else if (coord instanceof CoordinateTimeAbstract && coord.getSize() > 1)
                allAre1D = false;
            }
          }
        } else if (ds2dp.gctype == GribCollectionImmutable.Type.MRC || ds2dp.gctype == GribCollectionImmutable.Type.TwoD) {
          allAre1D = false;
        } */

        int groupIdx = 0;
        for (GribCollectionMutable.GroupGC g : ds2dp.groups) { // for each group in the partition
          GroupPartitions gs = groupMap.get(g.getGdsHash());
          if (gs == null) {
            gs = new GroupPartitions(ds2D.addGroupCopy(g), npart);
            groupMap.put(g.getGdsHash(), gs);
          }
          gs.componentGroups[countPartition] = g;
          gs.componentGroupIndex[countPartition] = groupIdx++;
        }
      } // close the gc
      countPartition++;
    } // loop over partition

    List<GroupPartitions> groupPartitions = new ArrayList<>(groupMap.values());
    result.masterRuntime = (CoordinateRuntime) runtimeAllBuilder.finish();
    if (result.isPartitionOfPartitions) // cache calendar dates for efficiency
      CoordinateTimeAbstract.cdf = new CalendarDateFactory(result.masterRuntime);

    if (!rangeOverlaps)
      ds2D.gctype = GribCollectionImmutable.Type.MRUTP;
      //else if (allAre1D)
      //  ds2D.gctype = GribCollectionImmutable.Type.MRSTP;
    else
      ds2D.gctype = GribCollectionImmutable.Type.TwoD;

    // create run2part: for each run, which partition to use
    result.run2part = new int[result.masterRuntime.getSize()];
    int partIdx = 0;
    for (CoordinateRuntime partRuntime : masterRuntimes) {
      for (Object val : partRuntime.getValues()) {
        int idx = result.masterRuntime.getIndex(val);
        result.run2part[idx] = partIdx;     // note that later partitions will override earlier if they have the same runtime
      }
      partIdx++;
    }

    // do each horiz group
    for (GroupPartitions gp : groupPartitions) {
      GribCollectionMutable.GroupGC resultGroup = gp.resultGroup;
      gp.makeVariableIndexPartitioned();

      String gname = resultGroup.getId();

      // for each partition in this gorup
      for (int partno = 0; partno < npart; partno++) {
        GribCollectionMutable.GroupGC group = gp.componentGroups[partno];
        if (group == null) {  // missing group in this partition
          f.format(" INFO canonical group %s not in partition %s%n", gname, result.getPartition(partno).getName());
          continue;
        }
        int groupIdx = gp.componentGroupIndex[partno];

        // for each variable in this Partition, add reference to it in the vip
        for (int varIdx = 0; varIdx < group.variList.size(); varIdx++) {
          GribCollectionMutable.VariableIndex vi = group.variList.get(varIdx);
          PartitionCollectionMutable.VariableIndexPartitioned vip = (PartitionCollectionMutable.VariableIndexPartitioned) resultGroup.findVariableByHash(vi);
          vip.addPartition(partno, groupIdx, varIdx, vi.ndups, vi.nrecords, vi.nmissing, vi);
        } // loop over variable
      } // loop over partition

      // each VariableIndexPartitioned now has its list of PartitionForVariable

      // overall set of unique coordinates
      boolean isDense = false; // (config != null) && "dense".equals(config.gribConfig.getParameter("CoordSys"));  // for now, assume non-dense
      CoordinateSharer sharify = new CoordinateSharer(isDense, logger);

      // for each variable, create union of coordinates across the partitions
      for (GribCollectionMutable.VariableIndex viResult : resultGroup.variList) {
        PartitionCollectionMutable.VariableIndexPartitioned vip = (PartitionCollectionMutable.VariableIndexPartitioned) viResult;
        vip.finish(); // create the SA, remove list LOOK, could do it differently

        // loop over partitions, make union coordinate; also time filter the intervals
        CoordinatePartitionUnionizer unionizer = new CoordinatePartitionUnionizer(viResult, intvMap, logger);
        for (int partno = 0; partno < npart; partno++) {
          GribCollectionMutable.GroupGC group = gp.componentGroups[partno];
          if (group == null) continue; // tolerate missing groups
          GribCollectionMutable.VariableIndex vi = group.findVariableByHash(viResult);
          if (vi == null) continue; // tolerate missing variables
          try {
            PartitionCollectionMutable.Partition part = ds2D.gctype.isUniqueTime() ? null : result.getPartition(partno);
            unionizer.addCoords(vi.getCoordinates(), part);
          } catch (IllegalStateException e) {
            logger.error(e.getMessage() + " on dataset " + name);
            return null;
          }
        }  // loop over partition

        viResult.coords = unionizer.finish();  // the viResult coordinates have been ortho/regularized
        sharify.addCoords(viResult.coords);
      } // loop over variable

      // create a list of common coordinates, put them into the group, and now variables just reference those by index
      sharify.finish();
      resultGroup.coords = sharify.getUnionCoords();

      // debug
      List<CoordinateTime2D> time2DCoords = new ArrayList<>();
      Map<CoordinateRuntime, CoordinateRuntime> runtimes = new HashMap<>();
      for (Coordinate coord : resultGroup.coords) {
        Coordinate.Type type = coord.getType();
        switch (type) {
          case runtime:
            CoordinateRuntime reftime = (CoordinateRuntime) coord;
            runtimes.put(reftime, reftime);
            break;

          case time2D:
            CoordinateTime2D t2d = (CoordinateTime2D) coord;
            time2DCoords.add(t2d);
            break;
        }
      }
      for (CoordinateTime2D t2d : time2DCoords) {
        CoordinateRuntime runtime2D = t2d.getRuntimeCoordinate();
        CoordinateRuntime runtime = runtimes.get(runtime2D);
        if (runtime == null)
          logger.warn("HEY assignRuntimeNames failed on {} group {}", t2d.getName(), resultGroup.getId());
      } // end debug

      for (GribCollectionMutable.VariableIndex viResult : resultGroup.variList) {
        // redo the variables against the shared coordinates
        viResult.coordIndex = sharify.reindex2shared(viResult.coords); // ok
        viResult.coords = null; // dont use anymore, now use coordIndex into group coordinates
      }

    } // loop over groups

    CoordinateTimeAbstract.cdf = null;
    return ds2D;
  }

  /* LOOK heres a place where one could "post process" and combine, instead of at coverage level.
  // this would benefit iosp, ie the netcdf API
  private void makeTime2runtime(GribCollectionMutable.Dataset ds2D, boolean isComplete) throws IOException {

    for (GribCollectionMutable.GroupGC group2D : ds2D.groups) {
      // for each time2D, add time2runtime
      for (Coordinate coord : group2D.coords) {
        if (coord instanceof CoordinateTime2D) {
          ((CoordinateTime2D) coord).makeTime2runtime(result.masterRuntime);
        }
      }
    }
  } */

  private void makeDatasetBest(GribCollectionMutable.Dataset ds2D, boolean isComplete) {
    GribCollectionMutable.Dataset dsBest = result.makeDataset(isComplete ? GribCollectionImmutable.Type.BestComplete : GribCollectionImmutable.Type.Best);

    int npart = result.getPartitionSize();

    // for each 2D group
    for (GribCollectionMutable.GroupGC group2D : ds2D.groups) {
      GribCollectionMutable.GroupGC groupB = dsBest.addGroupCopy(group2D);  // make copy of group, add to Best dataset
      groupB.isTwoD = false;

      // for each time2D, create the best time coordinates
      HashMap<Coordinate, CoordinateTimeAbstract> map2DtoBest = new HashMap<>(); // associate 2D coord with best
      CoordinateSharerBest sharer = new CoordinateSharerBest();
      for (Coordinate coord : group2D.coords) {
        if (coord instanceof CoordinateRuntime) continue; // skip it
        if (coord instanceof CoordinateTime2D) {
          CoordinateTimeAbstract best = ((CoordinateTime2D) coord).makeBestTimeCoordinate(result.masterRuntime);
          if (!isComplete) best = best.makeBestFromComplete();
          sharer.addCoordinate(best);
          map2DtoBest.put(coord, best);
        } else {
          sharer.addCoordinate(coord);
        }
      }
      groupB.coords = sharer.finish();  // these are the unique coords for group Best

      // transfer variables to Best group, set shared Coordinates
      for (GribCollectionMutable.VariableIndex vi2d : group2D.variList) {
        // copy vi2d and add to groupB
        PartitionCollectionMutable.VariableIndexPartitioned vip = result.makeVariableIndexPartitioned(groupB, vi2d, npart);
        vip.finish();

        // set shared coordinates
        List<Coordinate> newCoords = new ArrayList<>();
        for (Integer groupIndex : vi2d.coordIndex) {
          Coordinate coord2D = group2D.coords.get(groupIndex);
          if (coord2D instanceof CoordinateRuntime) continue; // skip runtime;
          if (coord2D instanceof CoordinateTime2D) {
            newCoords.add(map2DtoBest.get(coord2D)); // add the best coordinate for that CoordinateTime2D
          } else {
            newCoords.add(coord2D);
          }
        }
        vip.coordIndex = sharer.reindex(newCoords);
      }

    } // loop over groups
  }

  //////////////////////////////////////////////////////////////////////////////////////////////

  protected abstract String getMagicStart();

  protected abstract int getVersion();

  private GribCollectionWriter writer;

  // writing ncx2
  // The Groups are GribCollection.GroupGC, not GribXCollectionWriter.Group, so we cant inheritit most of the methods from GribXCollectionWriter

  /*
  MAGIC_START
  version
  sizeRecords
  VariableRecords (sizeRecords bytes)
  sizeIndex
  GribCollectionIndex (sizeIndex bytes)
  */
  protected boolean writeIndex(PartitionCollectionMutable pc, Formatter f) throws IOException {
    File idxFile = GribIndexCache.getFileOrCache(partitionManager.getIndexFilename(GribCdmIndex.NCX_SUFFIX));
    if (idxFile.exists()) {
      RandomAccessFile.eject(idxFile.getPath());
      if (!idxFile.delete())
        logger.error("gc2tp cant delete " + idxFile.getPath());
    }

    writer = new GribCollectionWriter(null, null);

    try (RandomAccessFile raf = new RandomAccessFile(idxFile.getPath(), "rw")) {
      raf.order(RandomAccessFile.BIG_ENDIAN);

      //// header message
      raf.write(getMagicStart().getBytes(CDM.utf8Charset));
      raf.writeInt(getVersion());
      raf.writeLong(0); // no record section

      /*
      message GribCollection {
        required string name = 1;         // must be unique - index filename is name.ncx
        required string topDir = 2;       // filenames are reletive to this
        repeated MFile mfiles = 3;        // list of grib MFiles
        repeated Dataset dataset = 4;
        repeated Gds gds = 5;             // unique Gds, shared amongst datasets

        required int32 center = 6;      // these 4 fields are to get a GribTable object
        required int32 subcenter = 7;
        required int32 master = 8;
        required int32 local = 9;       // grib1 table Version

        optional int32 genProcessType = 10;
        optional int32 genProcessId = 11;
        optional int32 backProcessId = 12;

        // repeated Parameter params = 20;      // not used
        FcConfig config = 21;
        uint64 startTime = 22; // calendar date, first valid time
        uint64 endTime = 23;   // calendar date, last valid time

        extensions 100 to 199;
      }

      extend GribCollection {
        repeated Partition partitions = 100;
      }
       */

      GribCollectionProto.GribCollection.Builder indexBuilder = GribCollectionProto.GribCollection.newBuilder();
      indexBuilder.setName(pc.getName());
      Path topDir = pc.directory.toPath();
      String pathS = StringUtil2.replace(topDir.toString(), '\\', "/");
      indexBuilder.setTopDir(pathS);

      // mfiles are the partition indexes
      int count = 0;
      for (PartitionCollectionMutable.Partition part : pc.partitions) {
        GribCollectionProto.MFile.Builder b = GribCollectionProto.MFile.newBuilder();
        String pathRS = makeReletiveFilename(pc, part); // reletive to pc.directory
        b.setFilename(pathRS);
        b.setLastModified(part.getLastModified());
        b.setLength(part.fileSize);
        b.setIndex(count++);
        indexBuilder.addMfiles(b.build());
      }

      indexBuilder.setCenter(pc.center);
      indexBuilder.setSubcenter(pc.subcenter);
      indexBuilder.setMaster(pc.master);
      indexBuilder.setLocal(pc.local);

      indexBuilder.setGenProcessId(pc.genProcessId);
      indexBuilder.setGenProcessType(pc.genProcessType);
      indexBuilder.setBackProcessId(pc.backProcessId);

      indexBuilder.setStartTime(pc.dateRange.getStart().getMillis());
      indexBuilder.setEndTime(pc.dateRange.getEnd().getMillis());

      indexBuilder.setMasterRuntime(writer.writeCoordProto(pc.masterRuntime));

      // dataset
      for (GribCollectionMutable.Dataset ds : pc.datasets)
        indexBuilder.addDataset(writeDatasetProto(pc, ds));

      // extensions
      if (pc.run2part != null) {
        for (int part : pc.run2part)
          indexBuilder.addRun2Part(part);
      }
      for (PartitionCollectionMutable.Partition part : pc.partitions)
        indexBuilder.addPartitions(writePartitionProto(pc, part));
      indexBuilder.setIsPartitionOfPartitions(pc.isPartitionOfPartitions);

      // write it out
      GribCollectionProto.GribCollection index = indexBuilder.build();
      byte[] b = index.toByteArray();
      NcStream.writeVInt(raf, b.length); // message size
      raf.write(b);  // message  - all in one gulp
      f.format("Grib2PartitionIndex= %d bytes file size =  %d bytes%n%n", b.length, raf.length());
    }

    return true;
  }

  private String makeReletiveFilename(PartitionCollectionMutable pc, PartitionCollectionMutable.Partition part) {
    Path topDir = pc.directory.toPath();
    Path partPath = new File(part.getDirectory(), part.getFilename()).toPath();
    Path pathRelative = topDir.relativize(partPath);
    return StringUtil2.replace(pathRelative.toString(), '\\', "/");
  }

  /*
  message Dataset {
    required Type type = 1;
    repeated Group groups = 2;
  }
   */
  private GribCollectionProto.Dataset writeDatasetProto(PartitionCollectionMutable pc, GribCollectionMutable.Dataset ds) throws IOException {
    GribCollectionProto.Dataset.Builder b = GribCollectionProto.Dataset.newBuilder();

    GribCollectionProto.Dataset.Type type = GribCollectionProto.Dataset.Type.valueOf(ds.gctype.toString());
    b.setType(type);

    for (GribCollectionMutable.GroupGC group : ds.groups)
      b.addGroups(writeGroupProto(pc, group));

    return b.build();
  }

  /*
  message Group {
    Gds gds = 1;                             // use this to build the HorizCoordSys
    repeated Variable variables = 2;         // list of variables
    repeated Coord coords = 3;               // list of coordinates
    repeated uint32 fileno = 4 [packed=true]; // the component files that are in this group, key into gc.mfiles
  }
   */
  private GribCollectionProto.Group writeGroupProto(PartitionCollectionMutable pc, GribCollectionMutable.GroupGC g) throws IOException {
    GribCollectionProto.Group.Builder b = GribCollectionProto.Group.newBuilder();

    b.setGds(GribCollectionWriter.writeGdsProto(g.horizCoordSys.getRawGds(), g.horizCoordSys.getPredefinedGridDefinition()));

    for (GribCollectionMutable.VariableIndex vb : g.variList) {
      b.addVariables(writeVariableProto((PartitionCollectionMutable.VariableIndexPartitioned) vb));
    }

    for (Coordinate coord : g.coords) {
      switch (coord.getType()) {
        case runtime:
          b.addCoords(writer.writeCoordProto((CoordinateRuntime) coord));
          break;
        case time:
          b.addCoords(writer.writeCoordProto((CoordinateTime) coord));
          break;
        case timeIntv:
          b.addCoords(writer.writeCoordProto((CoordinateTimeIntv) coord));
          break;
        case time2D:
          b.addCoords(writer.writeCoordProto((CoordinateTime2D) coord));
          break;
        case vert:
          b.addCoords(writer.writeCoordProto((CoordinateVert) coord));
          break;
        case ens:
          b.addCoords(writer.writeCoordProto((CoordinateEns) coord));
          break;
      }
    }

    if (g.filenose != null)
      for (Integer fileno : g.filenose)
        b.addFileno(fileno);

    return b.build();
  }

  /*
  message Variable {
   uint32 discipline = 1;
   bytes pds = 2;                   // raw pds
   repeated uint32 ids = 3 [packed=true];         // extra info not in pds; grib2 id section

   uint64 recordsPos = 4;  // offset of SparseArray message for this Variable
   uint32 recordsLen = 5;  // size of SparseArray message for this Variable

   repeated uint32 coordIdx = 6 [packed=true];    // indexes into Group.coords

   // optionally keep stats
   uint32 ndups = 8;
   uint32 nrecords = 9;
   uint32 missing = 10;

   // partition only
   repeated PartitionVariable partVariable = 100;
   }
   */
  private GribCollectionProto.Variable writeVariableProto(PartitionCollectionMutable.VariableIndexPartitioned vp) {

    GribCollectionProto.Variable.Builder b = GribCollectionProto.Variable.newBuilder();

    b.setDiscipline(vp.discipline);
    b.setPds(ByteString.copyFrom(vp.rawPds));

    // extra id info
    b.addIds(vp.center);
    b.addIds(vp.subcenter);

    b.setRecordsPos(vp.recordsPos);
    b.setRecordsLen(vp.recordsLen);

    for (int idx : vp.coordIndex)
      b.addCoordIdx(idx);

    b.setNdups(vp.ndups);
    b.setNrecords(vp.nrecords);
    b.setMissing(vp.nmissing);

    /* if (vp.twot != null) { // only for 2D
      for (int invCount : vp.twot.getCount())
        b.addInvCount(invCount);
    }

    if (vp.time2runtime != null) { // only for 1D
      for (int idx=0; idx < vp.time2runtime.getN(); idx++)
        b.addTime2Runtime(vp.time2runtime.get(idx));
    } */

    // extensions
    if (vp.nparts > 0 && vp.partnoSA != null) {
      for (int i = 0; i < vp.nparts; i++) // PartitionCollection.PartitionForVariable2D pvar : vp.getPartitionForVariable2D())
        b.addPartVariable(writePartitionVariableProto(vp.partnoSA.get(i), vp.groupnoSA.get(i), vp.varnoSA.get(i), vp.nrecords, vp.ndups, vp.nmissing));  // LOOK was it finished ??
    }

    return b.build();
  }

  /*
  message PartitionVariable {
    uint32 groupno = 1;
    uint32 varno = 2;
    uint32 flag = 3;
    uint32 partno = 4;

    // optionally keep stats
    uint32 ndups = 8;
    uint32 nrecords = 9;
    uint32 missing = 10;
  }
   */
  private GribCollectionProto.PartitionVariable writePartitionVariableProto(int partno, int groupno, int varno, int nrecords, int ndups, int nmissing) {
    GribCollectionProto.PartitionVariable.Builder pb = GribCollectionProto.PartitionVariable.newBuilder();
    pb.setPartno(partno);
    pb.setGroupno(groupno);
    pb.setVarno(varno);
    pb.setNdups(ndups);
    pb.setNrecords(nrecords);
    pb.setMissing(nmissing);

    return pb.build();
  }

  /*
message Partition {
  string name = 1;       // name is used in TDS - eg the subdirectory when generated by TimePartitionCollections
  string filename = 2;   // the gribCollection.ncx file, reletive to gc.
  string directory = 3;  // top directory NOT USED
  uint64 lastModified = 4;
  int64 length = 5;
  int64 partitionDate = 6;  // partition date added 11/25/14
  }
   */
  private GribCollectionProto.Partition writePartitionProto(PartitionCollectionMutable pc, PartitionCollectionMutable.Partition p) {
    GribCollectionProto.Partition.Builder b = GribCollectionProto.Partition.newBuilder();
    String pathRS = makeReletiveFilename(pc, p); // reletive to pc.directory
    b.setFilename(pathRS);
    b.setName(p.name);
    // b.setDirectory(p.directory);
    b.setLastModified(p.lastModified);
    b.setLength(p.fileSize);
    if (p.partitionDate != null)
      b.setPartitionDate(p.partitionDate.getMillis());  // LOOK what about calendar ??

    return b.build();
  }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.ft.fmrc;

import ucar.nc2.dataset.CoordinateAxis1D;

import java.util.List;
import java.util.ArrayList;

/**
 * Represents a ensemble coordinate shared among variables
 *
 * @author caron
 * @since Jan 12, 2010
 */
public class EnsCoord implements Comparable {
  private String name; //, units;
  private int id; // unique id for XML

  // ??
  public int ensembles;
  public int pdn;
  public int[] ensTypes;

  EnsCoord() {
  }

  EnsCoord(CoordinateAxis1D axis, int[] einfo) {
    this.name = axis.getFullName();
    this.ensembles = einfo[0];
    this.pdn = einfo[1];
    this.ensTypes = new int[this.ensembles];
    System.arraycopy(einfo, 2, ensTypes, 0, ensembles);
  }

  // copy constructor
  EnsCoord(ucar.nc2.ft.fmrc.EnsCoord ec) {
    this.name = ec.getName();
    this.id = ec.getId();
    this.ensembles = ec.getNEnsembles();
    this.pdn = ec.getPDN();
    this.ensTypes = ec.getEnsTypes().clone();
  }

  public int getId() {
    return id;
  }

  public void setId(int id) {
    this.id = id;
  }

  public String getName() {
    return name;
  }

  public void setName(String name) {
    this.name = name;
  }

  public int getNEnsembles() {
    return ensembles;
  }

  public void setNEnsembles(int ensembles) {
    this.ensembles = ensembles;
  }

  public int getPDN() {
    return pdn;
  }

  public void setPDN(int pdn) {
    this.pdn = pdn;
  }

  public int[] getEnsTypes() {
    return ensTypes;
  }

  public void setEnsTypes(int[] ensTypes) {
    this.ensTypes = ensTypes;
  }

  public int getSize() {
    return ensembles;
  }

  // LOOK
  public boolean equalsData(ucar.nc2.ft.fmrc.EnsCoord other) {

    if (ensembles != other.ensembles)
      return false;

    if (pdn != other.pdn)
      return false;

    for (int i = 0; i < ensTypes.length; i++) {
      if (ensTypes[i] != other.ensTypes[i])
        return false;
    }

    return true;
  }

  public int compareTo(Object o) {
    ucar.nc2.ft.fmrc.EnsCoord other = (ucar.nc2.ft.fmrc.EnsCoord) o;
    return name.compareTo(other.name);
  }

  ////////////////
  static public EnsCoord findEnsCoord(List<EnsCoord> ensCoords, EnsCoord want) {
    if (want == null) return null;

    for (EnsCoord ec : ensCoords) {
      if (want.equalsData(ec))
        return ec;
    }

    // make a new one
    EnsCoord result = new EnsCoord(want);
    ensCoords.add(result);
    return result;
  }

   /**
   * Extend result with all the values in the list of EnsCoord
   *
   * @param result extend this coord
   * @param ecList list of EnsCoord, may be empty
   */
  static public void normalize(EnsCoord result, List<EnsCoord> ecList) {
    List<EnsCoord> extra = new ArrayList<>();
    for (EnsCoord ec : ecList) {
      if (!result.equalsData(ec)) {
        // differences can only be greater
        extra.add(ec);
      }
    }
    if (extra.size() == 0)
      return;
    for (EnsCoord ec : extra) {
      if (ec.getNEnsembles() < result.getNEnsembles())
        continue;
      result = ec;
    }

  }
}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.grib.collection;

import com.google.protobuf.ByteString;
import thredds.inventory.MCollection;
import thredds.inventory.MFile;
import ucar.nc2.constants.CDM;
import ucar.nc2.grib.coord.Coordinate;
import ucar.nc2.grib.coord.CoordinateEns;
import ucar.nc2.grib.coord.CoordinateRuntime;
import ucar.nc2.grib.coord.CoordinateTime;
import ucar.nc2.grib.coord.CoordinateTime2D;
import ucar.nc2.grib.coord.CoordinateTimeIntv;
import ucar.nc2.grib.coord.CoordinateVert;
import ucar.nc2.grib.coord.SparseArray;
import ucar.nc2.grib.grib1.*;
import ucar.nc2.stream.NcStream;
import ucar.nc2.time.CalendarDate;
import ucar.nc2.time.CalendarDateRange;
import ucar.unidata.io.RandomAccessFile;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

/**
 * Grib1-specific ncx writer
 *
 * @author caron
 * @since 2/20/14
 */
class Grib1CollectionWriter extends GribCollectionWriter {

  public static final String MAGIC_START = "Grib1Collectio2Index";  // was Grib1CollectionIndex
  static final int minVersion = 1; // if less than this, force rewrite or at least do not read
  protected static final int version = 3;    // increment this as needed, must be backwards compatible through minVersion

  Grib1CollectionWriter(MCollection dcm, org.slf4j.Logger logger) {
    super(dcm, logger);
  }

  static class Group implements GribCollectionBuilder.Group {

    final Grib1SectionGridDefinition gdss;
    final int hashCode;
    final CalendarDate runtime;

    List<Grib1CollectionBuilder.VariableBag> gribVars;
    public List<Coordinate> coords;

    public final Set<Long> runtimes = new HashSet<>();
    public final List<Grib1Record> records = new ArrayList<>();
    Set<Integer> fileSet; // this is so we can show just the component files that are in this group

    Group(Grib1SectionGridDefinition gdss, int hashCode, CalendarDate runtime) {
      this.gdss = gdss;
      this.hashCode = hashCode;
      this.runtime = runtime;
    }

    @Override
    public CalendarDate getRuntime() {
      return runtime;
    }

    @Override
    public List<Coordinate> getCoordinates() {
      return coords;
    }

    @Override
    public Set<Long> getCoordinateRuntimes() {
      return runtimes;
    }
  }

    /*
   MAGIC_START
   version
   sizeRecords
   VariableRecords (sizeRecords bytes)
   sizeIndex
   GribCollectionIndex (sizeIndex bytes)
   */

  // indexFile is in the cache
  boolean writeIndex(String name, File idxFile, CoordinateRuntime masterRuntime, List<Group> groups,
      List<MFile> files,
      GribCollectionImmutable.Type type, CalendarDateRange dateRange) throws IOException {
    Grib1Record first = null; // take global metadata from here
    boolean deleteOnClose = false;

    if (idxFile.exists()) {
      RandomAccessFile.eject(idxFile.getPath());
      if (!idxFile.delete()) {
        logger.warn(" gc1 cant delete index file {}", idxFile.getPath());
      }
    }
    logger.debug(" createIndex for {}", idxFile.getPath());

    try (RandomAccessFile raf = new RandomAccessFile(idxFile.getPath(), "rw")) {
      raf.order(RandomAccessFile.BIG_ENDIAN);

      //// header message
      raf.write(MAGIC_START.getBytes(CDM.utf8Charset));
      raf.writeInt(version);
      long lenPos = raf.getFilePointer();
      raf.writeLong(0); // save space to write the length of the record section
      long countBytes = 0;
      int countRecords = 0;

      Set<Integer> allFileSet = new HashSet<>();
      for (Group g : groups) {
        g.fileSet = new HashSet<>();
        for (Grib1CollectionBuilder.VariableBag vb : g.gribVars) {
          if (first == null) {
            first = vb.first;
          }
          GribCollectionProto.SparseArray vr = writeSparseArray(vb, g.fileSet);
          byte[] b = vr.toByteArray();
          vb.pos = raf.getFilePointer();
          vb.length = b.length;
          raf.write(b);
          countBytes += b.length;
          countRecords += vb.coordND.getSparseArray().countNotMissing();
        }
        allFileSet.addAll(g.fileSet);
      }
      long bytesPerRecord = countBytes / ((countRecords == 0) ? 1 : countRecords);
      if (logger.isDebugEnabled()) {
        logger.debug("  write RecordMaps: bytes = {} records = {} bytesPerRecord={}", countBytes,
            countRecords, bytesPerRecord);
      }

      if (first == null) {
        deleteOnClose = true;
        throw new IOException("GribCollection " + name + " has no records");
      }

      long pos = raf.getFilePointer();
      raf.seek(lenPos);
      raf.writeLong(countBytes);
      raf.seek(pos); // back to the output.

    /*
    message GribCollection {
      string name = 1;         // must be unique - index filename is name.ncx
      string topDir = 2;       // MFile, Partition filenames are reletive to this
      repeated MFile mfiles = 3;        // list of grib MFiles
      repeated Dataset dataset = 4;
      repeated Gds gds = 5;             // unique Gds, shared amongst datasets
      Coord masterRuntime = 6;  // list of runtimes in this GC

      // LOOK these are taken from first record; not storing variants (!)
      int32 center = 7;      // these 4 fields are to get a GribCustomizer
      int32 subcenter = 8;
      int32 master = 9;
      int32 local = 10;       // grib1 table Version

      int32 genProcessType = 11;
      int32 genProcessId = 12;
      int32 backProcessId = 13;
      int32 version = 14;     // >= 3 for proto3 (5.0+)

      // repeated Parameter params = 20;      // not used
      FcConfig config = 21;
      uint64 startTime = 22; // calendar date, first valid time
      uint64 endTime = 23;   // calendar date, last valid time

      // extensions
      repeated Partition partitions = 100;
      bool isPartitionOfPartitions = 101;
      repeated uint32 run2part = 102 [packed=true];  // masterRuntime index to partition index
    }
     */

      GribCollectionProto.GribCollection.Builder indexBuilder = GribCollectionProto.GribCollection
          .newBuilder();
      indexBuilder.setName(name);
      indexBuilder.setTopDir(dcm.getRoot());
      indexBuilder.setVersion(currentVersion);

      // directory and mfile list
      File directory = new File(dcm.getRoot());
      List<GcMFile> gcmfiles = GcMFile.makeFiles(directory, files, allFileSet);
      for (GcMFile gcmfile : gcmfiles) {
        GribCollectionProto.MFile.Builder b = GribCollectionProto.MFile.newBuilder();
        b.setFilename(gcmfile.getName());
        b.setLastModified(gcmfile.getLastModified());
        b.setLength(gcmfile.getLength());
        b.setIndex(gcmfile.index);
        indexBuilder.addMfiles(b.build());
      }

      indexBuilder.setMasterRuntime(writeCoordProto(masterRuntime));

      //gds
      for (Group g : groups) {
        indexBuilder
            .addGds(writeGdsProto(g.gdss.getRawBytes(), g.gdss.getPredefinedGridDefinition()));
      }

      // the GC dataset
      indexBuilder.addDataset(writeDatasetProto(type, groups));

      // what about just storing first ??
      Grib1SectionProductDefinition pds = first.getPDSsection();
      indexBuilder.setCenter(pds.getCenter());
      indexBuilder.setSubcenter(pds.getSubCenter());
      indexBuilder.setLocal(pds.getTableVersion());
      indexBuilder.setMaster(0);
      indexBuilder.setGenProcessId(pds.getGenProcess());

      indexBuilder.setStartTime(dateRange.getStart().getMillis());
      indexBuilder.setEndTime(dateRange.getEnd().getMillis());

      GribCollectionProto.GribCollection index = indexBuilder.build();
      byte[] b = index.toByteArray();
      NcStream.writeVInt(raf, b.length); // message size
      raf.write(b);  // message  - all in one gulp

      logger.debug("  write GribCollectionIndex= {} bytes", b.length);
      logger.debug("  file size =  %d bytes", raf.length());
      return true;

    } finally {

      // remove it on failure
      if (deleteOnClose && !idxFile.delete()) {
        logger.error(" gc1 cant deleteOnClose index file {}", idxFile.getPath());
      }
    }
  }

  /*
message Record {
  uint32 fileno = 1;               // which GRIB file ? key into GC.fileMap
  uint64 pos = 2;                  // offset in GRIB file of the start of entire message
  uint64 bmsPos = 3;               // use alternate bms if non-zero (grib2 only)
  uint32 drsOffset = 4;            // offset of drs from pos (grib2 only)
}

// SparseArray only at the GCs (MRC and SRC) not at the Partitions
// dont need SparseArray in memory until someone wants to read from the variable
message SparseArray {
  repeated uint32 size = 2 [packed=true];     // multidim sizes = shape[]
  repeated uint32 track = 3 [packed=true];    // 1-based index into record list, 0 == missing
  repeated Record records = 4;                // List<Record>
  uint32 ndups = 5;                           // duplicates found when creating
}
 */
  private GribCollectionProto.SparseArray writeSparseArray(Grib1CollectionBuilder.VariableBag vb,
      Set<Integer> fileSet) {
    GribCollectionProto.SparseArray.Builder b = GribCollectionProto.SparseArray.newBuilder();
    SparseArray<Grib1Record> sa = vb.coordND.getSparseArray();
    for (int size : sa.getShape()) {
      b.addSize(size);
    }
    for (int track : sa.getTrack()) {
      b.addTrack(track);
    }

    for (Grib1Record gr : sa.getContent()) {
      GribCollectionProto.Record.Builder br = GribCollectionProto.Record.newBuilder();

      br.setFileno(gr.getFile());
      fileSet.add(gr.getFile());
      Grib1SectionIndicator is = gr.getIs();
      br.setStartPos(is.getStartPos()); // start of entire message

      b.addRecords(br);
    }

    b.setNdups(sa.getNdups());
    return b.build();
  }

  /*
  message Dataset {
    Type type = 1;
    repeated Group groups = 2;
   */
  private GribCollectionProto.Dataset writeDatasetProto(GribCollectionImmutable.Type type,
      List<Group> groups) {
    GribCollectionProto.Dataset.Builder b = GribCollectionProto.Dataset.newBuilder();

    GribCollectionProto.Dataset.Type ptype = GribCollectionProto.Dataset.Type
        .valueOf(type.toString());
    b.setType(ptype);

    for (Group group : groups) {
      b.addGroups(writeGroupProto(group));
    }

    return b.build();
  }

  /*
message Group {
  Gds gds = 1;                             // use this to build the HorizCoordSys
  repeated Variable variables = 2;         // list of variables
  repeated Coord coords = 3;               // list of coordinates
  repeated int32 fileno = 4 [packed=true]; // the component files that are in this group, key into gc.mfiles
}
 */
  private GribCollectionProto.Group writeGroupProto(Group g) {
    GribCollectionProto.Group.Builder b = GribCollectionProto.Group.newBuilder();

    b.setGds(writeGdsProto(g.gdss.getRawBytes(), g.gdss.getPredefinedGridDefinition()));

    for (Grib1CollectionBuilder.VariableBag vbag : g.gribVars) {
      b.addVariables(writeVariableProto(vbag));
    }

    for (Coordinate coord : g.coords) {
      switch (coord.getType()) {
        case runtime:
          b.addCoords(writeCoordProto((CoordinateRuntime) coord));
          break;
        case time:
          b.addCoords(writeCoordProto((CoordinateTime) coord));
          break;
        case timeIntv:
          b.addCoords(writeCoordProto((CoordinateTimeIntv) coord));
          break;
        case time2D:
          b.addCoords(writeCoordProto((CoordinateTime2D) coord));
          break;
        case vert:
          b.addCoords(writeCoordProto((CoordinateVert) coord));
          break;
        case ens:
          b.addCoords(writeCoordProto((CoordinateEns) coord));
          break;
      }
    }

    for (Integer aFileSet : g.fileSet) {
      b.addFileno(aFileSet);
    }

    return b.build();
  }

  /*
  message Variable {
     uint32 discipline = 1;
     bytes pds = 2;                   // raw pds
     repeated uint32 ids = 3 [packed=true];         // extra info not in pds; grib2 id section

     uint64 recordsPos = 4;  // offset of SparseArray message for this Variable
     uint32 recordsLen = 5;  // size of SparseArray message for this Variable

     repeated uint32 coordIdx = 6 [packed=true];    // indexes into Group.coords

     // optionally keep stats
     uint32 ndups = 8;
     uint32 nrecords = 9;
     uint32 missing = 10;

     // partition only
     repeated PartitionVariable partVariable = 100;
   }
   */
  private GribCollectionProto.Variable writeVariableProto(Grib1CollectionBuilder.VariableBag vb) {
    GribCollectionProto.Variable.Builder b = GribCollectionProto.Variable.newBuilder();

    b.setDiscipline(0);
    b.setPds(ByteString.copyFrom(vb.first.getPDSsection().getRawBytes()));

    b.setRecordsPos(vb.pos);
    b.setRecordsLen(vb.length);

    for (int idx : vb.coordIndex) {
      b.addCoordIdx(idx);
    }

    // keep stats
    SparseArray sa = vb.coordND.getSparseArray();
    if (sa != null) {
      b.setNdups(sa.getNdups());
      b.setNrecords(sa.countNotMissing());
      b.setMissing(sa.countMissing());
    }

    return b.build();
  }

}