/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package thredds.ui.monitor;

//import net.sf.ehcache.Cache;
//import net.sf.ehcache.Element;
import org.jfree.data.time.Minute;
import org.jfree.data.time.TimeSeries;
import thredds.logs.AccessLogParser;
import thredds.logs.LogCategorizer;
import thredds.logs.LogReader;
import ucar.nc2.ui.widget.BAMutil;
import ucar.nc2.ui.widget.IndependentWindow;
import ucar.nc2.ui.widget.PopupMenu;
import ucar.nc2.ui.widget.TextHistoryPane;
import ucar.nc2.units.TimeDuration;
import ucar.util.prefs.PreferencesExt;
import ucar.util.prefs.ui.BeanTable;

import javax.swing.*;
import javax.swing.event.ChangeEvent;
import javax.swing.event.ChangeListener;
import javax.swing.event.ListSelectionEvent;
import javax.swing.event.ListSelectionListener;
import java.awt.*;
import java.awt.event.ActionEvent;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
 * Access Logs in TdsMonitor.
 *
 * @author caron
 * @since Mar 26, 2009
 */
public class AccessLogTable extends JPanel {
  private PreferencesExt prefs;
  private DnsLookup dnsLookup;

  private ucar.util.prefs.ui.BeanTable logTable, userTable, datarootTable, serviceTable, clientTable;
  private JPanel timeSeriesPanel;

  private ArrayList<LogReader.Log> completeLogs;
  private ArrayList<LogReader.Log> restrictLogs;

  private boolean calcUser = true;
  private boolean calcRoot = true;
  private boolean calcService = true;
  private boolean calcClient = true;

  private JTabbedPane tabbedPanel;

  private TextHistoryPane infoTA;
  private IndependentWindow infoWindow;

  private JTextArea startDateField, endDateField;

  public AccessLogTable(JTextArea startDateField, JTextArea endDateField, PreferencesExt prefs, DnsLookup dnsLookup) {
      this.startDateField = startDateField;
    this.endDateField = endDateField;
    this.prefs = prefs;
    this.dnsLookup = dnsLookup;
    PopupMenu varPopup;

    logTable = new BeanTable(LogReader.Log.class, (PreferencesExt) prefs.node("Logs"), false);
    logTable.addListSelectionListener(new ListSelectionListener() {
      public void valueChanged(ListSelectionEvent e) {
        LogReader.Log log = (LogReader.Log) logTable.getSelectedBean();
        if (log == null) return;
        infoTA.setText(log.toString());
        infoWindow.show();
      }
    });
    varPopup = new PopupMenu(logTable.getJTable(), "Options");
    varPopup.addAction("Show", new AbstractAction() {
      public void actionPerformed(ActionEvent e) {
        LogReader.Log log = (LogReader.Log) logTable.getSelectedBean();
        if (log == null) return;
        Formatter f = new Formatter();
        log.toString(f);
        infoTA.setText(f.toString());
        infoWindow.show();
      }
    });
    varPopup.addAction("DNS Lookup", new AbstractAction() {
      public void actionPerformed(ActionEvent e) {
        LogReader.Log log = (LogReader.Log) logTable.getSelectedBean();
        if (log == null) return;
        try {
          infoTA.setText(log.getIp() + " = " + dnsLookup.reverseDNS(log.getIp()));
        } catch (Exception ee) {
          infoTA.setTextFromStackTrace(ee);
        }
        infoWindow.show();
      }
    });
    varPopup.addAction("Resend URL", new AbstractAction() {
      public void actionPerformed(ActionEvent e) {
        LogReader.Log log = (LogReader.Log) logTable.getSelectedBean();
        if (log == null) return;
        String urlString = log.getPath();
        AccessLogTable.this.firePropertyChange("UrlDump", null, "http://" + manager.getServer() + urlString);
      }
    });
    varPopup.addAction("Remove selected logs", new AbstractAction() {
      public void actionPerformed(ActionEvent e) {
        List all = logTable.getBeans();
        List selected = logTable.getSelectedBeans();
        all.removeAll(selected);
        logTable.setBeans(all);
      }
    });

    userTable = new BeanTable(User.class, (PreferencesExt) prefs.node("LogUser"), false);
    userTable.addListSelectionListener(new ListSelectionListener() {
      public void valueChanged(ListSelectionEvent e) {
        User accum = (User) userTable.getSelectedBean();
        if (accum == null) return;
        accum.run();
      }
    });
    PopupMenu varPopupU = new PopupMenu(userTable.getJTable(), "Options");
    varPopupU.addAction("User requests", new AbstractAction() {
      public void actionPerformed(ActionEvent e) {
        Accum accum = (Accum) userTable.getSelectedBean();
        if (accum == null) return;
        logTable.setBeans(accum.logs);
        tabbedPanel.setSelectedIndex(0);
      }
    });

    datarootTable = new BeanTable(Dataroot.class, (PreferencesExt) prefs.node("DataRoot"), false);
    datarootTable.addListSelectionListener(new ListSelectionListener() {
      public void valueChanged(ListSelectionEvent e) {
        Dataroot accum = (Dataroot) datarootTable.getSelectedBean();
        if (accum == null) return;
      }
    });
    PopupMenu varPopupR = new PopupMenu(datarootTable.getJTable(), "Options");
    varPopupR.addAction("User requests", new AbstractAction() {
      public void actionPerformed(ActionEvent e) {
        Accum accum = (Accum) datarootTable.getSelectedBean();
        if (accum == null) return;
        logTable.setBeans(accum.logs);
        tabbedPanel.setSelectedIndex(0);
      }
    });

    serviceTable = new BeanTable(Service.class, (PreferencesExt) prefs.node("Service"), false);
    serviceTable.addListSelectionListener(new ListSelectionListener() {
      public void valueChanged(ListSelectionEvent e) {
        Service accum = (Service) serviceTable.getSelectedBean();
        if (accum == null) return;
      }
    });
    ucar.nc2.ui.widget.PopupMenu varPopupS = new PopupMenu(serviceTable.getJTable(), "Options");
    varPopupS.addAction("User requests", new AbstractAction() {
      public void actionPerformed(ActionEvent e) {
        Accum accum = (Accum) serviceTable.getSelectedBean();
        if (accum == null) return;
        logTable.setBeans(accum.logs);
        tabbedPanel.setSelectedIndex(0);
      }
    });

    clientTable = new BeanTable(Service.class, (PreferencesExt) prefs.node("Service"), false);
    clientTable.addListSelectionListener(new ListSelectionListener() {
      public void valueChanged(ListSelectionEvent e) {
        Client accum = (Client) clientTable.getSelectedBean();
        if (accum == null) return;
      }
    });
    ucar.nc2.ui.widget.PopupMenu varPopupC = new PopupMenu(clientTable.getJTable(), "Options");
    varPopupC.addAction("User requests", new AbstractAction() {
      public void actionPerformed(ActionEvent e) {
        Accum accum = (Accum) clientTable.getSelectedBean();
        if (accum == null) return;
        logTable.setBeans(accum.logs);
        tabbedPanel.setSelectedIndex(0);
      }
    });

    timeSeriesPanel = new JPanel();
    //timeSeriesPanel.setLayout(new GridLayout(2, 2));
    timeSeriesPanel.setLayout(new BorderLayout());

    // the info window
    infoTA = new TextHistoryPane();
    infoWindow = new IndependentWindow("Extra Information", BAMutil.getImage("netcdfUI"), infoTA);
    infoWindow.setBounds((Rectangle) prefs.getBean("InfoWindowBounds", new Rectangle(300, 300, 800, 100)));

    // tabbed panes
    tabbedPanel = new JTabbedPane(JTabbedPane.TOP);
    tabbedPanel.addTab("LogTable", logTable);
    tabbedPanel.addTab("User", userTable);
    tabbedPanel.addTab("DataRoot", datarootTable);
    tabbedPanel.addTab("Service", serviceTable);
    tabbedPanel.addTab("Client", clientTable);
    tabbedPanel.addTab("TimeSeries", timeSeriesPanel);
    tabbedPanel.setSelectedIndex(0);

    tabbedPanel.addChangeListener(e -> {
        java.util.ArrayList<LogReader.Log> useBeans = (java.util.ArrayList<LogReader.Log>) logTable.getBeans();

        int idx = tabbedPanel.getSelectedIndex();
        String title = tabbedPanel.getTitleAt(idx);
        if (title.equals("User"))
          initUserLogs(useBeans);
        if (title.equals("DataRoot"))
          initDatarootLogs(useBeans);
        if (title.equals("Service"))
          initServiceLogs(useBeans);
        if (title.equals("Client"))
          initClientLogs(useBeans);
        if (title.equals("TimeSeries"))
          showTimeSeriesAll(useBeans);
    });

    setLayout(new BorderLayout());
    //add(topPanel, BorderLayout.NORTH);
    add(tabbedPanel, BorderLayout.CENTER);
  }

  public void exit() {
    if (executor != null)
      executor.shutdownNow();

    logTable.saveState(false);
    userTable.saveState(false);
    datarootTable.saveState(false);
    serviceTable.saveState(false);
    clientTable.saveState(false);
    prefs.putBeanObject("InfoWindowBounds", infoWindow.getBounds());
  }

  private SimpleDateFormat df = new java.text.SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
  private LogLocalManager manager;
  private java.util.List<LogLocalManager.FileDateRange> accessLogFiles = null;

  public void setLocalManager(LogLocalManager manager) {
    this.manager = manager;

    Date startDate = manager.getStartDate();
    Date endDate = manager.getEndDate();
    if (startDate != null)
      startDateField.setText(df.format(startDate));
    else
      startDateField.setText(df.format(new Date()));

    if (endDate != null)
      endDateField.setText(df.format(endDate));
    else
      endDateField.setText(df.format(new Date()));

    LogCategorizer.setRoots(manager.getRoots());
  }

  void showLogs(LogReader.LogFilter filter) {
    Date start = null, end = null;
    try {
      start = df.parse(startDateField.getText());
      end = df.parse(endDateField.getText());
      accessLogFiles = manager.getLocalFiles(start, end);
    } catch (Exception e) {
      e.printStackTrace();
      accessLogFiles = manager.getLocalFiles(null, null);
    }

    LogReader reader = new LogReader(new AccessLogParser());
    completeLogs = new ArrayList<>(30000);

    if ((start != null) && (end != null))
      filter = new LogReader.DateFilter(start.getTime(), end.getTime(), filter);

    try {
      long startElapsed = System.nanoTime();
      LogReader.Stats stats = new LogReader.Stats();

      for (LogLocalManager.FileDateRange fdr : accessLogFiles)
        reader.scanLogFile(fdr.f, new MyClosure(completeLogs), filter, stats);

      long elapsedTime = System.nanoTime() - startElapsed;
      System.out.printf(" setLogFile total= %d passed=%d%n", stats.total, stats.passed);
      System.out.printf(" elapsed=%f msecs %n", elapsedTime / (1000 * 1000.0));

    } catch (IOException ioe) {
      ioe.printStackTrace();
      return;
    }

    resetLogs();
  }

  void showInfo(Formatter f) {
    f.format(" Current time =   %s%n%n", new Date().toString());

    int n = 0;
    if (completeLogs != null) {
      n = completeLogs.size();
      f.format("Complete logs n=%d%n", n);
      f.format("  first log date= %s%n", completeLogs.get(0).getDate());
      f.format("   last log date= %s%n", completeLogs.get(n - 1).getDate());
    }
    List restrict = logTable.getBeans();
    if (restrict != null && (restrict.size() != n)) {
      f.format("%nRestricted logs n=%d%n", restrict.size());
    }
    if (accessLogFiles != null) {
        f.format("%nFiles used%n");
        for (LogLocalManager.FileDateRange fdr : accessLogFiles) {
            f.format(" %s [%s,%s]%n", fdr.f.getName(), fdr.start, fdr.end);
        }
    }
  }

  void resetLogs() {
    logTable.setBeans(completeLogs);
    tabbedPanel.setSelectedIndex(0);

    userTable.setBeans(new ArrayList());
    datarootTable.setBeans(new ArrayList());
    calcUser = true;
    calcRoot = true;
    calcService = true;
    calcClient = true;
    restrictLogs = completeLogs;

    /* int n = completeLogs.size();
    if (n > 0) {
      startDateField.setText(completeLogs.get(0).getDate());
      endDateField.setText(completeLogs.get(n-1).getDate());
    } */
  }

  void restrictLogs(String restrict) {

    restrictLogs = new ArrayList<LogReader.Log>(1000);
    for (LogReader.Log log : completeLogs) {
      String ip = log.getIp();
      if (ip.startsWith(restrict)) continue;
      restrictLogs.add(log);
    }

    logTable.setBeans(restrictLogs);
    tabbedPanel.setSelectedIndex(0);

    userTable.setBeans(new ArrayList());
    datarootTable.setBeans(new ArrayList());
    calcUser = true;
    calcRoot = true;
    calcService = true;
    calcClient = true;
  }

  ////////////////////////////////////////////////////////

  static class MyClosure implements LogReader.Closure {
    ArrayList<LogReader.Log> logs;

    MyClosure(ArrayList<LogReader.Log> logs) {
      this.logs = logs;
    }

    public void process(LogReader.Log log) {
      logs.add(log);
    }
  }

  ////////////////////////////////////////////////

  public class Accum {
    public String getName() {
      return name;
    }

    public long getMsecs() {
      return msecs;
    }

    public long getMsecsPerRequest() {
      return msecs / count;
    }

    public long getKbytes() {
      return bytes / 1000;
    }

    public int getCount() {
      return count;
    }

    ArrayList<LogReader.Log> logs = new ArrayList<>(100);
    String name;
    long msecs;
    long bytes;
    int count;

    public Accum() {
    }

    Accum(String name) {
      this.name = name;
    }

    void add(LogReader.Log log) {
      logs.add(log);
      count++;
      bytes += log.getBytes();
      msecs += log.getMsecs();
    }
  }

  public class User extends Accum implements Runnable {
    String ip, namer;

    public String getIp() {
      return ip;
    }

    public String getNameReverse() {
      if (name != null && namer == null) {
        StringBuffer sbuff = new StringBuffer();
        String[] p = name.split("\\.");
        for (int i = p.length - 1; i >= 0; i--) {
          sbuff.append(p[i]);
          if (i != 0) sbuff.append('.');
        }
        namer = sbuff.toString();
      }
      return namer;
    }

    public User() {
      super();
    }

    User(String ip) {
      this.ip = ip;
    }

    public void run() {
      if (name != null) return;
      try {
        long startElapsed = System.nanoTime();
        name = dnsLookup.reverseDNS(ip);
        long elapsedTime = System.nanoTime() - startElapsed;
        if (showDNStime) System.out.printf(" reverseDNS took=%f msecs %n", elapsedTime / (1000 * 1000.0));
      } catch (Throwable e) {
        name = e.getMessage();
      }
    }
  }

  public class Client extends Accum {

    public Client() {
      super();
    }

    Client(String client) {
      super(client);
    }

  }

  void initClientLogs(ArrayList<LogReader.Log> logs) {
    if (!calcClient) return;
    if (logs == null) return;

    HashMap<String, Client> map = new HashMap<String, Client>();

    for (LogReader.Log log : logs) {
      String clientName = log.getClient();
      if (clientName == null)  clientName = "";
      Client accum = map.get(clientName);
      if (accum == null) {
        accum = new Client(clientName);
        map.put(clientName, accum);
      }
      accum.add(log);
    }

    clientTable.setBeans(new ArrayList(map.values()));
    calcClient = false;
  }

  private boolean showDNStime = false;

  void initUserLogs(ArrayList<LogReader.Log> logs) {
    if (!calcUser) return;
    if (logs == null) return;

    HashMap<String, User> map = new HashMap<String, User>();

    for (LogReader.Log log : logs) {
      User accum = map.get(log.getIp());
      if (accum == null) {
        accum = new User(log.getIp());
        map.put(log.getIp(), accum);
      }
      accum.add(log);
    }

    userTable.setBeans(new ArrayList(map.values()));
    calcUser = false;
  }

  private ExecutorService executor = null;

  void showDNS() {

    if (null == executor) executor = Executors.newFixedThreadPool(3); // number of threads
    ArrayList<User> accums = (ArrayList<User>) userTable.getBeans();
    for (User a : accums) {
      executor.execute(a);
    }

    /* boolean ok = false;
    try {
      ok = executor.awaitTermination(3, TimeUnit.SECONDS);
    } catch (InterruptedException e) {
      e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
    }
    long elapsedTime = System.nanoTime() - startElapsed;
    System.out.printf(" reverseDNS took=%f msecs ok=%s %n", elapsedTime / (1000 * 1000.0), ok);  */
  }

  ////////////////////////////////////////////////

  public class Dataroot extends Accum {
    public Dataroot() {
      super();
    }

    Dataroot(String name) {
      super(name);
    }

  }

  void initDatarootLogs(ArrayList<LogReader.Log> logs) {
    if (!calcRoot) return;
    if (logs == null) return;

    HashMap<String, Dataroot> map = new HashMap<>();

    for (LogReader.Log log : logs) {
      String path = log.getPath();
      if (path == null) continue;
      String dataRoot = LogCategorizer.getDataroot(path, log.getStatus());
      Dataroot accum = map.get(dataRoot);
      if (accum == null) {
        accum = new Dataroot(dataRoot);
        map.put(dataRoot, accum);
      }
      accum.add(log);
    }

    datarootTable.setBeans(new ArrayList<>(map.values()));
    calcRoot = false;
  }

  ////////////////////////////////////////////////

  public class Service extends Accum {

    public Service() {
      super();
    }

    Service(String name) {
      super(name);
    }
  }

  void initServiceLogs(ArrayList<LogReader.Log> logs) {
    if (!calcService) return;
    if (logs == null) return;

    HashMap<String, Service> map = new HashMap<>();

    for (LogReader.Log log : logs) {
      String path = log.getPath();
      if (path == null) {
        System.out.printf("FAIL %s%n", log);
        continue;
      }
      String service = LogCategorizer.getService(path);
      Service accum = map.get(service);
      if (accum == null) {
        accum = new Service(service);
        map.put(service, accum);
      }
      accum.add(log);
    }

    serviceTable.setBeans(new ArrayList<>(map.values()));
    calcService = false;
  }

  //////////////////

  // construct the TImeSeries plot for the list of logs passed in

  private void showTimeSeriesAll(java.util.List<LogReader.Log> logs) {
    TimeSeries bytesSentData = new TimeSeries("Bytes Sent", Minute.class);
    TimeSeries timeTookData = new TimeSeries("Average Latency", Minute.class);
    TimeSeries nreqData = new TimeSeries("Number of Requests", Minute.class);

    String intervalS = "5 minute"; // interval.getText().trim();
    // if (intervalS.length() == 0) intervalS = "5 minute";
    long period = 1000 * 60 * 5;
    try {
      TimeDuration tu = new TimeDuration(intervalS);
      period = (long) (1000 * tu.getValueInSeconds());
    } catch (Exception e) {
      System.out.printf("Illegal Time interval=%s %n", intervalS);
    }

    long current = 0;
    long bytes = 0;
    long timeTook = 0;
    long total_count = 0;
    long count = 0;
    for (LogReader.Log log : logs) {
      long msecs = log.date;
      if (msecs - current > period) {
        if (current > 0) {
          total_count += count;
          addPoint(bytesSentData, timeTookData, nreqData, new Date(current), bytes, count, timeTook);
        }
        bytes = 0;
        count = 0;
        timeTook = 0;
        current = msecs;
      }
      bytes += log.getBytes();
      timeTook += log.getMsecs();
      count++;
    }
    if (count > 0)
        addPoint(bytesSentData, timeTookData, nreqData, new Date(current), bytes, count, timeTook);
    total_count += count;
    System.out.printf("showTimeSeriesAll: total_count = %d logs = %d%n", total_count, logs.size());

    MultipleAxisChart mc = new MultipleAxisChart("Access Logs", intervalS + " average", "Mbytes Sent", bytesSentData);
    mc.addSeries("Number of Requests", nreqData);
    mc.addSeries("Average Latency (secs)", timeTookData);
    mc.finish(new java.awt.Dimension(1000, 1000));

    //MultipleAxisChart mc = new MultipleAxisChart("Bytes Sent", "5 min average", "Mbytes/sec", bytesSentData);
    //Chart c2 = new Chart("Average Latency", "5 min average", "Millisecs", timeTookData);
    //Chart c3 = new Chart("Number of Requests/sec", "5 min average", "", nreqData);

    timeSeriesPanel.removeAll();
    timeSeriesPanel.add(mc);
  }

  void addPoint(TimeSeries bytesSentData, TimeSeries timeTookData, TimeSeries nreqData,
                Date date, long bytes, long count, long timeTook) {

    bytesSentData.add(new Minute(date), bytes / 1000. / 1000.);
    double latency = (double) timeTook / count / 1000.;
    //timeTookData.add(new Minute(date), (latency > 10*1000) ? 0 : latency); // note latency limited to 10 secs.
    timeTookData.add(new Minute(date), latency);
    nreqData.add(new Minute(date), (double) count);
  }

  static private void test(String ip) {
    StringBuilder sbuff = new StringBuilder();
    String[] p = ip.split("\\.");
    for (int i = p.length - 1; i >= 0; i--) {
      sbuff.append(p[i]);
      if (i != 0) sbuff.append('.');
    }
    String ipr = sbuff.toString();
    System.out.printf("%s == %s%n", ip, ipr);
  }

  static public void main(String args[]) {
    test("1.2.3.4");
    test("..1.2");
    test("..1.2..");

  }

}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2;

import ucar.ma2.Array;
import ucar.ma2.ArrayChar;
import ucar.ma2.DataType;
import ucar.ma2.ForbiddenConversionException;
import ucar.ma2.Index;
import ucar.nc2.constants.CDM;
import ucar.nc2.util.Indent;
import ucar.unidata.util.StringUtil2;

import java.nio.ByteBuffer;
import java.util.Formatter;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * An Attribute has a name and a value, used for associating arbitrary metadata with a Variable or a Group.
 * The value can be a one dimensional array of Strings or numeric values.
 *

 * Attributes are immutable after setImmutable().
 *
 * @author caron
 */

public class Attribute extends CDMNode
{

  static final String SPECIALPREFIX = "_";
  static final String[] SPECIALS = new String[]{
          CDM.NCPROPERTIES, CDM.ISNETCDF4, CDM.SUPERBLOCKVERSION,
          CDM.DAP4_LITTLE_ENDIAN, CDM.EDU_UCAR_PREFIX
  };

  /**
   * Turn a list into a map
   *
   * @param atts list of attributes
   * @return map of attributes by name
   */
  static public Map<String, Attribute> makeMap(List<Attribute> atts)
  {
    int size = (atts == null) ? 1 : atts.size();
    Map<String, Attribute> result = new HashMap<>(size);
    if(atts == null) return result;
    for(Attribute att : atts) result.put(att.getShortName(), att);
    return result;
  }

  static public boolean
  isspecial(Attribute a)
  {
    String nm = a.getShortName();
    if(nm.startsWith(SPECIALPREFIX)) {
      /* Check for selected special attributes */
      for(String s : SPECIALS) {
        if(nm.startsWith(s))
          return true; /* is special */
      }
    }
    return false; /* is not special */
  }
  ///////////////////////////////////////////////////////////////////////////////////

  /**
   * Get the data type of the Attribute value.
   *
   * @return DataType
   */
  public DataType getDataType() {
    return dataType;
  }

  public void setDataType(DataType dt)
  {
    this.dataType = dt;
  }

  public EnumTypedef getEnumType()
  {
      return this.enumtype;
  }

  public void setEnumType(EnumTypedef en)
  {
        this.enumtype = en;
  }

  /**
   * True if value is an array (getLength() > 1)
   *
   * @return if its an array.
   */
  public boolean isArray() {
    return (getLength() > 1);
  }

  /**
   * Get the length of the array of values
   *
   * @return number of elements in the array.
   */
  public int getLength() {
    return nelems;
  }

  /**
   * Get the value as an Array.
   *
   * @return Array of values.
   */
  public Array getValues() {
    if (values == null && svalue != null) {
      values = Array.factory(DataType.STRING, new int[]{1});
      values.setObject(values.getIndex(), svalue);
    }

    return values;
  }

  /**
   * Get the value as an Object.
   *
   * @param index which index
   * @return ith value as an Object.
   */
  public Object getValue(int index) {
    if (isString()) return getStringValue(index);
    return getNumericValue(index);
  }

  /**
   * True if value is of type String and not null.
   *
   * @return if its a String and not null.
   */
  public boolean isString() {
    return (dataType == DataType.STRING) && null != getStringValue();
  }

  /**
   * Retrieve String value; only call if isString() is true.
   *
   * @return String if this is a String valued attribute, else null.
   * @see Attribute#isString
   */
  public String getStringValue() {
    if (dataType != DataType.STRING) return null;
    return (svalue != null) ? svalue : _getStringValue(0);
  }

  /**
   * Retrieve ith String value; only call if isString() is true.
   *
   * @param index which index
   * @return ith String value (if this is a String valued attribute and index in range), else null.
   * @see Attribute#isString
   */
  public String getStringValue(int index) {
    if (dataType != DataType.STRING) return null;
    if ((svalue != null) && (index == 0)) return svalue;
    return _getStringValue(index);
  }

  private String _getStringValue(int index) {
    if ((index < 0) || (index >= nelems)) return null;
    return (String) values.getObject(index);
  }

  /**
   * Retrieve numeric value. Equivalent to getNumericValue(0)
   *
   * @return the first element of the value array, or null if its a String that cant be converted.
   */
  public Number getNumericValue() {
    return getNumericValue(0);
  }

  /// these deal with array-valued attributes

  /**
   * Retrieve a numeric value by index. If it's a String, it will try to parse it as a double.
   *
   * @param index the index into the value array.
   * @return Number value[index], or null if its a non-parseable String or
   * the index is out of range.
   */
  public Number getNumericValue(int index) {
    if ((index < 0) || (index >= nelems))
      return null;

    // LOOK can attributes be enum valued? for now, no
    switch (dataType) {
      case STRING:
        try {
          return new Double(getStringValue(index));
        } catch (NumberFormatException e) {
          return null;
        }
      case BYTE:
      case UBYTE:
        return values.getByte(index);
      case SHORT:
      case USHORT:
        return values.getShort(index);
      case INT:
      case UINT:
        return values.getInt(index);
      case FLOAT:
        return values.getFloat(index);
      case DOUBLE:
        return values.getDouble(index);
      case LONG:
      case ULONG:
        return values.getLong(index);
    }
    return null;
  }

    /**
     * CDL representation, not strict
     *
     * @return CDL representation
     */
    @Override
    public String toString () {
      return toString(false);
    }

    /**
     * CDL representation, may be strict
     * @param strict if true, create strict CDL, escaping names
     * @return CDL representation
     */

  public String toString(boolean strict) {
    Formatter f = new Formatter();
    writeCDL(f, strict, null);
    return f.toString();
  }

  /**
   * Write CDL representation into f
   *
   * @param f      write into this
   * @param strict if true, create strict CDL, escaping names
   */
  protected void writeCDL(Formatter f, boolean strict, String parentname) {
    if(strict && (isString() || this.getEnumType() != null))
      // Force type explicitly for string.
      f.format("string "); //note lower case and trailing blank
    if(strict && parentname != null) f.format(NetcdfFile.makeValidCDLName(parentname));
    f.format(":");
    f.format("%s", strict ? NetcdfFile.makeValidCDLName(getShortName()) : getShortName());
    if (isString()) {
      f.format(" = ");
      for(int i = 0; i < getLength(); i++) {
        if(i != 0) f.format(", ");
        String val = getStringValue(i);
        if(val != null)
          f.format("\"%s\"", encodeString(val));
      }
    } else if(getEnumType() != null) {
      f.format(" = ");
      for (int i = 0; i < getLength(); i++) {
        if(i != 0) f.format(", ");
        EnumTypedef en = getEnumType();
        String econst = getStringValue(i);
        Integer ecint = en.lookupEnumInt(econst);
        if(ecint == null)
           throw new ForbiddenConversionException("Illegal enum constant: "+econst);
        f.format("\"%s\"", encodeString(econst));
      }
    } else {
      f.format(" = ");
      for (int i = 0; i < getLength(); i++) {
        if (i != 0) f.format(", ");

        Number number = getNumericValue(i);
        if (dataType.isUnsigned()) {
          // 'number' is unsigned, but will be treated as signed when we print it below, because Java only has signed
          // types. If it is large enough ( >= 2^(BIT_WIDTH-1) ), its most-significant bit will be interpreted as the
          // sign bit, which will result in an invalid (negative) value being printed. To prevent that, we're going
          // to widen the number before printing it.
          number = DataType.widenNumber(number);
        }
        f.format("%s", number);

        if (dataType.isUnsigned()) {
          f.format("U");
        }

        if (dataType == DataType.FLOAT)
          f.format("f");
        else if (dataType == DataType.SHORT || dataType == DataType.USHORT) {
          f.format("S");
        } else if (dataType == DataType.BYTE || dataType == DataType.UBYTE) {
          f.format("B");
        } else if (dataType == DataType.LONG || dataType == DataType.ULONG) {
          f.format("L");
        }
      }
    }
  }

  private static char[] org = {'\b', '\f', '\n', '\r', '\t', '\\', '\'', '\"'};
  private static String[] replace = {"\\b", "\\f", "\\n", "\\r", "\\t", "\\\\", "\\\'", "\\\""};

  /**
   * Replace special characters '\t', '\n', '\f', '\r'.
   *
   * @param s string to quote
   * @return equivilent string replacing special chars
   */
  static public String encodeString(String s) {
    return StringUtil2.replace(s, org, replace);
  }

  ///////////////////////////////////////////////////////////////////////////////

  private String svalue; // optimization for common case of single String valued attribute
  private DataType dataType;
  private EnumTypedef enumtype = null;
  private int nelems; // can be 0 or greater
  private Array values;

  /**
   * Copy constructor
   *
   * @param name name of new Attribute
   * @param from copy value from here.
   */
  public Attribute(String name, Attribute from) {
    super(name);
    if (name == null) throw new IllegalArgumentException("Trying to set name to null on " + this);
    setDataType(from.dataType);
    setEnumType(from.enumtype);
    this.nelems = from.nelems;
    this.svalue = from.svalue;
    this.values = from.values;
    setImmutable();
  }

  /**
   * Create a String-valued Attribute.
   *
   * @param name name of Attribute
   * @param val  value of Attribute
   */
  public Attribute(String name, String val) {
    super(name);
    setDataType(DataType.STRING);
    if (name == null) throw new IllegalArgumentException("Trying to set name to null on " + this);
    setStringValue(val);
    setImmutable();
  }

  /**
   * Create a scalar numeric-valued Attribute.
   *
   * @param name name of Attribute
   * @param val  value of Attribute
   */
  public Attribute(String name, Number val) {
    this(name, val, false);
  }

  public Attribute(String name, Number val, boolean isUnsigned) {
    super(name);
    if (name == null) throw new IllegalArgumentException("Trying to set name to null on " + this);

    int[] shape = new int[1];
    shape[0] = 1;
    DataType dt = DataType.getType(val.getClass(), isUnsigned);
    setDataType(dt);
    Array vala = Array.factory(dt, shape);
    Index ima = vala.getIndex();
    vala.setObject(ima.set0(0), val);
    setValues(vala);
    setImmutable();
  }

  /**
   * Construct attribute with Array of values.
   *
   * @param name   name of attribute
   * @param values array of values.
   */
  public Attribute(String name, Array values) {
    this(name,values.getDataType());
    setValues(values);
    setImmutable();
  }

  /**
   * Construct an empty attribute with no values
   *
   * @param name
   * @param dataType
     */
  public Attribute(String name, DataType dataType)
  {
      this(name);
      setDataType(dataType);
  }

  public Attribute(String name, List values) {
    this(name, values, false);
  }

  /**
   * Construct attribute with list of String or Number values.
   * The list determines the attribute type
   * @param name   name of attribute
   * @param values list of values. must be String or Number, must all be the same type, and have at least 1 member
   * @param isUnsigned
   */
  public Attribute(String name, List values, boolean isUnsigned) {
    this(name);
    if(values == null || values.size() == 0)
	  throw new IllegalArgumentException("Cannot determine attribute's type");
    Class c = values.get(0).getClass();
    setDataType(DataType.getType(c, isUnsigned));
    setValues(values);
    setImmutable();
  }

  /**
   * A copy constructor using a ucar.unidata.util.Parameter.
   * Need to do this so ucar.unidata.geoloc package doesnt depend on ucar.nc2 library
   *
   * @param param copy info from here.
   */
  public Attribute(ucar.unidata.util.Parameter param) {
    this(param.getName());

    if (param.isString()) {
      setStringValue(param.getStringValue());

    } else {
      double[] values = param.getNumericValues();
      int n = values.length;
      Array vala = Array.factory(DataType.DOUBLE, new int[]{n}, values);
      setValues(vala);
    }
    setImmutable();
  }

  /**
   * set the value as a String, trimming trailing zeroes
   *
   * @param val value of Attribute
   */
  private void setStringValue(String val) {
    if (val == null)
      throw new IllegalArgumentException("Attribute value cannot be null");

    // get rid of trailing nul characters
    int len = val.length();
    while ((len > 0) && (val.charAt(len - 1) == 0))
      len--;
    if (len != val.length())
      val = val.substring(0, len);

    this.svalue = val;
    this.nelems = 1;
    this.dataType = DataType.STRING;

    //values = Array.factory(String.class, new int[]{1});
    //values.setObject(values.getIndex(), val);
    //setValues(values);
  }

  //////////////////////////////////////////
  // the following make this mutable, but its restricted to subclasses, namely DODSAttribute

  /**
   * Constructor. Must also set value
   *
   * @param name name of Attribute
   */
  protected Attribute(String name) {
    super(name);
    if (name == null) throw new IllegalArgumentException("Trying to set name to null on " + this);
  }

  /**
   * set the values from a list
   *
   * @param values
   */
  public void setValues(List values)
  {
    if(values == null || values.size() == 0)
	throw new IllegalArgumentException("Cannot determine attribute's type");
    int n = values.size();
    Class c = values.get(0).getClass();
    Object pa;

    if (c == String.class) {
      String[] va = new String[n];
      pa = va;
      for (int i = 0; i < n; i++) va[i] = (String) values.get(i);
    } else if (c == Integer.class) {
      int[] va = new int[n];
      pa = va;
      for (int i = 0; i < n; i++) va[i] = (Integer) values.get(i);
    } else if (c == Double.class) {
      double[] va = new double[n];
      pa = va;
      for (int i = 0; i < n; i++) va[i] = (Double) values.get(i);
    } else if (c == Float.class) {
      float[] va = new float[n];
      pa = va;
      for (int i = 0; i < n; i++) va[i] = (Float) values.get(i);
    } else if (c == Short.class) {
      short[] va = new short[n];
      pa = va;
      for (int i = 0; i < n; i++) va[i] = (Short) values.get(i);
    } else if (c == Byte.class) {
      byte[] va = new byte[n];
      pa = va;
      for (int i = 0; i < n; i++) va[i] = (Byte) values.get(i);
    } else if (c == Long.class) {
      long[] va = new long[n];
      pa = va;
      for (int i = 0; i < n; i++) va[i] = (Long) values.get(i);
    } else {
      throw new IllegalArgumentException("Unknown type for Attribute = " + c.getName());
    }
    setValues(Array.factory(this.dataType, new int[]{n}, pa));
  }

  /**
   * set the values from an Array
   *
   * @param arr value of Attribute
   */
  public void setValues(Array arr) {
    if (immutable) throw new IllegalStateException("Cant modify");

    if (arr == null) {
      dataType = DataType.STRING;
      return;
    }

    if (arr.getElementType() == char.class) { // turn CHAR into STRING
      ArrayChar carr = (ArrayChar) arr;
      if (carr.getRank() == 1) { // common case
        svalue = carr.getString();
        this.nelems = 1;
        this.dataType = DataType.STRING;
        return;
      }
      // otherwise its an array of Strings
      arr = carr.make1DStringArray();
    }

    // this should be a utility somewhere
    if (arr.getElementType() == ByteBuffer.class) { // turn OPAQUE into BYTE
      int totalLen = 0;
      arr.resetLocalIterator();
      while (arr.hasNext()) {
        ByteBuffer bb = (ByteBuffer) arr.next();
        totalLen += bb.limit();
      }
      byte[] ba = new byte[totalLen];
      int pos = 0;
      arr.resetLocalIterator();
      while (arr.hasNext()) {
        ByteBuffer bb = (ByteBuffer) arr.next();
        System.arraycopy(bb.array(), 0, ba, pos, bb.limit());
        pos += bb.limit();
      }
      arr = Array.factory(DataType.BYTE, new int[]{totalLen}, ba);
    }

    if (DataType.getType(arr) == DataType.OBJECT)
      throw new IllegalArgumentException("Cant set Attribute with type " + arr.getElementType());

    if (arr.getRank() > 1)
      arr = arr.reshape(new int[]{(int) arr.getSize()}); // make sure 1D

    this.values = arr;
    this.nelems = (int) arr.getSize();
    this.dataType = DataType.getType(arr);
  }

  /**
   * Set the name of this Attribute.
   * Attribute names are unique within a NetcdfFile's global set, and within a Variable's set.
   *
   * @param name name of attribute
   */
  public synchronized void setName(String name) {
    if (immutable) throw new IllegalStateException("Cant modify");
    setShortName(name);
  }

  /**
   * Instances which have same content are equal.
   */
  @Override
  public boolean equals(Object o) {
    if (this == o) return true;
    if ((o == null) || !(o instanceof Attribute)) return false;

    final Attribute att = (Attribute) o;

    String name = getShortName();
    if (!name.equals(att.getShortName())) return false;
    if (nelems != att.nelems) return false;
    if (!dataType.equals(att.dataType)) return false;

    if (isString())
      return att.getStringValue().equals(getStringValue());
    //if (svalue != null) return svalue.equals(att.getStringValue());

    if (values != null) {
      for (int i = 0; i < getLength(); i++) {
        int r1 = isString() ? getStringValue(i).hashCode() : getNumericValue(i).hashCode();
        int r2 = att.isString() ? att.getStringValue(i).hashCode() : att.getNumericValue(i).hashCode();
        if (r1 != r2) return false;
      }
    }

    return true;
  }

  /**
   * Override Object.hashCode() to implement equals.
   */
  @Override
  public int hashCode() {
    int result = 17;
    result = 37 * result + getShortName().hashCode();
    result = 37 * result + nelems;
    result = 37 * result + getDataType().hashCode();
    if (svalue != null)
      result = 37 * result + svalue.hashCode();
    else if (values != null) {
      for (int i = 0; i < getLength(); i++) {
        int h = isString() ? getStringValue(i).hashCode() : getNumericValue(i).hashCode();
        result = 37 * result + h;
      }
    }
    return result;
  }

  @Override
  public void hashCodeShow(Indent indent) {
    System.out.printf("%sAtt hash = %d%n", indent, hashCode());
    System.out.printf("%s shortName '%s' = %d%n", indent, getShortName(), getShortName() == null ? -1 : getShortName().hashCode());
    System.out.printf("%s nelems %s%n", indent, nelems);
    System.out.printf("%s dataType %s%n", indent, getDataType());
    if (svalue != null)
      System.out.printf("%s svalue %s = %s%n", indent, svalue, svalue.hashCode());
    else {
      indent.incr();
      for (int i = 0; i < getLength(); i++) {
        if (isString())
          System.out.printf("%s value %s = %s%n", indent, getStringValue(i), getStringValue(i).hashCode());
        else
          System.out.printf("%s value %s = %s%n", indent, getValue(i), getValue(i).hashCode());
      }
      indent.decr();
    }
  }
}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
// $Id: CollectionLevelScanner.java 63 2006-07-12 21:50:51Z edavis $
package thredds.cataloggen;

import thredds.catalog.*;
import thredds.catalog.util.CatalogUtils;
import thredds.crawlabledataset.*;

import java.util.*;
import java.io.IOException;

/**
 * CollectionLevelScanner maps between the CrawlableDataset realm and the
 * InvCatalog/InvDataset realm. It scans a single level of a dataset collection
 * and generates a catalog. The generated catalog contains InvCatalogRef
 * objects for all contained collection datasets.
 *
 *

 * Three different levels of the dataset collection must be provided to to
 * properly map from CrawlableDataset to InvCatalog/InvDataset:
 *

 *
 the collection level is the top of the data collection (the data root);

 *
 the catalog level is the level in the collection for which a catalog
 * is to be constructed; and

 *
 the current level (only different from catalog level when the resulting
 * single-level catalog will be used in the construction of a
 * multi-level catalog ) is the level in the
 * collection for which a catalog is currently being constructed.

 *

 *
 *
 Besides the three CrawlableDatasets that define the collection to be
 * cataloged, there are a variety of ways to modify or enhance the resulting
 * catalog. For more details, see the documentation for the various setters
 * ({@link #setCollectionId(String) setCollectionId()},
 * {@link #setIdentifier(CrawlableDatasetLabeler) setIdentifier()},
 * {@link #setNamer(CrawlableDatasetLabeler) setNamer()},
 * {@link #setDoAddDataSize(boolean) setDoAddDataSize()},
 * {@link #setSorter(CrawlableDatasetSorter) setSorter()} ,
 * {@link #setProxyDsHandlers(Map) setProxyDsHandlers()},
 * {@link #addChildEnhancer(thredds.cataloggen.DatasetEnhancer)
 * addChildEnhancer()}.
 *
 *
Example

 *
 *
 Here we'll look at the parameters used to construct a
 * CollectionLevelScanner and to generate a catalog for the following
 * request:
 *
 *
http://my.server:8080/thredds/ncep/nam/80km/catalog.xml

 *
 *
In the constuctor, we have:
 *

 *
 collection ID: "ncep";

 *
 collectionLevel.getPath(): "/my/data/collection/model/ncep";

 *
 catalogLevel.getPath(): "/my/data/collection/model/ncep/nam/80km";

 *
 currentLevel = null (so the catalogLevel is used);

 *
 filter and service: not really important for this example, so we'll
 * ignore them for now.

 *

 *
 The two datasets we'll use in the example are:
 *

 *
 childAtomicCrDs.getPath(): "/my/data/collection/model/ncep/nam/80km/20060208_1200_nam80km.grib"

 *
 childCollectionCrDs.getPath(): "/my/data/collection/model/ncep/nam/80km/2000archive"

 *

 *
 *
Following are the details on how the resulting InvDataset and
 * InvCatalogRef objects are created.
 *
 *

 *
 The name of a dataset element (and the xlink:title of a catalogRef
 * element) is the name of the corresponding CrawlableDataset. Example:
 *

 * <dataset name="20060208_1200_nam80km.grib"/>
 * <catalogRef xlink:title="2000archive"/>
 * </pre
 * where the values were determined as follows:
 *

 *
 name = childAtomicCrDs.getName()

 *
 xlink:title = childCollectionCrDs.getName()

 *

 *

 *
 *
 The ID of a catalog dataset element is the ID of the parent dataset
 * and the name of the corresponding CrawlableDataset seperated by a "/".
 * So, it ends up being the path of the corresponding CrawlableDataset from
 * the point where the collection CrawlableDataset path ends then prefixed
 * by the collectionId which is set using the setCollectionId() string. Example:
 *

 * <dataset name="20060208_1200_nam80km.grib" ID="NCEP/nam/80km/20060208_1200_nam80km.grib"/>
 * <catalogRef xlink:title="2000archive" ID="NCEP/nam/80km/2000archive" />
 *

 * where the values were determined as follows:
 *

 *
 ID = collectionId + childAtomicCrDs.getPath().substring( collectionLevel.getPath().length + 1)

 *
 ID = collectionId + childCollectionCrDs.getPath().substring( collectionLevel.getPath().length + 1)

 *

 *

 *
 *
 The urlPath of a dataset element is the collectionPath plus the path of
 * the corresponding CrawlableDataset starting at the point where the
 * collection CrawlableDataset path ends. Example:
 *

 * <dataset name="20060208_1200_nam80km.grib" ID="NCEP/nam/80km/20060208_1200_nam80km.grib"
 * urlPath="ncep/nam/80km/20060208_1200_nam80km.grib" />
 *

 * where the values were determined as follows:
 *

 *
 urlPath = collectionPath + "/" + childAtomicCrDs.getPath().substring( collectionLevel.getPath().length + 1)

 *

 *

 *
 *
 The xlink:href of a catalogRef element is the path of the corresponding
 * CrawlableDataset starting at the point where the catalogLevel
 * CrawlableDataset ends plus "/catalog.xml". Example:
 *
 *

 * <catalogRef xlink:title="2000archive" xlink:href="2000archive/catalog.xml"/>
 *

 * where the values were determined as follows:
 *

 *
 xlink:href = childCollectionCrDs.getPath().substring( catalogLevel.getPath().length() + 1 ) + "/catalog.xml"

 *

 *

 *

 *
 *
See {@link DatasetScanCatalogBuilder} for more details on how a
 * THREDDS server config file (catalog.xml) and the contained datasetScan
 * elements map into CollectionLevelScanner.
 *
 *
Multi-level Catalogs

 *
 *
Resulting single level catalogs can be used to construct multi-level
 * catalogs by replacing InvCatalogRef objects with the catalogs generated for
 * the corresponding CrawlableDataset objects. Construction of multi-level
 * catalogs is supported in several ways:

 *
 *

 *
 The getCatRefInfo() method provides access to the list of InvCatalogRef
 * objects and their corresponding CrawlableDataset objects.

 *
 In the CollectionLevelScanner constructor, the currentLevel parameter
 * indicates the level for which the current single-level catalog is to be
 * created rather than the top-level of the resulting multi-level catalog.

 *

 *
 *
NOTE: The {@link StandardCatalogBuilder} class is an example of using
 * ColletionLevelScanner to construct multi-level catalogs.

 *
 * @author edavis
 * @since Jun 14, 2005T12:41:23 PM
 */
public class CollectionLevelScanner
{
  // @todo Combine DatasetSource and DirectoryScanner into CollectionLevelScanner

  private final String collectionPath;
  private String collectionName;
  private String collectionId = null;

  private final CrawlableDataset collectionLevel;
  private final CrawlableDataset catalogLevel;
  private final CrawlableDataset currentLevel;

  private final InvService service;

  private final CrawlableDatasetFilter filter;

  private CrawlableDatasetSorter sorter;
  private Map proxyDsHandlers;

  private CrawlableDatasetLabeler identifier;
  private CrawlableDatasetLabeler namer;
  private boolean doAddDataSize = false;

  private boolean addLastModified = true;

  private List childEnhancerList;

  private InvDatasetImpl topLevelMetadataContainer;

  //private List allDsInfo;
  private List catRefInfo;
  private List atomicDsInfo;
  private List proxyDsInfo;

  private InvCatalogImpl genCatalog;

  // Allowed states:
  //   0 - constructed, setup OK;
  //   1 - currently scanning collection level;
  //   2 - scan completed and catalog generated;
  private int state;

  /**
   * Construct a CollectionLevelScanner.
   *
   *
 The collectionLevel and catalogLevel parameters are used to properly
   * determine the dataset urlPath. The catalogLevel must either be the
   * collectionLevel or be a decendent of the collectionLevel. The
   * currentLevel, if not null, must either be the catalogLevel or be a
   * decendent of the catalogLevel.

   *

   *
 If the service base is relative to the catalog (i.e., an empty
   * string), the urlPath needs to be relative to the catalog as well.
   * Therefore, the urlPath thus becomes the dataset path minus the
   * catalogLevel path.

   *
 If the service base is relative to the collection (absolute? relative
   * to the server?), e.g., "/thredds/dodsC/", the urlPath needs to be relative
   * to the collection as well. Therefore, the urlPath thus becomes the dataset
   * path minus the collectionLevel path.

   *

   *
 The currentLevel parameter indicates what level is to be scanned. It
   * is the same as the catalogLevel except for the case when catalogRefs are
   * not used for all collection levels. (The urlPath is still determined as
   * described above. Only the location of the datasets is changed.)

   *
   * @param collectionPath the path of the collection, used as the base of all resulting dataset@urlPath values (may be "", if null, "" is used).
   * @param collectionLevel the root of the collection to be cataloged (must not be a CrawlableDatasetAlias).
   * @param catalogLevel the location, within the collection, for which a catalog is being generated.
   * @param currentLevel the location, at or below the catalog level, which is to be scanned for datasets. Only necessary when multiple catalogs are to be aggregated. May be null. If null, assumed to be same as catalog level.
   * @param filter determines which CrawlableDatasets are accepted as part of the collection.
   * @param service the default service of all InvDatasets in the generated catalog.
   *
   * @throws IllegalArgumentException
   */
  public CollectionLevelScanner( String collectionPath,
                                 CrawlableDataset collectionLevel,
                                 CrawlableDataset catalogLevel,
                                 CrawlableDataset currentLevel,
                                 CrawlableDatasetFilter filter,
                                 InvService service )
  {
    this.collectionPath = ( collectionPath != null ? collectionPath : "");
    if ( collectionLevel == null ) throw new IllegalArgumentException( "The collection root must not be null.");
    // @todo Remove alias until sure how to handle things like ".scour*" being a regular file.
//    if ( CrawlableDatasetAlias.isAlias( collectionLevel.getPath() ) ) throw new IllegalArgumentException( "The collection root <" + collectionLevel.getPath() + "> must not be a CrawlableDatasetAlias.");
    if ( catalogLevel == null ) throw new IllegalArgumentException( "The catalog root must not be null.");
    if ( service == null )
      throw new IllegalArgumentException( "The service must not be null.");

    if ( ! catalogLevel.getPath().startsWith( collectionLevel.getPath() ) )
      throw new IllegalArgumentException( "The catalog root <" + catalogLevel.getPath() + "> must be under the collection root <" + collectionLevel.getPath() + ">." );
    if ( ! collectionLevel.isCollection() )
      throw new IllegalArgumentException( "The collection root <" + collectionLevel.getPath() + "> must be a collection." );
    if ( ! catalogLevel.isCollection() )
      throw new IllegalArgumentException( "The catalog root <" + catalogLevel.getPath() + "> must be a collection." );

    this.collectionLevel = collectionLevel;
    this.catalogLevel = catalogLevel;

    if ( currentLevel != null )
    {
      if ( ! currentLevel.isCollection() )
        throw new IllegalArgumentException( "The current root <" + currentLevel.getPath() + "> must be a collection." );
      if ( ! currentLevel.getPath().startsWith( catalogLevel.getPath() ) )
        throw new IllegalArgumentException( "The current root <" + currentLevel.getPath() + "> must be under the catalog root <" + catalogLevel.getPath() + ">." );
      this.currentLevel = currentLevel;
    }
    else
    {
      this.currentLevel = catalogLevel;
    }

    this.filter = filter;         // @todo Thread safe: copy instead of reference
    this.service = service;       // @todo Thread safe: copy instead of reference

    childEnhancerList = new ArrayList();

    // @todo Maybe postpone creation of list till know approx size
    catRefInfo = new ArrayList();
    atomicDsInfo = new ArrayList();
    proxyDsInfo = new ArrayList();

    collectionId = null;
    collectionName = null;

    state = 0;
  }

//  public CollectionLevelScanner( CrawlableDataset accessPath, CrawlableDataset accessPathHeader,
//                            CrawlableDatasetFilter filter, InvService service )
//  {
//    if ( accessPath == null ) throw new IllegalArgumentException( "The access path must not be null." );
//    if ( accessPathHeader == null ) throw new IllegalArgumentException( "The access path header must not be null." );
//    if ( service == null ) throw new IllegalArgumentException( "The service must not be null." );
//
//    if ( ! accessPath.getPath().startsWith( accessPathHeader.getPath() ) )
//      throw new IllegalArgumentException( "The accessPath <" + accessPath.getPath() + "> must start with the accessPathHeader <" + accessPathHeader.getPath() + ">." );
//    if ( ! accessPathHeader.isCollection() )
//      throw new IllegalArgumentException( "The accessPathHeader <" + accessPathHeader.getPath() + "> must be a collection." );
//    if ( ! accessPath.isCollection() )
//      throw new IllegalArgumentException( "The accessPath <" + accessPath.getPath() + "> must be a collection." );
//
//    this.collectionPath = ""; //accessPath.getPath().substring( accessPathHeader.getPath().length() );
//    this.collectionLevel = accessPathHeader;
//    this.catalogLevel = accessPath;
//
//    this.filter = matcher;         // @todo Thread safe: copy instead of reference
//    this.service = service;       // @todo Thread safe: copy instead of reference
//  }

  /** Copy constructor */
  public CollectionLevelScanner( CollectionLevelScanner cs )
  {
    this( cs.collectionPath, cs.collectionLevel, cs.catalogLevel, cs.currentLevel, cs.filter, cs.service );

    this.setSorter( cs.getSorter() );
    this.setProxyDsHandlers( cs.getProxyDsHandlers() );
    this.setCollectionId( cs.getCollectionId() );
    this.setCollectionName( cs.getCollectionName() );
    this.setIdentifier( cs.getIdentifier() );
    this.setNamer( cs.getNamer() );
    this.setDoAddDataSize( cs.getDoAddDataSize() );

    this.childEnhancerList.add( cs.getChildEnhancerList() );
  }

  public CrawlableDatasetSorter getSorter()
  {
    return sorter;
  }

  /**
   * Set the sorter with which to sort the list of child CrawlableDatasets.
   *
   * @param sorter the CrawlableDatasetSorter that will be used to sort the list of child CrawlableDatasets.
   */
  public void setSorter( CrawlableDatasetSorter sorter )
  {
    if ( state != 0 ) throw new IllegalStateException( "No further setup allowed, scan in progress or completed." );
    this.sorter = sorter;
  }

  public Map getProxyDsHandlers()
  {
    if ( proxyDsHandlers.isEmpty() )
      return Collections.EMPTY_MAP;
    else
      return Collections.unmodifiableMap( proxyDsHandlers );
  }
  public void setProxyDsHandlers( Map proxyDsHandlers )
  {
    if ( state != 0 ) throw new IllegalStateException( "No further setup allowed, scan in progress or completed." );
    if ( proxyDsHandlers == null )
      this.proxyDsHandlers = Collections.EMPTY_MAP;
    else
      this.proxyDsHandlers = new HashMap( proxyDsHandlers);
  }

  /**
   * Set the value of the base dataset ID. The value is used to construct
   * the value of the dataset@ID attribute for all datasets.
   *
   * @param collectionId
   */
  public void setCollectionId( String collectionId )
  {
    if ( state != 0 ) throw new IllegalStateException( "No further setup allowed, scan in progress or completed.");
    this.collectionId = collectionId;
  }

  protected String getCollectionId()
  {
    return( this.collectionId );
  }

  /**
   * Set the value of the collection Name. The value is used to name the
   * top-level dataset in the top-level collection catalog (that is, only when
   * the catalog level is the same as the collection level).
   *
   * @param collectionName
   */
  public void setCollectionName( String collectionName )
  {
    if ( state != 0 ) throw new IllegalStateException( "No further setup allowed, scan in progress or completed.");
    this.collectionName = collectionName;
  }

  protected String getCollectionName()
  {
    return( this.collectionName );
  }

  /**
   * Set the CrawlableDatasetLabeler used to determine the ID of the InvDataset
   * built during catalog generation. The labeler is applied to the
   * CrawlableDataset that corresponds to each InvDataset built.
   *
   * @param identifier
   */
  public void setIdentifier( CrawlableDatasetLabeler identifier )
  {
    if ( state != 0 ) throw new IllegalStateException( "No further setup allowed, scan in progress or completed." );
    this.identifier = identifier;
  }

  protected CrawlableDatasetLabeler getIdentifier()
  {
    return ( this.identifier );
  }

  /**
   * Set the CrawlableDatasetLabeler used to determine the name of each
   * InvDataset built during catalog generation. The labeler is applied to the
   * CrawlableDataset that corresponds to each InvDataset built.
   *
   * @param namer
   */
  public void setNamer( CrawlableDatasetLabeler namer )
  {
    if ( state != 0 ) throw new IllegalStateException( "No further setup allowed, scan in progress or completed." );
    this.namer = namer;
  }

  protected CrawlableDatasetLabeler getNamer()
  {
    return( this.namer );
  }

  /**
   * Determines if datasetSize metadata will be added to each InvDataset built
   * during catalog generation. The CrawlableDataset.length() method is used to
   * determine the size of the dataset.
   *
   * @param doAddDataSize
   */
  public void setDoAddDataSize( boolean doAddDataSize )
  {
    if ( state != 0 ) throw new IllegalStateException( "No further setup allowed, scan in progress or completed." );
    this.doAddDataSize = doAddDataSize;
  }

  protected boolean getDoAddDataSize()
  {
    return ( this.doAddDataSize );
  }

  /**
   * Add the given DatasetEnhancer to the list that will be applied to each of
   * the child datasets. The DatasetEnhancer only modify InvDataset objects but
   * can use the corresponding CrawlableDataset for information.
   *
   * @param childEnhancer
   */
  public void addChildEnhancer( DatasetEnhancer childEnhancer )
  {
    if ( state != 0 ) throw new IllegalStateException( "No further setup allowed, scan in progress or completed." );
    this.childEnhancerList.add( childEnhancer );
  }

  List getChildEnhancerList()
  {
    return Collections.unmodifiableList( childEnhancerList );
  }

  /**
   * Set the InvDatasetImpl that contains the metadata for the top level dataset.
   *
   * @param topLevelMetadataContainer
   */
  public void setTopLevelMetadataContainer( InvDatasetImpl topLevelMetadataContainer )
  {
    this.topLevelMetadataContainer = topLevelMetadataContainer;
  }

  /**
   * Scan the collection and gather information on contained datasets.
   *
   * @throws IOException if an I/O error occurs while locating the contained datasets.
   */
  public void scan() throws IOException
  {
    if ( state == 1 ) throw new IllegalStateException( "Scan already underway." );
    if ( state >= 2 ) throw new IllegalStateException( "Scan has already been generated." );
    state = 1;

    // Make sure proxyDsHandlers Map is not null.
    if ( proxyDsHandlers == null ) proxyDsHandlers = Collections.EMPTY_MAP;

    // Create a skeleton catalog.
    genCatalog = createSkeletonCatalog( currentLevel );
    InvDatasetImpl topInvDs = (InvDatasetImpl) genCatalog.getDatasets().get( 0 );

    // Get the datasets in this collection.
    List crDsList = currentLevel.listDatasets( this.filter );

    // Sort the datasets in this collection.
    // @todo Should we move sort to end of this method? As is, we can't use naming or enhancements to determine sort order.
    if ( sorter != null ) sorter.sort( crDsList );

    // Add the datasets to the catalog.
    for ( int i = 0; i < crDsList.size(); i++ )
    //for ( Iterator it = crDsList.iterator(); it.hasNext(); )
    {
      CrawlableDataset curCrDs = (CrawlableDataset) crDsList.get( i );

      InvDatasetImpl curInvDs = (InvDatasetImpl) createInvDatasetFromCrawlableDataset( curCrDs, topInvDs, null );

      // Add dataset info to appropriate lists.
      InvCrawlablePair dsInfo = new InvCrawlablePair( curCrDs, curInvDs );
      //allDsInfo.add( dsInfo );
      if ( curCrDs.isCollection() )
        catRefInfo.add( dsInfo );
      else
        atomicDsInfo.add( dsInfo );

      // Add current InvDataset to top dataset.
      topInvDs.addDataset( curInvDs );
    }

    // Tie up any loose ends in catalog with finish().
    ( (InvCatalogImpl) genCatalog ).finish();

    // Add proxy datasets to list (only if some atomic datasets in this collection).
    if ( atomicDsInfo.size() > 0 )
    {
      boolean anyProxiesAdded = false;
      for ( Iterator it = proxyDsHandlers.values().iterator(); it.hasNext(); )
      {
        // Get current
        ProxyDatasetHandler curProxy = (ProxyDatasetHandler) it.next();

        InvService proxyService = curProxy.getProxyDatasetService( currentLevel );
        if ( proxyService != null )
        {
          // Create proxy CrawlableDataset and corresponding InvDataset
          CrawlableDataset crDsToAdd = curProxy.createProxyDataset( currentLevel );
          InvDatasetImpl invDsToAdd = createInvDatasetFromCrawlableDataset( crDsToAdd, topInvDs, proxyService );

          // Add dataset info to appropriate lists.
          InvCrawlablePair dsInfo = new InvCrawlablePair( crDsToAdd, invDsToAdd );
          proxyDsInfo.add( dsInfo );

          // Add dataset to catalog
          int index = curProxy.getProxyDatasetLocation( currentLevel, topInvDs.getDatasets().size() );
          topInvDs.addDataset( index, (InvDatasetImpl) invDsToAdd );

          genCatalog.addService( proxyService );
          anyProxiesAdded = true;
        }
      }

      // Tie up any proxy dataset loose ends.
      if ( anyProxiesAdded ) ( (InvCatalogImpl) genCatalog ).finish();
    }

    // Add any top-level metadata.
    this.addTopLevelMetadata( genCatalog, true );

    state = 2;
    return;
  }

//  List getAllDsInfo()
//  {
//    if ( state != 2 ) throw new IllegalStateException( "Scan has not been performed." );
//    return Collections.unmodifiableList( allDsInfo );
//  }

  /**
   * Return a list of all the collection dataset objects (InvCatalogRef) and
   * their corresponding CrawlableDataset objects. Each item in the list is
   * an InvCrawlablePair.
   *
   * @return a list of InvCrawlablePairs
   */
  List getCatRefInfo()
  {
    if ( state != 2 ) throw new IllegalStateException( "Scan has not been performed." );
    return Collections.unmodifiableList( catRefInfo );
  }

  /**
   * Return a list of all the atomic dataset objects (InvDataset) and
   * their corresponding CrawlableDataset objects. Each item in the list is
   * an InvCrawlablePair.
   *
   * @return a list of InvCrawlablePairs
   */
  List getAtomicDsInfo()
  {
    if ( state != 2 ) throw new IllegalStateException( "Scan has not been performed." );
    return Collections.unmodifiableList( atomicDsInfo );
  }

  public InvCatalogImpl generateCatalog() throws IOException
  {
    if ( state != 2 ) throw new IllegalStateException( "Scan has not been performed.");
    return genCatalog;
  }

  /**
   * Generate the catalog for a resolver request of the given ProxyDatasetHandler.
   *
   * @param pdh the ProxyDatasetHandler corresponding to the resolver request.
   * @return the catalog for a resolver request of the given proxy dataset.
   * @throws IllegalStateException if this collection has not yet been scanned.
   * @throws IllegalArgumentException if the given ProxyDatasetHandler is not known by this CollectionLevelScanner.
   */
  public InvCatalogImpl generateProxyDsResolverCatalog( ProxyDatasetHandler pdh )
  {
    if ( state != 2 ) throw new IllegalStateException( "Scan has not been performed." );
    if ( ! proxyDsHandlers.containsValue( pdh )) throw new IllegalArgumentException( "Unknown ProxyDatasetHandler.");

    // Create a skeleton catalog.
    InvCatalogImpl catalog = createSkeletonCatalog( currentLevel );
    InvDatasetImpl topDs = (InvDatasetImpl) catalog.getDatasets().get( 0 );

    // Find actual dataset in the list of atomic dataset InvCrawlablePairs.
    InvCrawlablePair actualDsInfo = pdh.getActualDataset( atomicDsInfo );
    if ( actualDsInfo == null )
      return catalog; // TODO Test this case in TestDataRootHandler.
    InvDatasetImpl actualInvDs = (InvDatasetImpl) actualDsInfo.getInvDataset();

    actualInvDs.setName( pdh.getActualDatasetName( actualDsInfo, topDs.getName() ) );

    // Add current InvDataset to top dataset.
    catalog.removeDataset( topDs );
    catalog.addDataset( actualInvDs );
    // topDs.addDataset( actualInvDs );

    // Finish catalog.
    catalog.finish();

    // Add any top-level metadata.
    this.addTopLevelMetadata( catalog, false );

    return catalog;
  }

  private void addTopLevelMetadata( InvCatalog catalog, boolean isRegularCatalog )
  {
    if ( this.topLevelMetadataContainer == null )
      return;
    if ( ! catalogLevel.getPath().equals( currentLevel.getPath() ) )
      return;

    // Transfer all metadata from given metadata container to the top
    // InvDataset. This propagates any public metadata from the given
    // metadata container to all generated catalogs.
    InvDatasetImpl topInvDs = (InvDatasetImpl) catalog.getDataset();

    topInvDs.transferMetadata( this.topLevelMetadataContainer, true );

    // LOOK experimental datasetScan may have its own access elements
    for ( Iterator it = this.topLevelMetadataContainer.getAccess().iterator(); it.hasNext(); )
    {
      InvAccess invAccess = (InvAccess) it.next();
      topInvDs.addAccess( invAccess );
      InvService s = invAccess.getService();
      ( (InvCatalogImpl) catalog ).addService( s );
    }

    // If this is a collection level scan, set some special attributes
    // that transferMetadata() doesn't transfer.
    boolean isCollectionLevel = catalogLevel.getPath().equals( collectionLevel.getPath() );
    if ( isCollectionLevel && isRegularCatalog )
    {
      topInvDs.setHarvest( this.topLevelMetadataContainer.isHarvest() );
      topInvDs.setCollectionType( this.topLevelMetadataContainer.getCollectionType() );
    }

    // Finish catalog.
    ( (InvCatalogImpl) catalog ).finish();

  }

  private InvCatalogImpl createSkeletonCatalog( CrawlableDataset topCrDs )
  {
    InvCatalogImpl catalog = new InvCatalogImpl( null, null, null );

//    // Copy service to add to catalog.
//    InvService service = new InvService( this.service.getName(), this.service.getServiceType().toString(), this.service.getBase(), this.service.getSuffix(), this.service.getDescription() );
//    for ( Iterator it = this.service.getProperties().iterator(); it.hasNext(); )
//      service.addProperty( (InvProperty) it.next() );
//    for ( Iterator it = this.service.getServices().iterator(); it.hasNext(); )
//      service.addService( (InvService) it.next() );

    // Add service to catalog.
    catalog.addService( service );

    // Determine name of top-level dataset.
    String newName = null;
    if ( namer != null )
      newName = this.namer.getLabel( topCrDs );
    if ( newName == null )
      // If generating the collection level catalog, use the collection name.
      if ( collectionLevel.getPath().equals( catalogLevel.getPath() ) )
        newName = this.collectionName;
    if ( newName == null )
      newName = this.getName( topCrDs );

    // Create top-level dataset.
    InvDatasetImpl topDs = new InvDatasetImpl( null, newName, null, null, null );

    // Set ID of dataset.
    String newId = null;
    if ( identifier != null )
      newId = this.identifier.getLabel( topCrDs );
    else
      newId = this.getID( topCrDs );
    if ( newId != null )
      topDs.setID( newId );

    // Set the serviceName (inherited by all datasets) in top-level dataset.
    ThreddsMetadata tm = new ThreddsMetadata( false );
    tm.setServiceName( service.getName() );
    InvMetadata md = new InvMetadata( topDs, null, XMLEntityResolver.CATALOG_NAMESPACE_10, "", true, true, null, tm );
    ThreddsMetadata tm2 = new ThreddsMetadata( false );
    tm2.addMetadata( md );
    topDs.setLocalMetadata( tm2 );

    // Add top-level dataset to catalog.
    catalog.addDataset( topDs );

    return catalog;
  }

  private InvDatasetImpl createInvDatasetFromCrawlableDataset( CrawlableDataset crawlableDs, InvDatasetImpl parentInvDs, InvService service )
  {
    // Determine name of dataset.
    String newName = null;
    if ( namer != null )
      newName = this.namer.getLabel( crawlableDs );
    if ( newName == null )
      newName = this.getName( crawlableDs );

    InvDatasetImpl curInvDs;
    if ( crawlableDs.isCollection() )
    {
      // Collection dataset, create a catalogRef to be added to the catalog.
      curInvDs = new InvCatalogRef( parentInvDs, newName, this.getXlinkHref( crawlableDs ) );
    }
    else
    {
      // Atomic dataset, add a dataset to the catalog.
      curInvDs = new InvDatasetImpl( parentInvDs, newName, null,
                                     ( service != null ? service.getName() : null ),
                                     this.getUrlPath( crawlableDs, service ) );

      // Set dataSize of dataset (not applicable for collection datasets).
      if ( this.doAddDataSize && crawlableDs.length() != -1 )
      {
        curInvDs.setDataSize( crawlableDs.length() );
      }

      if ( this.addLastModified )
      {
        Date lastModDate = crawlableDs.lastModified();
        if ( lastModDate != null )
          curInvDs.setLastModifiedDate( lastModDate );
      }
    }

    // Set ID of dataset.
    String newId = null;
    if ( identifier != null )
      newId = this.identifier.getLabel( crawlableDs );
    else
      newId = this.getID( crawlableDs );
    if ( newId != null )
      curInvDs.setID( newId );

    // Enhance each child dataset.
    for ( Iterator it2 = childEnhancerList.iterator(); it2.hasNext(); )
    {
      ( (DatasetEnhancer) it2.next() ).addMetadata( curInvDs, crawlableDs );
    }

    return curInvDs;
  }

  // dataset name and xlink:title=
  //   - node.getName()
  private String getName( CrawlableDataset dataset )
  {
    // @todo Remove alias until sure how to handle things like ".scour*" being a regular file.
//    if ( ! CrawlableDatasetAlias.isAlias( currentLevel.getPath() ) )
//    {
      return ( dataset.getName() );
//    }
//    else
//    {
//      // @todo Move this functionality into CrawlableDatasetAlias
//      String path = dataset.getPath();
//      int curIndex = path.length();
//      for ( int i = 0; i < currentLevel.getName().split( "/" ).length; i++ )
//      {
//        curIndex = dataset.getPath().lastIndexOf( "/", curIndex - 1 );
//      }
//      return ( dataset.getPath().substring( curIndex + 1 ) );
//    }
  }

  // dataset ID=
  //   - parentId + "/" + dataset.getName() [when ID added after catalog built using dataset info]; or
  //   - baseID + "/" + node.getPath().substring( collectionLevel.length) [when ID added at catalog construction using node info]
  private String getID( CrawlableDataset dataset )
  {
    if ( dataset == null )
      return null;
    if ( collectionId == null )
      return null;

    int i = collectionLevel.getPath().length();
    String id = dataset.getPath().substring( i );
    if ( id.startsWith( "/" ))
      id = id.substring( 1 );
    if ( collectionId.equals( "" ) )
    {
      if ( id.equals( ""))
        return null;
      return id;
    }
    if ( id.equals( "") )
      return collectionId;

    return( collectionId + "/" + id );
  }

  // dataset urlPath=
  //   - node.getPath().substring( catalogLevel.length) [when service is relative to catalog URL]
  //   - collectionPath + "/" + node.getPath().substring( collectionLevel.length) [when service is absolute (i.e., relative to collection)]
  private String getUrlPath( CrawlableDataset dataset, InvService service )
  {
    InvService serviceForThisDs = service != null ? service : this.service;

    if ( serviceForThisDs.getBase().equals( "" )
         && ! serviceForThisDs.getServiceType().equals( ServiceType.COMPOUND ) )
    {
      // Service is relative to the catalog URL.
      String urlPath = dataset.getPath().substring( catalogLevel.getPath().length() );
      if ( urlPath.startsWith( "/" ))
        urlPath = urlPath.substring( 1 );
      return urlPath;
    }
    else
    {
      if ( serviceForThisDs.isRelativeBase() )
      {
        // Service is relative to the collection root.
        String relPath = dataset.getPath().substring( collectionLevel.getPath().length() );
        if ( relPath.startsWith( "/" ) )
          relPath = relPath.substring( 1 );
        return ( ( collectionPath.equals( "" ) ? "" : collectionPath + "/" ) + relPath );
      }
      else
      {
        // Service base URI is Absolute so don't use collectionPath.
        String relPath = dataset.getPath().substring( collectionLevel.getPath().length() );
        if ( relPath.startsWith( "/" ) )
          relPath = relPath.substring( 1 );
        return relPath;
      }
    }
  }

  // catalogRef href=
  //   - node.getPath().substring( catalogLevel.length) + "/catalog.xml"
  private String getXlinkHref( CrawlableDataset dataset )
  {
    // @todo Remove alias until sure how to handle things like ".scour*" being a regular file.
//    if ( ! CrawlableDatasetAlias.isAlias( catalogLevel.getPath()) )
//    {
    String path = dataset.getPath().substring( catalogLevel.getPath().length() );
    if ( path.startsWith( "/" ))
      path = path.substring( 1);
    if ( path.endsWith( "/"))
      path += "catalog.xml";
    else
      path += "/catalog.xml";
    return CatalogUtils.escapePathForURL(path);
//    }
//    else
//    {
//      // @todo Move this functionality into CrawlableDatasetAlias
//      String path = dataset.getPath();
//      int curIndex = path.length();
//      int numSegments = catalogLevel.getName().split( "/" ).length;
//      for ( int i = 0 ; i < numSegments; i++ )
//      {
//        curIndex = dataset.getPath().lastIndexOf( "/", curIndex - 1);
//      }
//      return( dataset.getPath().substring( curIndex + 1 ) + "/catalog.xml");
//    }
  }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.ft.point.remote;

import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.List;
import javax.annotation.Nonnull;
import com.google.protobuf.ByteString;
import com.google.protobuf.InvalidProtocolBufferException;
import ucar.ma2.ArrayStructureBB;
import ucar.ma2.DataType;
import ucar.ma2.StructureData;
import ucar.ma2.StructureDataDeep;
import ucar.ma2.StructureMembers;
import ucar.nc2.ft.DsgFeatureCollection;
import ucar.nc2.ft.PointFeature;
import ucar.nc2.ft.PointFeatureCollection;
import ucar.nc2.ft.PointFeatureIterator;
import ucar.nc2.ft.point.PointFeatureImpl;
import ucar.nc2.stream.NcStream;
import ucar.nc2.stream.NcStreamProto;
import ucar.nc2.time.CalendarDateUnit;
import ucar.unidata.geoloc.EarthLocation;
import ucar.unidata.geoloc.EarthLocationImpl;
import ucar.unidata.geoloc.Station;

/**
 * Defines the point stream format, along with pointStream.proto.
 *
 cd c:/dev/github/thredds/cdm/src/main/java
 protoc --proto_path=. --java_out=. ucar/nc2/ft/point/remote/pointStream.proto
 *
 * @author caron
 * @since Feb 16, 2009
 */
public class PointStream {
  public enum MessageType {
    Start, Header, Data, End, Error, Eos,
    StationList, PointFeatureCollection, PointFeature
  }

  static private final byte[] MAGIC_StationList = new byte[]{(byte) 0xfe, (byte) 0xfe, (byte) 0xef, (byte) 0xef};
  static private final byte[] MAGIC_PointFeatureCollection = new byte[]{(byte) 0xfa, (byte) 0xfa, (byte) 0xaf, (byte) 0xaf};
  static private final byte[] MAGIC_PointFeature = new byte[]{(byte) 0xf0, (byte) 0xf0, (byte) 0x0f, (byte) 0x0f};

  static private final boolean debug = false;

  static public MessageType readMagic(InputStream is) throws IOException {
    byte[] b = new byte[4];
    int done = NcStream.readFully(is, b);
    if (done != 4) return MessageType.Eos;

    if (test(b, MAGIC_PointFeature)) return MessageType.PointFeature;
    if (test(b, MAGIC_PointFeatureCollection)) return MessageType.PointFeatureCollection;
    if (test(b, MAGIC_StationList)) return MessageType.StationList;
    if (test(b, NcStream.MAGIC_START)) return MessageType.Start;
    if (test(b, NcStream.MAGIC_HEADER)) return MessageType.Header;
    if (test(b, NcStream.MAGIC_DATA)) return MessageType.Data;
    if (test(b, NcStream.MAGIC_END)) return MessageType.End;
    if (test(b, NcStream.MAGIC_ERR)) return MessageType.Error;
    return null;
  }

  static public int writeMagic(OutputStream out, MessageType type) throws IOException {
    switch (type) {
      case PointFeature:
        return NcStream.writeBytes(out, PointStream.MAGIC_PointFeature);
      case PointFeatureCollection:
        return NcStream.writeBytes(out, PointStream.MAGIC_PointFeatureCollection);
      case StationList:
        return NcStream.writeBytes(out, PointStream.MAGIC_StationList);
      case Start:
        return NcStream.writeBytes(out, NcStream.MAGIC_START);
      case End:
         return NcStream.writeBytes(out, NcStream.MAGIC_END);
      case Error:
        return NcStream.writeBytes(out, NcStream.MAGIC_ERR);
    }
    return 0;
  }

  private static boolean test(byte[] b, byte[] m) {
    if (b.length != m.length) return false;
    for (int i = 0; i < b.length; i++)
      if (b[i] != m[i]) return false;
    return true;
  }

  static public PointStreamProto.PointFeatureCollection encodePointFeatureCollection(
          String name, String timeUnitString, String altUnits, PointFeature pf) throws IOException {
    PointStreamProto.PointFeatureCollection.Builder builder = PointStreamProto.PointFeatureCollection.newBuilder();
    if (name == null)
      System.out.printf("HEY null pointstream name%n");
    builder.setName(name);
    builder.setTimeUnit(timeUnitString);

    if (altUnits != null) {
      builder.setAltUnit(altUnits);
    }

    StructureData sdata = pf.getDataAll();
    StructureMembers sm = sdata.getStructureMembers();
    for (StructureMembers.Member m : sm.getMembers()) {
      PointStreamProto.Member.Builder mbuilder = PointStreamProto.Member.newBuilder();
      mbuilder.setName(m.getName());
      if (null != m.getDescription())
        mbuilder.setDesc(m.getDescription());
      if (null != m.getUnitsString())
        mbuilder.setUnits(m.getUnitsString());
      mbuilder.setDataType(NcStream.convertDataType(m.getDataType()));
      mbuilder.setSection(NcStream.encodeSection(new ucar.ma2.Section(m.getShape())));
      builder.addMembers(mbuilder);
    }

    return builder.build();
  }

  static public PointStreamProto.PointFeature encodePointFeature(PointFeature pf) throws IOException {
    PointStreamProto.Location.Builder locBuilder = PointStreamProto.Location.newBuilder();
    locBuilder.setTime(pf.getObservationTime());
    locBuilder.setNomTime(pf.getNominalTime());

    EarthLocation loc = pf.getLocation();
    locBuilder.setLat(loc.getLatitude());
    locBuilder.setLon(loc.getLongitude());
    locBuilder.setAlt(loc.getAltitude());

    PointStreamProto.PointFeature.Builder builder = PointStreamProto.PointFeature.newBuilder();
    builder.setLoc(locBuilder);

    StructureData sdata = pf.getDataAll();
    ArrayStructureBB abb = StructureDataDeep.copyToArrayBB(sdata);
    ByteBuffer bb = abb.getByteBuffer();
    if (debug) {
      StructureMembers sm = sdata.getStructureMembers();
      int size = sm.getStructureSize();
      System.out.printf("encodePointFeature size= %d bb=%d%n", size, bb.position());
    }
    builder.setData(ByteString.copyFrom(bb.array()));
    List<Object> heap = abb.getHeap();
    if (heap != null) {
      for (Object ho : heap) {
        if (ho instanceof String)
          builder.addSdata((String) ho);
        else if (ho instanceof String[])
          builder.addAllSdata(Arrays.asList((String[]) ho));
        else
          throw new IllegalStateException("illegal object on heap = "+ho);
      }
    }
    return builder.build();
  }

  static public PointStreamProto.StationList encodeStations(List<Station> stnList) throws IOException {
    PointStreamProto.StationList.Builder stnBuilder = PointStreamProto.StationList.newBuilder();
    for (Station loc : stnList) {
      PointStreamProto.Station.Builder locBuilder = PointStreamProto.Station.newBuilder();

      locBuilder.setId(loc.getName());
      locBuilder.setLat(loc.getLatitude());
      locBuilder.setLon(loc.getLongitude());
      locBuilder.setAlt(loc.getAltitude());
      if (loc.getDescription() != null)
        locBuilder.setDesc(loc.getDescription());
      if (loc.getWmoId() != null)
        locBuilder.setWmoId(loc.getWmoId());

      stnBuilder.addStations(locBuilder);
    }

    return stnBuilder.build();
  }

  //////////////////////////////////////////////////////////////////
  // decoding
  // makes a PointFeature from the raw bytes of the protobuf message

  static class ProtobufPointFeatureMaker implements FeatureMaker {
    private CalendarDateUnit dateUnit;
    private StructureMembers sm;

    ProtobufPointFeatureMaker(PointStreamProto.PointFeatureCollection pfc) throws IOException {
      try {
        // LOOK No calendar
        dateUnit = CalendarDateUnit.of(null, pfc.getTimeUnit());
      } catch (Exception e) {
        e.printStackTrace();
        dateUnit = CalendarDateUnit.unixDateUnit;
      }

      sm = new StructureMembers(pfc.getName());
      for (PointStreamProto.Member m : pfc.getMembersList()) {
        String name = m.getName();
        String desc = m.getDesc().length() > 0 ? m.getDesc() : null;
        String units = m.getUnits().length() > 0 ? m.getUnits() : null;
        DataType dtype = NcStream.convertDataType(m.getDataType());
        int[] shape = NcStream.decodeSection(m.getSection()).getShape();

        sm.addMember(name, desc, units, dtype, shape);
      }
      ArrayStructureBB.setOffsets(sm);
    }

    @Override
    public PointFeature make(DsgFeatureCollection dsg, byte[] rawBytes) throws InvalidProtocolBufferException {
      PointStreamProto.PointFeature pfp = PointStreamProto.PointFeature.parseFrom(rawBytes);
      PointStreamProto.Location locp = pfp.getLoc();
      EarthLocationImpl location = new EarthLocationImpl(locp.getLat(), locp.getLon(), locp.getAlt());
      return new MyPointFeature(dsg, location, locp.getTime(), locp.getNomTime(), dateUnit, pfp);
    }

    private class MyPointFeature extends PointFeatureImpl {
      PointStreamProto.PointFeature pfp;

      MyPointFeature(DsgFeatureCollection dsg, EarthLocation location, double obsTime, double nomTime, CalendarDateUnit timeUnit, PointStreamProto.PointFeature pfp) {
        super(dsg, location, obsTime, nomTime, timeUnit);
        this.pfp = pfp;
      }

      @Nonnull
      @Override
      public StructureData getFeatureData() throws IOException {
        ByteBuffer bb = ByteBuffer.wrap(pfp.getData().toByteArray());
        ArrayStructureBB asbb = new ArrayStructureBB(sm, new int[]{1}, bb, 0);
        for (String s : pfp.getSdataList()) {
          asbb.addObjectToHeap(s);
        }
        return asbb.getStructureData(0);
      }

      @Nonnull
      @Override
      public StructureData getDataAll() throws IOException {
        return getFeatureData();
      }

      public String toString() {
        return location + " obs=" + obsTime + " nom=" + nomTime;
      }
    }
  }

  public static int write(PointFeatureCollection pointFeatCol, File outFile) throws IOException {
    String name = outFile.getCanonicalPath();
    String timeUnitString = pointFeatCol.getTimeUnit().getUdUnit();
    String altUnits = pointFeatCol.getAltUnits();
    PointFeatureIterator pointFeatIter = pointFeatCol.getPointFeatureIterator();

    try (BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(outFile))) {
      return write(out, pointFeatIter, name, timeUnitString, altUnits);
    } finally {
      pointFeatIter.close();
    }
  }

  // Adapted from thredds.server.cdmremote.PointWriter.WriterNcstream
  // Caller must iter.finish() and out.close().
  public static int write(OutputStream out, PointFeatureIterator pointFeatIter, String name, String timeUnitString,
          String altUnits) throws IOException {
    int numWritten = 0;

    while (pointFeatIter.hasNext()) {
      try {
        PointFeature pointFeat = pointFeatIter.next();

        if (numWritten == 0) {
          PointStreamProto.PointFeatureCollection protoPfc =
                  PointStream.encodePointFeatureCollection(name, timeUnitString, altUnits, pointFeat);
          byte[] data = protoPfc.toByteArray();

          PointStream.writeMagic(out, MessageType.PointFeatureCollection);
          NcStream.writeVInt(out, data.length);
          out.write(data);
        }

        PointStreamProto.PointFeature protoPointFeat = PointStream.encodePointFeature(pointFeat);
        byte[] data = protoPointFeat.toByteArray();

        PointStream.writeMagic(out, MessageType.PointFeature);
        NcStream.writeVInt(out, data.length);
        out.write(data);

        ++numWritten;
      } catch (Throwable t) {
        NcStreamProto.Error protoError =
                NcStream.encodeErrorMessage(t.getMessage() != null ? t.getMessage() : t.getClass().getName());
        byte[] data = protoError.toByteArray();

        PointStream.writeMagic(out, PointStream.MessageType.Error);
        NcStream.writeVInt(out, data.length);
        out.write(data);

        throw new IOException(t);
      }
    }

    PointStream.writeMagic(out, PointStream.MessageType.End);

    return numWritten;
  }
}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package thredds.ldm;

import ucar.nc2.iosp.bufr.Message;

import java.nio.channels.WritableByteChannel;
import java.nio.ByteBuffer;
import java.io.FileOutputStream;
import java.io.FileNotFoundException;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;

/**
 * Encapsolates writing BUFR message to one particular file.
 *
 * @author caron
 * @since Aug 19, 2008
 */
public class MessageWriter implements Callable<IndexerTask> {
  //private static String rootDir = "./";
  //static void setRootDir( String _rootDir) {
  //  rootDir = _rootDir;
  //}

  private final ConcurrentLinkedQueue<Message> q; // unbounded, threadsafe
  private final WritableByteChannel wbc;
  private final FileOutputStream fos;

  private final CompletionService<IndexerTask> executor;
  private final Indexer indexer;
  private final short fileno;

  private final AtomicBoolean isScheduled = new AtomicBoolean(false);
  private long lastModified;
  private long filePos = 0;

  //MessageWriter(CompletionService<IndexerTask> executor, Indexer indexer, short fileno, String fileout, Calendar mcal) throws FileNotFoundException {
  MessageWriter(CompletionService<IndexerTask> executor, Indexer indexer, File file, short fileno) throws FileNotFoundException {
    this.executor = executor;
    this.indexer = indexer;
    this.fileno = fileno;
    /* File dir = new File(rootDir + fileout);
    dir.mkdirs();

    String date = mcal.get( Calendar.YEAR)+"-"+(1+mcal.get( Calendar.MONTH))+"-"+mcal.get( Calendar.DAY_OF_MONTH);

    File file = new File(rootDir + fileout + "/"+fileout+"-"+date+  ".bufr");  */
    if (file.exists())
      filePos = file.length();

    fos = new FileOutputStream(file, true); // append
    wbc = fos.getChannel();
    q = new ConcurrentLinkedQueue<Message>();
  }

  // last time the file was written to
  public long getLastModified() { return lastModified; }

  // put a message on the queue, schedule writing if not already scheduled.
  void scheduleWrite(Message m) {
    q.add(m);
    if (!isScheduled.getAndSet(true)) {
      executor.submit( this);
    }
  }

  void close() throws IOException {
    if (wbc != null)
      wbc.close();
    if (fos != null)
      fos.close();
  }

  public IndexerTask call() throws IOException {
    IndexerTask task = new IndexerTask();
    task.indexer = this.indexer;
    if (this.indexer != null) {
      task.mlist = new ArrayList<Message>(10);
      task.fileno = this.fileno;
    }

    while (!q.isEmpty()) {
      Message m = q.remove();
      wbc.write(ByteBuffer.wrap(m.getHeader().getBytes()));
      wbc.write(ByteBuffer.wrap(m.getRawBytes()));

      if (this.indexer != null) { // track info we need to write the index
        m.setStartPos( filePos);
        filePos += m.getHeader().length() + m.getRawBytes().length;
        task.mlist.add(m);
      }
    }
    lastModified = System.currentTimeMillis();
    isScheduled.getAndSet(false);
    return task; // the result becomes available in the Future of the CompletionService
  }

}


/////////////////////////////////////////////////////////////////////////////
// This file is part of the "Java-DAP" project, a Java implementation
// of the OPeNDAP Data Access Protocol.
//
// Copyright (c) 2010, OPeNDAP, Inc.
// Copyright (c) 2002,2003 OPeNDAP, Inc.
//
// Author: James Gallagher <jgallagher@opendap.org>
//
// All rights reserved.
//
// Redistribution and use in source and binary forms,
// with or without modification, are permitted provided
// that the following conditions are met:
//
// - Redistributions of source code must retain the above copyright
//   notice, this list of conditions and the following disclaimer.
//
// - Redistributions in binary form must reproduce the above copyright
//   notice, this list of conditions and the following disclaimer in the
//   documentation and/or other materials provided with the distribution.
//
// - Neither the name of the OPeNDAP nor the names of its contributors may
//   be used to endorse or promote products derived from this software
//   without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
// PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
// TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
/////////////////////////////////////////////////////////////////////////////

package opendap.servlet.ascii;

import java.io.*;
import java.util.Enumeration;

import opendap.dap.*;

/**
 */
public class asciiGrid extends DGrid implements toASCII {

    private static boolean _Debug = false;

    /**
     * Constructs a new asciiGrid.
     */
    public asciiGrid() {
        this(null);
    }

    /**
     * Constructs a new asciiGrid with name n.
     *
     * @param n the name of the variable.
     */
    public asciiGrid(String n) {
        super(n);
    }

    /**
     * Returns a string representation of the variables value. This
     * is really foreshadowing functionality for Server types, but
     * as it may come in useful for clients it is added here. Simple
     * types (example: DFloat32) will return a single value. DConstuctor
     * and DVector types will be flattened. DStrings and DURL's will
     * have double quotes around them.
     *
     * @param addName is a flag indicating if the variable name should
     *                appear at the begining of the returned string.
     */
    public void toASCII(PrintWriter pw,
                        boolean addName,
                        String rootName,
                        boolean newLine) {

        if (_Debug)
            System.out.println("asciiGrid.toASCII(" + addName + ",'" + rootName + "')  getName(): " + getEncodedName());

        if (rootName != null)
            rootName += "." + getEncodedName();
        else
            rootName = getEncodedName();

        boolean firstPass = true;
        Enumeration e = getVariables();
        while (e.hasMoreElements()) {
            toASCII ta = (toASCII) e.nextElement();

            if (!newLine && !firstPass)
                pw.print(", ");

            ta.toASCII(pw, addName, rootName, newLine);

            firstPass = false;
        }

        if (newLine)
            pw.print("\n");

    }

    public String toASCIIAddRootName(PrintWriter pw, boolean addName, String rootName) {

        if (addName) {
            rootName = toASCIIFlatName(rootName);
            pw.print(rootName);
        }
        return (rootName);

    }

    public String toASCIIFlatName(String rootName) {

        StringBuilder s = new StringBuilder();
        boolean firstPass = true;
        Enumeration e = getVariables();
        while (e.hasMoreElements()) {
            toASCII ta = (toASCII) e.nextElement();
            if (!firstPass)
                s.append(", ");
            s.append(ta.toASCIIFlatName(rootName));
            firstPass = false;
        }

        return (s.toString());
    }

}