/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

// This is still experimental. Don't rely on any of these methods.
package thredds.server.radarServer2;

import com.google.common.base.Joiner;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.*;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.lang.UnsupportedOperationException;
import java.text.ParseException;
import java.util.*;

import org.springframework.web.servlet.HandlerMapping;
import thredds.client.catalog.*;
import thredds.client.catalog.builder.CatalogBuilder;
import thredds.client.catalog.builder.CatalogRefBuilder;
import thredds.client.catalog.builder.DatasetBuilder;
import thredds.client.catalog.tools.CatalogXmlWriter;
import thredds.server.admin.DebugCommands;
import thredds.server.config.TdsContext;
import thredds.server.config.ThreddsConfig;
import ucar.nc2.constants.CDM;
import ucar.nc2.time.CalendarDate;
import ucar.nc2.time.CalendarDateRange;
import ucar.nc2.time.CalendarPeriod;
import ucar.nc2.units.DateRange;
import ucar.nc2.units.DateType;
import ucar.nc2.units.TimeDuration;

import javax.annotation.PostConstruct;
import javax.servlet.http.HttpServletRequest;

/**
 * Serve up radar data in a way that makes it easy to query. Relevant query
 * data are pulled based on filename and directory structure to avoid needing
 * to crack open large numbers of files.
 *
 * @author rmay
 * @since 01/15/2014
 */

@Controller
@RequestMapping("/radarServer")
public class RadarServerController {
    Map<String, RadarDataInventory> data;
    static final String appName = "/thredds/";
    static final String entryPoint = "radarServer/";
    static final String URLbase = appName + entryPoint;
    static Map<String, List<RadarServerConfig.RadarConfigEntry.VarInfo>> vars;
    boolean enabled = false;

    @Autowired
    TdsContext tdsContext;

    @Autowired
    DebugCommands debugCommands;

    @ExceptionHandler(Exception.class)
    @ResponseBody
    public String handleException(Exception exc) {
        StringWriter sw = new StringWriter(5000);
        exc.printStackTrace(new PrintWriter(sw));
        return sw.toString();
    }

    @ExceptionHandler(UnsupportedOperationException.class)
    @ResponseBody
    public String badQuery(Exception exc) {
        return exc.getMessage();
    }

    public RadarServerController() {
    }

    void setupDebug() {
        DebugCommands.Category debugHandler = debugCommands.findCategory("RadarServer");
        DebugCommands.Action act = new DebugCommands.Action("showDatasets", "Show Datasets") {
            public void doAction(DebugCommands.Event e) {
                try {
                    for (Map.Entry<String, RadarDataInventory> ent : data.entrySet()) {
                        e.pw.println(ent.getKey());

                        RadarDataInventory di = ent.getValue();
                        if (di == null) {
                            e.pw.println("Dataset is null");
                            continue;
                        }
                        e.pw.printf("Collection Dir: %s%n", di.getCollectionDir().toString());
                        e.pw.printf("Last Update: %s%n", di.getLastUpdate());
                        e.pw.println("Dates:");
                        for (String item : di.listItems(RadarDataInventory.DirType.Date)) {
                            e.pw.println("\t" + item);
                        }
                        e.pw.println("Stations:");
                        for (String item : di.listItems(RadarDataInventory.DirType.Station)) {
                            e.pw.println("\t" + item);
                        }
                        e.pw.println("Vars:");
                        for (String item : di.listItems(RadarDataInventory.DirType.Variable)) {
                            e.pw.println("\t" + item);
                        }
                    }
                } catch (Exception ex) {
                    ex.printStackTrace(e.pw);
                }
            }
        };
        debugHandler.addAction(act);
    }

    @PostConstruct
    public void init() {
        enabled = ThreddsConfig.getBoolean("RadarServer.allow", false);
        if (!enabled) return;

        setupDebug();

        data = new TreeMap<>();
        vars = new TreeMap<>();
        String contentPath = tdsContext.getThreddsDirectory().getPath();
        List<RadarServerConfig.RadarConfigEntry> configs = RadarServerConfig.readXML(contentPath + "/radar/radarCollections.xml");
        for (RadarServerConfig.RadarConfigEntry conf : configs) {
            RadarDataInventory di = new RadarDataInventory(conf.dataPath,
                    conf.crawlItems);
            di.setName(conf.name);
            di.setDescription(conf.doc);

            for (String part: conf.layout.split("/")) {
                switch (part) {
                    case "STATION":
                        di.addStationDir();
                        break;
                    case "VARIABLE":
                        di.addVariableDir();
                        break;
                    default: // Assume date format
                        di.addDateDir(part);
                }
            }

            di.addFileTime(conf.dateParseRegex, conf.dateFmt);
            di.setNearestWindow(CalendarPeriod.of(1, CalendarPeriod.Field.Hour));

            // TODO: These needs to come from files instead
            di.setDataFormat(conf.dataFormat);
            di.setTimeCoverage(conf.timeCoverage);
            di.setGeoCoverage(conf.spatialCoverage);

            data.put(conf.urlPath, di);
            vars.put(conf.urlPath, conf.vars);
            StationList sl = di.getStationList();
            sl.loadFromXmlFile(contentPath + "/" + conf.stationFile);
        }
    }

    @RequestMapping(value="catalog.xml")
    @ResponseBody
    public HttpEntity<byte[]> topLevelCatalog() throws IOException
    {
        if (!enabled) return null;

        CatalogBuilder cb = new CatalogBuilder();
        cb.addService(new Service("radarServer", URLbase,
                "QueryCapability", null, null, new ArrayList<Service>(),
                new ArrayList<Property>(), null));
        cb.setName("THREDDS Radar Server");

        DatasetBuilder mainDB = new DatasetBuilder(null);
        mainDB.setName("Radar Data");

        for (Map.Entry<String, RadarDataInventory> ent: data.entrySet()) {
            RadarDataInventory di = ent.getValue();
            CatalogRefBuilder crb = new CatalogRefBuilder(mainDB);
            crb.setName(di.getName());
            crb.setTitle(di.getName());
            crb.setHref(ent.getKey() + "/dataset.xml");
            mainDB.addDataset(crb);
        }
        cb.addDataset(mainDB);

        CatalogXmlWriter writer = new CatalogXmlWriter();
        ByteArrayOutputStream os = new ByteArrayOutputStream(10000);
        writer.writeXML(cb.makeCatalog(), os);
        byte[] xmlBytes = os.toByteArray();

        HttpHeaders header = new HttpHeaders();
        header.setContentType(new MediaType("application", "xml"));
        header.setContentLength(xmlBytes.length);
        return new HttpEntity<>(xmlBytes, header);
    }

    // Old IDV code doesn't actually parse a catalog, but a custom XML file.
    // This code tweaks our catalog output to match.
    private String idvDatasetCatalog(String xml)
    {
        String ret = xml.replace("variables", "Variables");
        ret = ret.replace("timeCoverage", "TimeSpan");
        StringBuilder sub = new StringBuilder(ret.substring(0,
                ret.indexOf("<geospatialCoverage>")));
        sub.append("<LatLonBox>\n\t<north>90.0</north>\n\t<south>-90.0</south>");
        sub.append("\n\t<east>180.0</east>\n\t<west>-180.0</west></LatLonBox>");
        String endCoverage = "</geospatialCoverage>";
        sub.append(ret.substring(ret.indexOf(endCoverage) + endCoverage.length()));
        return sub.toString();
    }

    // Old IDV can't handle all that we put out as a time coverage. This
    // function forces the DateRange to use fixed times rather than, e.g.,
    // present and 14 days.
    private DateRange idvCompatibleRange(DateRange range)
    {
        CalendarDate start = range.getStart().getCalendarDate();
        CalendarDate end = range.getEnd().getCalendarDate();
        return new DateRange(start.toDate(), end.toDate());
    }

    private String parseDatasetFromURL(final HttpServletRequest request)
    {
        String match = (String) request.getAttribute(
                HandlerMapping.PATH_WITHIN_HANDLER_MAPPING_ATTRIBUTE);
        return match.substring(match.indexOf(entryPoint) + entryPoint.length());
    }

    private String parseDatasetFromURL(final HttpServletRequest request, String ending)
    {
        String match = (String) request.getAttribute(
                HandlerMapping.PATH_WITHIN_HANDLER_MAPPING_ATTRIBUTE);
        return match.substring(match.indexOf(entryPoint) + entryPoint.length(),
                match.indexOf(ending));
    }

    @RequestMapping(value="**/dataset.xml")
    @ResponseBody
    public HttpEntity<byte[]> datasetCatalog(final HttpServletRequest request) throws IOException
    {
        if (!enabled) return null;

        // Check the user-agent to try to guess if this request is coming from
        // the IDV--if so, we'll need to tweak the returned catalog XML.
        String agent = request.getHeader("user-agent");
        boolean makeIDVCatalog = false;
        if (agent != null && agent.startsWith("Java/1."))
            makeIDVCatalog = true;

        // Parse the request URL to get the name of the dataset that was
        // requested.
        String dataset = parseDatasetFromURL(request, "/dataset.xml");
        RadarDataInventory di = getInventory(dataset);

        CatalogBuilder cb = new CatalogBuilder();
        cb.addService(new Service("radarServer", URLbase,
                "DQC", null, null, new ArrayList<Service>(),
                new ArrayList<Property>(), null));
        cb.setName("Radar Data");

        DatasetBuilder mainDB = new DatasetBuilder(null);
        mainDB.setName(di.getName());
        mainDB.put(Dataset.Id, dataset);
        mainDB.put(Dataset.UrlPath, dataset);
        mainDB.put(Dataset.DataFormatType, di.getDataFormat());
        mainDB.put(Dataset.FeatureType, di.getFeatureType().toString());
        mainDB.put(Dataset.ServiceName, "radarServer");

        ThreddsMetadata tmd = new ThreddsMetadata();
        Map<String, Object> metadata = tmd.getFlds();
        metadata.put(Dataset.Documentation, new Documentation(null, null, null,
                "summary", di.getDescription()));

        RadarServerConfig.RadarConfigEntry.GeoInfo gi = di.getGeoCoverage();
        metadata.put(Dataset.GeospatialCoverage,
                new ThreddsMetadata.GeospatialCoverage(
                        new ThreddsMetadata.GeospatialRange(gi.eastWest.start,
                                gi.eastWest.size, 0.0, gi.eastWest.units),
                        new ThreddsMetadata.GeospatialRange(gi.northSouth.start,
                                gi.northSouth.size, 0.0, gi.northSouth.units),
                        new ThreddsMetadata.GeospatialRange(gi.upDown.start,
                                gi.upDown.size, 0.0, gi.upDown.units),
                        new ArrayList<ThreddsMetadata.Vocab>(), null));

        DateRange range = di.getTimeCoverage();
        if (makeIDVCatalog) range = idvCompatibleRange(range);
        metadata.put(Dataset.TimeCoverage, range);

        // TODO: Need to be able to get this from the inventory
        List<ThreddsMetadata.Variable> catalogVars = new ArrayList<>();
        for (RadarServerConfig.RadarConfigEntry.VarInfo vi: vars.get(dataset)) {
            catalogVars.add(new ThreddsMetadata.Variable(vi.name, null,
                    vi.vocabName, vi.units, null));
        }
        ThreddsMetadata.VariableGroup vg = new ThreddsMetadata.VariableGroup(
                "DIF", null, null, catalogVars);
        metadata.put(Dataset.VariableGroups, vg);

        mainDB.put(Dataset.ThreddsMetadataInheritable, tmd);

        cb.addDataset(mainDB);

        CatalogXmlWriter writer = new CatalogXmlWriter();
        ByteArrayOutputStream os = new ByteArrayOutputStream(10000);
        writer.writeXML(cb.makeCatalog(), os);

        byte[] xmlBytes;
        if (makeIDVCatalog) {
            String xml = os.toString(CDM.UTF8);
            xml = idvDatasetCatalog(xml);
            xmlBytes = xml.getBytes(CDM.utf8Charset);
        } else {
            xmlBytes = os.toByteArray();
        }

        HttpHeaders header = new HttpHeaders();
        header.setContentType(new MediaType("application", "xml"));
        header.setContentLength(xmlBytes.length);
        return new HttpEntity<>(xmlBytes, header);
    }

    @RequestMapping(value="**/{dataset}", params="station=all")
    @ResponseBody
    public StationList stations(@PathVariable String dataset,
                                final HttpServletRequest request)
    {
        if (!enabled) return null;
        dataset = parseDatasetFromURL(request);
        return listStations(dataset);
    }

    @RequestMapping(value="**/stations.xml")
    @ResponseBody
    public StationList stationsFile(final HttpServletRequest request)
    {
        if (!enabled) return null;
        String dataset = parseDatasetFromURL(request, "/stations.xml");
        return listStations(dataset);
    }

    StationList listStations(String dataset)
    {
        RadarDataInventory di = getInventory(dataset);
        return di.getStationList();
    }

    RadarDataInventory getInventory(String dataset)
    {
        return data.get(dataset);
    }

    HttpEntity<byte[]> simpleString(String str)
    {
        byte[] bytes = str.getBytes(CDM.utf8Charset);
        HttpHeaders header = new HttpHeaders();
        header.setContentType(new MediaType("application", "text"));
        header.setContentLength(bytes.length);
        return new HttpEntity<>(bytes, header);
    }

    @RequestMapping(value="**/{dataset}")
    HttpEntity<byte[]> handleQuery(@PathVariable String dataset,
            @RequestParam(value="stn", required=false) String[] stations,
            @RequestParam(value="longitude", required=false) Double lon,
            @RequestParam(value="latitude", required=false) Double lat,
            @RequestParam(value="east", required=false) Double east,
            @RequestParam(value="west", required=false) Double west,
            @RequestParam(value="north", required=false) Double north,
            @RequestParam(value="south", required=false) Double south,
            @RequestParam(value="time", required=false) String time,
            @RequestParam(value="time_start", required=false) String start,
            @RequestParam(value="time_end", required=false) String end,
            @RequestParam(value="time_duration", required=false) String period,
            @RequestParam(value="temporal", required=false) String temporal,
            @RequestParam(value="var", required=false) String[] vars,
            final HttpServletRequest request)
            throws ParseException, UnsupportedOperationException, IOException
    {
        if (!enabled) return null;

        dataset = parseDatasetFromURL(request);
        RadarDataInventory di = getInventory(dataset);
        if (di == null) {
            return simpleString("Could not find dataset: " + dataset);
        }

        RadarDataInventory.Query q = di.newQuery();
        try {
            if (!setTimeLimits(q, time, start, end, period, temporal))
                throw new UnsupportedOperationException("Either a single time " +
                        "argument, temporal=all, or a combination of time_start, " +
                        "time_end, and time_duration must be provided.");
        } catch (ParseException e) {
            throw new UnsupportedOperationException("Invalid time string passed: " + e.getMessage());
        }
        StringBuilder queryString = new StringBuilder();
        addQueryElement(queryString, "time", time);
        addQueryElement(queryString, "time_start", start);
        addQueryElement(queryString, "time_end", end);
        addQueryElement(queryString, "time_duration", period);
        addQueryElement(queryString, "temporal", temporal);

        if (stations == null) {
            try {
                stations = getStations(di.getStationList(), lon, lat, north,
                        south, east, west);
                addQueryElement(queryString, "longitude", lon);
                addQueryElement(queryString, "latitude", lat);
                addQueryElement(queryString, "north", north);
                addQueryElement(queryString, "south", south);
                addQueryElement(queryString, "east", east);
                addQueryElement(queryString, "west", west);
            } catch (UnsupportedOperationException e) {
                throw new UnsupportedOperationException("Either a list of " +
                        "stations, a lat/lon point, or a box defined by " +
                        "north, south, east, and west parameters must be " +
                        "provided.");
            }
        } else {
            addQueryElement(queryString, "stn", stations);
        }
        setStations(q, stations);

        if (di.needsVar()) {
            if (!setVariables(q, vars))
                throw new UnsupportedOperationException("One or more variables " +
                        "required.");
            addQueryElement(queryString, "var", vars);
        }

        return makeCatalog(dataset, di, q, queryString.toString());
    }

    private void addQueryElement(StringBuilder sb, String name,
                                 String[] values) {
        if (values != null) {
            addQueryElement(sb, name, Joiner.on(',').join(values));
        }
    }

    private void addQueryElement(StringBuilder sb, String name, Double value) {
        if (value != null) {
            addQueryElement(sb, name, value.toString());
        }
    }

    private void addQueryElement(StringBuilder sb, String name, String value) {
        if (value != null) {
            if (sb.length() > 0)
                sb.append('&');
            sb.append(name).append('=').append(value);
        }
    }

    private HttpEntity<byte[]> makeCatalog(String dataset,
                                           RadarDataInventory inv,
                                           RadarDataInventory.Query query,
                                           String queryString) throws
            IOException, ParseException
    {
        Collection<RadarDataInventory.Query.QueryResultItem> res = query.results();
        CatalogBuilder cb = new CatalogBuilder();

        // At least the IDV needs to have the trailing slash included
        if (!dataset.endsWith("/"))
            dataset += "/";

        Service dap = new Service("OPENDAP", "/thredds/dodsC/" + dataset,
                ServiceType.OPENDAP.toString(), ServiceType.OPENDAP.getDescription(), null,
                new ArrayList<Service>(), new ArrayList<Property>(), ServiceType.OPENDAP.getAccessType());
        Service cdmr = new Service("CDMRemote", "/thredds/cdmremote/" + dataset,
                ServiceType.CdmRemote.toString(), ServiceType.CdmRemote.getDescription(), null,
                new ArrayList<Service>(), new ArrayList<Property>(), ServiceType.CdmRemote.getAccessType());
        Service files = new Service("HTTPServer", "/thredds/fileServer/" + dataset,
                ServiceType.HTTPServer.toString(), ServiceType.HTTPServer.getDescription(), null,
                new ArrayList<Service>(), new ArrayList<Property>(), ServiceType.HTTPServer.getAccessType());
        cb.addService(new Service("RadarServices", "",
                ServiceType.Compound.toString(), ServiceType.Compound.getDescription(), null,
                Arrays.asList(dap, files, cdmr), new ArrayList<Property>(), ServiceType.Compound.getAccessType()));
        cb.setName("Radar " + inv.getName() + " datasets in near real time");

        DatasetBuilder mainDB = new DatasetBuilder(null);
        mainDB.setName("Radar" + inv.getName() +
                        " datasets for available stations and times");
        mainDB.put(Dataset.CollectionType, "TimeSeries");
        mainDB.put(Dataset.Id, queryString);

        ThreddsMetadata tmd = new ThreddsMetadata();
        Map<String, Object> metadata = tmd.getFlds();
        metadata.put(Dataset.DataFormatType, inv.getDataFormat());
        metadata.put(Dataset.FeatureType, inv.getFeatureType().toString());
        metadata.put(Dataset.ServiceName, "RadarServices");
        metadata.put(Dataset.Documentation, new Documentation(null, null, null,
                null, res.size() + " datasets found for query"));

        mainDB.put(Dataset.ThreddsMetadataInheritable, tmd);

        for (RadarDataInventory.Query.QueryResultItem i: res) {
            DatasetBuilder fileDB = new DatasetBuilder(mainDB);
            fileDB.setName(i.file.getFileName().toString());
            fileDB.put(Dataset.Id, String.valueOf(i.file.hashCode()));

            fileDB.put(Dataset.Dates, new DateType(i.time.toString(), null,
                    "start of ob", i.time.getCalendar()));

            // TODO: Does this need to be converted from the on-disk path
            // to a mapped url path?
            fileDB.put(Dataset.UrlPath,
                    inv.getCollectionDir().relativize(i.file).toString());
            mainDB.addDataset(fileDB);
        }

        cb.addDataset(mainDB);

        CatalogXmlWriter writer = new CatalogXmlWriter();
        ByteArrayOutputStream os = new ByteArrayOutputStream(10000);
        writer.writeXML(cb.makeCatalog(), os);
        byte[] xmlBytes = os.toByteArray();

        HttpHeaders header = new HttpHeaders();
        header.setContentType(new MediaType("application", "xml"));
        header.setContentLength(xmlBytes.length);
        return new HttpEntity<>(xmlBytes, header);
    }

    boolean setTimeLimits(RadarDataInventory.Query query, String timePoint,
                          String start, String end, String period,
                          String temporal)
            throws ParseException
    {
        CalendarDate time = parseTime(timePoint);
        if (time != null) {
            query.addDateRange(CalendarDateRange.of(time, time));
            return true;
        }

        CalendarDate timeStart = parseTime(start);
        CalendarDate timeEnd = parseTime(end);

        TimeDuration duration = null;
        if (period != null)
            duration = TimeDuration.parseW3CDuration(period);

        if (timeStart != null) {
            if (timeEnd != null) {
                query.addDateRange(CalendarDateRange.of(timeStart, timeEnd));
                return true;
            } else if (duration != null) {
                query.addDateRange(new CalendarDateRange(timeStart,
                        (long) duration.getValueInSeconds()));
                return true;
            }
        } else if (timeEnd != null && duration != null) {
            query.addDateRange(new CalendarDateRange(
                    timeEnd.add((long) -duration.getValueInSeconds(), CalendarPeriod.Field.Second),
                    (long) duration.getValueInSeconds()));
            return true;
        }

        if (temporal != null) {
            // Use null to indicate no range
            query.addDateRange(null);
            return true;
        }
        return false;
    }

    CalendarDate parseTime(String timeString)
    {
        if (timeString == null)
            return null;

        if (timeString.equalsIgnoreCase("present")) {
            return CalendarDate.present();
        } else {
            return CalendarDate.parseISOformat(null, timeString);
        }
    }

    void setStations(RadarDataInventory.Query query, String[] stations)
    {
        if (stations.length == 0)
            throw new UnsupportedOperationException("No stations " +
                    "found for specified coordinates.");
        for (String stid: stations)
            query.addCriteria(RadarDataInventory.DirType.Station, stid);
    }

    String[] getStations(StationList stations, Double lon, Double lat,
                         Double north, Double south, Double east, Double west)
    {
        if (lat != null && lon != null) {
            // Pull nearest station
            StationList.Station nearest = stations.getNearest(lon, lat);
            if (nearest == null) {
                throw new UnsupportedOperationException("No stations " +
                        "available to search for nearest.");
            }
            return new String[]{nearest.getStid()};
        } else if(north != null && south != null && east != null &&
                west != null) {
            // Pull all stations within box
            List<StationList.Station> inBox = stations.getStations(east, west,
                    north, south);
            List<String> stIds = new ArrayList<>(inBox.size());
            for(StationList.Station s: inBox) {
                stIds.add(s.getStid());
            }
            return stIds.toArray(new String[stIds.size()]);
        } else {
            throw new UnsupportedOperationException("Either station, " +
                    "a lat/lon point, or a box defined by north, " +
                    "south, east, and west parameters must be provided.");
        }
    }

    boolean setVariables(RadarDataInventory.Query query, String[] variables)
    {
        if (variables == null)
            return false;

        for (String var: variables)
            query.addCriteria(RadarDataInventory.DirType.Variable, var);

        return true;
    }
}


package ucar.nc2.ogc.om;

import net.opengis.gml.x32.TimePeriodDocument;
import net.opengis.om.x20.TimeObjectPropertyType;
import ucar.nc2.ft.StationTimeSeriesFeature;
import ucar.nc2.ogc.gml.NcTimePeriodType;

import java.io.IOException;

/**
 * Created by cwardgar on 3/6/14.
 */
public abstract class NcTimeObjectPropertyType {
    // wml2:Collection/wml2:observationMember/om:OM_Observation/om:phenomenonTime
    public static TimeObjectPropertyType initPhenomenonTime(
            TimeObjectPropertyType phenomenonTime, StationTimeSeriesFeature stationFeat) throws IOException {
        // gml:TimePeriod
        TimePeriodDocument timePeriodDoc = TimePeriodDocument.Factory.newInstance();
        NcTimePeriodType.initTimePeriod(timePeriodDoc.addNewTimePeriod(), stationFeat);
        phenomenonTime.set(timePeriodDoc);

        return phenomenonTime;
    }

    private NcTimeObjectPropertyType() { }
}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
// $Id: DMSPHeader.java 63 2006-07-12 21:50:51Z edavis $
package ucar.nc2.iosp.dmsp;

import ucar.nc2.Attribute;
import ucar.nc2.Dimension;
import ucar.nc2.constants.CDM;
import ucar.nc2.constants._Coordinate;

import java.io.IOException;
import java.util.*;
import java.text.SimpleDateFormat;
import java.text.ParseException;

/**
 * DMSP header parser
 * <p/>
 * User: edavis
 * Date: Aug 6, 2004
 * Time: 3:38:11 PM
 */
public class DMSPHeader
{
  private String[] header = null;
  private Map<String, String> headerInfo = new HashMap<>();
  private int headerSizeInBytes = 0;
  private int headerSizeInBytesGuess = 5000;

  private ucar.unidata.io.RandomAccessFile raFile;
  private ucar.nc2.NetcdfFile ncFile;
  private long actualSize;

  // File information
  private String fileIdAttName = "fileId";
  private Attribute fileIdAtt = null;

  private String datasetIdAttName = "datasetId";
  private Attribute datasetIdAtt = null;

  private int recordSizeInBytes = 0;

  private int numHeaderRecords = 0;
  private int numDataRecords = 0;

  private String numDataRecordsDimName = "numScans";
  private Dimension numDataRecordsDim = null;

  private int numArtificialDataRecords = 0;
  private int numRecords = 0;

  // Processing information
  private String suborbitHistoryAttName = "suborbitHistory";
  private Attribute suborbitHistoryAtt = null;

  private String processingSystemAttName = "processingSystem";
  private Attribute processingSystemAtt = null;

  private Date processingDate = null;

  private String processingDateAttName = "processingDate";
  private Attribute processingDateAtt = null;

  // Satellite information
  private String spacecraftIdAttName = "spacecraftId";
  private Attribute spacecraftIdAtt = null;

  private String noradIdAttName = "noradId";
  private Attribute noradIdAtt = null;

  // Orbit information
  private String startDateAttName = "startDate";
  private Attribute startDateAtt = null;
  private Date startDate = null;

  private String endDateAttName = "endDate";
  private Attribute endDateAtt = null;
  private Date endDate = null;

  private String startDateLocalAttName = "startDateLocal";
  private Attribute startDateLocalAtt = null;

  private String startTimeLocalAttName = "startTimeLocal";
  private Attribute startTimeLocalAtt = null;

  private String startLatitudeAttName = "startLatitude";
  private Attribute startLatitudeAtt = null;

  private String startLongitudeAttName = "startLongitude";
  private Attribute startLongitudeAtt = null;

  private String endLatitudeAttName = "endLatitude";
  private Attribute endLatitudeAtt = null;

  private String endLongitudeAttName = "endLongitude";
  private Attribute endLongitudeAtt = null;

  private String startSubsolarCoordsAttName = "startSubsolarCoords";
  private Attribute startSubsolarCoordsAtt = null;

  private String endSubsolarCoordsAttName = "endSubsolarCoords";
  private Attribute endSubsolarCoordsAtt = null;

  private String startLunarCoordsAttName = "startLunarCoords";
  private Attribute startLunarCoordsAtt = null;

  private String endLunarCoordsAttName = "endLunarCoords";
  private Attribute endLunarCoordsAtt = null;

  private String ascendingNodeAttName = "ascendingNode";
  private Attribute ascendingNodeAtt = null;

  private String nodeHeadingAttName = "nodeHeading";
  private Attribute nodeHeadingAtt = null;

  // Sensor information
  private int numSamplesPerBand = 0;
  private String numSamplesPerBandDimName = "numSamplesPerScan";
  private Dimension numSamplesPerBandDim = null;

  private String nominalResolutionAttName = "nominalResolution";
  private Attribute nominalResolutionAtt = null;

  private String bandsPerScanlineAttName = "bandsPerScanline";
  private Attribute bandsPerScanlineAtt = null;

  private String bytesPerSampleAttName = "bytesPerSample";
  private Attribute bytesPerSampleAtt = null;

  private String byteOffsetBand1AttName = "byteOffsetBand1";
  private Attribute byteOffsetBand1Att = null;

  private String byteOffsetBand2AttName = "byteOffsetBand2";
  private Attribute byteOffsetBand2Att = null;

  private String band1AttName = "band1";
  private Attribute band1Att = null;

  private String band2AttName = "band2";
  private Attribute band2Att = null;

  private String bandOrganizationAttName = "bandOrganization";
  private Attribute bandOrganizationAtt = null;

  private String thermalOffsetAttName = "thermalOffset";
  private Attribute thermalOffsetAtt = null;

  private String thermalScaleAttName = "thermalScale";
  private Attribute thermalScaleAtt = null;

  private String percentDaylightAttName = "percentDaylight";
  private Attribute percentDaylightAtt = null;

  private String percentFullMoonAttName = "percentFullMoon";
  private Attribute percentFullMoonAtt = null;

  private String percentTerminatorEvidentAttName = "percentTerminatorEvident";
  private Attribute percentTerminatorEvidentAtt = null;

  // QC information
  private String qcFlagsAttName = "qcFlags";
  private Attribute qcFlagsAtt = null;

  public Attribute getFileIdAtt() { return( this.fileIdAtt ); }
  public Attribute getDatasetIdAtt() { return( this.datasetIdAtt ); }
  public int getNumHeaderRecords() { return( this.numHeaderRecords ); }
  public int getNumDataRecords() { return( this.numDataRecords ); }
  public Dimension getNumDataRecordsDim() { return( this.numDataRecordsDim ); }
  public Dimension getNumSamplesPerBandDim() { return( this.numSamplesPerBandDim ); }
  public int getRecordSizeInBytes() { return( this.recordSizeInBytes ); }

  public Attribute getSuborbitHistoryAtt() { return( this.suborbitHistoryAtt ); }
  public Attribute getProcessingSystemAtt() { return( this.processingSystemAtt ); }
  public Attribute getProcessingDateAtt() { return( this.processingDateAtt ); }
  public Attribute getStartDateAtt() { return( this.startDateAtt ); }

  /** Check basic DMSP file validity of given random access file. */
  boolean isValidFile( ucar.unidata.io.RandomAccessFile raFile )
  {
    // @todo This method should not be called if read() has or will be called on this instance.
    this.raFile = raFile;
    try
    {
      this.actualSize = raFile.length();
    }
    catch ( IOException e )
    {
      return( false );
    }
    try
    {
      this.readHeaderFromFile( raFile );

      this.handleFileInformation();
      this.handleProcessingInformation();
      this.handleSatelliteInformation();
      this.handleSensorInformation();
    }
    catch ( IOException e )
    {
      return( false );
    }
    return( true );
  }

  void read( ucar.unidata.io.RandomAccessFile raFile, ucar.nc2.NetcdfFile ncFile )
          throws IOException
  {
    this.raFile = raFile;
    this.ncFile = ncFile;
    actualSize = this.raFile.length();

    // Read the header from the file
    this.readHeaderFromFile( raFile );

    // Deal with file/record information.
    this.handleFileInformation();

    this.ncFile.addAttribute( null, fileIdAtt );
    this.ncFile.addAttribute( null, datasetIdAtt );
    this.ncFile.addDimension( null, numDataRecordsDim );

    // Deal with processing/history information.
    this.handleProcessingInformation();

    this.ncFile.addAttribute( null, suborbitHistoryAtt);
    this.ncFile.addAttribute( null, processingSystemAtt);
    this.ncFile.addAttribute( null, processingDateAtt);

    // Deal with satellite information
    this.handleSatelliteInformation();

    this.ncFile.addAttribute( null, spacecraftIdAtt );
    this.ncFile.addAttribute( null, noradIdAtt );

    // Deal with orbit information
    this.handleOrbitInformation();

    this.ncFile.addAttribute( null, startDateAtt);
    this.ncFile.addAttribute( null, endDateAtt);
    this.ncFile.addAttribute( null, startDateLocalAtt);
    this.ncFile.addAttribute( null, startTimeLocalAtt);
    this.ncFile.addAttribute( null, startLatitudeAtt);
    this.ncFile.addAttribute( null, startLongitudeAtt);
    this.ncFile.addAttribute( null, endLatitudeAtt);
    this.ncFile.addAttribute( null, endLongitudeAtt);
    this.ncFile.addAttribute( null, startSubsolarCoordsAtt);
    this.ncFile.addAttribute( null, endSubsolarCoordsAtt);
    this.ncFile.addAttribute( null, startLunarCoordsAtt);
    this.ncFile.addAttribute( null, endLunarCoordsAtt);
    this.ncFile.addAttribute( null, ascendingNodeAtt);
    this.ncFile.addAttribute( null, nodeHeadingAtt);

    // Deal with sensor information
    this.handleSensorInformation();

    this.ncFile.addDimension( null, numSamplesPerBandDim );
    this.ncFile.addAttribute( null, nominalResolutionAtt );
    this.ncFile.addAttribute( null, bandsPerScanlineAtt );
    this.ncFile.addAttribute( null, bytesPerSampleAtt );
    this.ncFile.addAttribute( null, byteOffsetBand1Att );
    this.ncFile.addAttribute( null, byteOffsetBand2Att );
    this.ncFile.addAttribute( null, band1Att );
    this.ncFile.addAttribute( null, band2Att );
    this.ncFile.addAttribute( null, bandOrganizationAtt );
    this.ncFile.addAttribute( null, thermalOffsetAtt );
    this.ncFile.addAttribute( null, thermalScaleAtt );
    this.ncFile.addAttribute( null, percentDaylightAtt );
    this.ncFile.addAttribute( null, percentFullMoonAtt );
    this.ncFile.addAttribute( null, percentTerminatorEvidentAtt );

    // Deal with QC information
    this.handleQCInformation();

    this.ncFile.addAttribute( null, qcFlagsAtt );

    // Add some general metadata in global attributes.
    this.ncFile.addAttribute( null, new Attribute( "title",
                                                   new StringBuilder("NGDC archived ")
                                                   .append( datasetIdAtt.getStringValue())
                                                   .append( " data with start time ")
                                                   .append( startDateAtt.getStringValue())
                                                   .toString()));
    this.ncFile.addAttribute( null, new Attribute( "Convention", _Coordinate.Convention));

    // Add some THREDDS specific metadata in global attributes.
    this.ncFile.addAttribute( null, new Attribute( "thredds_creator", "DOD/USAF/SMC > Space and Missile Systems Center (SMC), U.S. Air Force, U.S. Department of Defense"));
    this.ncFile.addAttribute( null, new Attribute( "thredds_contributor", "DOC/NOAA/NESDIS/NGDC > National Geophysical Data Center, NESDIS, NOAA, U.S. Department of Commerce"));
    this.ncFile.addAttribute( null, new Attribute( "thredds_contributor_role", "archive"));
    this.ncFile.addAttribute( null, new Attribute( "thredds_publisher", "DOC/NOAA/NESDIS/NGDC > National Geophysical Data Center, NESDIS, NOAA, U.S. Department of Commerce"));
    this.ncFile.addAttribute( null, new Attribute( "thredds_publisher_url", "http://dmsp.ngdc.noaa.gov/"));
    this.ncFile.addAttribute( null, new Attribute( "thredds_publisher_email", "ngdc.dmsp@noaa.gov"));
    this.ncFile.addAttribute( null, new Attribute( "thredds_summary",
                                                   new StringBuilder("This dataset contains data from the DMSP ").append( spacecraftIdAtt.getStringValue())
                                                   .append( " satellite OLS instrument and includes both visible smooth and thermal smooth imagery with 2.7km resolution.")
                                                   .append( " The start time for this data is ").append( startDateAtt.getStringValue())
                                                   .append( " and the northerly equatorial crossing longitude is ").append( startLongitudeAtt.getNumericValue())
                                                   .append( ".  The DMSP satellite is a polar-orbiting satellite crossing the equator, depending on the satellite, at either dawn/dusk or noon/midnight.")
                                                   .append( " This data is in the NOAA/NGDC DMSP archive format.")
                                                   .toString()));
    this.ncFile.addAttribute( null, new Attribute( "thredds_history", ""));
    this.ncFile.addAttribute( null, new Attribute( "thredds_timeCoverage_start", startDateAtt.getStringValue()));
    this.ncFile.addAttribute( null, new Attribute( "thredds_timeCoverage_end", endDateAtt.getStringValue()));
    this.ncFile.addAttribute( null, new Attribute( "thredds_geospatialCoverage",
                                                   new StringBuilder("Polar orbit with northerly equatorial crossing at longitude ")
                                                   .append( ascendingNodeAtt.getNumericValue()).append( ".")
                                                   .toString()));

    // Set position of random access file to just after header record(s).
    this.raFile.seek( this.headerSizeInBytes );
  }

  /**
   * Read the header information from the file into name/value pairs.
   *
   * @param raFile - the file to be read.
   * @throws IOException if any problems reading the file (or validating the file).
   */
  private void readHeaderFromFile( ucar.unidata.io.RandomAccessFile raFile )
          throws IOException
  {
    long pos = 0;
    raFile.seek( pos );

    // Read in first record.
    this.headerSizeInBytes = raFile.length() > this.headerSizeInBytesGuess ?
                             this.headerSizeInBytesGuess : (int) raFile.length();
    byte[] b = new byte[this.headerSizeInBytes];
    if ( raFile.read(b) != this.headerSizeInBytes ) throw new IOException( "Invalid DMSP file: could not read first " + this.headerSizeInBytes + " bytes.");
    String fullHeader = new String( b, CDM.utf8Charset);
    // Make sure header starts with the proper item.
    if ( ! fullHeader.startsWith( HeaderInfoTitle.FILE_ID.toString() ) )
    {
      throw new IOException( "Invalid DMSP file: header does not start with \"" + HeaderInfoTitle.FILE_ID.toString() + "\"." );
    }
    // Make sure header contains end-of-header marker.
    int endOfHeaderIndex = fullHeader.indexOf( HeaderInfoTitle.END_HEADER.toString() );
    if ( endOfHeaderIndex == -1 )
    {
      throw new IOException( "Invalid DMSP file: header does not end with \"" + HeaderInfoTitle.END_HEADER.toString() + "\"." );
    }

    // Drop the end-of-header marker and the line feed ('\n') proceeding it.
    header = fullHeader.substring( 0, endOfHeaderIndex - 1 ).split( "\n" );

    int lineSeperatorIndex = 0;
    String curHeaderLine = null;
    String curHeaderTitle = null;
    String curHeaderValue = null;
    for (String aHeader : this.header) {
      curHeaderLine = aHeader.trim();
      lineSeperatorIndex = curHeaderLine.indexOf(':');
      if (lineSeperatorIndex == -1)
        throw new IOException("Invalid DMSP file: header line <" + curHeaderLine + "> contains no seperator <:>.");
      if (lineSeperatorIndex == 0)
        throw new IOException("Invalid DMSP file: header line <" + curHeaderLine + "> contains no title.");
      if (lineSeperatorIndex == curHeaderLine.length() - 1)
        throw new IOException("Invalid DMSP file: header line <" + curHeaderLine + "> contains no value.");

      curHeaderTitle = curHeaderLine.substring(0, lineSeperatorIndex).trim();
      curHeaderValue = curHeaderLine.substring(lineSeperatorIndex + 1).trim();
      if (curHeaderValue.equals(""))
        throw new IOException("Invalid DMSP file: header line <" + curHeaderLine + "> contains no value.");

      headerInfo.put(curHeaderTitle, curHeaderValue);
    }
  }

  /**
   * Parse the file header information about the file (e.g., file ID, dataset ID,
   * record size, number of records) and create netCDF attributes and dimensions
   * where appropriate.
   *
   * @throws IOException if any problems reading the file (or validating the file).
   */
  private void handleFileInformation()
          throws IOException
  {
    fileIdAtt = new Attribute( this.fileIdAttName, headerInfo.get(HeaderInfoTitle.FILE_ID.toString() ) );
    datasetIdAtt = new Attribute( this.datasetIdAttName, headerInfo.get(HeaderInfoTitle.DATA_SET_ID.toString() ) );
    recordSizeInBytes = Integer.parseInt( headerInfo.get( HeaderInfoTitle.RECORD_BYTES.toString() ) );
    numRecords = Integer.parseInt( headerInfo.get( HeaderInfoTitle.NUM_RECORDS.toString() ) );
    numHeaderRecords = Integer.parseInt( headerInfo.get( HeaderInfoTitle.NUM_HEADER_RECORDS.toString() ) );
    numDataRecords = Integer.parseInt( headerInfo.get( HeaderInfoTitle.NUM_DATA_RECORDS.toString() ) );
    numDataRecordsDim = new Dimension( this.numDataRecordsDimName, numDataRecords, true, true, false);
    numArtificialDataRecords = Integer.parseInt( headerInfo.get( HeaderInfoTitle.NUM_ARTIFICIAL_DATA_RECORDS.toString() ) );
    this.headerSizeInBytes = this.numHeaderRecords * this.recordSizeInBytes;
    if ( numRecords * ((long) this.recordSizeInBytes) != this.actualSize )
    {
      throw new IOException( "Invalid DMSP file: the number of records <" + this.numRecords + "> times the record size <" + this.recordSizeInBytes + "> does not equal the size of the file <" + this.actualSize + ">." );
    }
  }

  /**
   * Parse the processing/history information from the header.
   *
   * @throws IOException if any problems reading the file (or validating the file).
   */
  private void handleProcessingInformation()
          throws IOException
  {
    suborbitHistoryAtt = new Attribute(this.suborbitHistoryAttName,
            headerInfo.get(HeaderInfoTitle.SUBORBIT_HISTORY.toString()));
    processingSystemAtt = new Attribute(this.processingSystemAttName,
            headerInfo.get(HeaderInfoTitle.PROCESSING_SYSTEM.toString()));
    String processingDateString = headerInfo.get(HeaderInfoTitle.PROCESSING_DATE.toString());
    try
    {
      processingDate = DateFormatHandler.ALT_DATE_TIME.getDateFromDateTimeString( processingDateString );
    }
    catch ( ParseException e )
    {
      throw new IOException( "Invalid DMSP file: processing date string <" + processingDateString + "> not parseable: " + e.getMessage() );
    }
    processingDateAtt = new Attribute(
            this.processingDateAttName,
            DateFormatHandler.ISO_DATE_TIME.getDateTimeStringFromDate( processingDate ) );
  }

  /**
   * Parse the satellite information from the header.
   */
  private void handleSatelliteInformation()
  {
    spacecraftIdAtt = new Attribute(
            this.spacecraftIdAttName,
            headerInfo.get(HeaderInfoTitle.SPACECRAFT_ID.toString()));
    noradIdAtt = new Attribute(
            this.noradIdAttName,
            headerInfo.get(HeaderInfoTitle.NORAD_ID.toString()));
  }

  private void handleOrbitInformation() throws IOException
  {
    // Read the start date UTC information.
    String time = headerInfo.get(  HeaderInfoTitle.START_TIME_UTC.toString());
    String startDateTimeUTC = headerInfo.get(  HeaderInfoTitle.START_DATE_UTC.toString())
                              + "T" + time.substring( 0, time.indexOf( '.')+4) + "Z";
    try {
      this.startDate = DateFormatHandler.ISO_DATE_TIME.getDateFromDateTimeString( startDateTimeUTC);
    }
    catch ( ParseException e )
    {
      throw new IOException( "Invalid DMSP file: start date/time string <" + startDateTimeUTC + "> not parseable: " + e.getMessage() );
    }
    this.startDateAtt = new Attribute( this.startDateAttName, DateFormatHandler.ISO_DATE_TIME.getDateTimeStringFromDate( this.startDate));

    // Read the end date UTC information.
    time = headerInfo.get(  HeaderInfoTitle.END_TIME_UTC.toString());
    String endDateTimeUTC = headerInfo.get(  HeaderInfoTitle.END_DATE_UTC.toString())
                            + "T" + time.substring( 0, time.indexOf( '.')+4) + "Z";
    try {
      this.endDate = DateFormatHandler.ISO_DATE_TIME.getDateFromDateTimeString( endDateTimeUTC);
    }
    catch ( ParseException e )
    {
      throw new IOException( "Invalid DMSP file: end date/time string <" + endDateTimeUTC + "> not parseable: " + e.getMessage() );
    }
    this.endDateAtt = new Attribute( this.endDateAttName,
                                       DateFormatHandler.ISO_DATE_TIME.getDateTimeStringFromDate( this.endDate));

    // Read the local start/end date/time
    this.startDateLocalAtt = new Attribute( this.startDateLocalAttName, HeaderInfoTitle.START_DATE_LOCAL.toString());
    this.startTimeLocalAtt = new Attribute( this.startTimeLocalAttName, HeaderInfoTitle.START_TIME_LOCAL.toString());

    // Read the start latitude/longitude information.
    String startLatLon = headerInfo.get(  HeaderInfoTitle.START_LAT_LON.toString());
    String[] latLon = startLatLon.split( " ");
    Double lat, lon;
    if ( latLon.length != 2) throw new IOException( "Invalid DMSP file: start lat/lon <" + startLatLon + "> invalid.");
    try
    {
      lat = Double.valueOf( latLon[0]);
      lon = Double.valueOf( latLon[1]);
    }
    catch ( NumberFormatException e )
    {
      throw new IOException( "Invalid DMSP file: start lat/lon string <" + startLatLon + "> not parseable: " + e.getMessage() );
    }
    this.startLatitudeAtt = new Attribute( this.startLatitudeAttName, lat);
    this.startLongitudeAtt = new Attribute( this.startLongitudeAttName, lon);

    // Read the end latitude/longitude information.
    String endLatLon = headerInfo.get(  HeaderInfoTitle.END_LAT_LON.toString());
    latLon = endLatLon.split( " ");
    if ( latLon.length != 2) throw new IOException( "Invalid DMSP file: end lat/lon <" + endLatLon + "> invalid.");
    try
    {
      lat = Double.valueOf( latLon[0]);
      lon = Double.valueOf( latLon[1]);
    }
    catch ( NumberFormatException e )
    {
      throw new IOException( "Invalid DMSP file: end lat/lon string <" + endLatLon + "> not parseable: " + e.getMessage() );
    }
    this.endLatitudeAtt = new Attribute( this.endLatitudeAttName, lat);
    this.endLongitudeAtt = new Attribute( this.endLongitudeAttName, lon);

    // Read the start sub-solar coordinates.
    this.startSubsolarCoordsAtt = new Attribute( this.startSubsolarCoordsAttName, headerInfo.get(HeaderInfoTitle.START_SUBSOLAR_COORD.toString()));

    // Read the end sub-solar coordinates.
    this.endSubsolarCoordsAtt = new Attribute( this.endSubsolarCoordsAttName, headerInfo.get(HeaderInfoTitle.END_SUBSOLAR_COORD.toString()));

    // Read the start lunar coordinates.
    this.startLunarCoordsAtt = new Attribute( this.startLunarCoordsAttName, headerInfo.get(HeaderInfoTitle.START_LUNAR_COORD.toString()));

    // Read the end lunar coordinates.
    this.endLunarCoordsAtt = new Attribute( this.endLunarCoordsAttName, headerInfo.get(HeaderInfoTitle.END_LUNAR_COORD.toString()));

    // Read the ascending node.
    Double ascendingNode = Double.valueOf( headerInfo.get( HeaderInfoTitle.ASCENDING_NODE.toString()) );
    this.ascendingNodeAtt = new Attribute( this.ascendingNodeAttName, ascendingNode);

    Double nodeHeading = Double.valueOf( headerInfo.get( HeaderInfoTitle.NODE_HEADING.toString()) );
    this.nodeHeadingAtt = new Attribute( this.nodeHeadingAttName, nodeHeading);
  }

  /**
   * Parse the sensor information from the header.
   */
  private void handleSensorInformation()
  {
    numSamplesPerBand = Integer.parseInt( headerInfo.get( HeaderInfoTitle.SAMPLES_PER_BAND.toString()) );
    numSamplesPerBandDim = new Dimension(
            this.numSamplesPerBandDimName,
            numSamplesPerBand);

    // Read nominal resolution information
    nominalResolutionAtt = new Attribute( nominalResolutionAttName, headerInfo.get(HeaderInfoTitle.NOMINAL_RESOLUTION.toString()));

    // Read bands per scanlin information.
    bandsPerScanlineAtt = new Attribute( bandsPerScanlineAttName, Integer.valueOf(headerInfo.get(HeaderInfoTitle.BANDS_PER_SCANLINE.toString())));

    // Read bytes per smaple information
    bytesPerSampleAtt = new Attribute( bytesPerSampleAttName, Integer.valueOf(headerInfo.get(HeaderInfoTitle.BYTES_PER_SAMPLE.toString())));

    // Read byte offset for band 1 information.
    byteOffsetBand1Att = new Attribute( byteOffsetBand1AttName, Integer.valueOf(headerInfo.get(HeaderInfoTitle.BYTE_OFFSET_BAND_1.toString())));

    // Read byte offset for band 2 information.
    byteOffsetBand2Att = new Attribute( byteOffsetBand2AttName, Integer.valueOf(headerInfo.get(HeaderInfoTitle.BYTE_OFFSET_BAND_2.toString())));

    // Band 1 description
    band1Att = new Attribute( band1AttName, headerInfo.get( HeaderInfoTitle.BAND_1.toString()));

    // Band 2 description
    band2Att = new Attribute( band2AttName, headerInfo.get( HeaderInfoTitle.BAND_2.toString()));

    // Band organization
    bandOrganizationAtt = new Attribute( bandOrganizationAttName, headerInfo.get(HeaderInfoTitle.ORGANIZATION.toString()));

    // thermal offset
    thermalOffsetAtt = new Attribute( thermalOffsetAttName, headerInfo.get(HeaderInfoTitle.THERMAL_OFFSET.toString()));

    // thermal scale
    thermalScaleAtt = new Attribute( thermalScaleAttName, headerInfo.get(HeaderInfoTitle.THERMAL_SCALE.toString()));

    // percent daylight
    percentDaylightAtt = new Attribute( percentDaylightAttName, Double.valueOf(headerInfo.get(HeaderInfoTitle.PERCENT_DAYLIGHT.toString())));

    // percent full moon
    percentFullMoonAtt = new Attribute( percentFullMoonAttName, Double.valueOf(headerInfo.get(HeaderInfoTitle.PERCENT_FULL_MOON.toString())));

    // percent terminator evident
    percentTerminatorEvidentAtt = new Attribute( percentTerminatorEvidentAttName, Double.valueOf(headerInfo.get(HeaderInfoTitle.PERCENT_TERMINATOR_EVIDENT.toString())));
  }

  /**
   * Parse the sensor information from the header.
   */
  private void handleQCInformation()
  {
    // QC flags
    qcFlagsAtt = new Attribute( qcFlagsAttName, headerInfo.get(HeaderInfoTitle.QC_FLAGS.toString()));
  }

  /**
   * Return a string containing the header name/value pairs.
   *
   * @return a string of the file header name/value pairs.
   */
  protected String headerInfoDump()
  {
    StringBuilder retVal = new StringBuilder( );
    for ( String curHeaderTitle : this.headerInfo.keySet() ) {
      String curHeaderValue = this.headerInfo.get( curHeaderTitle );
      retVal.append( curHeaderTitle );
      retVal.append( ":::::" );
      retVal.append( curHeaderValue );
      retVal.append( ":::::\n" );
    }

    return( retVal.toString() );
  }

  /**
   * Return the header information for this file as a String.
   *
   * @return the string form (NGDC DMSP archive format) of the header information.
   */
  public String toString()
  {
    StringBuilder retVal = new StringBuilder();

    retVal.append( HeaderInfoTitle.FILE_ID.toString() );
    retVal.append( ": ");
    retVal.append( this.fileIdAtt.getStringValue());
    retVal.append( "\n");

    retVal.append( HeaderInfoTitle.DATA_SET_ID.toString() );
    retVal.append( ": " );
    retVal.append( this.datasetIdAtt.getStringValue() );
    retVal.append( "\n" );

    retVal.append( HeaderInfoTitle.RECORD_BYTES.toString() );
    retVal.append( ": " );
    retVal.append( this.recordSizeInBytes );
    retVal.append( "\n" );

    retVal.append( HeaderInfoTitle.NUM_HEADER_RECORDS.toString() );
    retVal.append( ": " );
    retVal.append( this.numHeaderRecords );
    retVal.append( "\n" );

    retVal.append( HeaderInfoTitle.NUM_RECORDS.toString() );
    retVal.append( ": " );
    retVal.append( this.numRecords );
    retVal.append( "\n" );

    retVal.append( HeaderInfoTitle.SUBORBIT_HISTORY.toString() );
    retVal.append( ": " );
    retVal.append( this.suborbitHistoryAtt.getStringValue() );
    retVal.append( "\n" );

    retVal.append( HeaderInfoTitle.PROCESSING_SYSTEM.toString() );
    retVal.append( ": " );
    retVal.append( this.processingSystemAtt.getStringValue() );
    retVal.append( "\n" );

    retVal.append( HeaderInfoTitle.PROCESSING_DATE.toString() );
    retVal.append( ": " );
    retVal.append( DateFormatHandler.ALT_DATE_TIME.getDateTimeStringFromDate( this.processingDate ) );
    retVal.append( "\n" );

    retVal.append( HeaderInfoTitle.SPACECRAFT_ID.toString() );
    retVal.append( ": " );
    retVal.append( this.spacecraftIdAtt.getStringValue() );
    retVal.append( "\n" );

    retVal.append( HeaderInfoTitle.NORAD_ID.toString() );
    retVal.append( ": " );
    retVal.append( this.noradIdAtt.getStringValue() );
    retVal.append( "\n" );

    return ( retVal.toString() );
  }

  /**
   * Class for dealing with date/time formats (from the header and for nc file).
   */
  static class DateFormatHandler
  {
    // Available date format handlers.
    public final static DateFormatHandler ISO_DATE = new DateFormatHandler( "yyyy-MM-dd");
    public final static DateFormatHandler ISO_TIME = new DateFormatHandler( "HH:mm:ss.SSSz" );
    public final static DateFormatHandler ISO_DATE_TIME = new DateFormatHandler( "yyyy-MM-dd\'T\'HH:mm:ss.SSS'Z'" );
    public final static DateFormatHandler ALT_DATE_TIME = new DateFormatHandler( "EEE MMM dd HH:mm:ss yyyy" );

    private String dateTimeFormatString = null;

    private DateFormatHandler( String dateTimeFormatString )
    {
      this.dateTimeFormatString = dateTimeFormatString;
    }

    public String getDateTimeFormatString() { return( this.dateTimeFormatString ); }

    /**
     * Return a java.util.Date given a date string using the date/time format string.
     *
     * @param dateTimeString - date/time string to be used to set java.util.Date.
     * @return The java.util.Date set by the given date/time string.
     */
    public Date getDateFromDateTimeString( String dateTimeString )
            throws ParseException
    {
      Date theDate = null;

      SimpleDateFormat dateFormat = new SimpleDateFormat( this.dateTimeFormatString, Locale.US );
      dateFormat.setTimeZone( TimeZone.getTimeZone( "GMT" ) );

      theDate = dateFormat.parse( dateTimeString );

      return ( theDate );
    }

    /**
     * Return the date/time string that represents the given a java.util.Date
     * in the format of this DataFormatHandler.
     *
     * @param date - the Date to be formatted into a date/time string.
     * @return The date/time string formatted from the given Date.
     */
    public String getDateTimeStringFromDate( Date date )
    {
      SimpleDateFormat dateFormat = new SimpleDateFormat( this.dateTimeFormatString, Locale.US );
      dateFormat.setTimeZone( TimeZone.getTimeZone( "GMT" ) );

      String dateString = dateFormat.format( date );

      return ( dateString );
    }
  }

  /**
   * Contains information about the NGDC DMSP archive file format's header information.
   */
  static class HeaderInfoTitle
  {
    private static java.util.Map<String, HeaderInfoTitle> hash = new java.util.HashMap<>( 20 );

    public final static HeaderInfoTitle FILE_ID = new HeaderInfoTitle( "file ID");                  // /dmsp/moby-1-3/subscriptions/IBAMA/1353226646955.tmp
    public final static HeaderInfoTitle DATA_SET_ID = new HeaderInfoTitle( "data set ID" );              // DMSP F14 OLS LS & TS
    public final static HeaderInfoTitle RECORD_BYTES = new HeaderInfoTitle( "record bytes" );             // 3040
    public final static HeaderInfoTitle NUM_HEADER_RECORDS = new HeaderInfoTitle( "number of header records" ); // 1
    public final static HeaderInfoTitle NUM_RECORDS = new HeaderInfoTitle( "number of records" );        // 692
    public final static HeaderInfoTitle SUBORBIT_HISTORY = new HeaderInfoTitle( "suborbit history" );         // F14200307192230.OIS (1,691)
    public final static HeaderInfoTitle PROCESSING_SYSTEM = new HeaderInfoTitle( "processing system" );        // v2.1b
    public final static HeaderInfoTitle PROCESSING_DATE = new HeaderInfoTitle( "processing date" );          // Sat Jul 19 19:33:23 2003
    public final static HeaderInfoTitle SPACECRAFT_ID = new HeaderInfoTitle( "spacecraft ID" );            // F14
    public final static HeaderInfoTitle NORAD_ID = new HeaderInfoTitle( "NORAD ID" );                 // 24753
    public final static HeaderInfoTitle START_DATE_UTC = new HeaderInfoTitle( "start date UTC" );           // 2003-07-19
    public final static HeaderInfoTitle START_TIME_UTC = new HeaderInfoTitle( "start time UTC" );           // 22:30:31.37112
    public final static HeaderInfoTitle END_DATE_UTC = new HeaderInfoTitle( "end date UTC" );             // 2003-07-19
    public final static HeaderInfoTitle END_TIME_UTC = new HeaderInfoTitle( "end time UTC" );             // 22:35:21.83694
    public final static HeaderInfoTitle START_DATE_LOCAL = new HeaderInfoTitle( "start date local" );         // 2003-07-19
    public final static HeaderInfoTitle START_TIME_LOCAL = new HeaderInfoTitle( "start time local" );         // 19:52:42.03518
    public final static HeaderInfoTitle START_LAT_LON = new HeaderInfoTitle( "start lat,lon" );            // 0.00 320.54
    public final static HeaderInfoTitle END_LAT_LON = new HeaderInfoTitle( "end lat,lon" );              // 16.99 316.69
    public final static HeaderInfoTitle START_SUBSOLAR_COORD = new HeaderInfoTitle( "start sub-solar coord" );    // 20.87 202.37
    public final static HeaderInfoTitle END_SUBSOLAR_COORD = new HeaderInfoTitle( "end sub-solar coord" );      // 20.87 201.16
    public final static HeaderInfoTitle START_LUNAR_COORD = new HeaderInfoTitle( "start lunar coord" );        // UNKNOWN
    public final static HeaderInfoTitle END_LUNAR_COORD = new HeaderInfoTitle( "end lunar coord" );          // UNKNOWN
    public final static HeaderInfoTitle START_DIRECTION = new HeaderInfoTitle("start direction");
    public final static HeaderInfoTitle QA_1_8 = new HeaderInfoTitle("QA 1/8");
    public final static HeaderInfoTitle QA_2_8 = new HeaderInfoTitle("QA 2/8");
    public final static HeaderInfoTitle QA_3_8 = new HeaderInfoTitle("QA 3/8");
    public final static HeaderInfoTitle QA_4_8 = new HeaderInfoTitle("QA 4/8");
    public final static HeaderInfoTitle QA_5_8 = new HeaderInfoTitle("QA 5/8");
    public final static HeaderInfoTitle QA_6_8 = new HeaderInfoTitle("QA 6/8");
    public final static HeaderInfoTitle QA_7_8 = new HeaderInfoTitle("QA 7/8");
    public final static HeaderInfoTitle QA_8_8 = new HeaderInfoTitle("QA 8/8");
    public final static HeaderInfoTitle DELTA_T = new HeaderInfoTitle("delta-t");
    public final static HeaderInfoTitle ASCENDING_NODE = new HeaderInfoTitle( "ascending node" );           // 320.55
    public final static HeaderInfoTitle NODE_HEADING = new HeaderInfoTitle( "node heading" );             // 8.64
    public final static HeaderInfoTitle EPHEMERIS_SOURCE = new HeaderInfoTitle( "ephemeris source" );         // NORAD
    public final static HeaderInfoTitle NUM_DATA_RECORDS = new HeaderInfoTitle( "number of data records" );   // 691
    public final static HeaderInfoTitle NUM_ARTIFICIAL_DATA_RECORDS = new HeaderInfoTitle( "number of artificial data records" ); // 0
    public final static HeaderInfoTitle NOMINAL_RESOLUTION = new HeaderInfoTitle( "nominal resolution" );                // 2.7 km
    public final static HeaderInfoTitle BANDS_PER_SCANLINE = new HeaderInfoTitle( "bands per scanline" );                // 2
    public final static HeaderInfoTitle SAMPLES_PER_BAND = new HeaderInfoTitle( "samples per band" );                  // 1465
    public final static HeaderInfoTitle BYTES_PER_SAMPLE = new HeaderInfoTitle( "bytes per sample" );                  // 1
    public final static HeaderInfoTitle BYTE_OFFSET_BAND_1 = new HeaderInfoTitle( "byte offset band 1" );                // 96
    public final static HeaderInfoTitle BYTE_OFFSET_BAND_2 = new HeaderInfoTitle( "byte offset band 2" );                // 1568
    public final static HeaderInfoTitle BAND_1 = new HeaderInfoTitle( "band 1" );                            // OLS Visible .4-1.1um
    public final static HeaderInfoTitle BAND_2 = new HeaderInfoTitle( "band 2" );                            // OLS Thermal 10.5-12.6um
    public final static HeaderInfoTitle ORGANIZATION = new HeaderInfoTitle( "organization" );                      // band interleaved by line
    public final static HeaderInfoTitle THERMAL_OFFSET = new HeaderInfoTitle( "thermal offset" );                    // 190.00 K
    public final static HeaderInfoTitle THERMAL_SCALE = new HeaderInfoTitle( "thermal scale" );                     // 0.47
    public final static HeaderInfoTitle QC_FLAGS = new HeaderInfoTitle( "QC flags" );                          // 0=not QC'ed  1=artificial  2=bad vis
    public final static HeaderInfoTitle PERCENT_DAYLIGHT = new HeaderInfoTitle( "% daylight" );                        // 0.0
    public final static HeaderInfoTitle PERCENT_FULL_MOON = new HeaderInfoTitle( "% full moon" );                       // 57.8
    public final static HeaderInfoTitle PERCENT_TERMINATOR_EVIDENT = new HeaderInfoTitle( "% terminator evident" );              // 0.0
    public final static HeaderInfoTitle END_HEADER = new HeaderInfoTitle( "end header" );

    private String HeaderInfoTitle;

    private HeaderInfoTitle( String title )
    {
      this.HeaderInfoTitle = title;
      hash.put( title, this );
    }

    /**
     * Find the HeaderInfoTitle that matches this title.
     *
     * @param title
     * @return HeaderInfoTitle or null if no match.
     */
    public static HeaderInfoTitle getTitle( String title )
    {
      if ( title == null )
      {
        return null;
      }
      return hash.get( title );
    }

    /**
     * Return the string title.
     */
    public String toString()
    {
      return HeaderInfoTitle;
    }

  }
}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.nc2.util.xml;

import org.jdom2.Element;
import org.jdom2.JDOMException;
import org.jdom2.input.SAXBuilder;

import java.io.IOException;

/**
 * Miscellaneous XML parsing methods
 * created Jul 3, 2007
 *
 * @author caron
 */
public class Parse {

  /**
   * Read an XML Document from a URL and return the root element.
   *
   * @param location the URL location
   * @return the root element of the Document
   * @throws java.io.IOException on read error
   */
  static public Element readRootElement(String location) throws IOException {
    org.jdom2.Document doc;
    try {
      SAXBuilder builder = new SAXBuilder();
      doc = builder.build(location);
    } catch (JDOMException e) {
      throw new IOException(e.getMessage());
    }

    return doc.getRootElement();
  }

  /**
   * Make sure that text is XML safe
   * @param text check this
   * @return original text if ok, else with bad characters removed
   */
  static public String cleanCharacterData(String text) {
    if (text == null) return null;

    boolean bad = false;
    for (int i = 0, len = text.length(); i < len; i++) {
      int ch = text.charAt(i);
      if (!org.jdom2.Verifier.isXMLCharacter(ch)) {
        bad = true;
        break;
      }
    }

    if (!bad) return text;

    StringBuilder sbuff = new StringBuilder(text.length());
    for (int i = 0, len = text.length(); i < len; i++) {
      int ch = text.charAt(i);
      if (org.jdom2.Verifier.isXMLCharacter(ch))
        sbuff.append((char) ch);
    }
    return sbuff.toString();
  }
}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.nc2.iosp.misc;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ucar.ma2.*;

import ucar.nc2.*;
import ucar.nc2.constants.AxisType;
import ucar.nc2.constants.CDM;
import ucar.nc2.util.CancelTask;

import ucar.unidata.io.RandomAccessFile;
//import ucar.unidata.util.StringUtil;

import java.io.IOException;

import java.nio.ByteBuffer;

import java.text.ParseException;

import java.text.SimpleDateFormat;

import java.util.ArrayList;
import java.util.Date;
import java.util.StringTokenizer;
import java.util.regex.Pattern;

/**
 * IOSP for the USPLN/NAPLN original and extended formats.
 *
 * @author caron, dmurray
 * @see "http://www.unidata.ucar.edu/data/lightning.html"
 */
public class Uspln extends AbstractLightningIOSP {
  static private final Logger logger = LoggerFactory.getLogger(Uspln.class);

  /*
    USPLN/NAPLN data format:

    Each 1 minute packet sent has an ASCII header, followed by a record for
    each lightning detection during the past 1 minute.

    Header
    The ASCII header provides information on the creation time of the one
    minute packet and ending date and time of the file.

    Sample Header:

    (original format)
    LIGHTNING-USPLN1,2004-10-11T20:45:02,2004-10-11T20:45:02

    (extended format)
    LIGHTNING-USPLN1EX,2004-10-11T20:45:02,2004-10-11T20:45:02

    Description:
    Name of Product: LIGHTNING-USPLN1
    Creation of 1 min Packet (yyyy-mm-ddThh:mm:ss): 2004-10-11T20:45:02
    Ending of 1 min Packet (yyyy-mm-ddThh:mm:ss): 2004-10-11T20:45:02

    NOTE: All times in UTC

    Stroke Record Following the header, an individual record is provided for
    each lightning stroke in a comma delimited format.

    Sample Stroke Records (original format):
    2004-10-11T20:44:02,32.6785331,-105.4344587,-96.1,1
    2004-10-11T20:44:05,21.2628231,-86.9596634,53.1,1
    2004-10-11T20:44:05,21.2967119,-86.9702106,50.3,1
    2004-10-11T20:44:06,19.9044769,-100.7082608,43.1,1
    2004-10-11T20:44:11,21.4523434,-82.5202274,-62.8,1
    2004-10-11T20:44:11,21.8155306,-82.6708778,80.9,1

    Sample Stroke Records (extended format):
    2004-10-11T20:44:02,32.6785331,-105.4344587,-96.1,0.5,0.25,0
    2004-10-11T20:44:05,21.2628231,-86.9596634,53.1,0.25,0.25,41
    2004-10-11T20:44:05,21.2967119,-86.9702106,50.3,0.25,0.25,78
    2004-10-11T20:44:06,19.9044769,-100.7082608,43.1,0.75,0.25,-51
    2004-10-11T20:44:11,21.4523434,-82.5202274,-62.8,0.25,0.25,-58
    2004-10-11T20:44:11,21.8155306,-82.6708778,80.9,0.25,0.25,-86

    Description:

    Stroke Date/Time (yyyy-mm-ddThh:mm:ss): 2004-10-11T20:44:02

    Stroke Latitude (deg): 32.6785331
    Stroke Longitude (deg): -105.4344587
    Stroke Amplitude (kAmps, see note below): -96.1

    (original format)
    Stroke Count (number of strokes per flash): 1
    Note: At the present time USPLN data are only provided in stroke format,
    so the stroke count will always be 1.

    (extended format)
    Error Ellipse Major Axis (km): 0.5
    Error Ellipse Minor Axis (km): 0.25
    Error Ellipse Major Axis Orientation (degrees): 0

    Notes about Decoding Stroke Amplitude
    The amplitude field is utilized to indicate the amplitude of strokes and
    polarity of strokes for all Cloud-to- Ground Strokes.

    For other types of detections this field is utilized to provide
    information on the type of stroke detected.

    The amplitude number for Cloud-to-Ground strokes provides the amplitude
    of the stroke and the sign (+/-) provides the polarity of the stroke.

    An amplitude of 0 indicates USPLN Cloud Flash Detections rather than
    Cloud-to-Ground detections.

    Cloud flash detections include cloud-to-cloud, cloud-to-air, and
    intra-cloud flashes.

    An amplitude of -999 or 999 indicates a valid cloud-to-ground stroke
    detection in which an amplitude was not able to be determined. Typically
    these are long-range detections.
  */

  /**
   * Magic string for determining if this is my type of file.
   */
  private static final String MAGIC = "LIGHTNING-.*(P|G)LN1";
  private Pattern pMAGIC = Pattern.compile(MAGIC);

  /**
   * Magic string for determining if this is my type of file.
   */
  private static final String MAGIC_OLD = "..PLN-LIGHTNING";
  private Pattern pMAGIC_OLD = Pattern.compile(MAGIC_OLD);

  /**
   * Magic string for determining if this is and extended type of file.
   */
  private static final String MAGIC_EX = ".*(GLN1|PLN1EX).*";
  private Pattern pMAGIC_EX = Pattern.compile(MAGIC_EX);

  /**
   * original time format
   */
  private static final String TIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ss";

  /**
   * extended time format
   */
  private static final String TIME_FORMAT_EX = "yyyy-MM-dd'T'HH:mm:ss.SSS";

  /**
   * is this extended data
   */
  private boolean isExtended = false;

  /**
   * offsets int the file
   */
  private long[] offsets;

  /**
   * max/min latitude
   */
  private double lat_min, lat_max;

  /**
   * max/min longitude
   */
  private double lon_min, lon_max;

  /**
   * max/min time
   */
  private double time_min, time_max;

  /**
   * The structure members
   */
  private StructureMembers sm;

  /**
   * the date format
   */
  private SimpleDateFormat isoDateFormat = null;

  /**
   * Check if this is a valid file for this IOServiceProvider.
   *
   * @param raf RandomAccessFile
   * @return true if valid.
   * @throws IOException if read error
   */
  public boolean isValidFile(RandomAccessFile raf) throws IOException {
    raf.seek(0);
    int n = MAGIC.length();
    if (raf.length() < n) {
      return false;
    }

    String got = raf.readString(n);

    return (pMAGIC.matcher(got).find() || pMAGIC_OLD.matcher(got).find());
  }

  /**
   * Open existing file, and populate ncfile with it. This method is only
   * called by the NetcdfFile constructor on itself. The provided NetcdfFile
   * object will be empty except for the location String and the
   * IOServiceProvider associated with this NetcdfFile object.
   *
   * @param raf        the file to work on, it has already passed the
   *                   isValidFile() test.
   * @param ncfile     add objects to this empty NetcdfFile
   * @param cancelTask used to monitor user cancellation; may be null.
   * @throws IOException if read error
   */
  public void open(RandomAccessFile raf, NetcdfFile ncfile, CancelTask cancelTask) throws IOException {
    super.open(raf, ncfile, cancelTask);

    isExtended = checkFormat();
    isoDateFormat = new SimpleDateFormat();
    isoDateFormat.setTimeZone(java.util.TimeZone.getTimeZone("GMT"));
    isoDateFormat.applyPattern(isExtended
            ? TIME_FORMAT_EX
            : TIME_FORMAT);
    Sequence seq = makeSequence(ncfile);
    ncfile.addVariable(null, seq);
    addLightningGlobalAttributes(ncfile);
    ncfile.finish();
    sm = seq.makeStructureMembers();
    ArrayStructureBB.setOffsets(sm);
  }

  protected Sequence makeSequence(NetcdfFile ncfile) {

    Sequence seq = new Sequence(ncfile, null, null, RECORD);

    Variable v = makeLightningVariable(ncfile, null, seq, TIME,
            DataType.DOUBLE, "",
            "time of stroke", null,
            secondsSince1970, AxisType.Time);
    seq.addMemberVariable(v);

    v = makeLightningVariable(ncfile, null, seq, LAT, DataType.DOUBLE,
            "", "latitude", "latitude", CDM.LAT_UNITS, AxisType.Lat);
    seq.addMemberVariable(v);

    v = makeLightningVariable(ncfile, null, seq, LON, DataType.DOUBLE,
            "", "longitude", "longitude",
            CDM.LON_UNITS, AxisType.Lon);
    seq.addMemberVariable(v);

    v = makeLightningVariable(ncfile, null, seq, SIGNAL, DataType.FLOAT,
            "", "signed peak amplitude (signal strength)", null, "kAmps", null);
    v.addAttribute(new Attribute(CDM.MISSING_VALUE, new Double(999)));
    seq.addMemberVariable(v);

    if (isExtended) {  // extended
      v = makeLightningVariable(ncfile, null, seq, MAJOR_AXIS,
              DataType.FLOAT, "",
              "error ellipse semi-major axis", null,
              "km", null);
      seq.addMemberVariable(v);

      v = makeLightningVariable(ncfile, null, seq, MINOR_AXIS,
              DataType.FLOAT, "",
              "error ellipse minor axis ", null,
              "km", null);
      seq.addMemberVariable(v);

      v = makeLightningVariable(
              ncfile, null, seq, ELLIPSE_ANGLE, DataType.INT, "",
              "error ellipse axis angle of orientation ", null, "degrees",
              null);
      seq.addMemberVariable(v);

    } else {  // original format
      v = makeLightningVariable(ncfile, null, seq, MULTIPLICITY,
              DataType.INT, "",
              "multiplicity [#strokes per flash]",
              null, "", null);
      seq.addMemberVariable(v);
    }
    return seq;
  }

  /**
   * Add the global attributes.
   *
   * @param ncfile the file to add to
   */
  protected void addLightningGlobalAttributes(NetcdfFile ncfile) {
    super.addLightningGlobalAttributes(ncfile);
    ncfile.addAttribute(null,
            new Attribute("title", "USPLN Lightning Data"));
    ncfile.addAttribute(null,
            new Attribute("file_format",
                    "USPLN1 " + (isExtended
                            ? "(extended)"
                            : "(original)")));

    /*
    ncfile.addAttribute(null,
                        new Attribute("time_coverage_start",
                                      time_min + " " + secondsSince1970));
    ncfile.addAttribute(null,
                        new Attribute("time_coverage_end",
                                      time_max + " " + secondsSince1970));

    ncfile.addAttribute(null,
                        new Attribute("geospatial_lat_min",
                                      new Double(lat_min)));
    ncfile.addAttribute(null,
                        new Attribute("geospatial_lat_max",
                                      new Double(lat_max)));

    ncfile.addAttribute(null,
                        new Attribute("geospatial_lon_min",
                                      new Double(lon_min)));
    ncfile.addAttribute(null,
                        new Attribute("geospatial_lon_max",
                                      new Double(lon_max)));
    */

  }

  /**
   * Read all the data and return the number of strokes
   *
   * @return true if extended format
   * @throws IOException if read error
   */
  private boolean checkFormat() throws IOException {

    raf.seek(0);
    boolean extended = false;
    while (true) {
      long offset = raf.getFilePointer();
      String line = raf.readLine();
      if (line == null) {
        break;
      }
      if (pMAGIC.matcher(line).find() || pMAGIC_OLD.matcher(line).find()) {
        extended = pMAGIC_EX.matcher(line).find();
        break;
      }
    }
    return extended;
  }

  /**
   * Read all the data and return the number of strokes
   *
   * @param raf the file to read
   * @return the number of strokes
   * @throws IOException           if read error
   * @throws NumberFormatException if problem parsing data
   * @throws ParseException        if parse problem
   */
  int readAllData(RandomAccessFile raf)
          throws IOException, NumberFormatException, ParseException {
    ArrayList offsetList = new ArrayList();

    java.text.SimpleDateFormat isoDateTimeFormat =
            new java.text.SimpleDateFormat(TIME_FORMAT);
    isoDateTimeFormat.setTimeZone(java.util.TimeZone.getTimeZone("GMT"));

    lat_min = 1000.0;
    lat_max = -1000.0;
    lon_min = 1000.0;
    lon_max = -1000.0;
    time_min = Double.POSITIVE_INFINITY;
    time_max = Double.NEGATIVE_INFINITY;

    raf.seek(0);
    int count = 0;
    boolean knowExtended = false;
    while (true) {
      long offset = raf.getFilePointer();
      String line = raf.readLine();
      if (line == null) {
        break;
      }
      if (pMAGIC.matcher(line).find() || pMAGIC_OLD.matcher(line).find()) {
        if (!knowExtended) {
          isExtended = pMAGIC_EX.matcher(line).find();
          if (isExtended) {
            isoDateTimeFormat.applyPattern(TIME_FORMAT_EX);
          }
          knowExtended = true;
        }
        continue;
      }

      // 2006-10-23T17:59:39,18.415434,-93.480526,-26.8,1             (original)
      // 2006-10-23T17:59:39,18.415434,-93.480526,-26.8,0.25,0.5,80   (extended)

      StringTokenizer stoker = new StringTokenizer(line, ",\r\n");
      while (stoker.hasMoreTokens()) {
        Date date = isoDateTimeFormat.parse(stoker.nextToken());
        double lat = Double.parseDouble(stoker.nextToken());
        double lon = Double.parseDouble(stoker.nextToken());
        double amp = Double.parseDouble(stoker.nextToken());
        int nstrokes = 1;
        double axisMaj = Double.NaN;
        double axisMin = Double.NaN;
        int orient = 0;
        if (isExtended) {
          axisMaj = Double.parseDouble(stoker.nextToken());
          axisMin = Double.parseDouble(stoker.nextToken());
          orient = Integer.parseInt(stoker.nextToken());
        } else {
          nstrokes = Integer.parseInt(stoker.nextToken());
        }

        Stroke s = isExtended
                ? new Stroke(date, lat, lon, amp, axisMaj,
                axisMin, orient)
                : new Stroke(date, lat, lon, amp, nstrokes);
        lat_min = Math.min(lat_min, s.lat);
        lat_max = Math.max(lat_max, s.lat);
        lon_min = Math.min(lon_min, s.lon);
        lon_max = Math.max(lon_max, s.lon);
        time_min = Math.min(time_min, s.secs);
        time_max = Math.max(time_max, s.secs);
      }

      offsetList.add(new Long(offset));
      count++;
    }

    offsets = new long[count];
    for (int i = 0; i < offsetList.size(); i++) {
      Long off = (Long) offsetList.get(i);
      offsets[i] = off.longValue();
    }

    //System.out.println("processed " + count + " records");
    return count;
  }

  /**
   * Read data from a top level Variable and return a memory resident Array.
   * This Array has the same element type as the Variable, and the requested shape.
   *
   * @param v2      a top-level Variable
   * @param section the section of data to read.
   *                There must be a Range for each Dimension in the variable, in order.
   *                Note: no nulls allowed. IOSP may not modify.
   * @return the requested data in a memory-resident Array
   * @throws IOException           if read error
   * @throws InvalidRangeException if invalid section
   * @see ucar.ma2.Range
   */
  public Array readData(Variable v2, Section section)
          throws IOException, InvalidRangeException {
    return new ArraySequence(sm, getStructureIterator(null, 0), nelems);
  }

  private int nelems = -1;

  /**
   * Get the structure iterator
   *
   * @param s          the Structure
   * @param bufferSize the buffersize
   * @return the data iterator
   * @throws java.io.IOException if problem reading data
   */
  public StructureDataIterator getStructureIterator(Structure s,
                                                    int bufferSize)
          throws java.io.IOException {
    return new UsplnSeqIter();
  }

  /**
   * Uspln Sequence Iterator
   *
   * @author Unidata Development Team
   */
  private class UsplnSeqIter implements StructureDataIterator {

    /**
     * the wrapped asbb
     */
    private ArrayStructureBB asbb = null;

    /**
     * number read
     */
    int numFlashes = 0;

    /**
     * Create a new one
     *
     * @throws IOException problem reading the file
     */
    UsplnSeqIter() throws IOException {
      raf.seek(0);
    }

    @Override
    public StructureDataIterator reset() {
      numFlashes = 0;
      try {
        raf.seek(0);
      } catch (IOException e) {
        throw new RuntimeException(e);
      }
      return this;
    }

    @Override
    public boolean hasNext() throws IOException {
      return readStroke();
    }

    @Override
    public StructureData next() throws IOException {
      numFlashes++;
      return asbb.getStructureData(0);
    }

    /**
     * Read the header
     *
     * @return true if okay
     * @throws IOException problem reading file
     */
    private boolean readStroke() throws IOException {
      String line = raf.readLine();
      if (line == null) {
        nelems = numFlashes; // track this for next time
        return false;
      }
      if (pMAGIC.matcher(line).find() || pMAGIC_OLD.matcher(line).find()) {
        return readStroke();
      }

      // 2006-10-23T17:59:39,18.415434,-93.480526,-26.8,1             (original)
      // 2006-10-23T17:59:39,18.415434,-93.480526,-26.8,0.25,0.5,80   (extended)
      Stroke stroke = null;

      StringTokenizer stoker = new StringTokenizer(line, ",\r\n");
      try {
        while (stoker.hasMoreTokens()) {
          Date date = isoDateFormat.parse(stoker.nextToken());
          double lat = Double.parseDouble(stoker.nextToken());
          double lon = Double.parseDouble(stoker.nextToken());
          double amp = Double.parseDouble(stoker.nextToken());
          int nstrokes = 1;
          double axisMaj = Double.NaN;
          double axisMin = Double.NaN;
          int orient = 0;
          if (isExtended) {
            axisMaj = Double.parseDouble(stoker.nextToken());
            axisMin = Double.parseDouble(stoker.nextToken());
            orient = Integer.parseInt(stoker.nextToken());
          } else {
            nstrokes = Integer.parseInt(stoker.nextToken());
          }

          stroke = isExtended
                  ? new Stroke(date, lat, lon, amp, axisMaj,
                  axisMin, orient)
                  : new Stroke(date, lat, lon, amp, nstrokes);
        }
      } catch (Exception e) {
        logger.error("bad header: {}", line.trim());
        return false;
      }

      if (stroke == null) {
        logger.error("bad header: {}", line.trim());
        return false;
      }

      byte[] data = new byte[sm.getStructureSize()];
      ByteBuffer bbdata = ByteBuffer.wrap(data);
      for (String mbrName : sm.getMemberNames()) {
        switch (mbrName) {
          case TIME:
            bbdata.putDouble(stroke.secs);
            break;
          case LAT:
            bbdata.putDouble(stroke.lat);
            break;
          case LON:
            bbdata.putDouble(stroke.lon);
            break;
          case SIGNAL:
            bbdata.putFloat((float) stroke.amp);
            break;
          case MULTIPLICITY:
            bbdata.putInt(stroke.n);
            break;
          case MAJOR_AXIS:
            bbdata.putFloat((float) stroke.axisMajor);
            break;
          case MINOR_AXIS:
            bbdata.putFloat((float) stroke.axisMinor);
            break;
          case ELLIPSE_ANGLE:
            bbdata.putInt(stroke.axisOrient);
            break;
        }
      }
      asbb = new ArrayStructureBB(sm, new int[]{1}, bbdata, 0);
      return true;
    }

    @Override
    public int getCurrentRecno() {
      return numFlashes - 1;
    }
  }

  /**
   * Get a unique id for this file type.
   *
   * @return registered id of the file type
   * @see "http://www.unidata.ucar.edu/software/netcdf-java/formats/FileTypes.html"
   */
  public String getFileTypeId() {
    return "USPLN";
  }

  /**
   * Get a human-readable description for this file type.
   *
   * @return description of the file type
   * @see "http://www.unidata.ucar.edu/software/netcdf-java/formats/FileTypes.html"
   */
  public String getFileTypeDescription() {
    return "US Precision Lightning Network";
  }

  /**
   * Get the version of this file type.
   *
   * @return version of the file type
   * @see "http://www.unidata.ucar.edu/software/netcdf-java/formats/FileTypes.html"
   */
  public String getFileTypeVersion() {
    return "1";
  }

  /**
   * Class to hold stroke information
   */
  private class Stroke {

    /**
     * the date as seconds since the epoch
     */
    double secs;

    /**
     * lat, lon and amplitude
     */
    double lat, lon, amp;

    /**
     * number of strokes
     */
    int n = 1;

    /**
     * major axis
     */
    double axisMajor;

    /**
     * minor axis
     */
    double axisMinor;

    /**
     * major axis orientation
     */
    int axisOrient;

    /**
     * Create a Stroke from the file data and time formatter
     *
     * @param offset            offset into the file
     * @param isoDateTimeFormat the time formatter
     * @throws IOException    problem reading the file
     * @throws ParseException problem parsing the file
     */
    Stroke(long offset, java.text.SimpleDateFormat isoDateTimeFormat)
            throws IOException, ParseException {
      raf.seek(offset);
      String line = raf.readLine();
      if ((line == null) || pMAGIC.matcher(line).find() || pMAGIC_OLD.matcher(line).find()) {
        throw new IllegalStateException();
      }

      StringTokenizer stoker = new StringTokenizer(line, ",\r\n");
      Date date =
              isoDateTimeFormat.parse(stoker.nextToken());
      makeSecs(date);
      lat = Double.parseDouble(stoker.nextToken());
      lon = Double.parseDouble(stoker.nextToken());
      amp = Double.parseDouble(stoker.nextToken());
      if (isExtended) {
        n = 1;
        axisMajor = Double.parseDouble(stoker.nextToken());
        axisMinor = Double.parseDouble(stoker.nextToken());
        axisOrient = Integer.parseInt(stoker.nextToken());
      } else {
        n = Integer.parseInt(stoker.nextToken());
      }

    }

    /**
     * Create a stroke from the info
     *
     * @param date The Date
     * @param lat  the latitude
     * @param lon  the longitude
     * @param amp  the amplitude
     * @param n    the number of strokes
     */
    Stroke(Date date, double lat, double lon, double amp, int n) {
      makeSecs(date);
      this.lat = lat;
      this.lon = lon;
      this.amp = amp;
      this.n = n;
    }

    /**
     * Create a stroke from the info
     *
     * @param date   The Date
     * @param lat    the latitude
     * @param lon    the longitude
     * @param amp    the amplitude
     * @param maj    error ellipse major axis
     * @param min    error ellipse minor axis
     * @param orient orientation of the major axis
     */
    Stroke(Date date, double lat, double lon, double amp, double maj,
           double min, int orient) {
      makeSecs(date);
      this.lat = lat;
      this.lon = lon;
      this.amp = amp;
      this.axisMajor = maj;
      this.axisMinor = min;
      this.axisOrient = orient;
    }

    /**
     * Make the date variable
     *
     * @param date the date
     */
    void makeSecs(Date date) {
      this.secs = (int) (date.getTime() / 1000);
    }

    /**
     * Get a String representation of this object
     *
     * @return the String representation
     */
    public String toString() {
      return (isExtended)
              ? lat + " " + lon + " " + amp + " " + axisMajor + "/"
              + axisMinor + " " + axisOrient
              : lat + " " + lon + " " + amp + " " + n;
    }

  }

}