#
# Changes
#
# 2022-09, Arjo Segers
#   Write files with zlib-compression, option to disable this.
#   Write files with variable packing (short integers and add_offset and scale_factor attributes), option to disable this.
#
# 2022-09, Arjo Segers
#   Fixed computation of packing parameters for array with constant value.
#
# 2023-08, Arjo Segers
#   Added "method" argument to CSO_Listing.Select().
#
# 2023, Lewis Blake, Arjo Segers
#   Formatted using "black".
#
# 2023-11, Arjo Segers
#   Added "CheckDir" method.
#
# 2024-01, Arjo Segers
#   Do not pack coordinates. Updated comment.
#

########################################################################
###
### help
###
########################################################################

"""
*******************
``cso_file`` module
*******************

Create and access file with satellite data extract.


Methods
=======

The following methods are defined:

* :py:meth:`CheckDir`
* :py:meth:`Pack_DataArray`


Class hierchy
=============

The classes are defined according to the following hierchy:

* :py:class:`.CSO_File`
* :py:class:`.CSO_Listing`


Methods and classes
===================
"""

# modules:
import logging


########################################################################
###
### tools
###
########################################################################


def CheckDir(filename):

    """
    Check if ``filename`` has a directory path;
    if so, create that directory if it does not exist yet.
    """

    # modules:
    import os

    # directory name, could be empty:
    dname = os.path.dirname(filename)
    # directory defined?
    if len(dname) > 0:
        # not present yet?
        if not os.path.isdir(dname):
            # create including subdirs:
            os.makedirs(dname)
        # endif # dname present
    # endif # dname defined


# enddef CheckDir


# *


def Pack_DataArray(da, dtype="i2"):

    """
    Set encoding parameters of :py:class:`xarray.DataArray` object ``da``;
    on return, the argument is changed in-place.

    Argument ``dtype`` defines the data type in which the array will be written;
    by default this is a short integer (``i2``, 2-bytes) instead of a float.
    Attributes are added that define how to decode values::

        short pressure(z) ;
		    pressure:add_offset   = 53257.7493494103 ;
		    pressure:scale_factor = 1.50569706659902 ;
		    pressure:_FillValue   = 32767s ;

    The actual values are recalculated using the formula::

      values = add_offset + scale_factor * values__short

    The packing parameters are computed using::

      scale_factor = (vmax - vmin)/( fmax - fmin)
      add_offset   = vmin - scale_factor * fmin

    where ``[vmin,vmax]`` is the range of input values,
    and ``[fmin,fmax]`` the range of possible values of the stored data type.

    The ``scale_factor`` should not become 0.0 since it is used for division during encoding.
    This will be the case when the input has the same value in all elements;
    in that case, the following parameters are used that will lead to zero packed values::

      scale_factor = 1.0
      add_offset   = vmin

    """

    # modules:
    import numpy

    # only floats ...
    if da.dtype in [numpy.float32, numpy.float64]:
        # value range, ignore nan's:
        vmin = numpy.nanmin(da.values)
        vmax = numpy.nanmax(da.values)
        # target data type could be integer or float:
        if dtype.startswith("i"):
            # use absolute minimum to represent nans:
            fill_value = numpy.iinfo(dtype).min
            # valid range outside nan:
            fmin = numpy.iinfo(dtype).min + 1
            fmax = numpy.iinfo(dtype).max
        else:
            logger.error('could not pack variables as "%s" type yet' % dtype)
            raise Exception
        # endif
        # packing parameters;
        # scale_factor should not be 0.0 since used for division during encoding,
        # therefore check if there is a value range at all ..
        if vmin < vmax:
            # range of values:
            scale_factor = (vmax - vmin) / (fmax - fmin)
            add_offset = vmin - scale_factor * fmin
        else:
            # single value only; set offset to this value, packed values will become 0:
            scale_factor = 1.0
            add_offset = vmin
        # endif
        # store type and packing parameters:
        da.encoding["dtype"] = dtype
        da.encoding["add_offset"] = add_offset
        da.encoding["scale_factor"] = scale_factor
        # store new fill value:
        da.encoding["_FillValue"] = fill_value
    # endif # floats


# enddef Pack_DataArray


########################################################################
###
### satellite extract file
###
########################################################################


class CSO_File(object):

    """
    Storage for satelite data extract.

    If optional ``filename`` is provided, the :py:meth:`.Read` method is
    called to read data from a file. If the optional ``varname`` list
    is specified too, only selected variables will be read.

    To create a new track, initialize without arguments and add
    new variables using the :py:meth:`.AddVariable` method.
    Only certain dimension names and combinations are supported.

    """

    def __init__(self, filename=None):

        """
        Initialize extract file.
        """

        # modules:
        import os
        import xarray

        # read?
        if filename is not None:

            # store:
            self.filename = filename

            # check ...
            if not os.path.isfile(self.filename):
                logging.error("file not found: %s" % self.filename)
                raise Exception
            # endif

            # access dataset:
            with xarray.open_dataset(self.filename) as self.ds:
                # load entire file:
                self.ds.load()
            # endwith # xarray

        else:

            # dummy:
            self.filename = "None"
            # init empty set:
            self.ds = xarray.Dataset()

        # endif  # filename provided

    # enddef __init__

    # *

    def Close(self):

        """
        Close dataset.
        """

        # close:
        self.ds.close()

    # enddef Close

    # *

    def AddDim(self, dname, length):

        """
        Add dimension.

        Arguments:

        * ``dname`` : (``str``) dimension name to be created
        * ``len`` : (``int``) dimension length

        """

        # store:
        self.ds.dims[dname] = length

    # enddef AddDim

    # *

    def GetDimLength(self, dname):

        """
        Return dimension length.
        """

        # check ..
        if dname not in self.ds.dims.keys():
            logging.error('no dimension "%s" defined' % dname)
            raise Exception
        # endif
        # get:
        return self.ds.dims[dname]

    # enddef GetDimLength

    # *

    def GetShape(self, dnames):

        """
        Return shape following specified dimension names.
        """

        # init result:
        shp = ()
        # loop:
        for dname in dnames:
            # add length:
            shp = shp + (self.GetDimLength(dname),)
        # endfor

        # ok
        return shp

    # enddef GetShape

    # *

    def AddCoord(self, cname, da):

        """
        Add coordinate variable.

        Arguments:

        * ``cname`` : coordinate name to be created
        * ``da`` : :py:class:`xarray.DataArray` object

        """

        # store:
        self.ds.coords[cname] = da

    # enddef AddCoord

    # *

    def GetCoords(self, cnames):

        """
        Return dict with coordinate variables as specified in ``cnames``.
        """

        # storage:
        coords = {}
        # loop:
        for cname in cnames:
            # check ...
            if cname not in self.ds.coords.keys():
                logging.error('coord "%s" not found' % cname)
                raise Exception
            # endif
            # copy:
            coords[cname] = self.ds.coords[cname]
        # endfor # cname

        # ok
        return coords

    # enddef GetCoords

    # *

    def AddTimeCoord(self, times, bounds=None):

        """
        Add time coordinate variable.

        Arguments:

        * ``times`` : time stamp series created with for example :py:meth:`pandas.date_range` method

        Optional arguments::

        * ``bounds=(times1,times2)`` : tupple or list with two timeseries that define the time bounds

        """

        # modules:
        import xarray
        import numpy

        # coordinate name:
        cname = "time"
        bname = cname + "_bounds"

        # time units:
        tunits = "seconds since %i-01-01 00:00:00" % times[0].year

        # init time attributes:
        attrs = {"standard_name": "time"}

        # bounds defined?
        if bounds is not None:
            # store bounds:
            self.ds[bname] = xarray.DataArray(numpy.array(bounds).T, dims=(cname, "nv"))
            # ensure output with correct units:
            self.ds[bname].encoding["units"] = tunits
            # write as float, no fill-value needed:
            self.ds[bname].encoding["dtype"] = "float64"
            self.ds[bname].encoding["_FillValue"] = None
            # store name for 'time' attributes:
            attrs["bounds"] = bname
        # endif

        # store time coordinate:
        self.ds.coords[cname] = xarray.DataArray(times, dims=(cname,), attrs=attrs)
        # ensure output with correct units:
        self.ds.coords[cname].encoding["units"] = tunits
        # write as float, no fill-value needed:
        self.ds.coords[cname].encoding["dtype"] = "float64"
        self.ds.coords[cname].encoding["_FillValue"] = None

    # enddef AddTimeCoord

    # *

    def AddVariable(self, vname, da):

        """
        Add variable field.

        Arguments:

        * ``vname`` : variable name to be created
        * ``da`` : :py:class:`xarray.DataArray` object

        """

        # store:
        self.ds[vname] = da

    # enddef AddVariable

    # *

    def AddDataVariable(self, vname, dnames, dtype="f4", attrs={}, values=None):

        """
        Add variable with specified name and dimensions.

        Arguments:

        * ``vname`` : variable name to be created
        * ``dnames`` : dimension names, e.g. ``('time','station')``

        Optional arguments:

        * ``dtype`` : datatype keyword
        * ``attrs`` : dict with variable attributes
        * ``values`` : array with data values
        """

        # modules:
        import xarray
        import numpy

        # check ...
        if "units" not in attrs.keys():
            logging.error('attributes for output variable "%s" have no "units"' % (vname))
            raise Exception
        # endif
        # check ...
        if ("long_name" not in attrs.keys()) and ("standard_name" not in attrs.keys()):
            logging.error(
                'attributes for output variable "%s" have neither "long_name" nor "standard_name"'
                % (vname)
            )
            raise Exception
        # endif

        # coordinates:
        coords = self.GetCoords(dnames)
        # no values provided? then create empty arrya:
        if values is None:
            # shape:
            shp = self.GetShape(dnames)
            # create:
            values = numpy.full(shp, numpy.nan, dtype=dtype)
        # endif
        # create variable (filled with no data?):
        self.ds[vname] = xarray.DataArray(values, dims=dnames, coords=coords, attrs=attrs)

    # enddef AddDataVariable

    # *

    def Write(
        self, filename, unlimited_dims=None, attrs=None, history=[], packed=True, complevel=1
    ):

        """
        Write data to provided ``filename``.

        Optional arguments:

        * ``attrs`` : dictionairy with global attributes
        * ``history`` : list of str values that describe how the content was created,
          this will be added to the global ``history`` attribute
        * ``unlimited_dims`` : names of unlimited dimesions

        Optional arguments to disable file size reduction measures:

        * ``packed`` : by default float arrays are written as short-integers accompanied with
          ``add_offset`` and ``scale_factor`` attributes; set to ``False`` to keep original data types.
        * ``complevel`` : zlib compression level, default 1 (of 9) to have decent compression without too much overhead;
          set to 0 to have no compression at all

        The following example shows the impact of packing and compression on file sizes;
        in this exmple, the combination of packing and deflation reduces file sizes
        with 83%%::

              301.009.424  S5p_RPRO_GLYOX_01130.nc
              111.648.738  S5p_RPRO_GLYOX_01130__complevel1.nc
              104.134.232  S5p_RPRO_GLYOX_01130__complevel9.nc
              153.334.891  S5p_RPRO_GLYOX_01130__packed.nc
               41.020.139  S5p_RPRO_GLYOX_01130__packed_complevel1.nc

        """

        # modules:
        import os
        import datetime
        import numpy

        # destination filename:
        if filename == None:
            filename = self.filename

        # create directory if necessary:
        dname = os.path.dirname(filename)
        if (len(dname) > 0) and (not os.path.isdir(dname)):
            os.makedirs(dname)

        # store global attributes:
        if attrs is not None:
            self.ds.attrs.update(attrs)

        # extend history:
        if len(history) > 0:
            # time step as "Mon 1 Jan 01:01:01 2001"
            tstamp = datetime.datetime.today().strftime("%a %b %d %H:%M:%S %Y")
            # create line:
            histline = str(tstamp)
            for hist in history:
                histline = histline + ", " + hist
            # endfor
            # extend?
            if "history" in self.ds.attrs.keys():
                self.ds.attrs["history"] = histline + "\n" + self.ds.attrs["history"]
            else:
                self.ds.attrs["history"] = histline
            # endif
        # endif

        # loop over coordinates:
        for cname in self.ds.coords.keys():
            # no fill value needed:
            self.ds.coords[cname].encoding["_FillValue"] = None
        # endfor

        # pack?
        if packed:
            # ... do not pack coordinates ...
            ## loop over coordinates:
            # for cname in self.ds.coords.keys():
            #    # set packing parameters:
            #    Pack_DataArray(self.ds[cname])
            ##endfor # variables
            # loop over variables:
            for vname in self.ds.keys():
                # set packing parameters:
                Pack_DataArray(self.ds[vname])
            # endfor # variables
        # endif # pack

        # set compression?
        if complevel > 0:
            # loop over variables:
            for vname in self.ds.keys():
                # enable zlib compression and set level:
                self.ds[vname].encoding["zlib"] = True
                self.ds[vname].encoding["complevel"] = complevel
            # endfor # variables
        # endif # comppress

        # write:
        self.ds.to_netcdf(filename, unlimited_dims=unlimited_dims)

    # enddef Write

    # *

    def GetDims(self, varname):

        """
        Return variable dimension names.
        """

        # check ...
        if varname not in self.ds.keys():
            logging.error('variable "%s" not available' % varname)
            if self.filename is not None:
                logging.error("filename: %s" % self.filename)
            raise Exeption
        # endif

        # ok
        return self.ds[varname].dims

    # enddef GetDims

    # *

    def GetData(self, varname):

        """
        Return data values and units for named variable.
        """

        # check ...
        if varname not in self.ds.keys():
            logging.error('variable "%s" not available' % varname)
            if self.filename is not None:
                logging.error("filename: %s" % self.filename)
            raise Exeption
        # endif
        # check ..
        if "units" not in self.ds[varname].attrs.keys():
            logging.error('no attribute "units" with varible "%s"' % varname)
            if self.filename is not None:
                logging.error("filename: %s" % self.filename)
            raise Exeption
        # endif

        # ok
        return self.ds[varname].values, self.ds[varname].attrs["units"]

    # enddef GetData

    # *

    def GetTrack(self, da):

        """
        Extract variable on track grid.

        Arguments:

        * ``da`` : :py:class:`xarray.DataArray` or variable name

        Return values:

        * ``xx``  : corner longitudes (nimage+1,npixel+1)
        * ``yy``  : corner latitudes (nimage+1,npixel+1)
        * ``values`` : masked array with values at pixel locations (nimage,npixel)
        * ``units`` : str units

        The corner locations are created using the following variables that are
        expected to be present:

        * ``track_longitude_bounds`` : 3D array of longitudes with dimensions ``('scan','pixel','corner')``
        * ``track_latitude_bounds``  : idem for latitudes
        * ``pixel`` : compress index that defines the ``('scan','pixel')`` location

        Example usage to obtain retrieval from *data* file that includes
        the track information::

            xx,yy,vcd_retr,vcd_units = dfile.GetTrack( 'vcd' )

        Example usage for simulated retrieval from *state* file::

            xx,yy,vcd_retr,vcd_units = dfile.GetTrack( sfile.ds['vcd'] )

        """

        # modules:
        import numpy

        # varialble:
        if type(da) == str:
            # should be variable in this fill:
            if da not in self.ds.keys():
                logging.error('variable "%s" not available in data set, available are:' % (da))
                logging.error(str(self.ds.keys()))
                raise Exception
            # endif
            # extract:
            da = self.ds[da]
        # endfor

        # variable dims:
        vdims = da.dims
        # check  ..
        if vdims[0] != "pixel":
            logging.error(
                "data array should have `pixel` as first dimenion, found: %s" % str(vdims)
            )
            raise Exception
        # endif

        # loop over required variables:
        for vname in ["track_longitude_bounds", "track_latitude_bounds"]:
            # check ...
            if vname not in self.ds.keys():
                logging.error('variable "%s" not available in data set' % (vname))
                raise Exception
            # endif
        # endfor

        # shape of track grid:
        nscan, npixel, ncorner = self.ds["track_longitude_bounds"].shape

        # init 2D corner fields:
        xx = numpy.zeros((nscan + 1, npixel + 1), float)
        yy = numpy.zeros((nscan + 1, npixel + 1), float)
        # no idea yet how to fill them correctly automatically,
        # for the moment assume the (converted!) OMI order:
        #
        #      ^ scan
        #       \        _ - o 2
        #         3 o -      \
        #                _ - o 1  ---> pixel
        #          0 o -
        #
        # figure of first pixel footprints to determine the order ..
        if False:
            # target file:
            figfile = "test-corners.png"
            # info ...
            logging.info("TESTING - create %s ..." % figfile)
            # modules:
            import matplotlib.pyplot as plt

            # new figure:
            fig = plt.figure()
            # new axis:
            ax = fig.add_axes([0.1, 0.1, 0.8, 0.8])
            # raster:
            ax.plot(
                self.ds["track_longitude_bounds"][0, 0, :],
                self.ds["track_latitude_bounds"][0, 0, :],
                "r-o",
                label="scan 0, pixel 0",
            )
            ax.plot(
                self.ds["track_longitude_bounds"][0, 0, 0],
                self.ds["track_latitude_bounds"][0, 0, 0],
                "rs",
                markersize=10,
            )
            ax.plot(
                self.ds["track_longitude_bounds"][0, 1, :],
                self.ds["track_latitude_bounds"][0, 1, :],
                "g-o",
                label="scan 0, pixel 1",
            )
            ax.plot(
                self.ds["track_longitude_bounds"][0, 1, 0],
                self.ds["track_latitude_bounds"][0, 1, 0],
                "gs",
                markersize=10,
            )
            ax.plot(
                self.ds["track_longitude_bounds"][0, 2, :],
                self.ds["track_latitude_bounds"][0, 2, :],
                "b-o",
                label="scan 0, pixel 2",
            )
            ax.plot(
                self.ds["track_longitude_bounds"][0, 2, 0],
                self.ds["track_latitude_bounds"][0, 2, 0],
                "bs",
                markersize=10,
            )
            ax.plot(
                self.ds["track_longitude_bounds"][1, 0, :],
                self.ds["track_latitude_bounds"][1, 0, :],
                "r-o",
                label="scan 1, pixel 0",
            )
            ax.plot(
                self.ds["track_longitude_bounds"][1, 0, 0],
                self.ds["track_latitude_bounds"][1, 0, 0],
                "rs",
                markersize=10,
            )
            # annote:
            ax.legend()
            # save:
            fig.savefig(figfile)
        # endif # testing
        #
        # copy longitudes:
        xx[0, 0:npixel] = self.ds["track_longitude_bounds"][0, :, 0]
        xx[0, npixel] = self.ds["track_longitude_bounds"][0, -1, 1]
        xx[1:, 0:npixel] = self.ds["track_longitude_bounds"][:, :, 3]
        xx[1:, npixel] = self.ds["track_longitude_bounds"][:, -1, 2]
        # copy latitudes:
        yy[0, 0:npixel] = self.ds["track_latitude_bounds"][0, :, 0]
        yy[0, npixel] = self.ds["track_latitude_bounds"][0, -1, 1]
        yy[1:, 0:npixel] = self.ds["track_latitude_bounds"][:, :, 3]
        yy[1:, npixel] = self.ds["track_latitude_bounds"][:, -1, 2]
        #
        # test ...
        if False:
            # target file:
            figfile = "test-raster.png"
            # info ...
            logging.info("TESTING - create %s ..." % figfile)
            # modules:
            import matplotlib.pyplot as plt

            # new figure:
            fig = plt.figure()
            # ad axes:
            ax = fig.add_axes([0.1, 0.1, 0.8, 0.8])
            # number of scans and pixels to be plotted:
            ns, np = 3, 4
            # mark original corner points:
            for k in range(4):
                ax.plot(
                    self.ds["track_longitude_bounds"][0:ns, 0:np, k],
                    self.ds["track_latitude_bounds"][0:ns, 0:np, k],
                    linestyle="None",
                    marker="o",
                    markeredgecolor="red",
                    markerfacecolor="None",
                )
            # endfor
            # add raster:
            for j in range(ns + 1):
                ax.plot(xx[j, 0 : np + 1], yy[j, 0 : np + 1], linestyle="-", color="black")
            # endfor
            for i in range(np + 1):
                ax.plot(xx[0 : ns + 1, i], yy[0 : ns + 1, i], linestyle="-", color="black")
            # endfor
            # save:
            fig.savefig(figfile)
            # testing ...
            raise Exception
        # endif # testing

        # compressed indices:
        pp = self.ds["pixel"]
        # uncompress:
        jj = numpy.floor(pp / npixel).astype(int)
        ii = numpy.floor(pp % npixel).astype(int)

        # switch:
        if len(da.shape) == 1:
            # init masked array:
            values = numpy.ma.array(
                data=numpy.zeros((nscan, npixel), float), mask=numpy.ones((nscan, npixel), bool)
            )
            # copy values:
            values.data[jj, ii] = da.data
            values.mask[jj, ii] = False
        #
        elif len(da.shape) == 2:
            # (npix,n) number of levels:
            n = da.shape[1]
            # init masked array:
            values = numpy.ma.array(
                data=numpy.zeros((nscan, npixel, n), float),
                mask=numpy.ones((nscan, npixel, n), bool),
            )
            # copy values:
            values.data[jj, ii, :] = da.data
            values.mask[jj, ii, :] = False
        #
        elif len(da.shape) == 3:
            # (npix,n,m) number of levels:
            n, m = da.shape[1:3]
            # init masked array:
            values = numpy.ma.array(
                data=numpy.zeros((nscan, npixel, n, m), float),
                mask=numpy.ones((nscan, npixel, n, m), bool),
            )
            # copy values:
            values.data[jj, ii, :, :] = da.data
            values.mask[jj, ii, :, :] = False
        #
        else:
            logging.error('unsupported shape %s for variable "%s"' % (str(shp), varname))
            raise Exception
        # endif

        # extract attributes:
        attrs = da.attrs

        # ok
        return xx, yy, values, attrs

    # enddef GetTrack

    # *

    def GetTimeRange(self):

        """
        Return time range of pixels in file.
        A variable ``time`` should be present for this.
        """

        # source variable:
        vname = "time"
        # check ...
        if vname not in self.ds.keys():
            logging.error('no time variable "%s" in file: %s' % (vname, self.filename))
            raise Exception
        # endif
        # bounds?
        if "bounds" in self.ds[vname].attrs.keys():
            # name of bounds variable:
            bname = self.ds[vname].attrs["bounds"]
            # check ...
            if bname not in self.ds.keys():
                logging.error('no time bounds variable "%s" in file: %s' % (vname, self.filename))
                raise Exception
            # endif
            # extract range from time interval bounds series:
            tmin = self.ds[bname].data.min()
            tmax = self.ds[bname].data.max()
        else:
            # extract range from time series:
            tmin = self.ds[vname].data.min()
            tmax = self.ds[vname].data.max()
        # endif

        # extact:
        return tmin, tmax

    # enddef GetTimeRange

    # *

    def GetTimeAverage(self, freq=None):

        """
        Return average of first and last time value.

        Optional arguments:

        * ``freq`` : round time to given fequency, for example '``60min``'.
        """

        # modules:
        import pandas

        # first and last:
        t1, t2 = self.GetTimeRange()
        # average:
        taver = t1 + (t2 - t1) / 2

        # round?
        if freq is not None:
            # round to frequency:
            taver = pandas.Timestamp(*taver.timetuple()).round(freq).to_pydatetime()
        # endif

        ## testing ...
        # tfmt = '%Y-%m-%d %H:%M'
        # print( 'xxx pixel times [ %s , %s ]' % (t1.strftime(tfmt),t2.strftime(tfmt)) )

        # ok
        return taver

    # enddef GetTimeAverage

    # *

    def GetAttr(self, name, quiet=False):

        """
        Return global attribute with provided name.
        If the attribtue is not present an error is raised,
        unless ``quiet=True``.
        """

        # check ...
        if name in self.ds.attrs.keys():
            # copy:
            return self.ds.attrs[name]
        else:
            # complain?
            if quiet:
                return None
            else:
                logging.error('attribute "%s" not found' % name)
                raise Exception
            # endif
        # endif

    # enddef GetAttr

    # *

    def SelectPixels(self, rcf, rckey, indent=""):

        """
        Apply filters specified in rcfile.

        Arguments:

        * ``rcf``       :  :py:class:`.RcFile` object with settings
        * ``rckey``     :  basename for rcfile keys, e.g. "s5p" for the example below

        Return values:

        * ``selected`` : boolean array with shape of track ``(pixel)`` which
          is True if a pixel passed all checks;
        * ``history``  : list of character string describing the selelections.

        Example configuration::

            ! Specifiy a list of filter names.
            ! For each name, specify the variable which values are used for testing,
            ! the type test, and eventually some thresholds or other settings for this type.
            ! The units of the thresholds should match with the units in the variables,
            ! the expected units have to be defined too.
            ! The examples below show possible types and their settings.

            ! filters:
            s5p.filters                        :  lons lats albedo quality detected valid

            ! select range of values:
            s5p.filter.lons.var                :  longitude
            s5p.filter.lons.type               :  minmax
            s5p.filter.lons.minmax             :  -15.0 35.0
            s5p.filter.lons.units              :  degrees_east

            ! select above a minimum:
            s5p.filter.lats.var                :  latitude
            s5p.filter.lats.type               :  minmax
            s5p.filter.lats.minmax             :  35.0 70.0
            s5p.filter.lats.units              :  degrees_north

            ! select up to a maximum:
            s5p.filter.albedo.var              :  albedo
            s5p.filter.albedo.type             :  max
            s5p.filter.albedo.max              :  0.3
            s5p.filter.albedo.units            :  1

            ! select from a minimum:
            s5p.filter.quality.var             :  qa_value
            s5p.filter.quality.type            :  min
            s5p.filter.quality.max             :  0.8
            s5p.filter.quality.units           :  1

            ! select on flag:
            s5p.filter.detected.var            :  detection_flag
            s5p.filter.detected.type           :  equal
            s5p.filter.detected.value          :  1

            ! select only values with data (no "_FillValue"):
            s5p.filter.valid.var               :  vmr
            s5p.filter.valid.type              :  valid

        """

        # modules:
        import xarray
        import numpy

        # info ...
        logging.info(indent + "pixel selection ...")

        # init selection, by default accept all:
        selected = numpy.ones((self.ds.dims["pixel"],), "bool")

        # init history:
        history = []

        # which filters ?
        filternames = rcf.get("%s.filters" % rckey).split()
        # loop over filters:
        for filtername in filternames:
            # info ...
            logging.info(indent + '  filter "%s" ...' % filtername)
            # filter key in rcfiles:
            fkey = "%s.filter.%s" % (rckey, filtername)
            # which variable ?
            varname = rcf.get(fkey + ".var")
            # check ..
            if varname not in self.ds.keys():
                logging.error('filter "%s" requires unknown variable "%s"' % (filtername, varname))
                raise Exception
            # endif
            # read variable as data array:
            da = self.ds[varname]
            # check ...
            if "units" not in da.attrs.keys():
                logging.error('variable "%s" has no "units" attribute ...' % varname)
                raise Exception
            # endif
            # extract units:
            vunits = da.attrs["units"]
            # which type ?
            filtertype = rcf.get(fkey + ".type")
            # switch:
            # ~ compare values:
            if filtertype == "equal":
                # read value:
                svalue = rcf.get(fkey + ".value")
                fvalue = float(svalue)
                # avoid warnings about comparing nan ...
                xdata = da.data.copy()
                if numpy.any(numpy.isnan(da.data)):
                    xdata[numpy.where(numpy.isnan(da.data))] = fvalue - 1
                # apply filter:
                selected = selected & (xdata == fvalue)
                # update history, also select on not-nan to avoid warnings from comparisons:
                history.append("selected pixels with '%s' == %s" % (varname, svalue))
            #
            # ~ minimum value:
            elif filtertype == "min":
                # read threshold:
                smin = rcf.get(fkey + ".min")
                fmin = float(smin)
                # check units ...
                funits = rcf.get(fkey + ".units")
                if funits != vunits:
                    logging.error(
                        'variable "%s" has units "%s" while threshold has units "%s"'
                        % (varname, vunits, funits)
                    )
                    raise Exception
                # endif
                # avoid warnings about comparing nan ...
                xdata = da.data.copy()
                if numpy.any(numpy.isnan(da.data)):
                    xdata[numpy.where(numpy.isnan(da.data))] = fmin - 1
                # apply filter, also select on not-nan to avoid warnings from comparisons:
                selected = selected & (xdata >= fmin)
                # update history:
                history.append("selected pixels with '%s' >= %s" % (varname, smin))
            #
            # ~ maximum value:
            elif filtertype == "max":
                # read threshold:
                smax = rcf.get(fkey + ".max")
                fmax = float(smax)
                # check units ...
                funits = rcf.get(fkey + ".units")
                if funits != vunits:
                    logging.error(
                        'variable "%s" has units "%s" while threshold has units "%s"'
                        % (varname, vunits, funits)
                    )
                    logging.error(
                        "value range : %f - %f" % (numpy.nanmin(values), numpy.nanmax(values))
                    )
                    raise Exception
                # endif
                # avoid warnings about comparing nan ...
                xdata = da.data.copy()
                if numpy.any(numpy.isnan(da.data)):
                    xdata[numpy.where(numpy.isnan(da.data))] = fmax + 1
                # apply filter, also select on not-nan to avoid warnings from comparisons:
                selected = selected & (xdata <= fmax)
                # update history:
                history.append("selected pixels with '%s' <= %s" % (varname, smax))
            #
            # ~ range of valid values:
            elif filtertype == "minmax":
                # read thresholds:
                smin, smax = rcf.get(fkey + ".minmax").split()
                fmin = float(smin)
                fmax = float(smax)
                # check units ...
                funits = rcf.get(fkey + ".units")
                if funits != vunits:
                    logging.error(
                        'variable "%s" has units "%s" while threshold has units "%s"'
                        % (varname, vunits, funits)
                    )
                    raise Exception
                # endif
                # avoid warnings about comparing nan ...
                xdata = da.data.copy()
                if numpy.any(numpy.isnan(da.data)):
                    xdata[numpy.where(numpy.isnan(da.data))] = fmin - 1
                # apply filter, also select on not-nan to avoid warnings from comparisons:
                selected = selected & (xdata >= fmin) & (xdata <= fmax)
                # update history:
                history.append("selected pixels with '%s' in [%s,%s]" % (varname, smin, smax))
            #
            # ~ only unmasked values:
            elif filtername == "valid":
                # only accept non-masked fields:
                selected = selected & (~numpy.isnan(da.data))
                # update history:
                history.append("selected pixels with valid '%s' values" % varname)
            #
            # ~ not yet ...
            else:
                logging.error("unsupported filter name : %s" % filtername)
                raise Exception
            # endif
        # endfor # filters

        ## if ncecessary, convert from a masked to a normal array:
        # if hasattr(selected,'mask') : selected = selected.filled(fill_value=False)

        # ok
        return selected, history

    # enddef SelectPixels


# endclass CSO_File


########################################################################
###
### table with extract files
###
########################################################################


class CSO_Listing(object):

    """
    Storage for table with file properties.
    The records describe properties of :py:class:`CSO_File` files,
    for example the time range of the pixels included.
    This could be used to quickly scan an archive for filenames
    with pixels within a requested interval.

    The content of the table file looks like::

        filename;start_time;end_time
        2007/06/RAL-IASI-CH4_20070601T003855.nc;2007-06-01 00:45:36.827000000;2007-06-01 02:23:57.567000000
        2007/06/RAL-IASI-CH4_20070601T022359.nc;2007-06-01 02:23:59.512000000;2007-06-01 04:05:57.328000000
        ...

    Arguments:

    * ``filename`` : listing file, content is read if this file is already present unless ``renew==True`` ;
      if ``exist==True`` the file should exist
    * ``renew`` : (optinal) if ``True``, an existing file is ignored

    """

    def __init__(self, filename, exist=False, renew=False, indent=""):

        """
        Initialize empty table or read existing.
        """

        # modules:
        import os
        import pandas

        # store:
        self.filename = filename
        # base path:
        self.dirname = os.path.dirname(self.filename)

        # csv style:
        self.sep = ";"
        # head for index column:
        self.index_label = "filename"

        # check ..
        if exist:
            if not os.path.isfile(filename):
                logging.error("listing file not found: %s" % filename)
                raise Exception
            # endif
        # endif

        # already present?
        if os.path.isfile(filename) and (not renew):

            # info ...
            logging.info(indent + "read listing %s ..." % self.filename)
            # read:
            self.df = pandas.read_csv(
                self.filename,
                sep=self.sep,
                index_col=self.index_label,
                parse_dates=["start_time", "end_time"],
                dtype="str",
            )

        else:

            # new table:
            self.df = pandas.DataFrame(columns=["start_time", "end_time"])

        # endif

    # enddef __init__

    # *

    def Close(self, indent=""):

        """
        Write table to file.
        """

        # info ...
        logging.info(indent + "save listing %s ..." % self.filename)
        # save:
        self.df.to_csv(self.filename, sep=self.sep, index_label=self.index_label)

    # enddef Close

    # *

    def __len__(self):

        """
        Returns number of records in table.
        """

        return len(self.df)

    # enddef __len__

    # *

    def GetRecord(self, irec):

        """
        Return record for corresponding 0-based row index.
        """

        # extract:
        rec = self.df.iloc[irec].copy()
        # add index values:
        rec[self.index_label] = self.df.index[irec]

        # ok
        return rec

    # enddef GetRecord

    # *

    def __contains__(self, key):

        """
        Returns True if key is in the table index.
        """

        return key in self.df.index

    # enddef __contains__

    # *

    def Cleanup(self, indent=""):

        """
        Remove records from table if filename is not present anymore.
        """

        # modules:
        import os

        # loop over records:
        for fname in self.df.index:
            # file missing?
            if not os.path.isfile(os.path.join(self.dirname, fname)):
                # info ...
                logging.info(indent + 'drop record "%s", file not present anymore ..' % fname)
                # remove record:
                self.df.drop(fname, inplace=True)
            # endif
        # endfor

    # enddef Cleanup

    # *

    def Check(self, filename, indent=""):

        """
        Return exception if no record for ``filename`` is present.
        """

        # modules:
        import os

        # relative:
        fname = os.path.relpath(filename, self.dirname)

        # check ..
        if fname not in self.df.index:
            logging.error('file "%s" is not a record in table: %s' % (fname, self.filename))
            raise Exception
        # endif

    # enddef Check

    # *

    def Update(self, filename, csf, tr=None, xcolumns=[], indent=""):

        """
        Add record (or replace) for ``filename`` with information
        from the ``csf`` :py:class:`CSO_File` object.

        The time range of pixels is read from the ``csf`` object,
        unless the :py:class:`datetime.datetime` tupple ``tr=(t1,t2)`` is supplied.

        The optional ``xcolumns`` specify a list of global attributes
        to be added as extra columns.
        """

        # modules:
        import os
        import numpy

        # timerange:
        if tr is None:
            # get from file:
            tr = csf.GetTimeRange()
        # endif
        # check, in some products the time values were found to be undefined ...
        if numpy.isnan(tr[0]) or numpy.isnan(tr[1]):
            logging.error("found undefined time range while updating listing record")
            raise Exception
        # endif
        # store:
        self.df.at[filename, "start_time"] = tr[0]
        self.df.at[filename, "end_time"] = tr[1]

        # extra:
        for name in xcolumns:
            # get value:
            avalue = csf.GetAttr(name)
            # copy:
            self.df.at[filename, name] = avalue
        # endfor # xcolumns

    # enddef Update

    # *

    def UpdateRecord(self, filename, data, indent=""):

        """
        Add record (or replace) for ``filename`` with columns and values defined in the ``data`` dict.
        """

        # modules:
        import os

        # loop over columns:
        for key in data.keys():
            self.df.at[filename, key] = data[key]
        # endfor

    # enddef UpdateRecord

    # *

    def Select(self, tr=None, method="overlap"):

        """
        Return dataframe with selected records.

        Optional arguments:

        * ``tr=(t1,t2)`` : select records that could be assigned to this timerange;
          requires the ``method`` keyword:

          * ``method='overlap'`` (default) : select records that have pixels overlapping with interval
          * ``method='middle'`` : select pixels with the middle of ``start_time`` and ``end_time``
            within the inerval ``(t1,t2]``
        """

        # modules:
        import pandas

        # init result with entire dataset:
        df = self.df

        # time range specified?
        if tr is not None:
            # convert if needed:
            t1, t2 = pandas.Timestamp(tr[0]), pandas.Timestamp(tr[1])
            # switch:
            if method == "overlap":
                # select:
                df = df[(df["start_time"] <= t2) & (df["end_time"] >= t1)]
            #
            elif method == "middle":
                # mid times:
                mid_time = df["start_time"] + (df["end_time"] - df["start_time"]) / 2
                # select:
                df = df[(mid_time > t1) & (mid_time <= t2)]
            #
            else:
                logging.error('unsupported method "%s" with argument ``tr``' % method)
                raise Exception
            # endif
        # endif

        # ok
        return df

    # enddef Select

    # *

    def Append(self, lst, path=""):

        """
        Append content of other listing.
        """

        # modules:
        import os
        import pandas

        # prefix paths?
        if len(path) > 0:
            # copy, make "filename" a normal column again:
            xdf = lst.df.reset_index()
            # add path:
            for key in ["filename", "filename_state"]:
                xdf[key] = os.path.join(path, xdf[key])
            # endfor
            # add rows to current rows, use the original index:
            self.df = pandas.concat((self.df, xdf.set_index(self.index_label)))
        else:
            # append:
            self.df = pandas.concat((self.df, lst.df))
        # endif

    # enddef Append


# endclass CSO_Listing


########################################################################
###
### end
###
########################################################################
