#
# Changes
# 
# 2022-09, Arjo Segers, Met-Norway
#   Write files with zlib-compression, option to disable this.
#   Write files with variable packing (short integers and add_offset and scale_factor attributes), option to disable this.
# 
# 2022-09, Arjo Segers, Met-Norway
#   Fixed computation of packing parameters for array with constant value.
#
# 2023-08, Arjo Segers
#   Added "method" argument to CSO_Listing.Select().
#

########################################################################
###
### help
###
########################################################################

"""
*******************
``cso_file`` module
*******************

Create and access file with satellite data extract.


Class hierchy
=============

The classes are defined according to the following hierchy:

* :py:class:`.CSO_File`
* :py:class:`.CSO_Listing`


Classes
=======
"""

# modules:
import logging



########################################################################
###
### tools
###
########################################################################
    

def Pack_DataArray( da, dtype='i2' ) :

    """
    Set encoding parameters of :py:class:`xarray.DataArray` object ``da``;
    on return, the argument is changed in-place.    

    Argument ``dtype`` defines the data type in which the array will be written;
    by default this is a short integer (``i2``, 2-bytes) instead of a float.
    Attributes are added that define how to decode values::
    
        short pressure(z) ;
		    pressure:add_offset   = 53257.7493494103 ;
		    pressure:scale_factor = 1.50569706659902 ;
		    pressure:_FillValue   = 32767s ;

    The actual values are recalculated using the formula::

      values = add_offset + scale_factor * values__short

    The packing parameters are computed using::

      scale_factor = (vmax - vmin)/( fmax - fmin)
      add_offset   = vmin - scale_factor * fmin

    where ``[vmin,vmax]`` is the range of input values,
    and ``[fmin,fmax]`` the range of possible values of the stored data type.
    
    The ``scale_factor`` should not become 0.0 since it is used for division during encoding.
    This will be the case when the input has the same value in all elements;
    in that case, the following parameters are used that will lead to zero packed values::
    
      scale_factor = 1.0
      add_offset   = vmin
    
    """
    
    # modules:
    import numpy

    # only floats ...
    if da.dtype in [numpy.float32,numpy.float64] :
        # value range, ignore nan's:
        vmin = numpy.nanmin( da.values )
        vmax = numpy.nanmax( da.values )
        # target data type could be integer or float:
        if dtype.startswith('i') :
            # use absolute minimum to represent nans:
            fill_value = numpy.iinfo(dtype).min
            # valid range outside nan:
            fmin       = numpy.iinfo(dtype).min+1
            fmax       = numpy.iinfo(dtype).max
        else :
            logger.error( 'could not pack variables as "%s" type yet' % dtype )
            raise Exception
        #endif
        # packing parameters; 
        # scale_factor should not be 0.0 since used for division during encoding,
        # therefore check if there is a value range at all ..
        if vmin < vmax :
            # range of values:
            scale_factor = (vmax - vmin)/( fmax - fmin)
            add_offset   = vmin - scale_factor * fmin
        else :
            # single value only; set offset to this value, packed values will become 0:
            scale_factor = 1.0
            add_offset   = vmin
        #endif
        # store type and packing parameters:
        da.encoding['dtype'       ] = dtype
        da.encoding['add_offset'  ] = add_offset
        da.encoding['scale_factor'] = scale_factor
        # store new fill value:
        da.encoding['_FillValue'  ] = fill_value
    #endif # floats

#enddef Pack_DataArray




########################################################################
###
### satellite extract file
###
########################################################################

class CSO_File( object ) :

    """
    Storage for satelite data extract.
    
    If optional ``filename`` is provided, the :py:meth:`.Read` method is
    called to read data from a file. If the optional ``varname`` list
    is specified too, only selected variables will be read.
    
    To create a new track, initialize without arguments and add
    new variables using the :py:meth:`.AddVariable` method.
    Only certain dimension names and combinations are supported.
    
    """
    
    def __init__( self, filename=None ) :
    
        """
        Initialize extract file.
        """
        
        # modules:
        import os
        import xarray
        
        # read?
        if filename is not None :
        
            # store:
            self.filename = filename
        
            # check ...
            if not os.path.isfile(self.filename) :
                logging.error( 'file not found: %s' % self.filename )
                raise Exception
            #endif
            
            # access dataset:
            with xarray.open_dataset( self.filename ) as self.ds :
                # load entire file:
                self.ds.load()
            #endwith # xarray
            
        else :
        
            # dummy:
            self.filename = 'None'
            # init empty set:
            self.ds = xarray.Dataset()
            
        #endif  # filename provided

    #enddef __init__
    
    # *
    
    def Close( self ) :
    
        """
        Close dataset.
        """
        
        # close:
        self.ds.close()
        
    #enddef Close

    # *
    
    def AddDim( self, dname, length ) :
    
        """
        Add dimension.
        
        Arguments:
        
        * ``dname`` : (``str``) dimension name to be created
        * ``len`` : (``int``) dimension length
    
        """
        
        # store:
        self.ds.dims[dname] = length

    #enddef AddDim

    # *
    
    def GetDimLength( self, dname ) :
    
        """
        Return dimension length.
        """
        
        # check ..
        if dname not in self.ds.dims.keys() :
            logging.error( 'no dimension "%s" defined' % dname )
            raise Exception
        #endif
        # get:
        return self.ds.dims[dname]

    #enddef GetDimLength

    # *
    
    def GetShape( self, dnames ) :
    
        """
        Return shape following specified dimension names.
        """
        
        # init result:
        shp = ()
        # loop:
        for dname in dnames :
            # add length:
            shp = shp+(self.GetDimLength(dname),)
        #endfor
        
        # ok
        return shp

    #enddef GetShape

    # *
    
    def AddCoord( self, cname, da ) :
    
        """
        Add coordinate variable.
        
        Arguments:
        
        * ``cname`` : coordinate name to be created
        * ``da`` : :py:class:`xarray.DataArray` object
    
        """
        
        # store:
        self.ds.coords[cname] = da

    #enddef AddCoord

    # *
    
    def GetCoords( self, cnames ) :
    
        """
        Return dict with coordinate variables as specified in ``cnames``.
        """
        
        # storage:
        coords = {}
        # loop:
        for cname in cnames :
            # check ...
            if cname not in self.ds.coords.keys() :
                logging.error( 'coord "%s" not found' % cname )
                raise Exception
            #endif
            # copy:
            coords[cname] = self.ds.coords[cname]
        #endfor # cname
        
        # ok
        return coords

    #enddef GetCoords

    # *
    
    def AddTimeCoord( self, times, bounds=None ) :
    
        """
        Add time coordinate variable.
        
        Arguments:
        
        * ``times`` : time stamp series created with for example :py:meth:`pandas.date_range` method
        
        Optional arguments::
        
        * ``bounds=(times1,times2)`` : tupple or list with two timeseries that define the time bounds
    
        """
        
        # modules:
        import xarray
        import numpy
        
        # coordinate name:
        cname = 'time'
        bname = cname+'_bounds'
        
        # time units:
        tunits = 'seconds since %i-01-01 00:00:00' % times[0].year

        # init time attributes:
        attrs = { 'standard_name' : 'time' }

        # bounds defined?
        if bounds is not None :
            # store bounds:
            self.ds[bname] = xarray.DataArray( numpy.array(bounds).T, dims=(cname,'nv') )
            # ensure output with correct units:
            self.ds[bname].encoding['units'] = tunits
            # write as float, no fill-value needed:
            self.ds[bname].encoding['dtype'     ] = 'float64'
            self.ds[bname].encoding['_FillValue'] = None
            # store name for 'time' attributes:
            attrs['bounds'] = bname
        #endif
        
        # store time coordinate:
        self.ds.coords[cname] = xarray.DataArray( times, dims=(cname,), attrs=attrs )
        # ensure output with correct units:
        self.ds.coords[cname].encoding['units'] = tunits
        # write as float, no fill-value needed:
        self.ds.coords[cname].encoding['dtype'     ] = 'float64'
        self.ds.coords[cname].encoding['_FillValue'] = None

    #enddef AddTimeCoord

    # *
    
    def AddVariable( self, vname, da ) :
    
        """
        Add variable field.
        
        Arguments:
        
        * ``vname`` : variable name to be created
        * ``da`` : :py:class:`xarray.DataArray` object
    
        """
        
        # store:
        self.ds[vname] = da

    #enddef AddVariable
    
    # *
    
    def AddDataVariable( self, vname, dnames, dtype='f4', attrs={}, values=None ) :

        """
        Add variable with specified name and dimensions.

        Arguments:

        * ``vname`` : variable name to be created
        * ``dnames`` : dimension names, e.g. ``('time','station')``

        Optional arguments:

        * ``dtype`` : datatype keyword
        * ``attrs`` : dict with variable attributes
        * ``values`` : array with data values
        """
        
        # modules:
        import xarray
        import numpy

        # check ...
        if 'units' not in attrs.keys() :
            logging.error( 'attributes for output variable "%s" have no "units"' % (vname) )
            raise Exception
        #endif
        # check ...
        if ('long_name' not in attrs.keys()) and ('standard_name' not in attrs.keys()) :
            logging.error( 'attributes for output variable "%s" have neither "long_name" nor "standard_name"' % (vname) )
            raise Exception
        #endif

        # coordinates:
        coords = self.GetCoords( dnames )
        # no values provided? then create empty arrya:
        if values is None :
            # shape:
            shp = self.GetShape( dnames )
            # create:
            values = numpy.full(shp,numpy.nan,dtype=dtype)
        #endif
        # create variable (filled with no data?):
        self.ds[vname] = xarray.DataArray( values, dims=dnames, coords=coords, attrs=attrs )

    #enddef AddDataVariable

    # *

    def Write( self, filename, unlimited_dims=None, attrs=None, history=[], 
                   packed=True, complevel=1 ) :
    
        """
        Write data to provided ``filename``.
        
        Optional arguments:
        
        * ``attrs`` : dictionairy with global attributes
        * ``history`` : list of str values that describe how the content was created, 
          this will be added to the global ``history`` attribute
        * ``unlimited_dims`` : names of unlimited dimesions
        
        Optional arguments to disable file size reduction measures:
        
        * ``packed`` : by default float arrays are written as short-integers accompanied with
          ``add_offset`` and ``scale_factor`` attributes; set to ``False`` to keep original data types.
        * ``complevel`` : zlib compression level, default 1 (of 9) to have decent compression without too much overhead;
          set to 0 to have no compression at all
          
        The following example shows the impact of packing and compression on file sizes;
        in this exmple, the combination of packing and deflation reduces file sizes
        with 83%%::
        
              301.009.424  S5p_RPRO_GLYOX_01130.nc
              111.648.738  S5p_RPRO_GLYOX_01130__complevel1.nc
              104.134.232  S5p_RPRO_GLYOX_01130__complevel9.nc
              153.334.891  S5p_RPRO_GLYOX_01130__packed.nc
               41.020.139  S5p_RPRO_GLYOX_01130__packed_complevel1.nc

        """
        
        # modules:
        import os
        import datetime
        import numpy
 
        # destination filename:
        if filename == None : filename = self.filename
        
        # create directory if necessary:
        dname = os.path.dirname( filename )
        if (len(dname) > 0) and (not os.path.isdir(dname)) : os.makedirs( dname )
        
        # store global attributes:
        if attrs is not None : self.ds.attrs.update( attrs )
        
        # extend history:
        if len(history) > 0 :
            # time step as "Mon 1 Jan 01:01:01 2001"
            tstamp = datetime.datetime.today().strftime('%a %b %d %H:%M:%S %Y' )
            # create line:
            histline = str(tstamp)
            for hist in history :
                histline = histline+', '+hist
            #endfor
            # extend?
            if 'history' in self.ds.attrs.keys() :
                self.ds.attrs['history'] = histline+'\n'+self.ds.attrs['history']
            else :
                self.ds.attrs['history'] = histline
            #endif
        #endif
        
        # loop over coordinates:
        for cname in self.ds.coords.keys() :
            # no fill value needed:
            self.ds.coords[cname].encoding['_FillValue'] = None
        #endfor

        # pack?
        if packed :
            # loop over coordinates:
            for cname in self.ds.coords.keys() :
                # set packing parameters:
                Pack_DataArray( self.ds[cname] )
            #endfor # variables
            # loop over variables:
            for vname in self.ds.keys() :
                # set packing parameters:
                Pack_DataArray( self.ds[vname] )
            #endfor # variables
        #endif # pack

        # set compression?
        if complevel > 0 :
            # loop over variables:
            for vname in self.ds.keys() :
                # enable zlib compression and set level:
                self.ds[vname].encoding['zlib'     ] = True
                self.ds[vname].encoding['complevel'] = complevel
            #endfor # variables
        #endif # comppress
        
        # write:
        self.ds.to_netcdf( filename, unlimited_dims=unlimited_dims )
        
    #enddef Write
    
    # *
    
    def GetDims( self, varname ) :
        
        """
        Return variable dimension names.
        """
        
        # check ...
        if varname not in self.ds.keys() :
            logging.error( 'variable "%s" not available' % varname )
            if self.filename is not None : logging.error( 'filename: %s' % self.filename )
            raise Exeption
        #endif
        
        # ok
        return self.ds[varname].dims
        
    #enddef GetDims
    
    # *
    
    def GetData( self, varname ) :
        
        """
        Return data values and units for named variable.
        """
        
        # check ...
        if varname not in self.ds.keys() :
            logging.error( 'variable "%s" not available' % varname )
            if self.filename is not None : logging.error( 'filename: %s' % self.filename )
            raise Exeption
        #endif
        # check ..
        if 'units' not in self.ds[varname].attrs.keys() :
            logging.error( 'no attribute "units" with varible "%s"' % varname )
            if self.filename is not None : logging.error( 'filename: %s' % self.filename )
            raise Exeption
        #endif
        
        # ok
        return self.ds[varname].values,self.ds[varname].attrs['units']
        
    #enddef GetData

    # *
    
    def GetTrack( self, da ) :
    
        """
        Extract variable on track grid.
        
        Arguments:
        
        * ``da`` : :py:class:`xarray.DataArray` or variable name
        
        Return values:
        
        * ``xx``  : corner longitudes (nimage+1,npixel+1)
        * ``yy``  : corner latitudes (nimage+1,npixel+1)
        * ``values`` : masked array with values at pixel locations (nimage,npixel)
        * ``units`` : str units

        The corner locations are created using the following variables that are
        expected to be present:
    
        * ``track_longitude_bounds`` : 3D array of longitudes with dimensions ``('scan','pixel','corner')``
        * ``track_latitude_bounds``  : idem for latitudes
        * ``pixel`` : compress index that defines the ``('scan','pixel')`` location
        
        Example usage to obtain retrieval from *data* file that includes
        the track information::
        
            xx,yy,vcd_retr,vcd_units = dfile.GetTrack( 'vcd' )
            
        Example usage for simulated retrieval from *state* file::
        
            xx,yy,vcd_retr,vcd_units = dfile.GetTrack( sfile.ds['vcd'] )
        
        """
         
        # modules:
        import numpy
        
        # varialble:
        if type(da) == str :
            # should be variable in this fill:
            if da not in self.ds.keys() :
                logging.error( 'variable "%s" not available in data set, available are:' % (da) )
                logging.error( str(self.ds.keys()) )
                raise Exception
            #endif
            # extract:
            da = self.ds[da]    
        #endfor
        
        # variable dims:
        vdims = da.dims
        # check  ..
        if vdims[0] != 'pixel':
            logging.error( 'data array should have `pixel` as first dimenion, found: %s' % str(vdims) )
            raise Exception
        #endif
        
        # loop over required variables:
        for vname in ['track_longitude_bounds','track_latitude_bounds'] :
            # check ...
            if vname not in self.ds.keys() :
                logging.error( 'variable "%s" not available in data set' % (vname) )
                raise Exception
            #endif
        #endfor
        
        # shape of track grid:
        nscan,npixel,ncorner = self.ds['track_longitude_bounds'].shape

        # init 2D corner fields:
        xx = numpy.zeros((nscan+1,npixel+1),float)
        yy = numpy.zeros((nscan+1,npixel+1),float)
        # no idea yet how to fill them correctly automatically,
        # for the moment assume the (converted!) OMI order:      
        #
        #      ^ scan
        #       \        _ - o 2
        #         3 o -      \     
        #                _ - o 1  ---> pixel
        #          0 o -
        #
        # figure of first pixel footprints to determine the order ..
        if False :
            # target file:
            figfile = 'test-corners.png'
            # info ...
            logging.info( 'TESTING - create %s ...' % figfile )
            # modules:
            import matplotlib.pyplot as plt
            # new figure:
            fig = plt.figure()
            # new axis:
            ax = fig.add_axes([0.1,0.1,0.8,0.8])
            # raster:
            ax.plot( self.ds['track_longitude_bounds'][0,0,:], 
                     self.ds['track_latitude_bounds' ][0,0,:], 'r-o', label='scan 0, pixel 0' )
            ax.plot( self.ds['track_longitude_bounds'][0,0,0], 
                     self.ds['track_latitude_bounds' ][0,0,0], 'rs', markersize=10 )
            ax.plot( self.ds['track_longitude_bounds'][0,1,:], 
                     self.ds['track_latitude_bounds' ][0,1,:], 'g-o', label='scan 0, pixel 1' )
            ax.plot( self.ds['track_longitude_bounds'][0,1,0], 
                     self.ds['track_latitude_bounds' ][0,1,0], 'gs', markersize=10 )
            ax.plot( self.ds['track_longitude_bounds'][0,2,:], 
                     self.ds['track_latitude_bounds' ][0,2,:], 'b-o', label='scan 0, pixel 2' )
            ax.plot( self.ds['track_longitude_bounds'][0,2,0], 
                     self.ds['track_latitude_bounds' ][0,2,0], 'bs', markersize=10 )
            ax.plot( self.ds['track_longitude_bounds'][1,0,:], 
                     self.ds['track_latitude_bounds' ][1,0,:], 'r-o', label='scan 1, pixel 0' )
            ax.plot( self.ds['track_longitude_bounds'][1,0,0], 
                     self.ds['track_latitude_bounds' ][1,0,0], 'rs', markersize=10 )
            # annote:
            ax.legend()
            # save:
            fig.savefig( figfile )
        #endif # testing
        #
        # copy longitudes:
        xx[ 0,0:npixel] = self.ds['track_longitude_bounds'][0, :,0]
        xx[ 0,  npixel] = self.ds['track_longitude_bounds'][0,-1,1]
        xx[1:,0:npixel] = self.ds['track_longitude_bounds'][:, :,3]
        xx[1:,  npixel] = self.ds['track_longitude_bounds'][:,-1,2]
        # copy latitudes:
        yy[ 0,0:npixel] = self.ds['track_latitude_bounds'][0, :,0]
        yy[ 0,  npixel] = self.ds['track_latitude_bounds'][0,-1,1]
        yy[1:,0:npixel] = self.ds['track_latitude_bounds'][:, :,3]
        yy[1:,  npixel] = self.ds['track_latitude_bounds'][:,-1,2]
        #
        # test ...
        if False :
            # target file:
            figfile = 'test-raster.png'
            # info ...
            logging.info( 'TESTING - create %s ...' % figfile )
            # modules:
            import matplotlib.pyplot as plt
            # new figure:
            fig = plt.figure()
            # ad axes:
            ax = fig.add_axes([0.1,0.1,0.8,0.8])
            # number of scans and pixels to be plotted:
            ns,np = 3,4
            # mark original corner points:
            for k in range(4) :
                ax.plot( self.ds['track_longitude_bounds'][0:ns,0:np,k], 
                         self.ds['track_latitude_bounds' ][0:ns,0:np,k], 
                         linestyle='None', marker='o', markeredgecolor='red', markerfacecolor='None' )
            #endfor
            # add raster:
            for j in range(ns+1) :
                ax.plot( xx[j,0:np+1], yy[j,0:np+1], linestyle='-', color='black' )
            #endfor
            for i in range(np+1) :
                ax.plot( xx[0:ns+1,i], yy[0:ns+1,i], linestyle='-', color='black' )
            #endfor
            # save:
            fig.savefig( figfile )
            # testing ...
            raise Exception
        #endif # testing
        
        # compressed indices:
        pp = self.ds['pixel']
        # uncompress:
        jj = numpy.floor( pp / npixel ).astype(int)
        ii = numpy.floor( pp % npixel ).astype(int)
        
        # switch:
        if len(da.shape) == 1 :
            # init masked array:
            values = numpy.ma.array( data=numpy.zeros((nscan,npixel),float),
                                     mask=numpy.ones ((nscan,npixel),bool ) )
            # copy values:
            values.data[jj,ii] = da.data
            values.mask[jj,ii] = False
        #
        elif len(da.shape) == 2 :
            # (npix,n) number of levels:
            n = da.shape[1]
            # init masked array:
            values = numpy.ma.array( data=numpy.zeros((nscan,npixel,n),float),
                                     mask=numpy.ones ((nscan,npixel,n),bool ) )
            # copy values:
            values.data[jj,ii,:] = da.data
            values.mask[jj,ii,:] = False
        #
        elif len(da.shape) == 3 :
            # (npix,n,m) number of levels:
            n,m = da.shape[1:3]
            # init masked array:
            values = numpy.ma.array( data=numpy.zeros((nscan,npixel,n,m),float),
                                     mask=numpy.ones ((nscan,npixel,n,m),bool ) )
            # copy values:
            values.data[jj,ii,:,:] = da.data
            values.mask[jj,ii,:,:] = False
        #
        else :
            logging.error( 'unsupported shape %s for variable "%s"' % (str(shp),varname) )
            raise Exception
        #endif
        
        # extract attributes:
        attrs = da.attrs
        
        # ok
        return xx,yy,values,attrs
        
    #enddef GetTrack

    # *
    
    def GetTimeRange( self ) :
    
        """
        Return time range of pixels in file.
        A variable ``time`` should be present for this.
        """
        
        # source variable:
        vname = 'time'
        # check ...
        if vname not in self.ds.keys() :
            logging.error( 'no time variable "%s" in file: %s' % (vname,self.filename) )
            raise Exception
        #endif
        # bounds?
        if 'bounds' in self.ds[vname].attrs.keys() :
            # name of bounds variable:
            bname = self.ds[vname].attrs['bounds']
            # check ...
            if bname not in self.ds.keys() :
                logging.error( 'no time bounds variable "%s" in file: %s' % (vname,self.filename) )
                raise Exception
            #endif
            # extract range from time interval bounds series:
            tmin = self.ds[bname].data.min()
            tmax = self.ds[bname].data.max()
        else :
            # extract range from time series:
            tmin = self.ds[vname].data.min()
            tmax = self.ds[vname].data.max()
        #endif

        # extact:
        return tmin,tmax
        
    #enddef GetTimeRange

    # *
    
    def GetTimeAverage( self, freq=None ) :
    
        """
        Return average of first and last time value.
        
        Optional arguments:
        
        * ``freq`` : round time to given fequency, for example '``60min``'.
        """
        
        # modules:
        import pandas
        
        # first and last:
        t1,t2 = self.GetTimeRange()
        # average:
        taver = t1 + (t2-t1)/2
        
        # round?
        if freq is not None :
            # round to frequency:
            taver = pandas.Timestamp(*taver.timetuple()).round(freq).to_pydatetime()
        #endif
        
        ## testing ...
        #tfmt = '%Y-%m-%d %H:%M'
        #print( 'xxx pixel times [ %s , %s ]' % (t1.strftime(tfmt),t2.strftime(tfmt)) )
        
        # ok
        return taver
        
    #enddef GetTimeAverage
    
    # *
    
    def GetAttr( self, name, quiet=False ) :
    
        """
        Return global attribute with provided name.
        If the attribtue is not present an error is raised,
        unless ``quiet=True``.
        """

        # check ...
        if name in self.ds.attrs.keys() :
            # copy:
            return self.ds.attrs[name]
        else :
            # complain?
            if quiet :
                return None
            else :
                logging.error( 'attribute "%s" not found' % name )
                raise Exception
            #endif
        #endif
        
    #enddef GetAttr
                    
    # *
    
    def SelectPixels( self, rcf, rckey, indent='' ) :
    
        """
        Apply filters specified in rcfile.
        
        Arguments:
        
        * ``rcf``       :  :py:class:`.RcFile` object with settings
        * ``rckey``     :  basename for rcfile keys, e.g. "s5p" for the example below
            
        Return values:
        
        * ``selected`` : boolean array with shape of track ``(pixel)`` which
          is True if a pixel passed all checks;
        * ``history``  : list of character string describing the selelections.
          
        Example configuration::
        
            ! Specifiy a list of filter names.
            ! For each name, specify the variable which values are used for testing,
            ! the type test, and eventually some thresholds or other settings for this type.
            ! The units of the thresholds should match with the units in the variables,
            ! the expected units have to be defined too.
            ! The examples below show possible types and their settings.

            ! filters:
            s5p.filters                        :  lons lats albedo quality detected valid

            ! select range of values:
            s5p.filter.lons.var                :  longitude
            s5p.filter.lons.type               :  minmax
            s5p.filter.lons.minmax             :  -15.0 35.0
            s5p.filter.lons.units              :  degrees_east

            ! select above a minimum:
            s5p.filter.lats.var                :  latitude
            s5p.filter.lats.type               :  minmax
            s5p.filter.lats.minmax             :  35.0 70.0
            s5p.filter.lats.units              :  degrees_north

            ! select up to a maximum:
            s5p.filter.albedo.var              :  albedo
            s5p.filter.albedo.type             :  max
            s5p.filter.albedo.max              :  0.3
            s5p.filter.albedo.units            :  1

            ! select from a minimum:
            s5p.filter.quality.var             :  qa_value
            s5p.filter.quality.type            :  min
            s5p.filter.quality.max             :  0.8
            s5p.filter.quality.units           :  1

            ! select on flag:
            s5p.filter.detected.var            :  detection_flag
            s5p.filter.detected.type           :  equal
            s5p.filter.detected.value          :  1

            ! select only values with data (no "_FillValue"):
            s5p.filter.valid.var               :  vmr
            s5p.filter.valid.type              :  valid
        
        """
        
        # modules:
        import xarray
        import numpy
        
        # info ...
        logging.info( indent+'pixel selection ...' )
        
        # init selection, by default accept all:
        selected = numpy.ones( (self.ds.dims['pixel'],), 'bool' )
        
        # init history:
        history = []

        # which filters ?
        filternames = rcf.get( '%s.filters' % rckey ).split()
        # loop over filters:
        for filtername in filternames :
            # info ...
            logging.info( indent+'  filter "%s" ...' % filtername )
            # filter key in rcfiles:
            fkey = '%s.filter.%s' % (rckey,filtername)
            # which variable ?
            varname = rcf.get(fkey+'.var')
            # check ..
            if varname not in self.ds.keys() :
                logging.error( 'filter "%s" requires unknown variable "%s"' % (filtername,varname) )
                raise Exception
            #endif
            # read variable as data array:
            da = self.ds[varname]
            # check ...
            if 'units' not in da.attrs.keys() :
                logging.error( 'variable "%s" has no "units" attribute ...' % varname )
                raise Exception
            #endif
            # extract units:
            vunits = da.attrs['units']
            # which type ?
            filtertype = rcf.get(fkey+'.type')
            # switch:
            #~ compare values:
            if filtertype == 'equal' :
                # read value:
                svalue = rcf.get(fkey+'.value')
                fvalue = float(svalue)
                # avoid warnings about comparing nan ...
                xdata = da.data.copy()
                if numpy.any( numpy.isnan(da.data) ) : xdata[numpy.where(numpy.isnan(da.data))] = fvalue - 1
                # apply filter:
                selected = selected & (xdata == fvalue)
                # update history, also select on not-nan to avoid warnings from comparisons:
                history.append( "selected pixels with '%s' == %s" % (varname,svalue) )
            #
            #~ minimum value:
            elif filtertype == 'min' :
                # read threshold:
                smin = rcf.get(fkey+'.min')
                fmin = float(smin)
                # check units ...
                funits = rcf.get(fkey+'.units')
                if funits != vunits :
                    logging.error( 'variable "%s" has units "%s" while threshold has units "%s"' % (varname,vunits,funits) )
                    raise Exception
                #endif
                # avoid warnings about comparing nan ...
                xdata = da.data.copy()
                if numpy.any( numpy.isnan(da.data) ) : xdata[numpy.where(numpy.isnan(da.data))] = fmin - 1
                # apply filter, also select on not-nan to avoid warnings from comparisons:
                selected = selected & (xdata >= fmin)
                # update history:
                history.append( "selected pixels with '%s' >= %s" % (varname,smin) )
            #
            #~ maximum value:
            elif filtertype == 'max' :
                # read threshold:
                smax = rcf.get(fkey+'.max')
                fmax = float(smax)
                # check units ...
                funits = rcf.get(fkey+'.units')
                if funits != vunits :
                    logging.error( 'variable "%s" has units "%s" while threshold has units "%s"' % (varname,vunits,funits) )
                    logging.error( 'value range : %f - %f' % (numpy.nanmin(values),numpy.nanmax(values)) )
                    raise Exception
                #endif
                # avoid warnings about comparing nan ...
                xdata = da.data.copy()
                if numpy.any( numpy.isnan(da.data) ) : xdata[numpy.where(numpy.isnan(da.data))] = fmax + 1
                # apply filter, also select on not-nan to avoid warnings from comparisons:
                selected = selected & (xdata <= fmax)
                # update history:
                history.append( "selected pixels with '%s' <= %s" % (varname,smax) )
            #
            #~ range of valid values:
            elif filtertype == 'minmax' :
                # read thresholds:
                smin,smax = rcf.get(fkey+'.minmax').split()
                fmin = float(smin)
                fmax = float(smax)
                # check units ...
                funits = rcf.get(fkey+'.units')
                if funits != vunits :
                    logging.error( 'variable "%s" has units "%s" while threshold has units "%s"' % (varname,vunits,funits) )
                    raise Exception
                #endif
                # avoid warnings about comparing nan ...
                xdata = da.data.copy()
                if numpy.any( numpy.isnan(da.data) ) : xdata[numpy.where(numpy.isnan(da.data))] = fmin - 1
                # apply filter, also select on not-nan to avoid warnings from comparisons:
                selected = selected & (xdata >= fmin) & (xdata <= fmax)
                # update history:
                history.append( "selected pixels with '%s' in [%s,%s]" % (varname,smin,smax) )
            #
            #~ only unmasked values:
            elif filtername == 'valid' :
                # only accept non-masked fields:
                selected = selected & (~ numpy.isnan(da.data))
                # update history:
                history.append( "selected pixels with valid '%s' values" % varname )
            #
            #~ not yet ...
            else :
                logging.error( 'unsupported filter name : %s' % filtername )
                raise Exception
            #endif
        #endfor # filters
        
        ## if ncecessary, convert from a masked to a normal array:
        #if hasattr(selected,'mask') : selected = selected.filled(fill_value=False)

        # ok
        return selected,history

    #enddef SelectPixels

#endclass CSO_File


########################################################################
###
### table with extract files
###
########################################################################

class CSO_Listing( object ) :

    """
    Storage for table with file properties.
    The records describe properties of :py:class:`CSO_File` files,
    for example the time range of the pixels included.
    This could be used to quickly scan an archive for filenames
    with pixels within a requested interval.
    
    The content of the table file looks like::
    
        filename;start_time;end_time
        2007/06/RAL-IASI-CH4_20070601T003855.nc;2007-06-01 00:45:36.827000000;2007-06-01 02:23:57.567000000
        2007/06/RAL-IASI-CH4_20070601T022359.nc;2007-06-01 02:23:59.512000000;2007-06-01 04:05:57.328000000
        ...

    Arguments:
    
    * ``filename`` : listing file, content is read if this file is already present unless ``renew==True`` ;
      if ``exist==True`` the file should exist
    * ``renew`` : (optinal) if ``True``, an existing file is ignored 
    
    """
    
    def __init__( self, filename, exist=False, renew=False, indent='' ) :
    
        """
        Initialize empty table or read existing.
        """
        
        # modules:
        import os
        import pandas
        
        # store: 
        self.filename = filename
        # base path:
        self.dirname = os.path.dirname(self.filename)
        
        # csv style:
        self.sep = ';'
        # head for index column:
        self.index_label = 'filename'
        
        # check ..
        if exist :
            if not os.path.isfile(filename) :
                logging.error( 'listing file not found: %s' % filename )
                raise Exception
            #endif
        #endif
        
        # already present?
        if os.path.isfile(filename) and (not renew) :

            # info ...
            logging.info( indent+'read listing %s ...' % self.filename )
            # read:
            self.df = pandas.read_csv( self.filename, sep=self.sep, 
                                         index_col=self.index_label,
                                         parse_dates=['start_time','end_time'],
                                         dtype='str' )

        else :

            # new table:
            self.df = pandas.DataFrame( columns=['start_time','end_time'] )
            
        #endif
        
    #enddef __init__
    
    # *
    
    def Close( self, indent='' ) :
    
        """
        Write table to file.
        """

        # info ...
        logging.info( indent+'save listing %s ...' % self.filename )
        # save:
        self.df.to_csv( self.filename, sep=self.sep, index_label=self.index_label )
        
    #enddef Close
    
    # *
    
    def __len__( self ) :
    
        """
        Returns number of records in table.
        """
        
        return len(self.df)
        
    #enddef __len__
    
    # *
    
    def GetRecord( self, irec ) :
    
        """
        Return record for corresponding 0-based row index.
        """
        
        # extract:
        rec = self.df.iloc[irec].copy()
        # add index values:
        rec[self.index_label] = self.df.index[irec]

        # ok
        return rec
        
    #enddef GetRecord
    
    # *
    
    def __contains__( self, key ) :
    
        """
        Returns True if key is in the table index.
        """
        
        return key in self.df.index
        
    #enddef __contains__

    # *
    
    def Cleanup( self, indent='' ) :
    
        """
        Remove records from table if filename is not present anymore.
        """
        
        # modules:
        import os
        
        # loop over records:
        for fname in self.df.index :
            # file missing?
            if not os.path.isfile( os.path.join(self.dirname,fname) ) :
                # info ...
                logging.info( indent+'drop record "%s", file not present anymore ..' % fname )
                # remove record:
                self.df.drop( fname, inplace=True )
            #endif
        #endfor
        
    #enddef Cleanup

    # *
    
    def Check( self, filename, indent='' ) :
    
        """
        Return exception if no record for ``filename`` is present.
        """
        
        # modules:
        import os
        
        # relative:
        fname = os.path.relpath( filename, self.dirname )
        
        # check ..
        if fname not in self.df.index :
            logging.error( 'file "%s" is not a record in table: %s' % (fname,self.filename) )
            raise Exception
        #endif                
        
    #enddef Check

    # *
    
    def Update( self, filename, csf, tr=None, xcolumns=[], indent='' ) :
    
        """
        Add record (or replace) for ``filename`` with information
        from the ``csf`` :py:class:`CSO_File` object.
        
        The time range of pixels is read from the ``csf`` object,
        unless the :py:class:`datetime.datetime` tupple ``tr=(t1,t2)`` is supplied.
        
        The optional ``xcolumns`` specify a list of global attributes 
        to be added as extra columns.
        """
        
        # modules:
        import os
        import numpy
        
        # timerange:
        if tr is None :
            # get from file:
            tr = csf.GetTimeRange()
        #endif
        # check, in some products the time values were found to be undefined ...
        if numpy.isnan(tr[0]) or numpy.isnan(tr[1]) :
            logging.error('found undefined time range while updating listing record')
            raise Exception
        #endif
        # store:
        self.df.at[filename,'start_time'] = tr[0]
        self.df.at[filename,'end_time'  ] = tr[1]

        # extra:
        for name in xcolumns :
            # get value:
            avalue = csf.GetAttr( name )
            # copy:
            self.df.at[filename,name] = avalue
        #endfor # xcolumns
        
    #enddef Update

    # *
    
    def UpdateRecord( self, filename, data, indent='' ) :
    
        """
        Add record (or replace) for ``filename`` with columns and values defined in the ``data`` dict.
        """
        
        # modules:
        import os
        
        # loop over columns:
        for key in data.keys() :
            self.df.at[filename,key] = data[key]
        #endfor
        
    #enddef UpdateRecord
    
    # *

    def Select(self, tr=None, method="overlap"):

        """
        Return dataframe with selected records.

        Optional arguments:

        * ``tr=(t1,t2)`` : select records that could be assigned to this timerange;
          requires the ``method`` keyword:
          
          * ``method='overlap'`` (default) : select records that have pixels overlapping with interval
          * ``method='middle'`` : select pixels with the middle of ``start_time`` and ``end_time``
            within the inerval ``(t1,t2]``
        """

        # modules:
        import pandas

        # init result with entire dataset:
        df = self.df

        # time range specified?
        if tr is not None:
            # convert if needed:
            t1, t2 = pandas.Timestamp(tr[0]), pandas.Timestamp(tr[1])
            # switch:
            if method == "overlap":
                # select:
                df = df[(df["start_time"] <= t2) & (df["end_time"] >= t1)]
            #
            elif method == "middle":
                # mid times:
                mid_time = df["start_time"] + (df["end_time"] - df["start_time"]) / 2
                # select:
                df = df[(mid_time > t1) & (mid_time <= t2)]
            #
            else:
                logging.error('unsupported method "%s" with argument ``tr``' % method)
                raise Exception
            # endif
        # endif

        # ok
        return df

    # enddef Select

    # *
    
    def Append( self, lst, path='' ) :
    
        """
        Append content of other listing.
        """
        
        # modules:
        import os
        import pandas
        
        # prefix paths?
        if len(path) > 0 :
            # copy, make "filename" a normal column again:
            xdf = lst.df.reset_index()
            # add path:
            for key in ['filename','filename_state'] :
                xdf[key] = os.path.join(path,xdf[key])
            #endfor
            # add rows to current rows, use the original index:
            self.df = pandas.concat( (self.df,xdf.set_index(self.index_label)) )
        else :
            # append:
            self.df = pandas.concat( (self.df,lst.df) )
        #endif
        
    #enddef Append
    

#endclass CSO_Listing



########################################################################
###
### end
###
########################################################################
