#
# Changes
# 
# 2022-09, Arjo Segers
#   Updated documentation of special conversions.
#
# 2022-09, Arjo Segers
#   Download files during conversion, eventually remove them when done.
# 
# 2022-09, Arjo Segers
#   Added flags to disable compression and packing of variables on output, 
#   by default both are enabled.
#
# 2023-08, Arjo Segers
#   Convert selected orbit files based on selection query.
#   Support combination of multiple inquiry files.
#

########################################################################
###
### help
###
########################################################################

"""
.. _cso-s5p:

******************
``cso_s5p`` module
******************

The :py:mod:`cso_s5p` module provides classes to convert S5p data
into a CSO format.


Class hierchy
=============

The classes and are defined according to the following hierchy:


* :py:class:`.S5p_File`
* :py:class:`.CSO_File`

  * :py:class:`.CSO_S5p_File`

* :py:class:`.UtopyaRc`

  * :py:class:`.CSO_S5p_Convert`
  * :py:class:`.CSO_S5p_Listing`



Classes
=======


"""


########################################################################
###
### modules
###
########################################################################

# modules:
import logging

# tools:
import utopya
import cso_file


########################################################################
###
### S5p file
###
########################################################################

class S5p_File( object ) :

    """
    Base class to access data in S5p file.
    
    Example of variables::

        PRODUCT/
            dimensions:
                scanline = 3245 ;
                ground_pixel = 450 ;
                corner = 4 ;
                time = 1 ;
                layer = 34 ;
            variables:
                int time(time) ;
                float latitude (time, scanline, ground_pixel) ;
  		                units = "degrees_east" ;
                float longitude(time, scanline, ground_pixel) ;
  		                units = "degrees_north" ;
                float nitrogendioxide_tropospheric_column(time, scanline, ground_pixel) ;
  		                units = "mol m-2" ;
  		                multiplication_factor_to_convert_to_molecules_percm2 = 6.02214e+19f ;
  	            float nitrogendioxide_tropospheric_column_precision(time, scanline, ground_pixel) ;
  		                units = "mol m-2" ;
  		                multiplication_factor_to_convert_to_molecules_percm2 = 6.022141e+19f ;
          	    float averaging_kernel(time, scanline, ground_pixel, layer) ;
  		                units = "1" ;
  	            ubyte qa_value(time, scanline, ground_pixel) ;
  		                units = "1" ;
  		                long_name = "data quality value" ;
  		                comment = "A continuous quality descriptor, varying between 0 (no data) and 1 (full quality data). Recommend to ignore data with qa_value < 0.5" ;
            SUPPORT_DATA/
                GEOLOCATIONS/
                    variables:
                        float solar_zenith_angle(time, scanline, ground_pixel) ;
      		                    units = "degree" ;
      	                float latitude_bounds   (time, scanline, ground_pixel, corner) ;
  		                        units = "degrees_east" ;
      	                float longitude_bounds  (time, scanline, ground_pixel, corner) ;
  		                        units = "degrees_north" ;
                INPUT_DATA/
                    variables:
      	                float surface_altitude(time, scanline, ground_pixel) ;
      		                    units = "m" ;
      	                float surface_pressure(time, scanline, ground_pixel) ;
      		                    units = "Pa" ;
      	                float surface_albedo(time, scanline, ground_pixel) ;
      		                    units = "1" ;
      	                float cloud_fraction_crb(time, scanline, ground_pixel) ;
      		                    units = "1" ;

    Arguments:
    
    * ``filename``    : name of input file

    """
    
    def __init__( self, filename ) :
    
        """
        Open data file.        
        """
        
        # modules:
        import os
        import xarray
        
        # store:
        self.filename = filename

        # check ...
        if not os.path.isfile(filename) :
            logging.error( 'file not found : %s' % filename )
            raise Exception
        #endif
        
        # open:
        with  xarray.open_dataset( self.filename, group='PRODUCT' ) as ds :

            # copy some dimensions for convenience:
            self.ntime  = ds.dims['time']
            self.nscan  = ds.dims['scanline']
            self.ngpix  = ds.dims['ground_pixel']
            # number 'atmoshpere' layers:
            if 'layer' in ds.dims.keys() :
                self.nlayer = ds.dims['layer']
            else :
                self.nlayer = None
            #endif

        #endwith ds
        
    #enddef __init__
    
    # *
    
    def GetData( self, path, units=None ) :
    
        """
        Extract data array from input file and perform some adhoc fixes:
        
        * Eventually insert units attribute if not present yet, this is then taken from the argument.
        * Convert data to target units if necessary.
        * Add a ``long_name`` attribute if not present yet.
        
        Arguments:
        
        * ``path`` : variable path in input
        
        Optional arguments:
        
        * ``units`` : target units
        
        Return values:
        
        * ``da`` : :py:xarray.DataArray` object
        """
        
        # modules:
        import os
        import xarray
        import time
    
        # split:
        if '/' in path :
            # split:
            group,vname = path.rsplit('/',1)
        else :
            # no group:
            group,vname = None,path
        #endif

        # somtimes fails ..
        maxtry = 5
        ntry = 0
        while True :
            # increase counter:
            ntry += 1
            # try to read:
            try :
                # access group:
                with xarray.open_dataset( self.filename, group=group ) as ds :
                    # check ..
                    if vname not in ds :
                        logging.error( 'no variable "%s" in file "%s"' % (path,self.filename) )
                        raise Exception
                    #endif
                    # extract variable:
                    da = ds[vname]
                    # not lazy ...
                    da.load()
                #endwith # ds
                # nothing wrong, leave try loop
                break
            except :
                # again?
                if ntry <= maxtry :
                    # info ...
                    logging.warning( 'attempt %i of %i to read group "%s" from "%s", try again ...' % (ntry,maxtry,group,self.filename) )
                    # wait ..
                    time.sleep(5)
                    # try again:
                    continue
                else :
                    logging.error( 'tried to read %i times, break ..' % maxtry )
                    raise Exception
                #endif
            #endtry # read group
        #endwhile # try to read
        
        # unit stuff?
        if units is not None :
        
            # set input units in case these are not present yet,
            # assume that what is provided in the settings is correct:
            if 'units' not in da.attrs.keys() : da.attrs['units'] = units

            ## info ...
            #logging.info( indent+'    input  range: [%f,%f] %s' % (da.min(),da.max(),da.attrs['units']) )

            # need to convert?
            if da.attrs['units'] != units :
                # conversion description:
                conversion = '%s -> %s' % (da.attrs['units'],units)
                # switch:
                if conversion in ['mol m-2 -> 1e15 mlc/cm2'] :
                    # check ...
                    aname = 'multiplication_factor_to_convert_to_molecules_percm2'
                    if aname not in da.attrs.keys() :
                        logging.error( 'attribute "%s" not found for conversion "%s"' % (aname,conversion) )
                        logging.error( 'variable : %s' % path )
                        logging.error( 'filename : %s' % self.filename )
                        raise Exception
                    #endif
                    # convert:
                    # mol/m2    (mlc/cm2)/(mol/m2)   1e15/1
                    da.data  *=  da.attrs[aname]   * 1.0e-15  # 1e15 mlc/cm2
                #
                elif conversion in ["1e-9 -> ppb" ] :
                    # no need to convert, only rename ...
                    pass
                #
                elif conversion in ["1 -> ppb" ] :
                    # volume mixing ratios:
                    #  1           ppb
                    da.data  *=  1.0e-9  # ppb
                    
                #
                #if conversion in ['deg -> degrees_east','deg -> degrees_north',
                #                  'Degrees -> degrees_east', 'Degrees -> degrees_north',
                #                  'NoUnits -> 1'] :
                #    # no conversion needed
                #    pass
                #elif conversion in ["molecules cm^-2 -> 1e15 cm**-2",
                #                    "molecules/cm^2 -> 1e15 cm**-2" ] :
                #    # apply factor:
                #    da.data = da.data * 1.0e-15
                #elif conversion in ["hPa -> Pa"] :
                #    # apply factor:
                #    da.data = da.data * 1.0e2
                #elif conversion in ["NoUnits -> %"] :
                #    # assume original is in [0,1], convert to % :
                #    da.data = da.data * 100.0
                else :
                    logging.error( 'unsupported conversion "%s" requested ...' % conversion )
                    raise Exception
                #endif
                # reset units:
                da.attrs['units'] = units
            #endif  # units differ
            
        #endif # units defined
        
        # adhoc fix: sometimes neither standard name or long name, 
        # cf checker gives a warning on this ...
        if ('standard_name' not in da.attrs.keys()) and ('long_name' not in da.attrs.keys()) :
            # use variable name as long name ...
            da.attrs['long_name'] = os.path.basename(path)
        #endif
        
        # ok
        return da
        
    #enddef GetData
                    
    # *
    
    def SelectPixels( self, rcf, rckey, indent='' ) :
    
        """
        Apply filters specified in rcfile.
        
        Arguments:
        
        * ``rcf``       :  :py:class:`.RcFile` object with settings
        * ``rckey``     :  basename for rcfile keys, e.g. "s5p" for the example below
            
        Return values:
        
        * ``selected`` : boolean array with shape of track ``(time,scan,pixel)`` which
          is True if a pixel passed all checks;
        * ``history``  : list of character string describing the selelections.
          
        Example configuration::
        
            ! Specifiy a list of filter names.
            ! For each name, specify the variable which values are used for testing,
            ! the type test, and eventually some thresholds or other settings for this type.
            ! The units of the thresholds should match with the units in the variables,
            ! the expected units have to be defined too.
            ! The examples below show possible types and their settings.

            ! filters:
            s5p.filters                        :  lons lats albedo valid

            ! select range of values:
            s5p.filter.lons.var                :  PRODUCT/longitude
            s5p.filter.lons.type               :  minmax
            s5p.filter.lons.minmax             :  -15.0 35.0
            s5p.filter.lons.units              :  degrees_east

            ! select above a minimum:
            s5p.filter.lats.var                :  PRODUCT/latitude
            s5p.filter.lats.type               :  minmax
            s5p.filter.lats.minmax             :  35.0 70.0
            s5p.filter.lats.units              :  degrees_north

            ! select below a maximum:
            s5p.filter.albedo.var              :  Data Fields/SurfaceAlbedo
            s5p.filter.albedo.type             :  max
            s5p.filter.albedo.max              :  0.3
            s5p.filter.albedo.units            :  1

            ! select only values with data (no "_FillValue"):
            s5p.filter.valid.var               :  Data Fields/NO2RetrievalTroposphericVerticalColumn
            s5p.filter.valid.type              :  valid
        
        """
        
        # modules:
        import xarray
        import numpy
        
        # info ...
        logging.info( indent+'pixel selection ...' )
   
        # init selection, by default accept all:
        selected = numpy.ones( (self.ntime,self.nscan,self.ngpix), 'bool' )
        
        # init history:
        history = []

        # which filters ?
        filternames = rcf.get( '%s.filters' % rckey ).split()
        # loop over filters:
        for filtername in filternames :
            # info ...
            logging.info( indent+'  filter "%s" ...' % filtername )
            # filter key in rcfiles:
            fkey = '%s.filter.%s' % (rckey,filtername)
            # which variable ?
            varname = rcf.get(fkey+'.var')
            # read variable as data array:
            da = self.GetData( varname )
            # which type ?
            filtertype = rcf.get(fkey+'.type')
            # switch:
            #~ compare values:
            if filtertype == 'equal' :
                # read value:
                svalue = rcf.get(fkey+'.value')
                fvalue = float(svalue)
                # avoid warnings about comparing nan ...
                xdata = da.data.copy()
                if numpy.any( numpy.isnan(da.data) ) : xdata[numpy.where(numpy.isnan(da.data))] = fvalue - 1
                # apply filter:
                selected = selected & (xdata == fvalue)
                # update history, also select on not-nan to avoid warnings from comparisons:
                history.append( "selected pixels with '%s' == %s" % (varname,svalue) )
            #
            elif filtertype == 'ratio_square' :
                try:
                    varname2 = rcf.get(fkey+'.var2')
                    varname3 = rcf.get(fkey+'.var3')
                except:
                    print('var2/var3')
                    raise ValueError
                # read other required fields
                da2 = self.GetData( varname2 )
                da3 = self.GetData( varname3 )
                fmax = float(rcf.get(fkey+'.max'))
                # only accept non-masked fields:
                selected = selected & (numpy.abs(numpy.sqrt(da2.data**2. + da3.data**2.)/da.data)<=fmax)
                selected = selected & (~numpy.isnan(numpy.abs(numpy.sqrt(da2.data**2. + da3.data**2.)/da.data)))
                # update history:
                history.append( "selected pixels with valid ratio 'sqrt(%s**2.+%s**2.)/%s' values" % (varname2,varname3,varname) )
            #~ minimum value:
            elif filtertype == 'min' :
                # read threshold:
                smin = rcf.get(fkey+'.min')
                fmin = float(smin)
                # extract input units:
                vunits = da.attrs['units']
                # check units ...
                funits = rcf.get(fkey+'.units')
                if funits != vunits :
                    logging.error( 'variable "%s" has units "%s" while threshold has units "%s"' % (varname,vunits,funits) )
                    raise Exception
                #endif
                # avoid warnings about comparing nan ...
                xdata = da.data.copy()
                if numpy.any( numpy.isnan(da.data) ) : xdata[numpy.where(numpy.isnan(da.data))] = fmin - 1
                # apply filter, also select on not-nan to avoid warnings from comparisons:
                selected = selected & (xdata >= fmin)
                # update history:
                history.append( "selected pixels with '%s' >= %s" % (varname,smin) )
            #
            #~ maximum value:
            elif filtertype == 'max' :
                # read threshold:
                smax = rcf.get(fkey+'.max')
                fmax = float(smax)
                # extract input units:
                vunits = da.attrs['units']
                # check units ...
                funits = rcf.get(fkey+'.units')
                if funits != vunits :
                    logging.error( 'variable "%s" has units "%s" while threshold has units "%s"' % (varname,vunits,funits) )
                    logging.error( 'value range : %f - %f' % (numpy.nanmin(values),numpy.nanmax(values)) )
                    raise Exception
                #endif
                # avoid warnings about comparing nan ...
                xdata = da.data.copy()
                if numpy.any( numpy.isnan(da.data) ) : xdata[numpy.where(numpy.isnan(da.data))] = fmax + 1
                # apply filter, also select on not-nan to avoid warnings from comparisons:
                selected = selected & (xdata <= fmax)
                # update history:
                history.append( "selected pixels with '%s' <= %s" % (varname,smax) )
            #
            #~ range of valid values:
            elif filtertype == 'minmax' :
                # read thresholds:
                smin,smax = rcf.get(fkey+'.minmax').split()
                fmin = float(smin)
                fmax = float(smax)
                # extract input units:
                vunits = da.attrs['units']
                # check units ...
                funits = rcf.get(fkey+'.units')
                if funits != vunits :
                    logging.error( 'variable "%s" has units "%s" while threshold has units "%s"' % (varname,vunits,funits) )
                    raise Exception
                #endif
                # avoid warnings about comparing nan ...
                xdata = da.data.copy()
                if numpy.any( numpy.isnan(da.data) ) : xdata[numpy.where(numpy.isnan(da.data))] = fmin - 1
                # apply filter, also select on not-nan to avoid warnings from comparisons:
                selected = selected & (xdata >= fmin) & (xdata <= fmax)
                # update history:
                history.append( "selected pixels with '%s' in [%s,%s]" % (varname,smin,smax) )
            #
            #~ only unmasked values:
            elif filtertype == 'valid' :
                # only accept non-masked fields:
                selected = selected & (~ numpy.isnan(da.data))
                # update history:
                history.append( "selected pixels with valid '%s' values" % varname )
            #
            #~ not yet ...
            else :
                logging.error( 'unsupported filter type : %s' % filtertype )
                raise Exception
            #endif
        #endfor # filters
        
        ## if ncecessary, convert from a masked to a normal array:
        #if hasattr(selected,'mask') : selected = selected.filled(fill_value=False)

        # ok
        return selected,history
    
    #enddef SelectPixels
        
#endclass S5p_File



########################################################################
###
### converted file
###
########################################################################


class CSO_S5p_File( cso_file.CSO_File ) :

    """
    Storage for CSO satellite file filled from S5p data.
    """
    
    def AddSelection( self, sfile, selected, rcf, rcbase, indent='' ) :
    
        """
        Add selected OMI to satellite extract file.
        
        Arguments:
        
        * ``sfile``    :  :py:class:`.S5p_File` object
        * ``selected``   :  boolean array as provided by :py:meth:`S5p_File.SelectPixels` method
        * ``rcf``        :  :py:class:`.RcFile` instance with settings
        * ``rcbase``      :  base of rcfile keys
            
        The first setting that is read is a list with variable names to be 
        created in the target file::

          <rcbase>.output.vars    :  longitude corner_longitudes \\
                                     latitude corner_latitudes \\
                                     vcd  ...

        For each variable, a series of settings should be specified that describe 
        how the variable should look like and how to create it.
        
        The first setting is a list of dimension names that define the shape of the variable.
        Supported dimensions are:
        
        * ``pixel``       : selected pixels
        * ``corner``      : number of footprint bounds (probably 4)
        * ``layer``       : number of layers in atmospheric profile (layers in kernel)
        * ``layeri``      : number of layer interfaces in atmospheric profile (``layer+1``)
        * ``retr``        : number of layers in retrieval product (1 for columns)
        * ``retr0``       : same as ``retr``, used for matrix dimensions ``(retr,retr0)`` to avoid
          repeated dimensions where the cf-checker complains about
        * ``track_scan``  : original scan index   in 2D track
        * ``track_pixel`` : original ground pixel in 2D track
        
        For a 1D variable with values per pixel the dimension setting is therefore::
        
            <rcbase>.output.var.longitude.dims    :   pixel
        
        For most variables it is sufficient to provide only the name of the original
        variable from which the data should be read::

          <rcbase>.output.var.longitude.from    :   Geolocation Fields/Longitude
        
        For some variables a special processing needs to be done.
        For these variables a key '``special``' is used which will enable the 
        correct conversion. The following specials are currently implemented:
        
        * ``track_longitude``          : longiudes at centers of original 2D track; requires a ``.from`` setting
        * ``track_latitude``           : latiudes  at centers of original 2D track; requires a ``.from`` setting
        * ``track_longitude_bounds``   : longiude bounds at centers of original 2D track; requires a ``.from`` setting
        * ``track_latitude_bounds``    : latiude  bounds at centers of original 2D track; requires a ``.from`` setting
        * ``ground_pixel``             : index of ground pixel in original 2D track; requires a ``.from`` setting
        * ``sum`` : create a variable as the sum over over layers; requires a ``.from`` setting
        * ``square`` : create a variable as the square of the input; requires a ``.from`` setting
        * ``time`` : create time stamps per pixel from a reference time and a time delta; requires settings::

            <rcbase>.output.var.time.tref      :   PRODUCT/time
            <rcbase>.output.var.time.dt        :   PRODUCT/delta_time
            
        * ``hybounds_to_pressure`` : form pressure from hybride sigma pressure coordinate,
          where the available hybride coefficients have shape ``('layer',2)``;
          requires settings::
        
            <rcbase>.output.var.pressure.sp       :   PRODUCT/SUPPORT_DATA/INPUT_DATA/surface_pressure
            <rcbase>.output.var.pressure.hyab     :   PRODUCT/tm5_constant_a
            <rcbase>.output.var.pressure.hybb     :   PRODUCT/tm5_constant_b
            <rcbase>.output.var.pressure.units    :   Pa
            
        * ``hymid_to_pressure`` : form pressure from hybride sigma pressure coordinate,
          where the available hybride coefficients are valid for the middle of the layers and 
          therefore have shape ``('layer')``;
          requires settings::

            <rcbase>.output.var.pressure.sp       :   PRODUCT/SUPPORT_DATA/INPUT_DATA/surface_pressure
            <rcbase>.output.var.pressure.hyam     :   PRODUCT/SUPPORT_DATA/INPUT_DATA/hyam
            <rcbase>.output.var.pressure.hybm     :   PRODUCT/SUPPORT_DATA/INPUT_DATA/hybm
            <rcbase>.output.var.pressure.units    :   Pa
            
        * ``sp_dp_to_pressure`` : form pressure from surface pressure and a constant pressure step per layer 
          (top at zero); requires settings::

            <rcbase>.output.var.pressure.sp       :   PRODUCT/SUPPORT_DATA/INPUT_DATA/surface_pressure
            <rcbase>.output.var.pressure.dp       :   PRODUCT/SUPPORT_DATA/INPUT_DATA/pressure_interval
            <rcbase>.output.var.pressure.units    :   Pa
        
        * ``kernel_trop`` : create averaging kernel for tropospheric column as the original kernel times
          the ratio between total-air-mass-factor and tropospheric-air-mass-factor:
        
          .. math::
             K_{tropo} ~=~ K~ AMF_{total}~/~{AMF}_{tropo}
             
          Required settings::
          
            <rcbase>.output.var.kernel.avk        :   PRODUCT/averaging_kernel
            <rcbase>.output.var.kernel.amf        :   PRODUCT/air_mass_factor_total
            <rcbase>.output.var.kernel.amft       :   PRODUCT/air_mass_factor_troposphere
            <rcbase>.output.var.kernel.troplayer  :   PRODUCT/tm5_tropopause_layer_index
           
          The variable specified by ``troplayer`` is used to reset 
          the higher layers of the kernel (strospher) to zero.
        
        * ``kernel_by_dh`` : convert a kernel in m (per layer) to a unitless kernel
          using divison by a constant layer height:
        
          .. math::
             A ~=~ A_m~/~dh
             
          The layer thinkness ``dh`` is taken from a ``layers`` coordinate that defines
          the middle of a layer for a regular grid; it is checked that the coordinate 
          defines a regular spacing.
             
          Required settings::
          
            <rcbase>.output.var.kernel.avk        :   PRODUCT/averaging_kernel
            <rcbase>.output.var.kernel.layer      :   PRODUCT/layer
           
        * ``square`` : create a variable as the square of the input; requires a ``.from`` setting.
           
        Optionally provide target units too. 
        In the (unlikely) case that the original variable has no ``units`` attribute,
        this setting is required to define the (assumed) units.
        If the provided units are different from the units in the original variable,
        the data is converted to the provided units::

          <rcbase>.output.var.longitude.units   :   degrees_east

        Optionally provide a dictionairy with attributes to be added.
        If the attribute value is ``None``, the attribute is removed if present from the input;
        this is sometimes needed if the CF compliance checker complains::
        
          !~ skip some attributes, cf-checker complains ...
          <rcbase>.output.var.qa_value.attrs         :   { 'valid_min' : None, 'valid_max' : None }


        """
        
        # modules:
        import os
        import numpy
        import xarray
        import pandas
        import scipy.interpolate
        
        # info ..
        logging.info( indent+'add pixels to satellite file ...' )
        
        # track size:
        track_ntime,track_nscan,track_npix = selected.shape

        # check ...
        if track_ntime != 1 :
            logging.error( 'only single time record supported, found %i' % track_ntime )
            logging.error( 'input file: %s' % sfile.filename )
            raise Exception
        #endif
        # time record:
        itime = 0
        
        # swap layers?
        swap_layers = rcf.get( rcbase+'.swap_layers', totype='bool' )

        # selection indices:
        iis,iip = numpy.where( selected[itime,:,:] )
        
        # selected scan lines:
        is0,is1 = iis.min(),iis.max()+1
        # scan line coordinate in track:
        track_scan = xarray.DataArray( numpy.arange(is0,is1,dtype='i2'), dims=('track_scan'), 
                                         attrs={ 'units' : '1', 'long_name' : 'scan line index within original track (0-based)' } )
        # store:
        self.AddCoord( 'track_scan', track_scan )
        
        # track pixel coordinate:
        track_pixel = xarray.DataArray( numpy.arange(track_npix,dtype='i2'), dims=('track_pixel'),
                                         attrs={ 'units' : '1', 'long_name' : 'pixel index within original track (0-based)' } )
        # store:
        self.AddCoord( 'track_pixel', track_pixel )
        
        # create pixel coordinate: 
        #   0-based index that represents the original scan and pixel,
        #   within the slab of selected scan lines;
        compressed_index = ( iis - is0 )*track_npix + iip
        # create coordinate data array ;
        # convert to int, otherwise cf-checker complains about the 'compress' attribute ..
        pixel = xarray.DataArray( numpy.array(compressed_index,dtype='i4'), dims=('pixel'),
                                    attrs={ 'units'     : '1', 
                                            'long_name' : 'pixel index', 
                                            'compress'  : 'track_scan track_pixel' } )
        # store:
        self.AddCoord( 'pixel', pixel )
        # store length:
        self.npixel = len(pixel)
        
        # number of retrieval layers:
        self.nretr = 1
        
        # target variables:
        varnames = rcf.get( rcbase+'.output.vars' ).split()
        
        # loop over target variables:
        for varname in varnames :
        
            # info ...
            logging.info( indent+'  add variable "%s" ...' % varname )

            # prefix for settings:
            vkey = '%s.output.var.%s' % (rcbase,varname)

            # target units:
            vunits = rcf.get( vkey+'.units', default='None' )
            if vunits == 'None' : vunits = None
            
            # target dimensions:
            vdims = tuple( rcf.get( vkey+'.dims' ).split() )

            # specials ?
            special = rcf.get( vkey+'.special', default='None' )
            # switch:
            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            if special in ['None','square'] :
            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            
                # name of original field:
                opath = rcf.get( vkey+'.from' )
                
                # optional data type:
                dtype = rcf.get( vkey+'.dtype', default='None' )
                if dtype == 'None' : dtype = None

                # extract data array:
                oda = sfile.GetData( opath, units=vunits )
                
                # check target dims:
                #~ single value per pixel:
                if vdims == ('pixel',) :
                
                    # extract 1D array from data that is only given per scan line,
                    # this is for example the case for the time array in some products:
                    if oda.dims in [('time','scanline')] :

                        # create 1D array with data from selected pixels, copy attributes:
                        da = xarray.DataArray( numpy.array(oda.data[itime,iis],dtype=dtype), 
                                                 dims=vdims, 
                                                 coords={ 'pixel' : pixel }, 
                                                 attrs=oda.attrs )

                    # extract 1D array of pixels from 2D track:
                    elif oda.dims in [('time','scanline','ground_pixel')] :

                        # create 1D array with data from selected pixels, copy attributes:
                        da = xarray.DataArray( numpy.array(oda.data[itime,iis,iip],dtype=dtype), 
                                                 dims=vdims, 
                                                 coords={ 'pixel' : pixel }, 
                                                 attrs=oda.attrs )

                    # same time value for each pixel:
                    elif oda.dims in [('time',)] :

                        # extract ref time:
                        tref = oda.data[itime]
                        # create 1D array with data from selected pixels, copy attributes:
                        da = xarray.DataArray( numpy.full((self.npixel,),tref), 
                                                 dims=vdims, 
                                                 coords={ 'pixel' : pixel }, 
                                                 attrs=oda.attrs )

                    else :
                        logging.error( 'could not convert from dims "%s" to target dims "%s"' % (str(oda.dims),str(vdims)) )
                        raise Exception
                    #endif
                    
                #~ retrieval profile per pixel:
                elif vdims == ('pixel','retr') :
                
                    # extract 1D array of pixels from 2D track:
                    if oda.dims in [('time','scanline','ground_pixel')] :

                        # check ...
                        if self.nretr != 1 :
                            logging.error( 'not implemented yet for number of retrieval layers %i' % self%nretr )
                            raise Exception
                        #endif

                        # create 1D array with data from selected pixels, copy attributes:
                        da = xarray.DataArray( numpy.array(oda.data[itime,iis,iip].reshape(self.npixel,self.nretr),dtype=dtype), 
                                                 dims=vdims, 
                                                 coords={ 'pixel' : pixel }, 
                                                 attrs=oda.attrs )

                    else :
                        logging.error( 'could not convert from dims "%s" to target dims "%s"' % (str(oda.dims),str(vdims)) )
                        raise Exception
                    #endif
                    
                #~ apriori profile per pixel:
                elif vdims == ('pixel','layer') :
                
                    # extract 1D array of profiles from 2D track:
                    if oda.dims in [('time','scanline','ground_pixel','layer')] :

                        # create 1D array with data from selected pixels, copy attributes:
                        da = xarray.DataArray( numpy.array(oda.data[itime,iis,iip,:],dtype=dtype), 
                                                 dims=vdims, 
                                                 coords={ 'pixel' : pixel }, 
                                                 attrs=oda.attrs )

                    else :
                        logging.error( 'could not convert from dims "%s" to target dims "%s"' % (str(oda.dims),str(vdims)) )
                        raise Exception
                    #endif

                #~ kernel per pixel:
                elif vdims == ('pixel','layer','retr') :
                
                    # extract 2D array of pixels:
                    if oda.dims in [('time','scanline','ground_pixel','layer')] :

                        # check ...
                        if self.nretr != 1 :
                            logging.error( 'not implemented yet for number of retrieval layers %i' % self%nretr )
                            raise Exception
                        #endif
                        # check ...
                        if sfile.nlayer is None :
                            logging.error( 'no layer dimension defined yet' )
                            raise Exception
                        #endif

                        # create data array with data from selected pixels, copy attributes:
                        da = xarray.DataArray( numpy.array(oda.data[itime,iis,iip,:].reshape(self.npixel,sfile.nlayer,self.nretr),dtype=dtype),
                                                 dims=vdims, 
                                                 coords={ 'pixel' : pixel }, 
                                                 attrs=oda.attrs )

                    else :
                        logging.error( 'could not convert from dims "%s" to target dims "%s"' % (str(oda.dims),str(vdims)) )
                        raise Exception
                    #endif

                #~ retrieval matrix per pixel:
                elif vdims == ('pixel','retr','retr0') :
                
                    # extract 1D array of pixels from 2D track:
                    if oda.dims in [('time','scanline','ground_pixel')] :

                        # check ...
                        if self.nretr != 1 :
                            logging.error( 'not implemented yet for number of retrieval layers %i' % self%nretr )
                            raise Exception
                        #endif

                        # create 1D array with data from selected pixels, copy attributes:
                        da = xarray.DataArray( numpy.array(oda.data[itime,iis,iip].reshape(self.npixel,self.nretr,self.nretr),dtype=dtype),
                                                 dims=vdims, 
                                                 coords={ 'pixel' : pixel }, 
                                                 attrs=oda.attrs )

                    else :
                        logging.error( 'could not convert from dims "%s" to target dims "%s"' % (str(oda.dims),str(vdims)) )
                        raise Exception
                    #endif

                #~ corners per pixel:
                elif vdims == ('pixel','corner') :
                
                    # bound arrays for longitude etc:
                    if oda.dims in [('time','scanline','ground_pixel','corner')] :

                        # create 1D array with data from selected pixels, copy attributes:
                        da = xarray.DataArray( numpy.array(oda.data[itime,iis,iip],dtype=dtype),
                                                 dims=vdims, 
                                                 coords={ 'pixel' : pixel },
                                                 attrs=oda.attrs )

                    else :
                        logging.error( 'could not convert from dims "%s" to target dims "%s"' % (str(oda.dims),str(vdims)) )
                        raise Exception
                    #endif

                else :
                    logging.error( 'unsupported target dims: %s' % str(vdims) )
                    logging.error( '  target variable   : %s' % varname )
                    raise Exception
                #endif
                
                # postprocessing:
                if special == 'square' :
                    # apply:
                    da.data = da.data**2
                    da.attrs['units'] = '(%s)**2' % da.attrs['units']
                #endif

            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            elif special == 'square_sum' :
            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            
                # name of original field:
                opath = rcf.get( vkey+'.from' )
                opath2 = rcf.get( vkey+'.from2' )
                
                # optional data type:
                dtype = rcf.get( vkey+'.dtype', default='None' )
                if dtype == 'None' : dtype = None

                # extract data array:
                oda = sfile.GetData( opath, units=vunits )
                oda2 = sfile.GetData( opath2, units=vunits )
                
                # check target dims:

                #~ retrieval matrix per pixel:
                if vdims == ('pixel','retr','retr0') :
                
                    # extract 1D array of pixels from 2D track:
                    if oda.dims in [('time','scanline','ground_pixel')] :

                        # check ...
                        if self.nretr != 1 :
                            logging.error( 'not implemented yet for number of retrieval layers %i' % self%nretr )
                            raise Exception
                        #endif

                        # create 1D array with data from selected pixels, copy attributes:
                        da = xarray.DataArray( oda.data[itime,iis,iip].reshape(self.npixel,self.nretr,self.nretr), 
                                                 dims=vdims, 
                                                 coords={ 'pixel' : pixel }, 
                                                 attrs=oda.attrs )
                        da2 = xarray.DataArray( oda2.data[itime,iis,iip].reshape(self.npixel,self.nretr,self.nretr), 
                                                 dims=vdims, 
                                                 coords={ 'pixel' : pixel }, 
                                                 attrs=oda.attrs )
                        da.data = numpy.sqrt(da.data**2 + da2.data**2)
                    else :
                        logging.error( 'could not convert from dims "%s" to target dims "%s"' % (str(oda.dims),str(vdims)) )
                        raise Exception
                    #endif

                else :
                    logging.error( 'unsupported target dims: %s' % str(vdims) )
                    logging.error( '  target variable   : %s' % varname )
                    raise Exception
                #endif
                
                # postprocessing:
                if special == 'square' :
                    # apply:
                    da.data = da.data**2
                    da.attrs['units'] = '(%s)**2' % da.attrs['units']
                #endif

            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            # time per pixel
            elif special == 'time-delta' :
            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            
                # Time variables in latest files:
                # 
                #    group: PRODUCT {
                #      dimensions:
                #          scanline = 3246 ;
                #          time = 1 ;
                #      variables:
                #          int time(time) ;
                #              time:units = "seconds since 2010-01-01 00:00:00" ;
                #              time:standard_name = "time" ;
                #              time:long_name = "reference time for the measurements" ;
                #          int delta_time(time, scanline) ;
                #              delta_time:long_name = "offset of start time of measurement relative to time_reference" ;
                #              delta_time:units = "milliseconds since 2019-06-01 00:00:00" ;
                #     	string time_utc(time, scanline) ;
                #              time_utc:long_name = "Time of observation as ISO 8601 date-time string" ;
                #              time_utc:_FillValue = "" ;
                #
                # Conversion from the "time_utc" strings failed; reading usually ok,
                # but writing to netcdf as floats with units "seconds since ..." seems difficult.
                #
                # In this example only one time per scan line.
                # For some files (SO2 from DLR) there is a time value for each pixel:
                #   
                #       int delta_time(time, scanline, ground_pixel) ;
                #
                # Older files have "delta_time" with different units:
                #
                #          int delta_time(time, scanline) ;
                #              delta_time:long_name = "offset from reference start time of measurement" ;
                #              delta_time:units = "milliseconds" ;
                #   
            
                # name of variable with reference time:
                opath = rcf.get( vkey+'.tref' )
                # extract data array:
                oda = sfile.GetData( opath )
                # extract ref time:
                tref = oda.data[itime]
            
                # name of variable with time step:
                opath = rcf.get( vkey+'.dt' )
                # extract data array:
                oda = sfile.GetData( opath )
                # check target dims:
                if vdims == ('pixel',) :
                    # extract per pixel:
                    if oda.dims in [('time','scanline')] :
                        dts = oda.data[itime,iis]
                    elif oda.dims in [('time','scanline','ground_pixel')] :
                        dts = oda.data[itime,iis,iip]
                    else :
                        logging.error( 'could not convert from dims "%s" to target dims "%s"' % (str(oda.dims),str(vdims)) )
                        raise Exception
                    #endif
                else :
                    logging.error( 'unsupported target dims: %s' % str(vdims) )
                    logging.error( '  target variable   : %s' % varname )
                    raise Exception
                #endif
                
                # adhoc: in CH4 files the dts seems a datetime already ...
                if type(dts[0]) == numpy.datetime64 :
                    # copy:
                    values = dts
                else :
                    # combine reference and delta:
                    values = tref + dts
                #endif
                
                ## base time for units "seconds since ..." as start of year:
                #tref = pandas.Timestamp( year=values[0].year, month=1, day=1, tz=0 )
            
                # create variable:
                da = xarray.DataArray( values, dims=vdims, 
                                         coords={ 'pixel' : pixel }, 
                                         attrs=oda.attrs )
                # ensure corrrect output:
                da.attrs['standard_name'] = 'time'
                da.attrs['long_name'    ] = 'time'
                da.encoding['calendar'] = 'standard'
                

            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            # full track, centers:
            elif special in ['track_longitude','track_latitude'] :
            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            
                # name of original field:
                opath = rcf.get( vkey+'.from' )
                # get data array:
                oda = sfile.GetData( opath, units=vunits )
                
                # check target dims:
                if vdims == ('track_scan','track_pixel') :
                
                    # check shape:
                    if oda.dims in [('time','scanline','ground_pixel')] :

                        # create 2D array with data from selected scan lines:
                        da = xarray.DataArray( oda.data[itime,is0:is1,:], 
                                                 dims=vdims, 
                                                 coords={ 'track_scan' : track_scan, 'track_pixel' : track_pixel },
                                                 attrs=oda.attrs )

                    else :
                        logging.error( 'could not convert from dims "%s" to target dims "%s"' % (str(oda.dims),str(vdims)) )
                        raise Exception
                    #endif
                else :
                    logging.error( 'unsupported target dims: %s' % str(vdims) )
                    logging.error( '  target variable   : %s' % varname )
                    raise Exception
                #endif

            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            # full track, corners:
            elif special in ['track_longitude_bounds','track_latitude_bounds'] :
            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            
                # name of original field:
                opath = rcf.get( vkey+'.from' )
                # get data array:
                oda = sfile.GetData( opath, units=vunits )
                
                # check target dims:
                if vdims == ('track_scan','track_pixel','corner') :
                
                    # check shape:
                    if oda.dims in [('time','scanline','ground_pixel','corner')] :

                        # create 2D array (+corners) with data from selected scan lines:
                        da = xarray.DataArray( oda.data[itime,is0:is1,:], 
                                                 dims=vdims, 
                                                 coords={ 'track_scan' : track_scan, 'track_pixel' : track_pixel },
                                                 attrs=oda.attrs )
                    else :
                        logging.error( 'could not convert from dims "%s" to target dims "%s"' % (str(oda.dims),str(vdims)) )
                        raise Exception
                    #endif
                else :
                    logging.error( 'unsupported target dims: %s' % str(vdims) )
                    logging.error( '  target variable   : %s' % varname )
                    raise Exception
                #endif

            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            # ground pixel :
            elif special in ['ground_pixel'] :
            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            
                # name of original field:
                opath = rcf.get( vkey+'.from' )
                # get data array:
                oda = sfile.GetData( opath, units=vunits )
                
                # check target dims:
                if vdims == ('track_pixel',) :
                
                    # check shape:
                    if oda.dims in [('ground_pixel',)] :
                        # create 2D array with data from selected scan lines:
                        da = xarray.DataArray( oda.data[:], 
                                                 dims=vdims, 
                                                 coords={'track_pixel' : track_pixel },
                                                 attrs=oda.attrs )
                    else :
                        logging.error( 'could not convert from dims "%s" to target dims "%s"' % (str(oda.dims),str(vdims)) )
                        raise Exception
                    #endif
                else :
                    logging.error( 'unsupported target dims: %s' % str(vdims) )
                    logging.error( '  target variable   : %s' % varname )
                    raise Exception
                #endif

            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            # sum over profile
            elif special == 'sum' :
            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            
                # name of original field:
                opath = rcf.get( vkey+'.from' )
                # get data array:
                oda = sfile.GetData( opath, units=vunits )
                
                # check target dims:
                if vdims == ('pixel',) :
                
                    # check shape:
                    if oda.dims in [('time','scanline','ground_pixel','layer')] :

                        # create 1D array with data from selected pixels, copy attributes:
                        da = xarray.DataArray( oda.data[itime,iis,iip,:].sum(axis=1), 
                                                 dims=vdims, 
                                                 coords={ 'pixel' : pixel },
                                                 attrs=oda.attrs )

                    else :
                        logging.error( 'could not convert from dims "%s" to target dims "%s"' % (str(oda.dims),str(vdims)) )
                        raise Exception
                    #endif

                else :
                    logging.error( 'unsupported target dims: %s' % str(vdims) )
                    logging.error( '  target variable   : %s' % varname )
                    raise Exception
                #endif

            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            # pressure from hybride coordinates;
            # hyab and hybb have shape ('layer','vertices') :
            elif special == 'hybounds_to_pressure' :
            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            
                # set attributes:
                attrs = {}
                attrs['units'    ] = vunits
                attrs['long_name'] = 'half level pressures'
            
                # original fields:
                odat = {}
                for key in ['sp','hyab','hybb'] :
                    # full name:
                    opath = rcf.get( '%s.output.var.%s.%s' % (rcbase,varname,key) )
                    # extract:
                    odat[key] = sfile.GetData( opath )
                #endfor
                
                # check ...
                if odat['hyab'].attrs['units'] != vunits :
                    logging.error( 'expecting hybride units hya "%s", found "%s"' % (vunits,odat['hyab'].attrs['units']) )
                    raise Exception
                #endif
                if odat['hybb'].attrs['units'] != '1' :
                    logging.error( 'expecting hybride units hyb "1", found "%s"' % odat['hybb'].attrs['units'] )
                    raise Exception
                #endif
                if odat['sp'].attrs['units'] !=  vunits :
                    logging.error( 'expecting hybride units sp "%s", found "%s"' % (vunits,odat['sp'].attrs['units']) )
                    raise Exception
                #endif

                # check target dims:
                if vdims == ('pixel','layeri') :

                    # extract per pixel:
                    if odat['sp'].dims in [('time','scanline','ground_pixel')] :
                        # extract pixels:
                        sp = odat['sp'].data[itime,iis,iip]
                    else :
                        logging.error( 'could not convert from dims "%s" to target dims "%s"' % (str(oda.dims),str(vdims)) )
                        raise Exception
                    #endif
                
                    # check ...
                    if sfile.nlayer is None :
                        logging.error( 'no layer dimension defined yet' )
                        raise Exception
                    #endif

                    # half level pressures:
                    hp = numpy.zeros((len(pixel),sfile.nlayer+1),dtype='f4')
                    # surface:
                    iz = 0
                    hp[:,iz] = odat['hyab'].data[iz,0] + odat['hybb'].data[iz,0] * sp 
                    # loop over layers:
                    for iz in range(1,sfile.nlayer+1) :
                        # top pressure:
                        hp[:,iz] = odat['hyab'].data[iz-1,1] + odat['hybb'].data[iz-1,1] * sp
                    #endfor

                    # create variable:
                    da = xarray.DataArray( hp, dims=('pixel','layeri'), 
                                                 coords={ 'pixel' : pixel }, 
                                                 attrs=attrs )

                else :
                    logging.error( 'unsupported target dims: %s' % str(vdims) )
                    logging.error( '  target variable   : %s' % varname )
                    raise Exception
                #endif
                
            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            # pressure from hybride coordinates;
            # hyam and hybm have shape ('layer'), 
            # interpolate/extrapolate to hyai and hybi ('layer_interface')
            elif special == 'hymid_to_pressure' :
            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            
                # set attributes:
                attrs = {}
                attrs['units'    ] = vunits
                attrs['long_name'] = 'half level pressures'
            
                # original fields:
                odat = {}
                for key in ['sp','hyam','hybm'] :
                    # full name:
                    opath = rcf.get( '%s.output.var.%s.%s' % (rcbase,varname,key) )
                    # extract:
                    odat[key] = sfile.GetData( opath )
                #endfor
                
                # check ...
                if odat['hyam'].attrs['units'] != vunits :
                    logging.error( 'expecting hybride units hya "%s", found "%s"' % (vunits,odat['hyam'].attrs['units']) )
                    raise Exception
                #endif
                if odat['hybm'].attrs['units'] != '1' :
                    logging.error( 'expecting hybride units hyb "1", found "%s"' % odat['hybm'].attrs['units'] )
                    raise Exception
                #endif
                if odat['sp'].attrs['units'] !=  vunits :
                    logging.error( 'expecting hybride units sp "%s", found "%s"' % (vunits,odat['sp'].attrs['units']) )
                    raise Exception
                #endif
                
                # check target dims:
                if vdims == ('pixel','layeri') :

                    # extract per pixel:
                    if odat['sp'].dims in [('time','scanline','ground_pixel')] :
                        # extract pixels:
                        sp = odat['sp'].data[itime,iis,iip]
                    else :
                        logging.error( 'could not convert from dims "%s" to target dims "%s"' % (str(odat['sp'].dims),str(vdims)) )
                        raise Exception
                    #endif
                
                    # extract, sometimes data with time dimension (1):
                    hyam = odat['hyam'].data.squeeze()
                    hybm = odat['hybm'].data.squeeze()                
                    # extend to interfaces, edges are not extrapolated but will be reset below:
                    nlay = len(hyam)
                    hyai = numpy.interp( numpy.arange(nlay+1)-0.5, numpy.arange(nlay), hyam )
                    hybi = numpy.interp( numpy.arange(nlay+1)-0.5, numpy.arange(nlay), hybm )
                    # a is zero at surface (use surface pressre) and top (space) ...
                    hyai[ 0] = 0.0
                    hyai[-1] = 0.0
                    # direction?
                    if hybm[0] > hybm[-1] :
                        hybi[ 0] = 1.0   # surface
                        hybi[-1] = 0.0   # space
                    else :
                        hybi[-1] = 1.0   # surface
                        hybi[ 0] = 0.0   # space
                    #endif

                    # check ...
                    if sfile.nlayer is None :
                        logging.error( 'no layer dimension defined yet' )
                        raise Exception
                    #endif

                    # half level pressures:
                    hp = numpy.zeros((len(pixel),sfile.nlayer+1),dtype='f4')
                    # loop over layers:
                    for iz in range(sfile.nlayer+1) :
                        # top pressure:
                        hp[:,iz] = hyai[iz] + hybi[iz] * sp
                    #endfor

                    # create variable:
                    da = xarray.DataArray( hp, dims=('pixel','layeri'), 
                                                 coords={ 'pixel' : pixel }, 
                                                 attrs=attrs )

                else :
                    logging.error( 'unsupported target dims: %s' % str(vdims) )
                    logging.error( '  target variable   : %s' % varname )
                    raise Exception
                #endif
                
            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            # pressure at boundaries from mid pressure
            elif special == 'pmid_to_pressure' :
            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            
                # set attributes:
                attrs = {}
                attrs['units'    ] = vunits
                attrs['long_name'] = 'half level pressures'
            
                # original fields:
                odat = {}
                for key in ['pmid'] :
                    # full name:
                    opath = rcf.get( '%s.output.var.%s.%s' % (rcbase,varname,key) )
                    # extract:
                    odat[key] = sfile.GetData( opath )
                #endfor
                
                # check ...
                if odat['pmid'].attrs['units'] !=  vunits :
                    logging.error( 'expecting pressure units pmid "%s", found "%s"' % (vunits,odat['pmid'].attrs['units']) )
                    raise Exception
                #endif
                
                # check target dims:
                if vdims == ('pixel','layeri') :

                    # extract per pixel:
                    if odat['pmid'].dims in [('time','scanline','ground_pixel','layer')] :
                        # extract pixels:
                        pmid = odat['pmid'].data[itime,iis,iip,:]
                    else :
                        logging.error( 'could not convert from dims "%s" to target dims "%s"' % (str(odat['pmid'].dims),str(vdims)) )
                        raise Exception
                    #endif
                
                    # check ...
                    if sfile.nlayer is None :
                        logging.error( 'no layer dimension defined yet' )
                        raise Exception
                    #endif
                    
                    # define interpolator with index ax,
                    # apply with half-level indices:
                    hp = numpy.array( 
                                scipy.interpolate.interp1d(
                                        numpy.arange(sfile.nlayer), pmid, fill_value='extrapolate'
                                    )( numpy.arange(sfile.nlayer+1)-0.5 ),
                            dtype='f4' )

                    # create variable:
                    da = xarray.DataArray( hp, dims=('pixel','layeri'), 
                                                 coords={ 'pixel' : pixel }, 
                                                 attrs=attrs )

                else :
                    logging.error( 'unsupported target dims: %s' % str(vdims) )
                    logging.error( '  target variable   : %s' % varname )
                    raise Exception
                #endif
                
            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            # pressure at half levels using:
            # - surface pressure
            # - same pressure step for each layer
            # - top zero
            elif special == 'sp_dp_to_pressure' :
            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            
                # set attributes:
                attrs = {}
                attrs['units'    ] = vunits
                attrs['long_name'] = 'half level pressures'
            
                # original fields:
                odat = {}
                for key in ['sp','dp'] :
                    # full name:
                    opath = rcf.get( '%s.output.var.%s.%s' % (rcbase,varname,key) )
                    # extract:
                    odat[key] = sfile.GetData( opath )
                    # check ...
                    if odat[key].attrs['units'] !=  vunits :
                        logging.error( 'expecting hybride units %s "%s", found "%s"' % (key,vunits,odat[key].attrs['units']) )
                        raise Exception
                    #endif
                #endif
                
                # check target dims:
                if vdims == ('pixel','layeri') :

                    # extract per pixel:
                    if odat['sp'].dims in [('time','scanline','ground_pixel')] :
                        # extract pixels:
                        sp = odat['sp'].data[itime,iis,iip]
                    else :
                        logging.error( 'could not convert from dims "%s" to target dims "%s"' % (str(odat['sp'].dims),str(vdims)) )
                        raise Exception
                    #endif

                    # extract per pixel:
                    if odat['dp'].dims in [('time','scanline','ground_pixel')] :
                        # extract pixels:
                        dp = odat['dp'].data[itime,iis,iip]
                    else :
                        logging.error( 'could not convert from dims "%s" to target dims "%s"' % (str(odat['dp'].dims),str(vdims)) )
                        raise Exception
                    #endif

                    # check ...
                    if sfile.nlayer is None :
                        logging.error( 'no layer dimension defined yet' )
                        raise Exception
                    #endif

                    # half level pressures:
                    hp = numpy.zeros((len(pixel),sfile.nlayer+1),dtype='f4')
                    # top, numbering from top to bottom:
                    hp[:,0] = 0
                    # loop over layers:
                    for iz in range(1,sfile.nlayer+1) :
                        hp[:,iz] = hp[:,iz-1] + dp
                    #endfor

                    # create variable:
                    da = xarray.DataArray( hp, dims=('pixel','layeri'), 
                                                 coords={ 'pixel' : pixel }, 
                                                 attrs=attrs )

                else :
                    logging.error( 'unsupported target dims: %s' % str(vdims) )
                    logging.error( '  target variable   : %s' % varname )
                    raise Exception
                #endif
                
            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            # pressure at half levels using:
            # - pressures at bottom of layer
            # - top zero
            elif special == 'pbottom_to_pressure' :
            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            
                # set attributes:
                attrs = {}
                attrs['units'    ] = vunits
                attrs['long_name'] = 'half level pressures'
            
                # original fields:
                odat = {}
                for key in ['pbottom'] :
                    # full name:
                    opath = rcf.get( '%s.output.var.%s.%s' % (rcbase,varname,key) )
                    # extract:
                    odat[key] = sfile.GetData( opath )
                    # check ...
                    if odat[key].attrs['units'] !=  vunits :
                        logging.error( 'expecting hybride units %s "%s", found "%s"' % (key,vunits,odat[key].attrs['units']) )
                        raise Exception
                    #endif
                #endif
                
                # check target dims:
                if vdims == ('pixel','layeri') :

                    # extract per pixel:
                    if odat['pbottom'].dims in [('time','scanline','ground_pixel','layer')] :
                        # extract pixels:
                        pbottom = odat['pbottom'].data[itime,iis,iip,:]
                    else :
                        logging.error( 'could not convert from dims "%s" to target dims "%s"' % (str(odat['sp'].dims),str(vdims)) )
                        raise Exception
                    #endif

                    # check ...
                    if sfile.nlayer is None :
                        logging.error( 'no layer dimension defined yet' )
                        raise Exception
                    #endif

                    # half level pressures:
                    hp = numpy.zeros((len(pixel),sfile.nlayer+1),dtype='f4')
                    # top, numbering from top to bottom:
                    hp[:,0] = 0
                    # copy bottom pressures:
                    hp[:,1:] = pbottom
                    
                    # create variable:
                    da = xarray.DataArray( hp, dims=('pixel','layeri'), 
                                                 coords={ 'pixel' : pixel }, 
                                                 attrs=attrs )

                else :
                    logging.error( 'unsupported target dims: %s' % str(vdims) )
                    logging.error( '  target variable   : %s' % varname )
                    raise Exception
                #endif
                
            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            # tropospheric kernel
            #   A_trop = A * M / M_trop
            elif special == 'kernel_trop' :
            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            
                # original fields:
                odat = {}
                for key in ['avk','amf','amft'] :
                    # full name:
                    opath = rcf.get( '%s.output.var.%s.%s' % (rcbase,varname,key) )
                    # extract:
                    odat[key] = sfile.GetData( opath )
                #endfor
                
                # set attributes:
                attrs = {}
                attrs['units'    ] = vunits
                attrs['long_name'] = 'averaging kernel'

                # check target dims:
                if vdims == ('pixel','layer','retr') :
                
                    # storage for extract along pixels index:
                    pdat = {}
                    # extract subset:
                    for key in ['avk'] :
                        # check dims:
                        if odat[key].dims == ('time','scanline','ground_pixel','layer') :
                            pdat[key] = odat[key].data[itime,iis,iip,:]
                        else:
                            logging.error( 'unsupported dimnames "%s" for special "%s"' % (odat[key].dims,special) )
                            raise Exception
                        #endif
                    #endfor
                    for key in ['amf','amft'] :
                        pdat[key] = odat[key].data[itime,iis,iip]
                    #endfor
                    
                    # check ...
                    if self.nretr != 1 :
                        logging.error( 'not implemented yet for number of retrieval layers %i' % self%nretr )
                        raise Exception
                    #endif
                    # dummy:
                    iretr = 0

                    # check ...
                    if sfile.nlayer is None :
                        logging.error( 'no layer dimension defined yet' )
                        raise Exception
                    #endif

                    # convert, result should have shape (pixels,levels,retr) :
                    avk = numpy.zeros( (len(pixel),sfile.nlayer,self.nretr), dtype='f4' )
                    # loop over input layers:
                    for k in range(sfile.nlayer) :
                        # fill kernel:
                        avk[:,k,iretr] = pdat['avk'][:,k] * pdat['amf'] / pdat['amft']
                    #endfor

                    # is troposphere layer defined ?
                    key = 'troplayer'
                    opath = rcf.get( vkey+'.'+key )
                    # defined ?
                    if len(opath) > 0 :
                        # extract data:
                        odat[key] = sfile.GetData( opath )
                        # loop over pixels:
                        for ipix in range(len(pixel)) :
                            # layer index (1-based?), could be nan ..
                            trpl = odat[key].data[itime,iis[ipix],iip[ipix]]
                            # defined?
                            if not numpy.isnan(trpl) :
                                # convert to index:
                                trpl = int(trpl)
                                # check ..
                                if (trpl < 1) or (trpl > sfile.nlayer) :
                                    #logging.warning( 'found tropo layer %i outside range 1,..,%i' % (trpl,sfile.nlayer) )
                                    pass
                                else :
                                   # reset to zero above layer holding tropopause:
                                   avk[ipix,trpl:,iretr] = 0.0
                                #endif
                            #endif
                        #endfor
                    #endif

                    # create variable:
                    da = xarray.DataArray( avk, dims=vdims, 
                                                 coords={ 'pixel' : pixel }, 
                                                 attrs=odat['avk'].attrs )

                else :
                    logging.error( 'unsupported target dims: %s' % str(vdims) )
                    logging.error( '  target variable   : %s' % varname )
                    raise Exception
                #endif
                
            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            # kernel scaled by layer thickness:
            #   kernel := kernel / dh
            elif special == 'kernel_by_dh' :
            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            
                # original fields:
                opath = {}
                odat = {}
                for key in ['avk','layer'] :
                    # full name:
                    opath[key] = rcf.get( '%s.output.var.%s.%s' % (rcbase,varname,key) )
                    # extract:
                    odat[key] = sfile.GetData( opath[key] )
                #endfor
                
                # check ...
                if odat['avk'].attrs['units'] != odat['layer'].attrs['units'] :
                    logging.error( 'could not scale kernel by dh; "%s" has units "%s", while "%s" has units "%s"' % \
                                      (opath['kernel'],odat['avk'].attrs['units'],opath['layer'],odat['layer'].attrs['units']) )
                    raise Exception
                #endif
                
                # set attributes:
                attrs = {}
                attrs['units'    ] = vunits
                attrs['long_name'] = 'averaging kernel'

                # check target dims:
                if vdims == ('pixel','layer','retr') :
                
                    # storage for extract along pixels index:
                    pdat = {}
                    # extract kernel:
                    key = 'avk'
                    if odat[key].dims == ('time','scanline','ground_pixel','layer') :
                        pdat[key] = odat[key].data[itime,iis,iip,:]
                    else:
                        logging.error( 'unsupported dimnames "%s" for special "%s"' % (odat[key].dims,special) )
                        raise Exception
                    #endif
                    # extract mid of layers (dummy coordinate?)
                    hmid = odat['layer'].values

                    # thickness:
                    dhs = abs( hmid[1:] - hmid[:-1] )
                    if abs( dhs.max() - dhs.min() ) > 1.0 :
                        logging.error( 'expected constant layer thickness; mid of layers defined:' )
                        for k in range(len(hmid)) :
                            logging.error( '%i6 %10.2f' % (k+1,hmid[k]) )
                        #endfor
                        raise Exception
                    #endif
                    # average:
                    dh = dhs.mean()
                    
                    # check ...
                    if self.nretr != 1 :
                        logging.error( 'not implemented yet for number of retrieval layers %i' % self%nretr )
                        raise Exception
                    #endif
                    # dummy:
                    iretr = 0
                    
                    # check ...
                    if sfile.nlayer is None :
                        logging.error( 'no layer dimension defined yet' )
                        raise Exception
                    #endif
                    
                    # convert, result should have shape (pixels,levels,retr) :
                    avk = numpy.zeros( (len(pixel),sfile.nlayer,self.nretr), dtype='f4' )
                    avk[:,:,iretr] = pdat['avk'] / dh

                    # create variable:
                    da = xarray.DataArray( avk, dims=vdims, 
                                                 coords={ 'pixel' : pixel }, 
                                                 attrs=attrs )

                else :
                    logging.error( 'unsupported target dims: %s' % str(vdims) )
                    logging.error( '  target variable   : %s' % varname )
                    raise Exception
                #endif

            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            else :
                logging.error( 'unsupported special "%s" for variable "%s"' % (special,varname) )
                raise Exception
            #endif  # special
            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            
            # need to swap layers?
            if swap_layers :
                # loop over dims:
                for dim in vdims :
                    # level dim or not?
                    if dim in ['layer','layeri','retr','retri','retr0'] :
                        # index:
                        idim = vdims.index(dim)
                        # swap:
                        da.data = numpy.flip( da.data, axis=idim )
                    #
                    elif dim in ['pixel','corner','track_scan','track_pixel'] :
                        pass
                    #
                    else :
                        logging.error( 'could not decide if "%s" is a level dimension or not' % dim )
                        raise Exception
                    #endif
                #endfor # dims
            #endif # swap layers
            
            # adhoc attributes ...
            attrs = rcf.get( vkey+'.attrs', totype='dict', default={} )
            # loop:
            for key in attrs.keys() :
                # remove if None:
                if attrs[key] is None :
                    # remove if present:
                    if key in da.attrs.keys() : da.attrs.pop(key)
                else :
                    # update:
                    da.attrs[key] = attrs[key]
                #endif
            #endfor
            
            # add variable to target data set:
            self.AddVariable( varname, da )
        
        #endfor # target variables                

    #enddef AddSelection

#endclass CSO_S5p_File





########################################################################
###
### convert
###
########################################################################

class CSO_S5p_Convert( utopya.UtopyaRc ) :

    """
    Convert raw S5p observations to CSO format.
    During conversion, a variable and pixel selection could be applied.
    
    This version will also download source files if not available yet,
    and (optionally) remove them after conversion.
    This is useful in case storage capacity is limitted and the entire archive 
    of source files cannot be permanently mirrored.
    
    Arguments:

    * ``rcfile``, ``rcbase``, ``env``  :  settings file, prefix for keys, and environment dictionairy 
    
    A time range is read to select the files to be converted::
    
      <rcbase>.timerange.start        :  2018-06-01 00:00
      <rcbase>.timerange.end          :  2018-06-03 23:59
    
    The input files are searched in a table created by an *inquire* class,
    for example :py:class:`CSO_SciHub_Inquire <cso_scihub.CSO_SciHub_Inquire>` 
    or :py:class:`CSO_PAL_Inquire <cso_pal.CSO_PAL_Inquire>`.
    These have scanned the archives to examine which processings and versions are available,
    and stored the result in a csv file.
    Specify the name of the csv file; this might contain templates for a date
    that is taken from another key::
    
        ! listing of available source files,
        ! created by 'inquire-s5phub' job:
        <rcbase>.inquire.file                   :  /data/Copernicus/S5p/Copernicus_S5P_NO2_%Y-%m-%d.csv
        !! date used in filename, leave empty for today:
        !<rcbase>.inquire.filedate               :  2022-01-28

    Specify the directory where the input files are to be searched,
    or where to download them to if not present yet.
    A flag is used to decide whether downloaded files should be removed immediately after conversion::

        ! target dir for downloads:
        <rcbase>.input.dir                      :  /data/Copernicus/S5P/%{processing}/NO2/%Y/%m

        ! remove downloaded input files after convert?
        <rcbase>.downloads.cleanup              :  False

    The input files keep the same name as used in the SciHub archive, for example::
    
        /data/Copernicus/S5P/OFFL/NO2/2018/07/S5P_OFFL_L2__NO2____20180701T005930_20180701T024100_03698_01_010002_20180707T022838.nc
                                                                    start_time      end_time      orbit 

    Specify which processings should be converted; other processings listed in the inquirerd
    table are ignored::

        ! list of processings:
        <rcbase>.processings                    :  OFFL RPRO

    Similar specify a list of processor versions:

        ! list of processor versions, empty for all:
        <rcbase>.processor_versions             :  020301
        
    Sometimes a file cannot be converted, for example because it is corrupted or could not be downloaded at all.
    Specify an (optional) blacklist::

        ! skip some input files:
        <rcbase>.blacklist         :  S5P_PAL__L2__NO2____20190806T022006_20190806T040136_09388_01_020301_20211110T020511.nc

    If an input file should be converted, it is read into a :py:class:`.S5p_File` object.
    The :py:meth:`SelectPixels <S5p_File.SelectPixels>` method is called to select pixels based 
    on critera defined in the settings; see its documentation for how to configure the pixel selection.
    This method als returns a history line to desribe the selection, which will be added as
    attribute to the output file.
    
    The output file is created as an :py:class:`.CSO_S5p_File` object.
    It's :py:meth:`AddSelection <.CSO_S5p_File.AddSelection>` method is called with the input object as argument,
    and this will copy the selected pixels for variables specified in the settings.
    The :py:meth:`Write <.CSO_File.Write>` method creates the file.
    
    The name of the output files should be specified with a directory and filename;
    the later could include a template for the orbit number::
    
        ! output filename:
        ! - times are taken from mid of selection, rounded to hours
        ! - replace templates with column values of listing, for example:
        !      %{orbit}, %{processing}, ...
        <rcbase>.output.filename       :  /Scratch/CSO/S5p/RPRO/NO2/Europe/%Y/%m/S5p_RPRO_NO2_%{orbit}.nc

    A flag is read to decide if existing output files should be renewed or kept::
    
       <rcbase>.renew                  :  True              

    Global attributes for the target file should be specified with::

        ! global attributes:
        <rcbase>.output.attrs               :  format Conventions author institution email
        !
        <rcbase>.output.attr.format         :  1.0
        <rcbase>.output.attr.Conventions    :  CF-1.7
        <rcbase>.output.attr.author         :  T. Emplate
        <rcbase>.output.attr.institution    :  CSO
        <rcbase>.output.attr.email          :  t.emplate@cso.org
        
    To reduce file size, by default all variables are written to file as short-integers (2 bytes) 
    accompanied by ``add_offset`` and ``scale_factor`` attributes. 
    A flag is available to disable packing.
    In addition, ``zlib`` compression is enabled.
    The default compression level is 1 (out of 9); set the following flag to a higher level
    to have stronger compression (on expense of computation time), or set to 0 to disable compression::
    
        ! pack floats as shorts:
        <rcbase>.output.packed              :  True
        ! zlib compression level (default 1, 0 for no compression):
        <rcbase>.output.complevel           :  1

    For testing an (optional) whitelist could be provided with output filenames (no path);
    if defined, only the listed files will be created::

        ! testing: create only these files:
        <rcbase>.whitelist         :  S5p_RPRO_NO2_123456.nc

    """
    
    def __init__( self, rcfile, rcbase='', env={}, indent='' ) :
    
        """
        Convert data.
        """
                        
        # modules:
        import os
        import datetime
        import fnmatch
        import pandas
        
        # tools:
        import cso_file
        import cso_scihub
        import utopya

        # info ...
        logging.info( indent+'' )
        logging.info( indent+'** convert S5p data' )
        logging.info( indent+'' )

        # init base object:
        utopya.UtopyaRc.__init__( self, rcfile=rcfile, rcbase=rcbase, env=env )
        
        # renew output?
        renew = self.GetSetting( 'renew', totype='bool' )
        
        # time range:
        t1 = self.GetSetting( 'timerange.start', totype='datetime' )
        t2 = self.GetSetting( 'timerange.end'  , totype='datetime' )
        
        # info ...
        tfmt = '%Y-%m-%d %H:%M'
        logging.info( indent+'timerange: [%s,%s]' % (t1.strftime(tfmt),t2.strftime(tfmt)) )

        # inquire tables:
        filename__templates = self.GetSetting( 'inquire.file' ).split(';')
        # time stamp in file?
        filedates = self.GetSetting( 'inquire.filedate', default='' )
        if len(filedates) == 0 :
            filedates = ['']*len(filename__templates)
        elif len(filedates) != len(filename__templates) :
            logging.error( 'number of filedates should match with number of listing file names:' )
            logging.error( '  inquire.file      : %s' % self.GetSetting( 'inquire.file' ) )
            logging.error( '  inquire.filedate  : %s' % self.GetSetting( 'inquire.filedate' ) )
            raise Exception
        #endif
        # init storage:
        dfs = []
        # loop:
        for ifile in range(len(filename__templates)) :
            # current:
            filename__template = filename__templates[ifile].strip()
            filedate = filedates[ifile].strip()
            # time to use for templates:
            if len(filedate) > 0 :
                t0 = datetime.datetime.strptime( filedate, '%Y-%m-%d' )
            else :
                t0 = default=datetime.datetime.now()
            #endif
            # expand time templates
            filename = t0.strftime( filename__template )
            # check ..
            if not os.path.isfile(filename) :
                logging.error( 'inquire table not found: %s' % filename )
                raise Exception
            #endif
            # info ..
            logging.info( indent+'read inquire table: %s' % filename )
            # read:
            dfs.append( pandas.read_csv( filename, sep=';', skip_blank_lines=True,
                                           parse_dates=['start_time','end_time'],
                                           dtype='str' ) )
        #endfor # filenames
        # combine:
        df = pandas.concat( dfs )
        # sort by filename:
        df = df.sort_values( 'filename' )

        # info ...
        logging.info( indent+'number of files : %i' % len(df) )
        
        ## selected proecessings:
        #processings = self.GetSetting( 'processings' ).split()
        ## selected processor versions:
        #processor_versions = self.GetSetting( 'processor_versions' ).split()
        
        # list of ';' seperated selection expression:
        #     (%{processor_version} == '020400') & (%{processing} == 'RPRO') ; ...
        line = self.GetSetting( 'selection' )
        # replace templates:
        #     (rec['processor_version'] == '020400') & (rec['processing'] == 'RPRO') ; ...
        for key in df.keys() :
            line = line.replace( "%{"+key+"}", "rec['"+key+"']" )
        #endfor
        # split:
        selections = line.split(';')
        # info ..
        logging.info( 'selection criteria (first with matching orbit is used):' )
        for selection in selections :
            logging.info( '  %s' % selection.strip() )
        #endif

        # skip some?
        blacklist = self.GetSetting( 'blacklist', default='' ).split()
        # select some specific files?
        whitelist = self.GetSetting( 'whitelist', default='' ).split()
        
        # path to store download:
        input_dir__template = self.GetSetting( 'input.dir' )
        # cleanup?
        downloads_cleanup = self.GetSetting( 'downloads.cleanup', 'bool' )

        # target file:
        output_filename__template = self.GetSetting( 'output.filename' )
        
        # attributes
        attrs = {}
        attrnames = self.GetSetting( 'output.attrs' ).split()
        for key in attrnames :
            attrs[key] = self.GetSetting( 'output.attr.%s' % (key) )
        #endfor
        
        # pack floats as shorts?
        packed = self.GetSetting( 'output.packed', 'bool' )
        # compression level:
        complevel = self.GetSetting( 'output.complevel', 'int' )

        # select records with start time inside time range:
        xdf = df[ (df['start_time'] >= t1) & (df['start_time'] <= t2) ]
        # info ..
        logging.info( indent+'found %i orbits with overlap of time range ..' % len(xdf) )
        
        # oribit labels:
        orbits = xdf['orbit'].unique()
            
        # info ..
        logging.info( indent+'loop over orbits overlapping with time interval ...' )
        # loop over orbits:
        for orbit in orbits :
            # info ...
            logging.info( indent+'  orbit "%s" ...' % orbit )

            # search for other records for same orbit:
            odf = xdf[ xdf['orbit'] == orbit ]
            
            # storage for status label: "selected", "blacklisted", ...
            filestatus = {}
            # no match yet ..
            seleted = []
            # loop over selection criteria,
            # this should give either none or a single file:
            for selection in selections :
                # make empty again:
                selected = []
                # loop over records:
                for indx,rec in odf.iterrows() :
                    # skip?
                    if os.path.basename(rec['filename']) in blacklist : 
                        filestatus[rec['filename']] = 'blacklisted'
                        continue
                    #endif
                    # evaluate expression including 'rec[key]' values:
                    if eval( selection ) :
                        selected.append( rec['filename'] )
                        filestatus[rec['filename']] = 'selected'
                    #endif
                #endfor # records
                # exactly one? then leave:
                if len(selected) == 1 :
                    break
                elif len(selected) > 1 :
                    logging.error( 'found more than one orbit file matching selection: %s' % selection )
                    for fname in selected :
                        logging.error( '  %s' % fname )
                    #endfor
                    raise Exception
                #endif  # number found
            #endfor # selection criteria

            # info ...
            logging.info( indent+'    available file(s):' )
            # loop:
            for fname in odf['filename'] :
                line = fname
                if fname in filestatus.keys() : line = line+' ['+filestatus[fname]+']'
                logging.info( indent+'      '+line )
            #endfor
            
            # no match?
            if len(selected) == 0 :
                # info ...
                logging.warning( '    no match with any selection criterium; next orbit ...' )
                ## degug:
                #for selection in selections :
                #    logging.warning( '    %s' % selection.strip() )
                #logging.warning( '  record:' )
                #for key in rec.keys() :
                #    logging.warning( '    "%s" : "%s"' % (key,rec[key]) )
                # next orbit:
                continue
            #endif
                    
            # start time of orbit:
            t0 = rec['start_time']
            
            # target file:
            output_filename = t0.strftime( output_filename__template )
            for key in rec.keys() :
                if type(rec[key]) == str :
                    output_filename = output_filename.replace( '%{'+key+'}', rec[key] )
                #endif
            #endfor
            # info ...
            logging.info( '    target file: %s' % output_filename )
            
            # selected productions?
            if len(whitelist) > 0 :
                if os.path.basename(output_filename) not in whitelist :
                    logging.info( '      not in whitelist, skip ...' )
                    continue
                #endif
            #endif
            
            # output dir:
            dname = os.path.dirname( output_filename )
            if len(dname) > 0 :
                if not os.path.isdir(dname) : os.makedirs( dname )
            #endif
            
            # message file:
            fname,ext = os.path.splitext( output_filename )
            output_msgfile = fname+'.msg'
            
            # messsage present?
            if os.path.isfile(output_msgfile) :
                # info ..
                logging.info( '    message file present:' )
                # read:
                with open(output_msgfile,'r') as f :
                    lines = f.readlines()
                #endwith
                # info..
                for line in lines :
                    logging.info( '      %s' % line )
                #endfor
                # info ..
                logging.info( '    do not try to create again ...:' )
                # do not create ..
                create = False
            #~ not present yet, create file?
            elif not os.path.isfile(output_filename) :
                # info ...
                logging.info( '    create new file ...' )
                # always create:
                create = True
            #~ renew?
            elif renew :
                # info ...
                logging.info( '    renew file ...' )
                # always create:
                create = True
            #~ already a version present:
            elif os.path.isfile(output_filename) :
                # there might be duplicated processings, with different processing times;
                # filenames are sorted, so current is probably newer
                # open existing file:
                csf = CSO_S5p_File( output_filename )
                # original orbit as attribute?
                orbit_file = csf.GetAttr( 'orbit_file', quiet=True )
                if orbit_file is None :
                    # info ...
                    logging.info( '    replace existing file, no "orbit_file" attribute yet ...' )
                    # attribute not found, re-create to have this attribute:
                    create = True
                elif fname <= orbit_file :
                    # info ...
                    logging.info( '    keep existing file, based on same or newer source file ...' )
                    # same file, no need to re-create:
                    create = False
                else :
                    # info ...
                    logging.info( '    replace existing file, based on older processing: %s' % orbit_file )
                    # replace:
                    create = True
                #endif
            #endif
            
            # create?
            if create :
            
                # keep list of downloaded files:
                downloads = []
                
                # input dir:
                input_dir = t0.strftime( input_dir__template )
                # replace templates:
                for key in rec.keys() :
                    if type(rec[key]) == str :
                        input_dir = input_dir.replace( '%{'+key+'}', rec[key] )
                    #endif
                #endfor
                # full path:
                input_file = os.path.join( input_dir, rec['filename'] )

                # info ..
                logging.info( '    input file: %s' % input_file )
                # check ..
                if not os.path.isfile( input_file ) :
                    # info ..
                    logging.info( '      not present yet, download ...' )
                    # download ...
                    cso_scihub.CSO_SciHub_DownloadFile( rec['href'], input_file )
                    # store name:
                    downloads.append( input_file )
                #endif
                    
                # download might have failed ..
                if not os.path.isfile(input_file) :
                    # info ..
                    logging.warning( indent+'      message input file, write message file ...' )
                    # write message file:
                    with open(output_msgfile,'w') as f :
                        f.write( 'missing file: %s' % input_file )
                    #endwith
                    # next:
                    continue
                #endif

                # info ...
                logging.info( indent+'    open ...' )
                # read file, sometimes files are corrupted ...
                try :
                    sfile = S5p_File( input_file )
                except :
                    # info ..
                    logging.warning( indent+'      could not open input file, write message file ...' )
                    # write message file:
                    with open(output_msgfile,'w') as f :
                        f.write( 'could not open file: %s' % input_file )
                    #endwith
                    # cleanup?
                    if downloads_cleanup and (input_file in downloads) :
                        logging.warning( indent+'      remove downloaded %s ...' % input_file )
                        os.remove( input_file )
                    #endif
                    # next:
                    continue
                #endtry

                # apply selections, return bool mask and list of history lines:
                selected,history = sfile.SelectPixels( self.rcf, self.rcbase, indent='    ' )
                # count:
                nselected = selected.sum()

                # info ...
                logging.info( indent+'      selected %i of %i pixels' % (nselected,selected.size) )
                # any ?
                if nselected == 0 :

                    # info ..
                    logging.warning( indent+'        no pixels selected, write message file ...' )
                    # write message file:
                    with open(output_msgfile,'w') as f :
                        f.write( 'no pixels selected in: %s' % input_file )
                    #endwith

                else :

                    # info ...
                    logging.info( indent+'    create %s ...' % output_filename )
                    # init:
                    csf = CSO_S5p_File()
                    # add:
                    csf.AddSelection( sfile, selected, self.rcf, self.rcbase, indent=indent+'    ' )
                    # update history:
                    history.append( 'added %i pixels from %s' % (nselected,os.path.basename(input_file)) )
                    # update attributes:
                    for key in ['orbit','processing','processor_version','collection'] :
                        attrs[key] = rec[key]
                    #endfor
                    attrs['orbit_file'] = input_file
                    # write:
                    csf.Write( filename=output_filename, attrs=attrs, history=history, 
                                 packed=packed, complevel=complevel )
                    
                    ## testing ...
                    #logging.warning( indent+'break after first created file ...' )
                    #break

                #endif  # any selected
                
                # cleanup?
                if downloads_cleanup and (input_file in downloads) :
                    logging.warning( indent+'      remove downloaded %s ...' % input_file )
                    os.remove( input_file )
                #endif

            #endif # create

            ## testing ...
            #logging.warning( 'break after first orbit' )
            #break

        #endfor  # input files
        
        # info ...
        logging.info( indent+'' )
        logging.info( indent+'** end convert' )
        logging.info( indent+'' )

    #enddef __init__

#endclass CSO_S5p_Convert




########################################################################
###
### create listing file
###
########################################################################

class CSO_S5p_Listing( utopya.UtopyaRc ) :

    """
    Create *listing* file for converted orbit files.
    
    A *listing* file contains the names of the converted orbit files,
    and the time range of pixels in the file::

        filename                     ;start_time                   ;end_time                     ;orbit
        2018/06/S5p_RPRO_NO2_03272.nc;2018-06-01T01:32:46.673000000;2018-06-01T01:36:12.948000000;03272
        2018/06/S5p_RPRO_NO2_03273.nc;2018-06-01T03:12:53.649000000;2018-06-01T03:17:43.082000000;03273
        2018/06/S5p_RPRO_NO2_03274.nc;2018-06-01T04:52:43.586000000;2018-06-01T04:59:12.377000000;03274
        :

    This file will be used by the observation operator to selects orbits with pixels valid for 
    a desired time range.
    
    In the settings, define the name of the file to be created::

        ! csv file that will hold records per file with:
        ! - timerange of pixels in file
        ! - orbit number
        <rcbase>.file        :  /Scratch/CSO/S5p/RPRO/NO2/Europe/listing.csv
        
    An existing listing file is not replaced,
    unless the following flag is set::

        ! renew table?
        <rcbase>.renew           :  True
        
    Orbit files are searched within a timerange::
    
        <rcbase>.timerange.start        :  2018-06-01 00:00
        <rcbase>.timerange.end          :  2018-06-03 23:59

    Specify filename filters to search for orbit files;
    the patterns are relative to the basedir of the listing file,
    and might contain templates for the time values.
    Multiple patterns could be defined; if for a certain orbit number more than one
    file is found, the first match is used.
    This could be explored to create a listing that combines reprocessed data
    with near-real-time data::

        <rcbase>.patterns            :  RPRO/NO2/Europe/%Y/%m/S5p_*.nc \
                                        OFFL/NO2/Europe/%Y/%m/S5p_*.nc

    Usually the time range is read from the file, but in case the file does
    not have a time accordinate, then the following flag might be used to
    force that the time that matches with the filename is used::
    
        ! adhoc: use the time for which file is valid as timerange;
        ! this is used for the synthetic S4 data that have no time record ...
        <rcbase>.use_t              :  True

    The ``orbit`` column in the listing is extra, and is read from global attributes;
    the list of extra columns is defined with::
    
        ! extra columns to be added, read from global attributes:
        <rcbase>.xcolumns           :  orbit

    """
    
    def __init__( self, rcfile, rcbase='', env={}, indent='' ) :
    
        """
        Convert data.
        """
                        
        # modules:
        import os
        import datetime
        import glob
        
        # tools:
        import cso_file

        # info ...
        logging.info( indent+'' )
        logging.info( indent+'** create listing file' )
        logging.info( indent+'' )

        # init base object:
        utopya.UtopyaRc.__init__( self, rcfile=rcfile, rcbase=rcbase, env=env )
        
        # renew output?
        renew = self.GetSetting( 'renew', totype='bool' )

        # table file to be written:
        lst_file = self.GetSetting( 'file' )
        # create?
        if (not os.path.isfile(lst_file)) or renew :        
            # info ..
            logging.info( indent+'create %s ...' % lst_file )
        
            # time range:
            t1 = self.GetSetting( 'timerange.start', totype='datetime' )
            t2 = self.GetSetting( 'timerange.end'  , totype='datetime' )        
            # info ...
            tfmt = '%Y-%m-%d %H:%M'
            logging.info( indent+'  timerange: [%s,%s]' % (t1.strftime(tfmt),t2.strftime(tfmt)) )
            
            # base directory:
            bdir = os.path.dirname( lst_file )
            # create?
            if len(bdir) > 0 :
                if not os.path.isdir(bdir) : os.makedirs( bdir )
            #endif
            # current directory?
            if len(bdir) == 0 : bdir = '.'
            # info ...
            logging.info( indent+'  base directory: %s ...' % bdir )
        
            # initiallize for (re)creation:
            listing = cso_file.CSO_Listing( lst_file, indent=indent+'    ' )

            # info ...
            logging.info( indent+'  cleanup records if necessary ...' )
            # remove entries that do not exist anymore:
            listing.Cleanup( indent=indent+'    ' )

            # filename pattern templates:
            pattern_templates = self.GetSetting( 'patterns' ).split()
            # collection of scanned patterns:
            patterns = []
            
            # adhoc: use time of file as time range?
            use_t = self.GetSetting( 'use_t', totype='bool', default=False )
            
            # extra columns to be added:
            xcolumns = self.GetSetting( 'xcolumns' ).split()
            
            # loop over days:
            t = t1
            while t <= t2 :

                # loop over patterns:
                for pattern_template in pattern_templates :
                    # expand time values:
                    pattern = t.strftime(pattern_template)
                    # skip if already scanned ...
                    if pattern in patterns : continue
                    # store:
                    patterns.append( pattern )
                    # info ...
                    logging.info( indent+'scan %s ...' % pattern )
                    
                    # list relative to basedir:
                    cwd = os.getcwd()
                    os.chdir( bdir )
                    fnames = glob.glob( pattern )
                    os.chdir( cwd )
                    # empty ?
                    if len(fnames) == 0 :
                        logging.info( indent+'  empty ..' )
                        continue
                    #endif
                    # sort in place:
                    fnames.sort()
                    # loop over files:
                    for fname in fnames :
                        # absolute path:
                        filename = os.path.join(bdir,fname)
                        # already in table?
                        if fname in listing :
                            # info ...
                            logging.info( indent+'  keep entry %s ...' % fname )
                        else :
                            # open orbit file:
                            csf = CSO_S5p_File( filename=filename )
                            # get orbit from attribute:
                            orbit = csf.GetAttr( 'orbit', quiet=True )
                            # already present?
                            if ('orbit' in listing.df.keys()) and (orbit in listing.df['orbit'].values) :
                                # info ...
                                logging.info( indent+'  skip file  %s (existing orbit "%s") ...' % (fname,orbit) )
                            else :
                                # info ...
                                logging.info( indent+'  add entry %s ...' % fname )
                                # timerange argument:
                                if use_t :
                                    # use current time instead of reading from file:
                                    tr = (t,t)
                                else :
                                    # read time range from file:
                                    tr = None
                                #endif
                                # add record to listing for this file, columns are:
                                #  - start_time, end_time
                                #  - extra columns specified by "xcolumns"
                                listing.Update( fname, csf, tr=tr, xcolumns=xcolumns, indent=indent+'    ' )
                            #endif

                        #endif
                    #endfor # filenames

                #endfor # patterns
                
                ## testing ...
                #break
                        
                # next
                t = t + datetime.timedelta(0,3600)
            #endwhile
        
            # save:
            listing.Close( indent=indent+'  ' )

        else :      
            # info ..
            logging.info( indent+'keep %s ...' % lst_file )
        #endif

        # info ...
        logging.info( indent+'' )
        logging.info( indent+'** end listing' )
        logging.info( indent+'' )
        
    #enddef __init__
    
#endclass CSO_S5p_Listing


########################################################################
###
### end
###
########################################################################

