
########################################################################
###
### help
###
########################################################################

"""
.. _cso-tropomi-superobs:

*******************************
``cso_tropomi_superobs`` module
*******************************

The :py:mod:`cso_tropomi_superobs` module provides classes to convert
TROPOMI super-observations (level-3 gridded product) into a CSO format.


Class hierchy
=============

The classes and are defined according to the following hierchy:

.. NOT YET
    * :py:class:`.TROPOMI_SuperObs_File`
    * :py:class:`.CSO_File`

      * :py:class:`.CSO_S5p_File`

    * :py:class:`.UtopyaRc`

      * :py:class:`.CSO_S5p_Convert`
      * :py:class:`.CSO_S5p_Listing`



Classes
=======


"""


########################################################################
###
### modules
###
########################################################################

# modules:
import logging

# tools:
import utopya
import cso_file


########################################################################
###
### S5p file
###
########################################################################

class S5p_SuperObs_File( object ) :

    """
    Base class to access data in S5p file.
    
    Example of content::

        netcdf s5p_no2_rpro_v1.2.2_superobs_ll_0.5deg_02833 {
        dimensions:
	        lon = 720 ;
	        lat = 360 ;
	        layer = 34 ;
	        vertices = 2 ;
        variables:
	        float lon(lon) ;
		        lon:standard_name = "longitude" ;
		        lon:long_name = "longitude" ;
		        lon:units = "degrees_east" ;
	        float lat(lat) ;
		        lat:standard_name = "latitude" ;
		        lat:long_name = "latitude" ;
		        lat:units = "degrees_north" ;
	        float layer(layer) ;
		        layer:standard_name = "layer" ;
		        layer:long_name = "layer" ;
		        layer:units = "1" ;
	        float no2_superobs(lat, lon) ;
		        no2_superobs:standard_name = "nitrogen_dioxide_tropospheric_column_superobservation" ;
		        no2_superobs:long_name = "NO2 tropospheric column superobservation, TROPOMI sensor" ;
		        no2_superobs:units = "1e-6 mol m^-2" ;
		        no2_superobs:_FillValue = 9.96921e+36f ;
	        float no2_superobs_uncertainty(lat, lon) ;
		        no2_superobs_uncertainty:standard_name = "nitrogen_dioxide_tropospheric_column_superobservation_uncertainty" ;
		        no2_superobs_uncertainty:long_name = "NO2 tropospheric column superobservation uncertainty, TROPOMI sensor, including representation error" ;
		        no2_superobs_uncertainty:units = "1e-6 mol m^-2" ;
		        no2_superobs_uncertainty:_FillValue = 9.96921e+36f ;
	        float no2_superobs_uncertainty_representation(lat, lon) ;
		        no2_superobs_uncertainty_representation:standard_name = "nitrogen_dioxide_tropospheric_column_superobservation_representativity_uncertainty" ;
		        no2_superobs_uncertainty_representation:long_name = "NO2 tropospheric column superobservation uncertainty, TROPOMI sensor, only representation error" ;
		        no2_superobs_uncertainty_representation:units = "1e-6 mol m^-2" ;
		        no2_superobs_uncertainty_representation:_FillValue = 9.96921e+36f ;
	        float average_uncertainty(lat, lon) ;
		        average_uncertainty:standard_name = "nitrogen_dioxide_tropospheric_column_average_uncertainty" ;
		        average_uncertainty:long_name = "NO2 tropospheric column superobservation uncertainty, TROPOMI sensor, average over individual observation" ;
		        average_uncertainty:units = "1e-6 mol m^-2" ;
		        average_uncertainty:_FillValue = 9.96921e+36f ;
	        float surface_pressure(lat, lon) ;
		        surface_pressure:standard_name = "surface_pressure" ;
		        surface_pressure:long_name = "superobservation surface pressure" ;
		        surface_pressure:units = "Pa" ;
		        surface_pressure:_FillValue = 9.96921e+36f ;
	        float cloud_radiance_fraction(lat, lon) ;
		        cloud_radiance_fraction:standard_name = "cloud_radiance_fraction" ;
		        cloud_radiance_fraction:long_name = "superobservation cloud radiance fraction" ;
		        cloud_radiance_fraction:units = "1" ;
		        cloud_radiance_fraction:_FillValue = 9.96921e+36f ;
	        float kernel_troposphere(layer, lat, lon) ;
		        kernel_troposphere:standard_name = "averaging_kernel_troposphere" ;
		        kernel_troposphere:long_name = "NO2 tropospheric column averaging kernel of the superobservation, TROPOMI sensor" ;
		        kernel_troposphere:units = "1" ;
		        kernel_troposphere:_FillValue = 9.96921e+36f ;
	        float covered_area_fraction(lat, lon) ;
		        covered_area_fraction:standard_name = "covered_area_fraction" ;
		        covered_area_fraction:long_name = "Fraction of the grid box covered by satellite observations" ;
		        covered_area_fraction:units = "1" ;
		        covered_area_fraction:_FillValue = 9.96921e+36f ;
	        float no2_scd_geo_superobs(lat, lon) ;
		        no2_scd_geo_superobs:standard_name = "nitrogen_dioxide_slant_amfgeo_superobservation" ;
		        no2_scd_geo_superobs:long_name = "NO2 slant column divided by AMFgeo superobservation, TROPOMI sensor" ;
		        no2_scd_geo_superobs:units = "1e-6 mol m^-2" ;
		        no2_scd_geo_superobs:_FillValue = 9.96921e+36f ;
	        float no2_strat_superobs(lat, lon) ;
		        no2_strat_superobs:standard_name = "nitrogen_dioxide_stratospheric_column_superobservation" ;
		        no2_strat_superobs:long_name = "NO2 stratospheric column superobservation, TROPOMI sensor" ;
		        no2_strat_superobs:units = "1e-6 mol m^-2" ;
		        no2_strat_superobs:_FillValue = 9.96921e+36f ;
	        float amf_trop_superobs(lat, lon) ;
		        amf_trop_superobs:standard_name = "tropospheric_air_mass_factor_superobservation" ;
		        amf_trop_superobs:long_name = "NO2 tropospheric air mass factor, TROPOMI sensor" ;
		        amf_trop_superobs:units = "1" ;
		        amf_trop_superobs:_FillValue = 9.96921e+36f ;
	        float amf_strat_superobs(lat, lon) ;
		        amf_strat_superobs:standard_name = "stratospheric_air_mass_factor_superobservation" ;
		        amf_strat_superobs:long_name = "NO2 stratospheric air mass factor, TROPOMI sensor" ;
		        amf_strat_superobs:units = "1" ;
		        amf_strat_superobs:_FillValue = 9.96921e+36f ;
	        float tm5_constant_a(layer, vertices) ;
		        tm5_constant_a:standard_name = "tm5_constant_a" ;
		        tm5_constant_a:long_name = "TM5 hybrid A coefficient at upper and lower interface levels" ;
		        tm5_constant_a:units = "Pa" ;
		        tm5_constant_a:_FillValue = 9.96921e+36f ;
	        float tm5_constant_b(layer, vertices) ;
		        tm5_constant_b:standard_name = "tm5_constant_b" ;
		        tm5_constant_b:long_name = "TM5 hybrid B coefficient at upper and lower interface levels" ;
		        tm5_constant_b:units = "1" ;
		        tm5_constant_b:_FillValue = 9.96921e+36f ;

        // global attributes:
		        :authors = "Henk Eskes, eskes@knmi.nl" ;
		        :institution = "Royal Netherlands Meteorological Institute (KNMI)" ;
		        :product_version = "v 1.2.2, reprocessing March 2019, RPRO" ;
		        :project = "S5P, phase E2" ;
		        :reference = "http://www.tropomi.eu/" ;
		        :source = "TROPOMI / Sentinel-5P" ;
		        :title = "Nitrogen dioxide (NO2) column data, superobservations, regular lat-lon, 0.5x0.5 degree" ;
		        :vertical_column_processor_version = "TM5-MP-Domino3 v3.5.3" ;
		        :date_created = "2020-02-24T11:08:45Z" ;
		        :time_coverage_start = "2018-05-01T02:03:58Z" ;
		        :time_coverage_end = "2018-05-01T02:56:14Z" ;
		        :filter_min_QA_value = 0.75f ;
		        :filter_max_QA_value = 1.1f ;
		        :data_version = "TM3-MP-Domino v3.5.3, reprocessing March 2019, version 1.2.2" ;
		        :S5P_NO2_file_name = "S5P_RPRO_L2__NO2____20180501T014123_20180501T032451_02833_01_010202_20190201T165715.nc" ;
		        :S5P_NO2_orbit_number = 2833 ;
		        :S5P_NO2_history = "2019-02-01 17:26:08 f_s5pops tropnll2dp /mnt/data1/storage_offl_l2/cache_offl_l2/WORKING-565374376/JobOrder.565372716.xml; 2019-02-27 08:50:05 TM5-MP-DOMINO offline" ;
		        :S5P_NO2_references = "http://www.tropomi.eu/data-products/nitrogen-dioxide" ;
		        :S5P_NO2_processor_version = "1.2.2" ;
		        :S5P_NO2_identifier_product_doi = "10.5270/S5P-s4ljg54" ;
		        :S5P_NO2_date_created = "2019-02-01T17:17:00Z" ;
		        :S5P_NO2_date_modified = "2019-02-27T08:50:05Z" ;
		        :superobs_setting_modelStratVcdError = 0.15f ;
		        :superobs_setting_minimal_coverage = 0.5f ;
		        :superobs_setting_correlation_factor = 0.15f ;
		        :superobs_setting_meanVariabilityFactor = 0.5f ;
        }

    Arguments:
    
    * ``filename``    : name of input file

    """
    
    def __init__( self, filename ) :
    
        """
        Open data file.        
        """
        
        # modules:
        import os
        import xarray
        
        # store:
        self.filename = filename

        # check ...
        if not os.path.isfile(filename) :
            logging.error( 'file not found : %s' % filename )
            raise Exception
        #endif
        
        # open:
        with  xarray.open_dataset( self.filename ) as ds :

            # copy some dimensions for convenience:
            #self.ntime  = ds.dims['time']
            #self.nscan  = ds.dims['scanline']
            #self.ngpix  = ds.dims['ground_pixel']
            self.nlon  = ds.dims['lon']
            self.nlat  = ds.dims['lat']
            # number 'atmoshpere' layers:
            if 'layer' in ds.dims.keys() :
                self.nlayer = ds.dims['layer']
            else :
                self.nlayer = None
            #endif
            
            # copy global attributes:
            self.attrs = ds.attrs

        #endwith ds
        
    #enddef __init__
    
    # *
    
    def GetData( self, vname, units=None ) :
    
        """
        Extract data array from input file and perform some adhoc fixes:
        
        * Eventually insert units attribute if not present yet, this is then taken from the argument.
        * Convert data to target units if necessary.
        * Add a ``long_name`` attribute if not present yet.
        
        Arguments:
        
        * ``vname`` : variable in in input
        
        Optional arguments:
        
        * ``units`` : target units
        
        Return values:
        
        * ``da`` : :py:xarray.DataArray` object
        """
        
        # modules:
        import os
        import xarray
    
        # access group:
        with xarray.open_dataset( self.filename ) as ds :
            # check ..
            if vname not in ds :
                logging.error( 'no variable "%s" in file "%s"' % (vname,self.filename) )
                raise Exception
            #endif
            # extract variable:
            da = ds[vname]
            # not lazy ...
            da.load()
        #endwith # xarray
        
        # unit stuff?
        if units is not None :
        
            # set input units in case these are not present yet,
            # assume that what is provided in the settings is correct:
            if 'units' not in da.attrs.keys() : da.attrs['units'] = units

            ## info ...
            #logging.info( indent+'    input  range: [%f,%f] %s' % (da.min(),da.max(),da.attrs['units']) )

            # need to convert?
            if da.attrs['units'] != units :
                # conversion description:
                conversion = '%s -> %s' % (da.attrs['units'],units)
                # switch:
                if conversion in ['1e-6 mol m^-2 -> mol m-2'] :
                    # apply factor:
                    da.data = da.data * 1.0e-6
                else :
                    logging.error( 'unsupported conversion "%s" requested ...' % conversion )
                    raise Exception
                #endif
                # reset units:
                da.attrs['units'] = units
            #endif  # units differ
            
        #endif # units defined
        
        # adhoc fix: sometimes neither standard name or long name, 
        # cf checker gives a warning on this ...
        if ('standard_name' not in da.attrs.keys()) and ('long_name' not in da.attrs.keys()) :
            # use variable name as long name ...
            da.attrs['long_name'] = os.path.basename(path)
        #endif
        
        # ok
        return da
        
    #enddef GetData
    
    # *
    
    def GetTimeRange( self ) :
    
        """
        Return timerange ``(t1,t2)`` extracted from attributes.
        """
        
        # modules:
        import datetime
        
        # attribute names:
        attr_tr1 = 'time_coverage_start'
        attr_tr2 = 'time_coverage_end'
        # check ...
        for aname in [attr_tr1,attr_tr2] :
            if aname not in self.attrs.keys() :
                logging.error( 'attribute "%s" not found in: %s' % (aname,self.filename) )
                raise Exception
            #endif
        #endfor

        # read time range from attributes:
        tfmt = '%Y-%m-%dT%H:%M:%SZ'
        tr = ( datetime.datetime.strptime(self.attrs[attr_tr1],tfmt), \
               datetime.datetime.strptime(self.attrs[attr_tr2],tfmt)  )
               
        # ok
        return tr
        
    #enddef GetTimeRange
        
                    
    # *
    
    def SelectPixels( self, rcf, rckey, indent='' ) :
    
        """
        Apply filters specified in rcfile.
        
        Arguments:
        
        * ``rcf``       :  :py:class:`.RcFile` object with settings
        * ``rckey``     :  basename for rcfile keys, e.g. "s5p" for the example below
            
        Return values:
        
        * ``selected`` : boolean array with shape of track ``(time,scan,pixel)`` which
          is True if a pixel passed all checks;
        * ``history``  : list of character string describing the selelections.
          
        Example configuration::
        
            ! Specifiy a list of filter names.
            ! For each name, specify the variable which values are used for testing,
            ! the type test, and eventually some thresholds or other settings for this type.
            ! The units of the thresholds should match with the units in the variables,
            ! the expected units have to be defined too.
            ! The examples below show possible types and their settings.

            ! filters:
            s5p.filters                        :  lons lats albedo valid

            ! select range of values:
            s5p.filter.lons.var                :  PRODUCT/longitude
            s5p.filter.lons.type               :  minmax
            s5p.filter.lons.minmax             :  -15.0 35.0
            s5p.filter.lons.units              :  degrees_east

            ! select above a minimum:
            s5p.filter.lats.var                :  PRODUCT/latitude
            s5p.filter.lats.type               :  minmax
            s5p.filter.lats.minmax             :  35.0 70.0
            s5p.filter.lats.units              :  degrees_north

            ! select below a maximum:
            s5p.filter.albedo.var              :  Data Fields/SurfaceAlbedo
            s5p.filter.albedo.type             :  max
            s5p.filter.albedo.max              :  0.3
            s5p.filter.albedo.units            :  1

            ! select only values with data (no "_FillValue"):
            s5p.filter.valid.var               :  Data Fields/NO2RetrievalTroposphericVerticalColumn
            s5p.filter.valid.type              :  valid
        
        """
        
        # modules:
        import xarray
        import numpy
        
        # info ...
        logging.info( indent+'pixel selection ...' )
   
        # init selection, by default accept all:
        selected = numpy.ones( (self.nlat,self.nlon), 'bool' )
        
        # init history:
        history = []

        # which filters ?
        filternames = rcf.get( '%s.filters' % rckey ).split()
        # loop over filters:
        for filtername in filternames :
            # info ...
            logging.info( indent+'  filter "%s" ...' % filtername )
            # filter key in rcfiles:
            fkey = '%s.filter.%s' % (rckey,filtername)
            # which variable ?
            varname = rcf.get(fkey+'.var')
            # read variable as data array:
            da = self.GetData( varname )
            # extract units:
            vunits = da.attrs['units']
            # which type ?
            filtertype = rcf.get(fkey+'.type')
            # switch:
            #~ compare values:
            if filtertype == 'equal' :
                # read value:
                svalue = rcf.get(fkey+'.value')
                fvalue = float(svalue)
                # avoid warnings about comparing nan ...
                xdata = da.data.copy()
                if numpy.any( numpy.isnan(da.data) ) : xdata[numpy.where(numpy.isnan(da.data))] = fvalue - 1
                # apply filter:
                selected = selected & (xdata == fvalue)
                # update history, also select on not-nan to avoid warnings from comparisons:
                history.append( "selected pixels with '%s' == %s" % (varname,svalue) )
            #
            #~ minimum value:
            elif filtertype == 'min' :
                # read threshold:
                smin = rcf.get(fkey+'.min')
                fmin = float(smin)
                # check units ...
                funits = rcf.get(fkey+'.units')
                if funits != vunits :
                    logging.error( 'variable "%s" has units "%s" while threshold has units "%s"' % (varname,vunits,funits) )
                    raise Exception
                #endif
                # avoid warnings about comparing nan ...
                xdata = da.data.copy()
                if numpy.any( numpy.isnan(da.data) ) : xdata[numpy.where(numpy.isnan(da.data))] = fmin - 1
                # apply filter, also select on not-nan to avoid warnings from comparisons:
                selected = selected & (xdata >= fmin)
                # update history:
                history.append( "selected pixels with '%s' >= %s" % (varname,smin) )
            #
            #~ maximum value:
            elif filtertype == 'max' :
                # read threshold:
                smax = rcf.get(fkey+'.max')
                fmax = float(smax)
                # check units ...
                funits = rcf.get(fkey+'.units')
                if funits != vunits :
                    logging.error( 'variable "%s" has units "%s" while threshold has units "%s"' % (varname,vunits,funits) )
                    logging.error( 'value range : %f - %f' % (numpy.nanmin(values),numpy.nanmax(values)) )
                    raise Exception
                #endif
                # avoid warnings about comparing nan ...
                xdata = da.data.copy()
                if numpy.any( numpy.isnan(da.data) ) : xdata[numpy.where(numpy.isnan(da.data))] = fmax + 1
                # apply filter, also select on not-nan to avoid warnings from comparisons:
                selected = selected & (xdata <= fmax)
                # update history:
                history.append( "selected pixels with '%s' <= %s" % (varname,smax) )
            #
            #~ range of valid values:
            elif filtertype == 'minmax' :
                # read thresholds:
                smin,smax = rcf.get(fkey+'.minmax').split()
                fmin = float(smin)
                fmax = float(smax)
                # check units ...
                funits = rcf.get(fkey+'.units')
                if funits != vunits :
                    logging.error( 'variable "%s" has units "%s" while threshold has units "%s"' % (varname,vunits,funits) )
                    raise Exception
                #endif
                # avoid warnings about comparing nan ...
                xdata = da.data.copy()
                if numpy.any( numpy.isnan(da.data) ) : xdata[numpy.where(numpy.isnan(da.data))] = fmin - 1
                # apply filter, also select on not-nan to avoid warnings from comparisons:
                selected = selected & (xdata >= fmin) & (xdata <= fmax)
                # update history:
                history.append( "selected pixels with '%s' in [%s,%s]" % (varname,smin,smax) )
            #
            #~ only unmasked values:
            elif filtername == 'valid' :
                # only accept non-masked fields:
                selected = selected & (~ numpy.isnan(da.data))
                # update history:
                history.append( "selected pixels with valid '%s' values" % varname )
            #
            #~ not yet ...
            else :
                logging.error( 'unsupported filter name : %s' % filtername )
                raise Exception
            #endif
        #endfor # filters
        
        ## if ncecessary, convert from a masked to a normal array:
        #if hasattr(selected,'mask') : selected = selected.filled(fill_value=False)

        # ok
        return selected,history
    
    #enddef SelectPixels
        
#endclass S5p_SuperObs_File



########################################################################
###
### converted file
###
########################################################################


class CSO_S5p_SuperObs_File( cso_file.CSO_File ) :

    """
    Storage for CSO satellite file filled from S5p data.
    """
    
    def AddSelection( self, sfile, selected, rcf, rcbase, indent='' ) :
    
        """
        Add selected OMI to satellite extract file.
        
        Arguments:
        
        * ``sfile``      :  :py:class:`.S5p_File` object
        * ``selected``   :  boolean array as provided by :py:meth:`S5p_File.SelectPixels` method
        * ``rcf``        :  :py:class:`.RcFile` instance with settings
        * ``rcbase``     :  base of rcfile keys
            
        The first setting that is read is a list with variable names to be 
        created in the target file::

          <rcbase>.output.vars    :  longitude corner_longitudes \\
                                     latitude corner_latitudes \\
                                     vcd  ...

        For each variable, a series of settings should be specified that describe 
        how the variable should look like and how to create it.
        
        The first setting is a list of dimension names that define the shape of the variable.
        Supported dimensions are:
        
        * ``pixel``       : selected pixels
        * ``corner``      : number of footprint bounds (probably 4)
        * ``layer``       : number of layers in atmospheric profile (layers in kernel)
        * ``layeri``      : number of layer interfaces in atmospheric profile (``layer+1``)
        * ``retr``        : number of layers in retrieval product (1 for columns)
        * ``retr0``       : same as ``retr``, used for matrix dimensions ``(retr,retr0)`` to avoid
          repeated dimensions where the cf-checker complains about
        
        For a 1D variable with values per pixel the dimension setting is therefore::
        
            <rcbase>.output.var.longitude.dims    :   pixel
        
        For most variables it is sufficient to provide only the name of the original
        variable from which the data should be read::

          <rcbase>.output.var.longitude.from    :   lon

        For some variables a special processing needs to be done.
        For these variables a key '``special``' is used which will enable the 
        correct conversion. The following specials are currently implemented:
        
        * ``longitude_to_bounds``   : longiude bounds interpolated/extrpolated from centers; requires a ``.from`` setting
        * ``latitude_to_bounds``    : latiude  bounds interpolated/extrpolated from centers; requires a ``.from`` setting
            
        * ``hybounds_to_pressure`` : form pressure from hybride sigma pressure coordinate,
          where the available hybride coefficients have shape ``('layer','vertices')``;
          requires settings::
        
            <rcbase>.output.var.pressure.sp       :   PRODUCT/SUPPORT_DATA/INPUT_DATA/surface_pressure
            <rcbase>.output.var.pressure.hyab     :   PRODUCT/tm5_constant_a
            <rcbase>.output.var.pressure.hybb     :   PRODUCT/tm5_constant_b
            <rcbase>.output.var.pressure.units    :   Pa
        
        * ``kernel_trop_to_nla`` : 1-based index of lowest layer where tropo-spheric kernel is non-zero.
          Required settings::
          
            <rcbase>.output.var.kernel.kernel_trop  :   kernel_troposphere
           
        * ``square`` : create a variable as the square of the input; requires a ``.from`` setting.
           
        Optionally provide target units too. 
        In the (unlikely) case that the original variable has no ``units`` attribute,
        this setting is required to define the (assumed) units.
        If the provided units are different from the units in the original variable,
        the data is converted to the provided units::

          <rcbase>.output.var.longitude.units   :   degrees_east

        Optionally provide a dictionairy with attributes to be added.
        If the attribute value is ``None``, the attribute is removed if present from the input;
        this is sometimes needed if the CF compliance checker complains::
        
          !~ skip some attributes, cf-checker complains ...
          <rcbase>.output.var.qa_value.attrs         :   { 'valid_min' : None, 'valid_max' : None }


        """
        
        # modules:
        import os
        import numpy
        import xarray
        import pandas
        import scipy.interpolate
        
        # info ..
        logging.info( indent+'add pixels to satellite file ...' )
        
        # track size:
        nlat,nlon = selected.shape

        # swap layers?
        swap_layers = rcf.get( rcbase+'.swap_layers', totype='bool' )

        # selection indices:
        jj,ii = numpy.where( selected )
        
        # scan line coordinate is "lat" ..
        track_scan = xarray.DataArray( numpy.arange(nlat,dtype='i4'), dims=('track_scan'), 
                                         attrs={ 'units' : '1', 'long_name' : 'scan line index (0-based)' } )
        # store:
        self.AddCoord( 'track_scan', track_scan )
        
        # track pixel coordinate is "lon" ..
        track_pixel = xarray.DataArray( numpy.arange(nlon,dtype='i4'), dims=('track_pixel'),
                                         attrs={ 'units' : '1', 'long_name' : 'pixel index track (0-based)' } )
        # store:
        self.AddCoord( 'track_pixel', track_pixel )
        
        # create pixel coordinate:
        #   0-based index that represents the original latitude and longitude
        compressed_index = jj*nlon + ii
        # create coordinate data array ;
        # convert to int, otherwise cf-checker complains about the 'compress' attribute ..
        pixel = xarray.DataArray( numpy.array(compressed_index,dtype='i4'), dims=('pixel'),
                                    attrs={ 'units'     : '1', 
                                            'long_name' : 'pixel index', 
                                            'compress'  : 'track_scan track_pixel' } )
        # store:
        self.AddCoord( 'pixel', pixel )
        # store length:
        self.npixel = len(pixel)
        
        # number of retrieval layers:
        self.nretr = 1
        
        # target variables:
        varnames = rcf.get( rcbase+'.output.vars' ).split()
        
        # loop over target variables:
        for varname in varnames :
        
            # info ...
            logging.info( indent+'  add variable "%s" ...' % varname )

            # prefix for settings:
            vkey = '%s.output.var.%s' % (rcbase,varname)

            # target units:
            vunits = rcf.get( vkey+'.units', default='None' )
            if vunits == 'None' : vunits = None
            
            # target dimensions:
            vdims = tuple( rcf.get( vkey+'.dims' ).split() )

            # specials ?
            special = rcf.get( vkey+'.special', default='None' )
            # switch:
            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            if special in ['None','square'] :
            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            
                # name of original field:
                opath = rcf.get( vkey+'.from' )
                
                # optional data type:
                dtype = rcf.get( vkey+'.dtype', default='None' )
                if dtype == 'None' : dtype = None

                # extract data array:
                oda = sfile.GetData( opath, units=vunits )
                
                # check target dims:
                #~ single value per pixel:
                if vdims == ('pixel',) :
                
                    # extract 1D array from data that is only given per longitude:
                    if oda.dims in [('lon',)] :

                        # create 1D array with data from selected pixels, copy attributes:
                        da = xarray.DataArray( numpy.array(oda.data[ii],dtype=dtype), 
                                                 dims=vdims, 
                                                 coords={ 'pixel' : pixel }, 
                                                 attrs=oda.attrs )
                
                    # extract 1D array from data that is only given per latitude:
                    elif oda.dims in [('lat',)] :

                        # create 1D array with data from selected pixels, copy attributes:
                        da = xarray.DataArray( numpy.array(oda.data[jj],dtype=dtype), 
                                                 dims=vdims, 
                                                 coords={ 'pixel' : pixel }, 
                                                 attrs=oda.attrs )

                    else :
                        logging.error( 'could not convert from dims "%s" to target dims "%s"' % (str(oda.dims),str(vdims)) )
                        raise Exception
                    #endif
                    
                #~ retrieval profile per pixel:
                elif vdims == ('pixel','retr') :
                
                    # extract 1D array of pixels from 2D track:
                    if oda.dims in [('lat','lon')] :

                        # check ...
                        if self.nretr != 1 :
                            logging.error( 'not implemented yet for number of retrieval layers %i' % self%nretr )
                            raise Exception
                        #endif

                        # create 1D array with data from selected pixels, copy attributes:
                        da = xarray.DataArray( numpy.array(oda.data[jj,ii].reshape(self.npixel,self.nretr),dtype=dtype), 
                                                 dims=vdims, 
                                                 coords={ 'pixel' : pixel }, 
                                                 attrs=oda.attrs )

                    else :
                        logging.error( 'could not convert from dims "%s" to target dims "%s"' % (str(oda.dims),str(vdims)) )
                        raise Exception
                    #endif
                    
                #~ kernel per pixel:
                elif vdims == ('pixel','layer','retr') :
                
                    # extract 2D array of pixels:
                    if oda.dims in [('layer','lat','lon')] :

                        # check ...
                        if self.nretr != 1 :
                            logging.error( 'not implemented yet for number of retrieval layers %i' % self%nretr )
                            raise Exception
                        #endif
                        # check ...
                        if sfile.nlayer is None :
                            logging.error( 'no layer dimension defined yet' )
                            raise Exception
                        #endif

                        # create data array with data from selected pixels, copy attributes:
                        da = xarray.DataArray( numpy.array(oda.data[:,jj,ii].T.reshape(self.npixel,sfile.nlayer,self.nretr),dtype=dtype),
                                                 dims=vdims, 
                                                 coords={ 'pixel' : pixel }, 
                                                 attrs=oda.attrs )

                    else :
                        logging.error( 'could not convert from dims "%s" to target dims "%s"' % (str(oda.dims),str(vdims)) )
                        raise Exception
                    #endif

                #~ retrieval matrix per pixel:
                elif vdims == ('pixel','retr','retr0') :
                
                    # extract 1D array of pixels from 2D track:
                    if oda.dims in [('lat','lon')] :

                        # check ...
                        if self.nretr != 1 :
                            logging.error( 'not implemented yet for number of retrieval layers %i' % self%nretr )
                            raise Exception
                        #endif

                        # create 1D array with data from selected pixels, copy attributes:
                        da = xarray.DataArray( numpy.array(oda.data[jj,ii].reshape(self.npixel,self.nretr,self.nretr),dtype=dtype),
                                                 dims=vdims, 
                                                 coords={ 'pixel' : pixel }, 
                                                 attrs=oda.attrs )

                    else :
                        logging.error( 'could not convert from dims "%s" to target dims "%s"' % (str(oda.dims),str(vdims)) )
                        raise Exception
                    #endif
                    
                #~ track
                elif vdims == ('track_scan','track_pixel') :
                
                    # storage:
                    values = numpy.zeros((nlat,nlon))

                    # copy 1D array into 2D track:
                    if oda.dims in [('lon',)] :
                        # copy:
                        for ilat in range(nlat) :
                            values[ilat,:] = oda.data
                        #endfor
                    #
                    elif oda.dims in [('lat',)] :
                        # copy:
                        for ilon in range(nlon) :
                            values[:,ilon] = oda.data
                        #endfor
                    else :
                        logging.error( 'could not convert from dims "%s" to target dims "%s"' % (str(oda.dims),str(vdims)) )
                        raise Exception
                    #endif

                    # create 1D array with data from selected pixels, copy attributes:
                    da = xarray.DataArray( numpy.array(values,dtype=dtype), 
                                             dims=vdims,
                                             coords={ 'track_scan' : track_scan, 'track_pixel' : track_pixel },
                                             attrs=oda.attrs )

                else :
                    logging.error( 'unsupported target dims: %s' % str(vdims) )
                    logging.error( '  target variable   : %s' % varname )
                    raise Exception
                #endif
                
                # postprocessing:
                if special == 'square' :
                    # apply:
                    da.data = da.data**2
                    da.attrs['units'] = '(%s)**2' % da.attrs['units']
                #endif

            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            # longitude bounds, interpolated/extrapolated from centers:
            elif special in ['longitude_to_bounds'] :
            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            
                # name of original field:
                opath = rcf.get( vkey+'.from' )
                # get data array:
                oda = sfile.GetData( opath, units=vunits )
                
                # check shape:
                if oda.dims in [('lon',)] :

                    # interpolation/extrapolation function:
                    fix = scipy.interpolate.interp1d( numpy.arange(nlon), oda.data, fill_value='extrapolate' )
                    # apply:
                    blons = fix( numpy.arange(nlon+1)-0.5 )

                    # corners:
                    #    3      2
                    #     o----o
                    #     |    |
                    #     o----o
                    #    0      1

                    # check target dims:
                    if vdims == ('pixel','corner') :
                
                        # storage for corner values:
                        values = numpy.zeros((self.npixel,4))
                        # fill:
                        values[:,0] = blons[ii  ]
                        values[:,1] = blons[ii+1]
                        values[:,2] = blons[ii+1]
                        values[:,3] = blons[ii  ]

                        # create 2D array (+corners) with data from selected scan lines:
                        da = xarray.DataArray( values, dims=vdims, 
                                                 coords={ 'pixel' : pixel },
                                                 attrs=oda.attrs )
                    #
                    elif vdims == ('track_scan','track_pixel','corner') :
                
                        # storage for corner values:
                        values = numpy.zeros((nlat,nlon,4))
                        # fill:
                        for ilat in range(nlat) :
                            values[ilat,:,0] = blons[:-1]
                            values[ilat,:,1] = blons[ 1:]
                            values[ilat,:,2] = blons[ 1:]
                            values[ilat,:,3] = blons[:-1]
                        #endfor

                        # create 2D array (+corners) with data from selected scan lines:
                        da = xarray.DataArray( values, dims=vdims, 
                                                 coords={ 'track_scan' : track_scan, 'track_pixel' : track_pixel },
                                                 attrs=oda.attrs )
                    #
                    else :
                        logging.error( 'unsupported target dims: %s' % str(vdims) )
                        logging.error( '  target variable   : %s' % varname )
                        raise Exception
                    #endif

                else :
                    logging.error( 'could not convert from dims "%s" to target dims "%s"' % (str(oda.dims),str(vdims)) )
                    raise Exception
                #endif

            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            # latitude bounds, interpolated/extrapolated from centers:
            elif special in ['latitude_to_bounds'] :
            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            
                # name of original field:
                opath = rcf.get( vkey+'.from' )
                # get data array:
                oda = sfile.GetData( opath, units=vunits )
                
                # check shape:
                if oda.dims in [('lat',)] :

                    # interpolation/extrapolation function:
                    fix = scipy.interpolate.interp1d( numpy.arange(nlat), oda.data, fill_value='extrapolate' )
                    # apply:
                    blats = fix( numpy.arange(nlat+1)-0.5 )

                    # corners:
                    #    3      2
                    #     o----o
                    #     |    |
                    #     o----o
                    #    0      1

                    # check target dims:
                    if vdims == ('pixel','corner') :
                
                        # storage for corner values:
                        values = numpy.zeros((self.npixel,4))
                        # fill:
                        values = numpy.zeros((self.npixel,4))
                        values[:,0] = blats[jj  ]
                        values[:,1] = blats[jj  ]
                        values[:,2] = blats[jj+1]
                        values[:,3] = blats[jj+1]

                        # create 2D array (+corners) with data from selected scan lines:
                        da = xarray.DataArray( values, dims=vdims, 
                                                 coords={ 'pixel' : pixel },
                                                 attrs=oda.attrs )
                    #
                    elif vdims == ('track_scan','track_pixel','corner') :
                
                        # storage for corner values:
                        values = numpy.zeros((nlat,nlon,4))
                        # fill:
                        for ilon in range(nlon) :
                            values[:,ilon,0] = blats[:-1]
                            values[:,ilon,1] = blats[:-1]
                            values[:,ilon,2] = blats[ 1:]
                            values[:,ilon,3] = blats[ 1:]
                        #endfor

                        # create 2D array (+corners) with data from selected scan lines:
                        da = xarray.DataArray( values, dims=vdims, 
                                                 coords={ 'track_scan' : track_scan, 'track_pixel' : track_pixel },
                                                 attrs=oda.attrs )
                    #
                    else :
                        logging.error( 'unsupported target dims: %s' % str(vdims) )
                        logging.error( '  target variable   : %s' % varname )
                        raise Exception
                    #endif

                else :
                    logging.error( 'could not convert from dims "%s" to target dims "%s"' % (str(oda.dims),str(vdims)) )
                    raise Exception
                #endif

            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            # pressure from hybride coordinates;
            # hyab and hybb have shape ('layer','vertices') :
            elif special == 'hybounds_to_pressure' :
            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            
                # set attributes:
                attrs = {}
                attrs['units'    ] = vunits
                attrs['long_name'] = 'half level pressures'
            
                # original fields:
                odat = {}
                for key in ['sp','hyab','hybb'] :
                    # full name:
                    opath = rcf.get( '%s.output.var.%s.%s' % (rcbase,varname,key) )
                    # extract:
                    odat[key] = sfile.GetData( opath )
                #endfor
                
                # check ...
                if odat['hyab'].attrs['units'] != vunits :
                    logging.error( 'expecting hybride units hya "%s", found "%s"' % (vunits,odat['hyab'].attrs['units']) )
                    raise Exception
                #endif
                if odat['hybb'].attrs['units'] != '1' :
                    logging.error( 'expecting hybride units hyb "1", found "%s"' % odat['hybb'].attrs['units'] )
                    raise Exception
                #endif
                if odat['sp'].attrs['units'] !=  vunits :
                    logging.error( 'expecting hybride units sp "%s", found "%s"' % (vunits,odat['sp'].attrs['units']) )
                    raise Exception
                #endif

                # check target dims:
                if vdims == ('pixel','layeri') :

                    # extract per pixel:
                    if odat['sp'].dims in [('lat','lon')] :
                        # extract pixels:
                        sp = odat['sp'].data[jj,ii]
                    else :
                        logging.error( 'could not convert from dims "%s" to target dims "%s"' % (str(oda.dims),str(vdims)) )
                        raise Exception
                    #endif
                
                    # check ...
                    if sfile.nlayer is None :
                        logging.error( 'no layer dimension defined yet' )
                        raise Exception
                    #endif

                    # half level pressures:
                    hp = numpy.zeros((len(pixel),sfile.nlayer+1),dtype='f4')
                    # surface:
                    iz = 0
                    hp[:,iz] = odat['hyab'].data[iz,0] + odat['hybb'].data[iz,0] * sp 
                    # loop over layers:
                    for iz in range(1,sfile.nlayer+1) :
                        # top pressure:
                        hp[:,iz] = odat['hyab'].data[iz-1,1] + odat['hybb'].data[iz-1,1] * sp
                    #endfor

                    # create variable:
                    da = xarray.DataArray( hp, dims=('pixel','layeri'), 
                                                 coords={ 'pixel' : pixel }, 
                                                 attrs=attrs )

                else :
                    logging.error( 'unsupported target dims: %s' % str(vdims) )
                    logging.error( '  target variable   : %s' % varname )
                    raise Exception
                #endif
                
            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            # count number of troposphere layers from tropospheric kernel
            elif special == 'kernel_trop_to_nla' :
            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            
                # original field:
                vname = rcf.get( '%s.output.var.%s.%s' % (rcbase,varname,'kernel_trop') )
                # read:
                kernel_trop = sfile.GetData( vname )
                # check source dims:
                if kernel_trop.dims not in [('layer','lat','lon')] :
                    logging.error( 'unsupported source dims: %s' % str(kernel_trop.dims) )
                    logging.error( '  source variable   : %s' % vname )
                    raise Exception
                #endif
                # convert to pixel array:
                avk = kernel_trop.data[:,jj,ii].T   # ('pixel','layer')

                # check target dims:
                if vdims != ('pixel','retr') :
                    logging.error( 'unsupported target dims: %s' % str(vdims) )
                    logging.error( '  target variable   : %s' % varname )
                    raise Exception
                #endif
                # check ...
                if self.nretr != 1 :
                    logging.error( 'not implemented yet for number of retrieval layers %i' % self%nretr )
                    raise Exception
                #endif
                
                # 1-based index of layer where kernel is non-zero:
                nla = numpy.zeros( (self.npixel,self.nretr), dtype='i2' )
                # loop over pixels:
                for ipix in range(self.npixel) :
                    # search last index with non-zero kernel value,
                    # convert to 1-based index:
                    nla[ipix,0] = numpy.where( abs(avk[ipix,:]) > 0.0 )[0].max() + 1
                #endfor
                
                # set attributes:
                attrs = {}
                attrs['units'    ] = '1'
                attrs['long_name'] = 'number of tropospheric layers in profile'

                # create data array with data from selected pixels, copy attributes:
                da = xarray.DataArray( nla, dims=vdims, 
                                         coords={ 'pixel' : pixel }, 
                                         attrs=attrs )


            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            else :
                logging.error( 'unsupported special "%s" for variable "%s"' % (special,varname) )
                raise Exception
            #endif  # special
            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            
            # need to swap layers?
            if swap_layers :
                # loop over dims:
                for dim in vdims :
                    # level dim or not?
                    if dim in ['layer','layeri','retr','retri','retr0'] :
                        # index:
                        idim = vdims.index(dim)
                        # swap:
                        da.data = numpy.flip( da.data, axis=idim )
                    #
                    elif dim in ['pixel','corner','track_scan','track_pixel'] :
                        pass
                    #
                    else :
                        logging.error( 'could not decide if "%s" is a level dimension or not' % dim )
                        raise Exception
                    #endif
                #endfor # dims
            #endif # swap layers
            
            # adhoc attributes ...
            attrs = rcf.get( vkey+'.attrs', totype='dict', default={} )
            # loop:
            for key in attrs.keys() :
                # remove if None:
                if attrs[key] is None :
                    # remove if present:
                    if key in da.attrs.keys() : da.attrs.pop(key)
                else :
                    # update:
                    da.attrs[key] = attrs[key]
                #endif
            #endfor
            
            # add variable to target data set:
            self.AddVariable( varname, da )
        
        #endfor # target variables                

    #enddef AddSelection

#endclass CSO_S5p_SuperObs_File





########################################################################
###
### convert
###
########################################################################

class CSO_S5p_SuperObs_Convert( utopya.UtopyaRc ) :

    """
    Convert TROPOMI Super-Observations to CSO format.
    During conversion, a variable and pixel selection could be applied.
    
    Arguments:

    * ``rcfile``, ``rcbase``, ``env``  :  settings file, prefix for keys, and environment dictionairy 
    
    A time range is read to select the files to be converted::
    
      <rcbase>.timerange.start        :  2018-06-01 00:00
      <rcbase>.timerange.end          :  2018-06-03 23:59

    The files are searched in subdirectories including templates for time fields;
    also specify filename patterns to be matched::

        ! input directory:
        <rcbase>.files.dir         :  /Scratch/TROPOMI/NO2/superobs/%Y%m_0.5deg/compressed

        ! filename filters (space seperated list):
        <rcbase>.files.filters     :  s5p_no2_rpro_v1.2.2_superobs_ll_0.5deg_*.nc

    The following form is expected for the filenames::

        s5p_no2_rpro_v1.2.2_superobs_ll_0.5deg_02833.nc

    The name of the target files should be specified with a directory and filename;
    the later could include a template for the orbit number::

        ! output directory and filename:
        ! - times are taken from mid of selection, rounded to hours
        ! - use '%{orbit}' for orbit number
        <rcbase>.output.dir             :  /Scratch/CSO-data/S5p/NO2/superobs/%Y/%m
        <rcbase>.output.filename        :  S5p_NO2_superobs_%{orbit}.nc
    
    A flag is read to decide if existing files should be renewed or kept:
    
      <rcbase>.renew                  :  True              

    If an input file should be converted, it is read into a :py:class:`.S5p_SuperObs_File` object.
    The :py:meth:`SelectPixels <S5p_SuperObs_File.SelectPixels>` method is called to select pixels based 
    on critera defined in the settings. This method returns a history line to desribe the selection.
    
    The target file is created as an :py:class:`.CSO_S5p_SuperObs_File` object.
    It's :py:meth:`AddSelection <.CSO_S5p_SuperObs_File.AddSelection>` method is called with the input object as argument,
    and this will copy the selected pixels for variables specified in the settings.
    The :py:meth:`Write <.CSO_File.Write>` method creates the file.
    
    Global attributes for the target file should be specified with::

        ! global attributes:
        <rcbase>.output.attrs               :  format Conventions author institution email
        !
        <rcbase>.output.attr.format         :  1.0
        <rcbase>.output.attr.Conventions    :  CF-1.7
        <rcbase>.output.attr.author         :  Your Name
        <rcbase>.output.attr.institution    :  CSO
        <rcbase>.output.attr.email          :  Your.Name@cso.org
    
    In addition, the global attribtues from the original file are copied too,
    preceeded by a specified prefix::

        ! copy global attributes? specify filename filters:
        <rcbase>.output.gattrs.filters      :  *
        ! prefix for new attributes:
        <rcbase>.output.gattrs.prefix       :  source__


    """
    
    
    def __init__( self, rcfile, rcbase='', env={}, indent='' ) :
    
        """
        Convert data.
        """
                        
        # modules:
        import os
        import datetime
        import fnmatch

        # info ...
        logging.info( indent+'' )
        logging.info( indent+'** convert TROPOMI Super-Observations' )
        logging.info( indent+'' )

        # init base object:
        utopya.UtopyaRc.__init__( self, rcfile=rcfile, rcbase=rcbase, env=env )
        
        # renew output?
        renew = self.GetSetting( 'renew', totype='bool' )
        
        # time range:
        t1 = self.GetSetting( 'timerange.start', totype='datetime' )
        t2 = self.GetSetting( 'timerange.end'  , totype='datetime' )
        
        # info ...
        tfmt = '%Y-%m-%d %H:%M'
        logging.info( indent+'timerange: [%s,%s]' % (t1.strftime(tfmt),t2.strftime(tfmt)) )

        # input directory:
        inputdir = self.GetSetting( 'files.dir' )

        # filename filters:
        fnfilters    = self.GetSetting( 'files.filters' ).split()
        #tarfnfilters = self.GetSetting( 'files.tarfilters', default='' ).split()

        # info ...
        logging.info( indent+'input directory template:' )
        logging.info( indent+'  %s' % inputdir )
        logging.info( indent+'collect input files available for timerange ...' )
        # init list with input filenames:
        filenames = []
        # times associated with files:
        times = {}
        # content of input directories is listed,
        # name of directory changes in time:
        indir_prev = None
        # loop over time range:
        t = t1
        while t <= t2 :        
            # info ...
            logging.info( indent+'  %s ...' % t.strftime('%Y-%m-%d') )
            # resolve time templates:
            indir = t.strftime(inputdir)
            # new?
            if indir != indir_prev :
                # present ?
                if os.path.isdir( indir ) :
                    # list directory:
                    fnames = os.listdir( indir )
                    # sort (in place)
                    fnames.sort()
                    # loop over files:
                    for fname in fnames :
                        # filter:
                        for fnfilter in fnfilters :
                            match = fnmatch.fnmatch( fname, fnfilter )
                            if match : break
                        #endfor
                        if not match : continue
                        # info ...
                        logging.info( indent+'    %s ...' % fname )
                        # store:
                        infile = os.path.join(indir,fname)
                        if infile not in filenames :
                            filenames.append( infile )
                            times[infile] = t
                        #endif
                    #endfor # file
                    # reset:
                    indir_prev = indir
                #endif # present
            #endif # new dir
            # next:
            t = t + datetime.timedelta(1)
        #endwhile

        # info ...
        logging.info( indent+'number of files : %i' % len(filenames) )

        # target file:
        output_dir               = self.GetSetting( 'output.dir' )
        output_filename_template = self.GetSetting( 'output.filename' )
        
        # attributes
        attrs = {}
        attrnames = self.GetSetting( 'output.attrs' ).split()
        for key in attrnames :
            attrs[key] = self.GetSetting( 'output.attr.%s' % (key) )
        #endfor
        
        # filters for global attributes to be copied:
        gattrs_filters = self.GetSetting( 'output.gattrs.filters' ).split()
        # prefix to be added to copy:
        gattrs_prefix = self.GetSetting( 'output.gattrs.prefix' )

        # info ..
        logging.info( indent+'loop over files ...' )
        # loop:
        for filename in filenames :

            # basename:
            fname = os.path.basename( filename )
            
            # info ...
            logging.info( indent+'  file %s ...' % fname )
            
            ## testing ...
            #if '11540' not in fname : continue

            # filenames:
            #     s5p_no2_rpro_v1.2.2_superobs_ll_0.5deg_02833.nc
            bname = os.path.basename(filename).replace('.nc','')
            # split:
            platformname,species,processingmode,processor,level,grid,resol,orbit = bname.split('_')
            
            # .. open now already, timerange could not be extracted from filename ...
            # info ...
            logging.info( indent+'    open ...' )
            # read file, sometimes files are corrupted ...
            try :
                sfile = S5p_SuperObs_File( filename )
            except :
                logging.warning( 'could not open file, skip: %s' % filename )
                continue
            #endtry

            # time range:
            tr = sfile.GetTimeRange()
            # use start as reference time for filenames etc:
            t = tr[0]
            # info ...
            logging.info( indent+'    start time of pixels: %s' % t.strftime('%Y-%m-%d %H:%M:%S') )
            # filter ...
            if (t < t1) or (t > t2) :
                logging.info( indent+'    outside timerange, skip ..' )
                continue
            #endif
            
            # target dir:
            outdir = t.strftime(output_dir)
            # target file:
            outfile = output_filename_template
            outfile = outfile.replace( '%{orbit}', orbit )
            outfile = t.strftime(outfile)
            # combine:
            output_filename = os.path.join( outdir, outfile )

            # default:
            create = False            
            # create file?
            if not os.path.isfile(output_filename) :
                # info ...
                logging.info( '    create new file ...' )
                # always create:
                create = True
            #~ renew?
            elif renew :
                # info ...
                logging.info( '    renew file ...' )
                # always create:
                create = True
            #endif
            
            # create?
            if create :

                # apply selections, return bool mask and list of history lines:
                selected,history = sfile.SelectPixels( self.rcf, self.rcbase, indent='    ' )
                # count:
                nselected = selected.sum()

                # info ...
                logging.info( indent+'      selected %i of %i pixels' % (nselected,selected.size) )
                # any ?
                if nselected > 0 :

                    # info ...
                    logging.info( indent+'    create %s ...' % output_filename )
                    # init:
                    csf = CSO_S5p_SuperObs_File()
                    # add:
                    csf.AddSelection( sfile, selected, self.rcf, self.rcbase, indent=indent+'    ' )
                    # update history:
                    history.append( 'added %i pixels from %s' % (nselected,os.path.basename(fname)) )

                    # update attributes:
                    attrs['orbit'     ] = orbit
                    attrs['orbit_file'] = fname
                    
                    # loop over global attribute filters:
                    for afilter in gattrs_filters :
                        # loop over original attributes:
                        for aname in sfile.attrs.keys() :
                            # selected?
                            if fnmatch.fnmatch( aname, afilter ) :
                                # copy:
                                attrs[gattrs_prefix+aname] = sfile.attrs[aname]
                            #endif
                        #endfor # attributes
                    #endfor # gattr filters
                    
                    # write:
                    csf.Write( filename=output_filename, attrs=attrs, history=history )
                    
                #endif  # any selected
                
            else :
                # info ..
                logging.info( indent+'    keep   %s ...' % output_filename )
            #endif

            ## testing ...
            #break

        #endfor  # input files
        
        # info ...
        logging.info( indent+'' )
        logging.info( indent+'** end convert' )
        logging.info( indent+'' )
        
    #enddef __init__

#endclass CSO_S5p_SuperObs_Convert




########################################################################
###
### create listing file
###
########################################################################

class CSO_S5p_SuperObs_Listing( utopya.UtopyaRc ) :

    """
    Create *listing* file for converted orbit files.
    
    A *listing* file contains the names of the converted orbit files,
    and the time range of pixels in the file::

        filename                     ;start_time                   ;end_time                     ;orbit
        2018/06/S5p_RPRO_NO2_03272.nc;2018-06-01T01:32:46.673000000;2018-06-01T01:36:12.948000000;03272
        2018/06/S5p_RPRO_NO2_03273.nc;2018-06-01T03:12:53.649000000;2018-06-01T03:17:43.082000000;03273
        2018/06/S5p_RPRO_NO2_03274.nc;2018-06-01T04:52:43.586000000;2018-06-01T04:59:12.377000000;03274
        :

    This file will be used by the observation operator to selects orbits with pixels valid for 
    a desired time range.
    
    In the settings, define the name of the file to be created::

        ! csv file that will hold records per file with:
        ! - timerange of pixels in file
        ! - orbit number
        <rcbase>.file        :  /Scratch/CSO/S5p/RPRO/NO2/Europe/listing.csv
        
    An existing listing file is not replaced,
    unless the following flag is set::

        ! renew table?
        <rcbase>.renew           :  True
        
    Orbit files are searched within a timerange::
    
        <rcbase>.timerange.start        :  2018-06-01 00:00
        <rcbase>.timerange.end          :  2018-06-03 23:59

    Specify filename filters to search for orbit files;
    the patterns are relative to the basedir of the listing file,
    and might contain templates for the time values.
    Multiple patterns could be defined; if for a certain orbit number more than one
    file is found, the first match is used.
    This could be explored to create a listing that combines reprocessed data
    with near-real-time data::

        <rcbase>.patterns            :  RPRO/NO2/Europe/%Y/%m/S5p_*.nc \
                                        OFFL/NO2/Europe/%Y/%m/S5p_*.nc

    Usually the time range is read from the file, but in case the file does
    not have a time accordinate, then the following flag might be used to
    force that the time that matches with the filename is used::
    
        ! adhoc: use the time for which file is valid as timerange;
        ! this is used for the synthetic S4 data that have no time record ...
        <rcbase>.use_t              :  True
    
    Alternatively, the time range could be read from the global attributes if 
    it is specified there::

        ! adhoc: use attributes to extract timerange from?
        <listing>.use_attrs           :  True
        ! if True, define names:
        <listing>.attr_tr1            :  source__time_coverage_start
        <listing>.attr_tr2            :  source__time_coverage_end

    The ``orbit`` column in the listing is extra, and is read from global attributes;
    the list of extra columns is defined with::
    
        ! extra columns to be added, read from global attributes:
        <rcbase>.xcolumns           :  orbit

    """
    
    def __init__( self, rcfile, rcbase='', env={}, indent='' ) :
    
        """
        Convert data.
        """
                        
        # modules:
        import os
        import datetime
        import glob
        
        # tools:
        import cso_file

        # info ...
        logging.info( indent+'' )
        logging.info( indent+'** create listing file' )
        logging.info( indent+'' )

        # init base object:
        utopya.UtopyaRc.__init__( self, rcfile=rcfile, rcbase=rcbase, env=env )
        
        # renew output?
        renew = self.GetSetting( 'renew', totype='bool' )

        # table file to be written:
        lst_file = self.GetSetting( 'file' )
        # create?
        if (not os.path.isfile(lst_file)) or renew :        
            # info ..
            logging.info( indent+'create %s ...' % lst_file )
        
            # time range:
            t1 = self.GetSetting( 'timerange.start', totype='datetime' )
            t2 = self.GetSetting( 'timerange.end'  , totype='datetime' )        
            # info ...
            tfmt = '%Y-%m-%d %H:%M'
            logging.info( indent+'  timerange: [%s,%s]' % (t1.strftime(tfmt),t2.strftime(tfmt)) )
            
            # base directory:
            bdir = os.path.dirname( lst_file )
            # create?
            if len(bdir) > 0 :
                if not os.path.isdir(bdir) : os.makedirs( bdir )
            #endif
            # current directory?
            if len(bdir) == 0 : bdir = '.'
            # info ...
            logging.info( indent+'  base directory: %s ...' % bdir )
        
            # initiallize for (re)creation:
            listing = cso_file.CSO_Listing( lst_file, indent=indent+'    ' )

            # info ...
            logging.info( indent+'  cleanup records if necessary ...' )
            # remove entries that do not exist anymore:
            listing.Cleanup( indent=indent+'    ' )

            # filename pattern templates:
            pattern_templates = self.GetSetting( 'patterns' ).split()
            # collection of scanned patterns:
            patterns = []
            
            # adhoc: use time of file as time range?
            use_t = self.GetSetting( 'use_t', totype='bool' )
            
            # adhoc: use attributes to extract time?
            use_attrs = self.GetSetting( 'use_attrs', totype='bool' )
            # enabled?
            if use_attrs :
                # read names:
                attr_tr1 = self.GetSetting( 'attr_tr1' )
                attr_tr2 = self.GetSetting( 'attr_tr2' )
            #endif
            
            # extra columns to be added:
            xcolumns = self.GetSetting( 'xcolumns' ).split()
            
            # loop over days:
            t = t1
            while t <= t2 :

                # loop over patterns:
                for pattern_template in pattern_templates :
                    # expand time values:
                    pattern = t.strftime(pattern_template)
                    # skip if already scanned ...
                    if pattern in patterns : continue
                    # store:
                    patterns.append( pattern )
                    # info ...
                    logging.info( indent+'scan %s ...' % pattern )
                    
                    # list relative to basedir:
                    cwd = os.getcwd()
                    os.chdir( bdir )
                    fnames = glob.glob( pattern )
                    os.chdir( cwd )
                    # empty ?
                    if len(fnames) == 0 :
                        logging.info( indent+'  empty ..' )
                        continue
                    #endif
                    # sort in place:
                    fnames.sort()
                    # loop over files:
                    for fname in fnames :
                        # absolute path:
                        filename = os.path.join(bdir,fname)
                        # already in table?
                        if fname in listing :
                            # info ...
                            logging.info( indent+'  keep entry %s ...' % fname )
                        else :
                            # open orbit file:
                            csf = cso_file.CSO_File( filename=filename )
                            # get orbit from attribute:
                            orbit = csf.GetAttr( 'orbit', quiet=True )
                            # already present?
                            if ('orbit' in listing.df.keys()) and (orbit in listing.df['orbit'].values) :
                                # info ...
                                logging.info( indent+'  skip file  %s (existing orbit "%s") ...' % (fname,orbit) )
                            else :
                                # info ...
                                logging.info( indent+'  add entry %s ...' % fname )
                                # timerange argument:
                                if use_t :
                                    # use current time instead of reading from file:
                                    tr = (t,t)
                                    #
                                elif use_attrs :
                                    # check ...
                                    for aname in [attr_tr1,attr_tr2] :
                                        if aname not in csf.ds.attrs.keys() :
                                            logging.error( 'attribute "%s" not found in: %s' % (aname,filename) )
                                            raise Exception
                                        #endif
                                    #endfor
                                    # read time range from attributes:
                                    tfmt = '%Y-%m-%dT%H:%M:%SZ'
                                    tr = ( datetime.datetime.strptime(csf.ds.attrs[attr_tr1],tfmt), \
                                           datetime.datetime.strptime(csf.ds.attrs[attr_tr2],tfmt)  )
                                    #
                                else :
                                    # time range will be extracted from 'time' coordinate ..
                                    tr = None
                                #endif
                                # add record to listing for this file, columns are:
                                #  - start_time, end_time
                                #  - extra columns specified by "xcolumns"
                                listing.Update( fname, csf, tr=tr, xcolumns=xcolumns, indent=indent+'    ' )
                            #endif

                        #endif
                    #endfor # filenames

                #endfor # patterns
                
                ## testing ...
                #break
                        
                # next
                t = t + datetime.timedelta(0,3600)
            #endwhile
        
            # save:
            listing.Close( indent=indent+'  ' )

        else :      
            # info ..
            logging.info( indent+'keep %s ...' % lst_file )
        #endif

        # info ...
        logging.info( indent+'' )
        logging.info( indent+'** end listing' )
        logging.info( indent+'' )
        
    #enddef __init__
    
#endclass CSO_S5p_SuperObs_Listing


########################################################################
###
### end
###
########################################################################

