#
# Changes
#
# 2023-08, Arjo Segers
#   Reformatted using 'black'.
#

########################################################################
###
### help
###
########################################################################

"""
.. _cso-tropomi-superobs:

*******************************
``cso_tropomi_superobs`` module
*******************************

The :py:mod:`cso_tropomi_superobs` module provides classes to convert
TROPOMI super-observations (level-3 gridded product) into a CSO format.


Class hierchy
=============

The classes and are defined according to the following hierchy:

.. NOT YET
    * :py:class:`.TROPOMI_SuperObs_File`
    * :py:class:`.CSO_File`

      * :py:class:`.CSO_S5p_File`

    * :py:class:`.UtopyaRc`

      * :py:class:`.CSO_S5p_Convert`
      * :py:class:`.CSO_S5p_Listing`



Classes
=======


"""


########################################################################
###
### modules
###
########################################################################

# modules:
import logging

# tools:
import utopya
import cso_file


########################################################################
###
### S5p file
###
########################################################################


class S5p_SuperObs_File(object):

    """
    Base class to access data in S5p file.
    
    Example of content::

        netcdf s5p_no2_rpro_v1.2.2_superobs_ll_0.5deg_02833 {
        dimensions:
	        lon = 720 ;
	        lat = 360 ;
	        layer = 34 ;
	        vertices = 2 ;
        variables:
	        float lon(lon) ;
		        lon:standard_name = "longitude" ;
		        lon:long_name = "longitude" ;
		        lon:units = "degrees_east" ;
	        float lat(lat) ;
		        lat:standard_name = "latitude" ;
		        lat:long_name = "latitude" ;
		        lat:units = "degrees_north" ;
	        float layer(layer) ;
		        layer:standard_name = "layer" ;
		        layer:long_name = "layer" ;
		        layer:units = "1" ;
	        float no2_superobs(lat, lon) ;
		        no2_superobs:standard_name = "nitrogen_dioxide_tropospheric_column_superobservation" ;
		        no2_superobs:long_name = "NO2 tropospheric column superobservation, TROPOMI sensor" ;
		        no2_superobs:units = "1e-6 mol m^-2" ;
		        no2_superobs:_FillValue = 9.96921e+36f ;
	        float no2_superobs_uncertainty(lat, lon) ;
		        no2_superobs_uncertainty:standard_name = "nitrogen_dioxide_tropospheric_column_superobservation_uncertainty" ;
		        no2_superobs_uncertainty:long_name = "NO2 tropospheric column superobservation uncertainty, TROPOMI sensor, including representation error" ;
		        no2_superobs_uncertainty:units = "1e-6 mol m^-2" ;
		        no2_superobs_uncertainty:_FillValue = 9.96921e+36f ;
	        float no2_superobs_uncertainty_representation(lat, lon) ;
		        no2_superobs_uncertainty_representation:standard_name = "nitrogen_dioxide_tropospheric_column_superobservation_representativity_uncertainty" ;
		        no2_superobs_uncertainty_representation:long_name = "NO2 tropospheric column superobservation uncertainty, TROPOMI sensor, only representation error" ;
		        no2_superobs_uncertainty_representation:units = "1e-6 mol m^-2" ;
		        no2_superobs_uncertainty_representation:_FillValue = 9.96921e+36f ;
	        float average_uncertainty(lat, lon) ;
		        average_uncertainty:standard_name = "nitrogen_dioxide_tropospheric_column_average_uncertainty" ;
		        average_uncertainty:long_name = "NO2 tropospheric column superobservation uncertainty, TROPOMI sensor, average over individual observation" ;
		        average_uncertainty:units = "1e-6 mol m^-2" ;
		        average_uncertainty:_FillValue = 9.96921e+36f ;
	        float surface_pressure(lat, lon) ;
		        surface_pressure:standard_name = "surface_pressure" ;
		        surface_pressure:long_name = "superobservation surface pressure" ;
		        surface_pressure:units = "Pa" ;
		        surface_pressure:_FillValue = 9.96921e+36f ;
	        float cloud_radiance_fraction(lat, lon) ;
		        cloud_radiance_fraction:standard_name = "cloud_radiance_fraction" ;
		        cloud_radiance_fraction:long_name = "superobservation cloud radiance fraction" ;
		        cloud_radiance_fraction:units = "1" ;
		        cloud_radiance_fraction:_FillValue = 9.96921e+36f ;
	        float kernel_troposphere(layer, lat, lon) ;
		        kernel_troposphere:standard_name = "averaging_kernel_troposphere" ;
		        kernel_troposphere:long_name = "NO2 tropospheric column averaging kernel of the superobservation, TROPOMI sensor" ;
		        kernel_troposphere:units = "1" ;
		        kernel_troposphere:_FillValue = 9.96921e+36f ;
	        float covered_area_fraction(lat, lon) ;
		        covered_area_fraction:standard_name = "covered_area_fraction" ;
		        covered_area_fraction:long_name = "Fraction of the grid box covered by satellite observations" ;
		        covered_area_fraction:units = "1" ;
		        covered_area_fraction:_FillValue = 9.96921e+36f ;
	        float no2_scd_geo_superobs(lat, lon) ;
		        no2_scd_geo_superobs:standard_name = "nitrogen_dioxide_slant_amfgeo_superobservation" ;
		        no2_scd_geo_superobs:long_name = "NO2 slant column divided by AMFgeo superobservation, TROPOMI sensor" ;
		        no2_scd_geo_superobs:units = "1e-6 mol m^-2" ;
		        no2_scd_geo_superobs:_FillValue = 9.96921e+36f ;
	        float no2_strat_superobs(lat, lon) ;
		        no2_strat_superobs:standard_name = "nitrogen_dioxide_stratospheric_column_superobservation" ;
		        no2_strat_superobs:long_name = "NO2 stratospheric column superobservation, TROPOMI sensor" ;
		        no2_strat_superobs:units = "1e-6 mol m^-2" ;
		        no2_strat_superobs:_FillValue = 9.96921e+36f ;
	        float amf_trop_superobs(lat, lon) ;
		        amf_trop_superobs:standard_name = "tropospheric_air_mass_factor_superobservation" ;
		        amf_trop_superobs:long_name = "NO2 tropospheric air mass factor, TROPOMI sensor" ;
		        amf_trop_superobs:units = "1" ;
		        amf_trop_superobs:_FillValue = 9.96921e+36f ;
	        float amf_strat_superobs(lat, lon) ;
		        amf_strat_superobs:standard_name = "stratospheric_air_mass_factor_superobservation" ;
		        amf_strat_superobs:long_name = "NO2 stratospheric air mass factor, TROPOMI sensor" ;
		        amf_strat_superobs:units = "1" ;
		        amf_strat_superobs:_FillValue = 9.96921e+36f ;
	        float tm5_constant_a(layer, vertices) ;
		        tm5_constant_a:standard_name = "tm5_constant_a" ;
		        tm5_constant_a:long_name = "TM5 hybrid A coefficient at upper and lower interface levels" ;
		        tm5_constant_a:units = "Pa" ;
		        tm5_constant_a:_FillValue = 9.96921e+36f ;
	        float tm5_constant_b(layer, vertices) ;
		        tm5_constant_b:standard_name = "tm5_constant_b" ;
		        tm5_constant_b:long_name = "TM5 hybrid B coefficient at upper and lower interface levels" ;
		        tm5_constant_b:units = "1" ;
		        tm5_constant_b:_FillValue = 9.96921e+36f ;

        // global attributes:
		        :authors = "Henk Eskes, eskes@knmi.nl" ;
		        :institution = "Royal Netherlands Meteorological Institute (KNMI)" ;
		        :product_version = "v 1.2.2, reprocessing March 2019, RPRO" ;
		        :project = "S5P, phase E2" ;
		        :reference = "http://www.tropomi.eu/" ;
		        :source = "TROPOMI / Sentinel-5P" ;
		        :title = "Nitrogen dioxide (NO2) column data, superobservations, regular lat-lon, 0.5x0.5 degree" ;
		        :vertical_column_processor_version = "TM5-MP-Domino3 v3.5.3" ;
		        :date_created = "2020-02-24T11:08:45Z" ;
		        :time_coverage_start = "2018-05-01T02:03:58Z" ;
		        :time_coverage_end = "2018-05-01T02:56:14Z" ;
		        :filter_min_QA_value = 0.75f ;
		        :filter_max_QA_value = 1.1f ;
		        :data_version = "TM3-MP-Domino v3.5.3, reprocessing March 2019, version 1.2.2" ;
		        :S5P_NO2_file_name = "S5P_RPRO_L2__NO2____20180501T014123_20180501T032451_02833_01_010202_20190201T165715.nc" ;
		        :S5P_NO2_orbit_number = 2833 ;
		        :S5P_NO2_history = "2019-02-01 17:26:08 f_s5pops tropnll2dp /mnt/data1/storage_offl_l2/cache_offl_l2/WORKING-565374376/JobOrder.565372716.xml; 2019-02-27 08:50:05 TM5-MP-DOMINO offline" ;
		        :S5P_NO2_references = "http://www.tropomi.eu/data-products/nitrogen-dioxide" ;
		        :S5P_NO2_processor_version = "1.2.2" ;
		        :S5P_NO2_identifier_product_doi = "10.5270/S5P-s4ljg54" ;
		        :S5P_NO2_date_created = "2019-02-01T17:17:00Z" ;
		        :S5P_NO2_date_modified = "2019-02-27T08:50:05Z" ;
		        :superobs_setting_modelStratVcdError = 0.15f ;
		        :superobs_setting_minimal_coverage = 0.5f ;
		        :superobs_setting_correlation_factor = 0.15f ;
		        :superobs_setting_meanVariabilityFactor = 0.5f ;
        }

    Arguments:
    
    * ``filename``    : name of input file

    """

    def __init__(self, filename):

        """
        Open data file.        
        """

        # modules:
        import os
        import xarray

        # store:
        self.filename = filename

        # check ...
        if not os.path.isfile(filename):
            logging.error("file not found : %s" % filename)
            raise Exception
        # endif

        # open:
        with xarray.open_dataset(self.filename) as ds:

            # copy some dimensions for convenience:
            # self.ntime  = ds.dims['time']
            # self.nscan  = ds.dims['scanline']
            # self.ngpix  = ds.dims['ground_pixel']
            self.nlon = ds.dims["lon"]
            self.nlat = ds.dims["lat"]
            # number 'atmoshpere' layers:
            if "layer" in ds.dims.keys():
                self.nlayer = ds.dims["layer"]
            else:
                self.nlayer = None
            # endif

            # copy global attributes:
            self.attrs = ds.attrs

        # endwith ds

    # enddef __init__

    # *

    def GetData(self, vname, units=None):

        """
        Extract data array from input file and perform some adhoc fixes:
        
        * Eventually insert units attribute if not present yet, this is then taken from the argument.
        * Convert data to target units if necessary.
        * Add a ``long_name`` attribute if not present yet.
        
        Arguments:
        
        * ``vname`` : variable in in input
        
        Optional arguments:
        
        * ``units`` : target units
        
        Return values:
        
        * ``da`` : :py:xarray.DataArray` object
        """

        # modules:
        import os
        import xarray

        # access group:
        with xarray.open_dataset(self.filename) as ds:
            # check ..
            if vname not in ds:
                logging.error('no variable "%s" in file "%s"' % (vname, self.filename))
                raise Exception
            # endif
            # extract variable:
            da = ds[vname]
            # not lazy ...
            da.load()
        # endwith # xarray

        # unit stuff?
        if units is not None:

            # set input units in case these are not present yet,
            # assume that what is provided in the settings is correct:
            if "units" not in da.attrs.keys():
                da.attrs["units"] = units

            ## info ...
            # logging.info( indent+'    input  range: [%f,%f] %s' % (da.min(),da.max(),da.attrs['units']) )

            # need to convert?
            if da.attrs["units"] != units:
                # conversion description:
                conversion = "%s -> %s" % (da.attrs["units"], units)
                # switch:
                if conversion in ["1e-6 mol m^-2 -> mol m-2"]:
                    # apply factor:
                    da.data = da.data * 1.0e-6
                else:
                    logging.error('unsupported conversion "%s" requested ...' % conversion)
                    raise Exception
                # endif
                # reset units:
                da.attrs["units"] = units
            # endif  # units differ

        # endif # units defined

        # adhoc fix: sometimes neither standard name or long name,
        # cf checker gives a warning on this ...
        if ("standard_name" not in da.attrs.keys()) and ("long_name" not in da.attrs.keys()):
            # use variable name as long name ...
            da.attrs["long_name"] = os.path.basename(path)
        # endif

        # ok
        return da

    # enddef GetData

    # *

    def GetTimeRange(self):

        """
        Return timerange ``(t1,t2)`` extracted from attributes.
        """

        # modules:
        import datetime

        # attribute names:
        attr_tr1 = "time_coverage_start"
        attr_tr2 = "time_coverage_end"
        # check ...
        for aname in [attr_tr1, attr_tr2]:
            if aname not in self.attrs.keys():
                logging.error('attribute "%s" not found in: %s' % (aname, self.filename))
                raise Exception
            # endif
        # endfor

        # read time range from attributes:
        tfmt = "%Y-%m-%dT%H:%M:%SZ"
        tr = (
            datetime.datetime.strptime(self.attrs[attr_tr1], tfmt),
            datetime.datetime.strptime(self.attrs[attr_tr2], tfmt),
        )

        # ok
        return tr

    # enddef GetTimeRange

    # *

    def SelectPixels(self, rcf, rckey, indent=""):

        """
        Apply filters specified in rcfile.
        
        Arguments:
        
        * ``rcf``       :  :py:class:`.RcFile` object with settings
        * ``rckey``     :  basename for rcfile keys, e.g. "s5p" for the example below
            
        Return values:
        
        * ``selected`` : boolean array with shape of track ``(time,scan,pixel)`` which
          is True if a pixel passed all checks;
        * ``history``  : list of character string describing the selelections.
          
        Example configuration::
        
            ! Specifiy a list of filter names.
            ! For each name, specify the variable which values are used for testing,
            ! the type test, and eventually some thresholds or other settings for this type.
            ! The units of the thresholds should match with the units in the variables,
            ! the expected units have to be defined too.
            ! The examples below show possible types and their settings.

            ! filters:
            s5p.filters                        :  lons lats albedo valid

            ! select range of values:
            s5p.filter.lons.var                :  PRODUCT/longitude
            s5p.filter.lons.type               :  minmax
            s5p.filter.lons.minmax             :  -15.0 35.0
            s5p.filter.lons.units              :  degrees_east

            ! select above a minimum:
            s5p.filter.lats.var                :  PRODUCT/latitude
            s5p.filter.lats.type               :  minmax
            s5p.filter.lats.minmax             :  35.0 70.0
            s5p.filter.lats.units              :  degrees_north

            ! select below a maximum:
            s5p.filter.albedo.var              :  Data Fields/SurfaceAlbedo
            s5p.filter.albedo.type             :  max
            s5p.filter.albedo.max              :  0.3
            s5p.filter.albedo.units            :  1

            ! select only values with data (no "_FillValue"):
            s5p.filter.valid.var               :  Data Fields/NO2RetrievalTroposphericVerticalColumn
            s5p.filter.valid.type              :  valid
        
        """

        # modules:
        import xarray
        import numpy

        # info ...
        logging.info(indent + "pixel selection ...")

        # init selection, by default accept all:
        selected = numpy.ones((self.nlat, self.nlon), "bool")

        # init history:
        history = []

        # which filters ?
        filternames = rcf.get("%s.filters" % rckey).split()
        # loop over filters:
        for filtername in filternames:
            # info ...
            logging.info(indent + '  filter "%s" ...' % filtername)
            # filter key in rcfiles:
            fkey = "%s.filter.%s" % (rckey, filtername)
            # which variable ?
            varname = rcf.get(fkey + ".var")
            # read variable as data array:
            da = self.GetData(varname)
            # extract units:
            vunits = da.attrs["units"]
            # which type ?
            filtertype = rcf.get(fkey + ".type")
            # switch:
            # ~ compare values:
            if filtertype == "equal":
                # read value:
                svalue = rcf.get(fkey + ".value")
                fvalue = float(svalue)
                # avoid warnings about comparing nan ...
                xdata = da.data.copy()
                if numpy.any(numpy.isnan(da.data)):
                    xdata[numpy.where(numpy.isnan(da.data))] = fvalue - 1
                # apply filter:
                selected = selected & (xdata == fvalue)
                # update history, also select on not-nan to avoid warnings from comparisons:
                history.append("selected pixels with '%s' == %s" % (varname, svalue))
            #
            # ~ minimum value:
            elif filtertype == "min":
                # read threshold:
                smin = rcf.get(fkey + ".min")
                fmin = float(smin)
                # check units ...
                funits = rcf.get(fkey + ".units")
                if funits != vunits:
                    logging.error(
                        'variable "%s" has units "%s" while threshold has units "%s"'
                        % (varname, vunits, funits)
                    )
                    raise Exception
                # endif
                # avoid warnings about comparing nan ...
                xdata = da.data.copy()
                if numpy.any(numpy.isnan(da.data)):
                    xdata[numpy.where(numpy.isnan(da.data))] = fmin - 1
                # apply filter, also select on not-nan to avoid warnings from comparisons:
                selected = selected & (xdata >= fmin)
                # update history:
                history.append("selected pixels with '%s' >= %s" % (varname, smin))
            #
            # ~ maximum value:
            elif filtertype == "max":
                # read threshold:
                smax = rcf.get(fkey + ".max")
                fmax = float(smax)
                # check units ...
                funits = rcf.get(fkey + ".units")
                if funits != vunits:
                    logging.error(
                        'variable "%s" has units "%s" while threshold has units "%s"'
                        % (varname, vunits, funits)
                    )
                    logging.error(
                        "value range : %f - %f" % (numpy.nanmin(values), numpy.nanmax(values))
                    )
                    raise Exception
                # endif
                # avoid warnings about comparing nan ...
                xdata = da.data.copy()
                if numpy.any(numpy.isnan(da.data)):
                    xdata[numpy.where(numpy.isnan(da.data))] = fmax + 1
                # apply filter, also select on not-nan to avoid warnings from comparisons:
                selected = selected & (xdata <= fmax)
                # update history:
                history.append("selected pixels with '%s' <= %s" % (varname, smax))
            #
            # ~ range of valid values:
            elif filtertype == "minmax":
                # read thresholds:
                smin, smax = rcf.get(fkey + ".minmax").split()
                fmin = float(smin)
                fmax = float(smax)
                # check units ...
                funits = rcf.get(fkey + ".units")
                if funits != vunits:
                    logging.error(
                        'variable "%s" has units "%s" while threshold has units "%s"'
                        % (varname, vunits, funits)
                    )
                    raise Exception
                # endif
                # avoid warnings about comparing nan ...
                xdata = da.data.copy()
                if numpy.any(numpy.isnan(da.data)):
                    xdata[numpy.where(numpy.isnan(da.data))] = fmin - 1
                # apply filter, also select on not-nan to avoid warnings from comparisons:
                selected = selected & (xdata >= fmin) & (xdata <= fmax)
                # update history:
                history.append("selected pixels with '%s' in [%s,%s]" % (varname, smin, smax))
            #
            # ~ only unmasked values:
            elif filtername == "valid":
                # only accept non-masked fields:
                selected = selected & (~numpy.isnan(da.data))
                # update history:
                history.append("selected pixels with valid '%s' values" % varname)
            #
            # ~ not yet ...
            else:
                logging.error("unsupported filter name : %s" % filtername)
                raise Exception
            # endif
        # endfor # filters

        ## if ncecessary, convert from a masked to a normal array:
        # if hasattr(selected,'mask') : selected = selected.filled(fill_value=False)

        # ok
        return selected, history

    # enddef SelectPixels


# endclass S5p_SuperObs_File


########################################################################
###
### converted file
###
########################################################################


class CSO_S5p_SuperObs_File(cso_file.CSO_File):

    """
    Storage for CSO satellite file filled from S5p data.
    """

    def AddSelection(self, sfile, selected, rcf, rcbase, indent=""):

        """
        Add selected OMI to satellite extract file.
        
        Arguments:
        
        * ``sfile``      :  :py:class:`.S5p_File` object
        * ``selected``   :  boolean array as provided by :py:meth:`S5p_File.SelectPixels` method
        * ``rcf``        :  :py:class:`.RcFile` instance with settings
        * ``rcbase``     :  base of rcfile keys
            
        The first setting that is read is a list with variable names to be 
        created in the target file::

          <rcbase>.output.vars    :  longitude corner_longitudes \\
                                     latitude corner_latitudes \\
                                     vcd  ...

        For each variable, a series of settings should be specified that describe 
        how the variable should look like and how to create it.
        
        The first setting is a list of dimension names that define the shape of the variable.
        Supported dimensions are:
        
        * ``pixel``       : selected pixels
        * ``corner``      : number of footprint bounds (probably 4)
        * ``layer``       : number of layers in atmospheric profile (layers in kernel)
        * ``layeri``      : number of layer interfaces in atmospheric profile (``layer+1``)
        * ``retr``        : number of layers in retrieval product (1 for columns)
        * ``retr0``       : same as ``retr``, used for matrix dimensions ``(retr,retr0)`` to avoid
          repeated dimensions where the cf-checker complains about
        
        For a 1D variable with values per pixel the dimension setting is therefore::
        
            <rcbase>.output.var.longitude.dims    :   pixel
        
        For most variables it is sufficient to provide only the name of the original
        variable from which the data should be read::

          <rcbase>.output.var.longitude.from    :   lon

        For some variables a special processing needs to be done.
        For these variables a key '``special``' is used which will enable the 
        correct conversion. The following specials are currently implemented:
        
        * ``longitude_to_bounds``   : longiude bounds interpolated/extrpolated from centers; requires a ``.from`` setting
        * ``latitude_to_bounds``    : latiude  bounds interpolated/extrpolated from centers; requires a ``.from`` setting
            
        * ``hybounds_to_pressure`` : form pressure from hybride sigma pressure coordinate,
          where the available hybride coefficients have shape ``('layer','vertices')``;
          requires settings::
        
            <rcbase>.output.var.pressure.sp       :   PRODUCT/SUPPORT_DATA/INPUT_DATA/surface_pressure
            <rcbase>.output.var.pressure.hyab     :   PRODUCT/tm5_constant_a
            <rcbase>.output.var.pressure.hybb     :   PRODUCT/tm5_constant_b
            <rcbase>.output.var.pressure.units    :   Pa
        
        * ``kernel_trop_to_nla`` : 1-based index of lowest layer where tropo-spheric kernel is non-zero.
          Required settings::
          
            <rcbase>.output.var.kernel.kernel_trop  :   kernel_troposphere
           
        * ``square`` : create a variable as the square of the input; requires a ``.from`` setting.
           
        Optionally provide target units too. 
        In the (unlikely) case that the original variable has no ``units`` attribute,
        this setting is required to define the (assumed) units.
        If the provided units are different from the units in the original variable,
        the data is converted to the provided units::

          <rcbase>.output.var.longitude.units   :   degrees_east

        Optionally provide a dictionairy with attributes to be added.
        If the attribute value is ``None``, the attribute is removed if present from the input;
        this is sometimes needed if the CF compliance checker complains::
        
          !~ skip some attributes, cf-checker complains ...
          <rcbase>.output.var.qa_value.attrs         :   { 'valid_min' : None, 'valid_max' : None }


        """

        # modules:
        import os
        import numpy
        import xarray
        import pandas
        import scipy.interpolate

        # info ..
        logging.info(indent + "add pixels to satellite file ...")

        # track size:
        nlat, nlon = selected.shape

        # swap layers?
        swap_layers = rcf.get(rcbase + ".swap_layers", totype="bool")

        # selection indices:
        jj, ii = numpy.where(selected)

        # scan line coordinate is "lat" ..
        track_scan = xarray.DataArray(
            numpy.arange(nlat, dtype="i4"),
            dims=("track_scan"),
            attrs={"units": "1", "long_name": "scan line index (0-based)"},
        )
        # store:
        self.AddCoord("track_scan", track_scan)

        # track pixel coordinate is "lon" ..
        track_pixel = xarray.DataArray(
            numpy.arange(nlon, dtype="i4"),
            dims=("track_pixel"),
            attrs={"units": "1", "long_name": "pixel index track (0-based)"},
        )
        # store:
        self.AddCoord("track_pixel", track_pixel)

        # create pixel coordinate:
        #   0-based index that represents the original latitude and longitude
        compressed_index = jj * nlon + ii
        # create coordinate data array ;
        # convert to int, otherwise cf-checker complains about the 'compress' attribute ..
        pixel = xarray.DataArray(
            numpy.array(compressed_index, dtype="i4"),
            dims=("pixel"),
            attrs={"units": "1", "long_name": "pixel index", "compress": "track_scan track_pixel"},
        )
        # store:
        self.AddCoord("pixel", pixel)
        # store length:
        self.npixel = len(pixel)

        # number of retrieval layers:
        self.nretr = 1

        # target variables:
        varnames = rcf.get(rcbase + ".output.vars").split()

        # loop over target variables:
        for varname in varnames:

            # info ...
            logging.info(indent + '  add variable "%s" ...' % varname)

            # prefix for settings:
            vkey = "%s.output.var.%s" % (rcbase, varname)

            # target units:
            vunits = rcf.get(vkey + ".units", default="None")
            if vunits == "None":
                vunits = None

            # target dimensions:
            vdims = tuple(rcf.get(vkey + ".dims").split())

            # specials ?
            special = rcf.get(vkey + ".special", default="None")
            # switch:
            # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            if special in ["None", "square"]:
                # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

                # name of original field:
                opath = rcf.get(vkey + ".from")

                # optional data type:
                dtype = rcf.get(vkey + ".dtype", default="None")
                if dtype == "None":
                    dtype = None

                # extract data array:
                oda = sfile.GetData(opath, units=vunits)

                # check target dims:
                # ~ single value per pixel:
                if vdims == ("pixel",):

                    # extract 1D array from data that is only given per longitude:
                    if oda.dims in [("lon",)]:

                        # create 1D array with data from selected pixels, copy attributes:
                        da = xarray.DataArray(
                            numpy.array(oda.data[ii], dtype=dtype),
                            dims=vdims,
                            coords={"pixel": pixel},
                            attrs=oda.attrs,
                        )

                    # extract 1D array from data that is only given per latitude:
                    elif oda.dims in [("lat",)]:

                        # create 1D array with data from selected pixels, copy attributes:
                        da = xarray.DataArray(
                            numpy.array(oda.data[jj], dtype=dtype),
                            dims=vdims,
                            coords={"pixel": pixel},
                            attrs=oda.attrs,
                        )

                    else:
                        logging.error(
                            'could not convert from dims "%s" to target dims "%s"'
                            % (str(oda.dims), str(vdims))
                        )
                        raise Exception
                    # endif

                # ~ retrieval profile per pixel:
                elif vdims == ("pixel", "retr"):

                    # extract 1D array of pixels from 2D track:
                    if oda.dims in [("lat", "lon")]:

                        # check ...
                        if self.nretr != 1:
                            logging.error(
                                "not implemented yet for number of retrieval layers %i"
                                % self
                                % nretr
                            )
                            raise Exception
                        # endif

                        # create 1D array with data from selected pixels, copy attributes:
                        da = xarray.DataArray(
                            numpy.array(
                                oda.data[jj, ii].reshape(self.npixel, self.nretr), dtype=dtype
                            ),
                            dims=vdims,
                            coords={"pixel": pixel},
                            attrs=oda.attrs,
                        )

                    else:
                        logging.error(
                            'could not convert from dims "%s" to target dims "%s"'
                            % (str(oda.dims), str(vdims))
                        )
                        raise Exception
                    # endif

                # ~ kernel per pixel:
                elif vdims == ("pixel", "layer", "retr"):

                    # extract 2D array of pixels:
                    if oda.dims in [("layer", "lat", "lon")]:

                        # check ...
                        if self.nretr != 1:
                            logging.error(
                                "not implemented yet for number of retrieval layers %i"
                                % self
                                % nretr
                            )
                            raise Exception
                        # endif
                        # check ...
                        if sfile.nlayer is None:
                            logging.error("no layer dimension defined yet")
                            raise Exception
                        # endif

                        # create data array with data from selected pixels, copy attributes:
                        da = xarray.DataArray(
                            numpy.array(
                                oda.data[:, jj, ii].T.reshape(
                                    self.npixel, sfile.nlayer, self.nretr
                                ),
                                dtype=dtype,
                            ),
                            dims=vdims,
                            coords={"pixel": pixel},
                            attrs=oda.attrs,
                        )

                    else:
                        logging.error(
                            'could not convert from dims "%s" to target dims "%s"'
                            % (str(oda.dims), str(vdims))
                        )
                        raise Exception
                    # endif

                # ~ retrieval matrix per pixel:
                elif vdims == ("pixel", "retr", "retr0"):

                    # extract 1D array of pixels from 2D track:
                    if oda.dims in [("lat", "lon")]:

                        # check ...
                        if self.nretr != 1:
                            logging.error(
                                "not implemented yet for number of retrieval layers %i"
                                % self
                                % nretr
                            )
                            raise Exception
                        # endif

                        # create 1D array with data from selected pixels, copy attributes:
                        da = xarray.DataArray(
                            numpy.array(
                                oda.data[jj, ii].reshape(self.npixel, self.nretr, self.nretr),
                                dtype=dtype,
                            ),
                            dims=vdims,
                            coords={"pixel": pixel},
                            attrs=oda.attrs,
                        )

                    else:
                        logging.error(
                            'could not convert from dims "%s" to target dims "%s"'
                            % (str(oda.dims), str(vdims))
                        )
                        raise Exception
                    # endif

                # ~ track
                elif vdims == ("track_scan", "track_pixel"):

                    # storage:
                    values = numpy.zeros((nlat, nlon))

                    # copy 1D array into 2D track:
                    if oda.dims in [("lon",)]:
                        # copy:
                        for ilat in range(nlat):
                            values[ilat, :] = oda.data
                        # endfor
                    #
                    elif oda.dims in [("lat",)]:
                        # copy:
                        for ilon in range(nlon):
                            values[:, ilon] = oda.data
                        # endfor
                    else:
                        logging.error(
                            'could not convert from dims "%s" to target dims "%s"'
                            % (str(oda.dims), str(vdims))
                        )
                        raise Exception
                    # endif

                    # create 1D array with data from selected pixels, copy attributes:
                    da = xarray.DataArray(
                        numpy.array(values, dtype=dtype),
                        dims=vdims,
                        coords={"track_scan": track_scan, "track_pixel": track_pixel},
                        attrs=oda.attrs,
                    )

                else:
                    logging.error("unsupported target dims: %s" % str(vdims))
                    logging.error("  target variable   : %s" % varname)
                    raise Exception
                # endif

                # postprocessing:
                if special == "square":
                    # apply:
                    da.data = da.data ** 2
                    da.attrs["units"] = "(%s)**2" % da.attrs["units"]
                # endif

            # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            # longitude bounds, interpolated/extrapolated from centers:
            elif special in ["longitude_to_bounds"]:
                # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

                # name of original field:
                opath = rcf.get(vkey + ".from")
                # get data array:
                oda = sfile.GetData(opath, units=vunits)

                # check shape:
                if oda.dims in [("lon",)]:

                    # interpolation/extrapolation function:
                    fix = scipy.interpolate.interp1d(
                        numpy.arange(nlon), oda.data, fill_value="extrapolate"
                    )
                    # apply:
                    blons = fix(numpy.arange(nlon + 1) - 0.5)

                    # corners:
                    #    3      2
                    #     o----o
                    #     |    |
                    #     o----o
                    #    0      1

                    # check target dims:
                    if vdims == ("pixel", "corner"):

                        # storage for corner values:
                        values = numpy.zeros((self.npixel, 4))
                        # fill:
                        values[:, 0] = blons[ii]
                        values[:, 1] = blons[ii + 1]
                        values[:, 2] = blons[ii + 1]
                        values[:, 3] = blons[ii]

                        # create 2D array (+corners) with data from selected scan lines:
                        da = xarray.DataArray(
                            values, dims=vdims, coords={"pixel": pixel}, attrs=oda.attrs
                        )
                    #
                    elif vdims == ("track_scan", "track_pixel", "corner"):

                        # storage for corner values:
                        values = numpy.zeros((nlat, nlon, 4))
                        # fill:
                        for ilat in range(nlat):
                            values[ilat, :, 0] = blons[:-1]
                            values[ilat, :, 1] = blons[1:]
                            values[ilat, :, 2] = blons[1:]
                            values[ilat, :, 3] = blons[:-1]
                        # endfor

                        # create 2D array (+corners) with data from selected scan lines:
                        da = xarray.DataArray(
                            values,
                            dims=vdims,
                            coords={"track_scan": track_scan, "track_pixel": track_pixel},
                            attrs=oda.attrs,
                        )
                    #
                    else:
                        logging.error("unsupported target dims: %s" % str(vdims))
                        logging.error("  target variable   : %s" % varname)
                        raise Exception
                    # endif

                else:
                    logging.error(
                        'could not convert from dims "%s" to target dims "%s"'
                        % (str(oda.dims), str(vdims))
                    )
                    raise Exception
                # endif

            # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            # latitude bounds, interpolated/extrapolated from centers:
            elif special in ["latitude_to_bounds"]:
                # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

                # name of original field:
                opath = rcf.get(vkey + ".from")
                # get data array:
                oda = sfile.GetData(opath, units=vunits)

                # check shape:
                if oda.dims in [("lat",)]:

                    # interpolation/extrapolation function:
                    fix = scipy.interpolate.interp1d(
                        numpy.arange(nlat), oda.data, fill_value="extrapolate"
                    )
                    # apply:
                    blats = fix(numpy.arange(nlat + 1) - 0.5)

                    # corners:
                    #    3      2
                    #     o----o
                    #     |    |
                    #     o----o
                    #    0      1

                    # check target dims:
                    if vdims == ("pixel", "corner"):

                        # storage for corner values:
                        values = numpy.zeros((self.npixel, 4))
                        # fill:
                        values = numpy.zeros((self.npixel, 4))
                        values[:, 0] = blats[jj]
                        values[:, 1] = blats[jj]
                        values[:, 2] = blats[jj + 1]
                        values[:, 3] = blats[jj + 1]

                        # create 2D array (+corners) with data from selected scan lines:
                        da = xarray.DataArray(
                            values, dims=vdims, coords={"pixel": pixel}, attrs=oda.attrs
                        )
                    #
                    elif vdims == ("track_scan", "track_pixel", "corner"):

                        # storage for corner values:
                        values = numpy.zeros((nlat, nlon, 4))
                        # fill:
                        for ilon in range(nlon):
                            values[:, ilon, 0] = blats[:-1]
                            values[:, ilon, 1] = blats[:-1]
                            values[:, ilon, 2] = blats[1:]
                            values[:, ilon, 3] = blats[1:]
                        # endfor

                        # create 2D array (+corners) with data from selected scan lines:
                        da = xarray.DataArray(
                            values,
                            dims=vdims,
                            coords={"track_scan": track_scan, "track_pixel": track_pixel},
                            attrs=oda.attrs,
                        )
                    #
                    else:
                        logging.error("unsupported target dims: %s" % str(vdims))
                        logging.error("  target variable   : %s" % varname)
                        raise Exception
                    # endif

                else:
                    logging.error(
                        'could not convert from dims "%s" to target dims "%s"'
                        % (str(oda.dims), str(vdims))
                    )
                    raise Exception
                # endif

            # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            # pressure from hybride coordinates;
            # hyab and hybb have shape ('layer','vertices') :
            elif special == "hybounds_to_pressure":
                # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

                # set attributes:
                attrs = {}
                attrs["units"] = vunits
                attrs["long_name"] = "half level pressures"

                # original fields:
                odat = {}
                for key in ["sp", "hyab", "hybb"]:
                    # full name:
                    opath = rcf.get("%s.output.var.%s.%s" % (rcbase, varname, key))
                    # extract:
                    odat[key] = sfile.GetData(opath)
                # endfor

                # check ...
                if odat["hyab"].attrs["units"] != vunits:
                    logging.error(
                        'expecting hybride units hya "%s", found "%s"'
                        % (vunits, odat["hyab"].attrs["units"])
                    )
                    raise Exception
                # endif
                if odat["hybb"].attrs["units"] != "1":
                    logging.error(
                        'expecting hybride units hyb "1", found "%s"' % odat["hybb"].attrs["units"]
                    )
                    raise Exception
                # endif
                if odat["sp"].attrs["units"] != vunits:
                    logging.error(
                        'expecting hybride units sp "%s", found "%s"'
                        % (vunits, odat["sp"].attrs["units"])
                    )
                    raise Exception
                # endif

                # check target dims:
                if vdims == ("pixel", "layeri"):

                    # extract per pixel:
                    if odat["sp"].dims in [("lat", "lon")]:
                        # extract pixels:
                        sp = odat["sp"].data[jj, ii]
                    else:
                        logging.error(
                            'could not convert from dims "%s" to target dims "%s"'
                            % (str(oda.dims), str(vdims))
                        )
                        raise Exception
                    # endif

                    # check ...
                    if sfile.nlayer is None:
                        logging.error("no layer dimension defined yet")
                        raise Exception
                    # endif

                    # half level pressures:
                    hp = numpy.zeros((len(pixel), sfile.nlayer + 1), dtype="f4")
                    # surface:
                    iz = 0
                    hp[:, iz] = odat["hyab"].data[iz, 0] + odat["hybb"].data[iz, 0] * sp
                    # loop over layers:
                    for iz in range(1, sfile.nlayer + 1):
                        # top pressure:
                        hp[:, iz] = (
                            odat["hyab"].data[iz - 1, 1] + odat["hybb"].data[iz - 1, 1] * sp
                        )
                    # endfor

                    # create variable:
                    da = xarray.DataArray(
                        hp, dims=("pixel", "layeri"), coords={"pixel": pixel}, attrs=attrs
                    )

                else:
                    logging.error("unsupported target dims: %s" % str(vdims))
                    logging.error("  target variable   : %s" % varname)
                    raise Exception
                # endif

            # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            # count number of troposphere layers from tropospheric kernel
            elif special == "kernel_trop_to_nla":
                # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

                # original field:
                vname = rcf.get("%s.output.var.%s.%s" % (rcbase, varname, "kernel_trop"))
                # read:
                kernel_trop = sfile.GetData(vname)
                # check source dims:
                if kernel_trop.dims not in [("layer", "lat", "lon")]:
                    logging.error("unsupported source dims: %s" % str(kernel_trop.dims))
                    logging.error("  source variable   : %s" % vname)
                    raise Exception
                # endif
                # convert to pixel array:
                avk = kernel_trop.data[:, jj, ii].T  # ('pixel','layer')

                # check target dims:
                if vdims != ("pixel", "retr"):
                    logging.error("unsupported target dims: %s" % str(vdims))
                    logging.error("  target variable   : %s" % varname)
                    raise Exception
                # endif
                # check ...
                if self.nretr != 1:
                    logging.error(
                        "not implemented yet for number of retrieval layers %i" % self % nretr
                    )
                    raise Exception
                # endif

                # 1-based index of layer where kernel is non-zero:
                nla = numpy.zeros((self.npixel, self.nretr), dtype="i2")
                # loop over pixels:
                for ipix in range(self.npixel):
                    # search last index with non-zero kernel value,
                    # convert to 1-based index:
                    nla[ipix, 0] = numpy.where(abs(avk[ipix, :]) > 0.0)[0].max() + 1
                # endfor

                # set attributes:
                attrs = {}
                attrs["units"] = "1"
                attrs["long_name"] = "number of tropospheric layers in profile"

                # create data array with data from selected pixels, copy attributes:
                da = xarray.DataArray(nla, dims=vdims, coords={"pixel": pixel}, attrs=attrs)

            # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            else:
                logging.error('unsupported special "%s" for variable "%s"' % (special, varname))
                raise Exception
            # endif  # special
            # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

            # need to swap layers?
            if swap_layers:
                # loop over dims:
                for dim in vdims:
                    # level dim or not?
                    if dim in ["layer", "layeri", "retr", "retri", "retr0"]:
                        # index:
                        idim = vdims.index(dim)
                        # swap:
                        da.data = numpy.flip(da.data, axis=idim)
                    #
                    elif dim in ["pixel", "corner", "track_scan", "track_pixel"]:
                        pass
                    #
                    else:
                        logging.error('could not decide if "%s" is a level dimension or not' % dim)
                        raise Exception
                    # endif
                # endfor # dims
            # endif # swap layers

            # adhoc attributes ...
            attrs = rcf.get(vkey + ".attrs", totype="dict", default={})
            # loop:
            for key in attrs.keys():
                # remove if None:
                if attrs[key] is None:
                    # remove if present:
                    if key in da.attrs.keys():
                        da.attrs.pop(key)
                else:
                    # update:
                    da.attrs[key] = attrs[key]
                # endif
            # endfor

            # add variable to target data set:
            self.AddVariable(varname, da)

        # endfor # target variables

    # enddef AddSelection


# endclass CSO_S5p_SuperObs_File


########################################################################
###
### convert
###
########################################################################


class CSO_S5p_SuperObs_Convert(utopya.UtopyaRc):

    """
    Convert TROPOMI Super-Observations to CSO format.
    During conversion, a variable and pixel selection could be applied.
    
    Arguments:

    * ``rcfile``, ``rcbase``, ``env``  :  settings file, prefix for keys, and environment dictionairy 
    
    A time range is read to select the files to be converted::
    
      <rcbase>.timerange.start        :  2018-06-01 00:00
      <rcbase>.timerange.end          :  2018-06-03 23:59

    The files are searched in subdirectories including templates for time fields;
    also specify filename patterns to be matched::

        ! input directory:
        <rcbase>.files.dir         :  /Scratch/TROPOMI/NO2/superobs/%Y%m_0.5deg/compressed

        ! filename filters (space seperated list):
        <rcbase>.files.filters     :  s5p_no2_rpro_v1.2.2_superobs_ll_0.5deg_*.nc

    The following form is expected for the filenames::

        s5p_no2_rpro_v1.2.2_superobs_ll_0.5deg_02833.nc

    The name of the target files should be specified with a directory and filename;
    the later could include a template for the orbit number::

        ! output directory and filename:
        ! - times are taken from mid of selection, rounded to hours
        ! - use '%{orbit}' for orbit number
        <rcbase>.output.dir             :  /Scratch/CSO-data/S5p/NO2/superobs/%Y/%m
        <rcbase>.output.filename        :  S5p_NO2_superobs_%{orbit}.nc
    
    A flag is read to decide if existing files should be renewed or kept:
    
      <rcbase>.renew                  :  True              

    If an input file should be converted, it is read into a :py:class:`.S5p_SuperObs_File` object.
    The :py:meth:`SelectPixels <S5p_SuperObs_File.SelectPixels>` method is called to select pixels based 
    on critera defined in the settings. This method returns a history line to desribe the selection.
    
    The target file is created as an :py:class:`.CSO_S5p_SuperObs_File` object.
    It's :py:meth:`AddSelection <.CSO_S5p_SuperObs_File.AddSelection>` method is called with the input object as argument,
    and this will copy the selected pixels for variables specified in the settings.
    The :py:meth:`Write <.CSO_File.Write>` method creates the file.
    
    Global attributes for the target file should be specified with::

        ! global attributes:
        <rcbase>.output.attrs               :  format Conventions author institution email
        !
        <rcbase>.output.attr.format         :  1.0
        <rcbase>.output.attr.Conventions    :  CF-1.7
        <rcbase>.output.attr.author         :  Your Name
        <rcbase>.output.attr.institution    :  CSO
        <rcbase>.output.attr.email          :  Your.Name@cso.org
    
    In addition, the global attribtues from the original file are copied too,
    preceeded by a specified prefix::

        ! copy global attributes? specify filename filters:
        <rcbase>.output.gattrs.filters      :  *
        ! prefix for new attributes:
        <rcbase>.output.gattrs.prefix       :  source__


    """

    def __init__(self, rcfile, rcbase="", env={}, indent=""):

        """
        Convert data.
        """

        # modules:
        import os
        import datetime
        import fnmatch

        # info ...
        logging.info(indent + "")
        logging.info(indent + "** convert TROPOMI Super-Observations")
        logging.info(indent + "")

        # init base object:
        utopya.UtopyaRc.__init__(self, rcfile=rcfile, rcbase=rcbase, env=env)

        # renew output?
        renew = self.GetSetting("renew", totype="bool")

        # time range:
        t1 = self.GetSetting("timerange.start", totype="datetime")
        t2 = self.GetSetting("timerange.end", totype="datetime")

        # info ...
        tfmt = "%Y-%m-%d %H:%M"
        logging.info(indent + "timerange: [%s,%s]" % (t1.strftime(tfmt), t2.strftime(tfmt)))

        # input directory:
        inputdir = self.GetSetting("files.dir")

        # filename filters:
        fnfilters = self.GetSetting("files.filters").split()
        # tarfnfilters = self.GetSetting( 'files.tarfilters', default='' ).split()

        # info ...
        logging.info(indent + "input directory template:")
        logging.info(indent + "  %s" % inputdir)
        logging.info(indent + "collect input files available for timerange ...")
        # init list with input filenames:
        filenames = []
        # times associated with files:
        times = {}
        # content of input directories is listed,
        # name of directory changes in time:
        indir_prev = None
        # loop over time range:
        t = t1
        while t <= t2:
            # info ...
            logging.info(indent + "  %s ..." % t.strftime("%Y-%m-%d"))
            # resolve time templates:
            indir = t.strftime(inputdir)
            # new?
            if indir != indir_prev:
                # present ?
                if os.path.isdir(indir):
                    # list directory:
                    fnames = os.listdir(indir)
                    # sort (in place)
                    fnames.sort()
                    # loop over files:
                    for fname in fnames:
                        # filter:
                        for fnfilter in fnfilters:
                            match = fnmatch.fnmatch(fname, fnfilter)
                            if match:
                                break
                        # endfor
                        if not match:
                            continue
                        # info ...
                        logging.info(indent + "    %s ..." % fname)
                        # store:
                        infile = os.path.join(indir, fname)
                        if infile not in filenames:
                            filenames.append(infile)
                            times[infile] = t
                        # endif
                    # endfor # file
                    # reset:
                    indir_prev = indir
                # endif # present
            # endif # new dir
            # next:
            t = t + datetime.timedelta(1)
        # endwhile

        # info ...
        logging.info(indent + "number of files : %i" % len(filenames))

        # target file:
        output_dir = self.GetSetting("output.dir")
        output_filename_template = self.GetSetting("output.filename")

        # attributes
        attrs = {}
        attrnames = self.GetSetting("output.attrs").split()
        for key in attrnames:
            attrs[key] = self.GetSetting("output.attr.%s" % (key))
        # endfor

        # filters for global attributes to be copied:
        gattrs_filters = self.GetSetting("output.gattrs.filters").split()
        # prefix to be added to copy:
        gattrs_prefix = self.GetSetting("output.gattrs.prefix")

        # info ..
        logging.info(indent + "loop over files ...")
        # loop:
        for filename in filenames:

            # basename:
            fname = os.path.basename(filename)

            # info ...
            logging.info(indent + "  file %s ..." % fname)

            ## testing ...
            # if '11540' not in fname : continue

            # filenames:
            #     s5p_no2_rpro_v1.2.2_superobs_ll_0.5deg_02833.nc
            bname = os.path.basename(filename).replace(".nc", "")
            # split:
            (
                platformname,
                species,
                processingmode,
                processor,
                level,
                grid,
                resol,
                orbit,
            ) = bname.split("_")

            # .. open now already, timerange could not be extracted from filename ...
            # info ...
            logging.info(indent + "    open ...")
            # read file, sometimes files are corrupted ...
            try:
                sfile = S5p_SuperObs_File(filename)
            except:
                logging.warning("could not open file, skip: %s" % filename)
                continue
            # endtry

            # time range:
            tr = sfile.GetTimeRange()
            # use start as reference time for filenames etc:
            t = tr[0]
            # info ...
            logging.info(indent + "    start time of pixels: %s" % t.strftime("%Y-%m-%d %H:%M:%S"))
            # filter ...
            if (t < t1) or (t > t2):
                logging.info(indent + "    outside timerange, skip ..")
                continue
            # endif

            # target dir:
            outdir = t.strftime(output_dir)
            # target file:
            outfile = output_filename_template
            outfile = outfile.replace("%{orbit}", orbit)
            outfile = t.strftime(outfile)
            # combine:
            output_filename = os.path.join(outdir, outfile)

            # default:
            create = False
            # create file?
            if not os.path.isfile(output_filename):
                # info ...
                logging.info("    create new file ...")
                # always create:
                create = True
            # ~ renew?
            elif renew:
                # info ...
                logging.info("    renew file ...")
                # always create:
                create = True
            # endif

            # create?
            if create:

                # apply selections, return bool mask and list of history lines:
                selected, history = sfile.SelectPixels(self.rcf, self.rcbase, indent="    ")
                # count:
                nselected = selected.sum()

                # info ...
                logging.info(
                    indent + "      selected %i of %i pixels" % (nselected, selected.size)
                )
                # any ?
                if nselected > 0:

                    # info ...
                    logging.info(indent + "    create %s ..." % output_filename)
                    # init:
                    csf = CSO_S5p_SuperObs_File()
                    # add:
                    csf.AddSelection(
                        sfile, selected, self.rcf, self.rcbase, indent=indent + "    "
                    )
                    # update history:
                    history.append(
                        "added %i pixels from %s" % (nselected, os.path.basename(fname))
                    )

                    # update attributes:
                    attrs["orbit"] = orbit
                    attrs["orbit_file"] = fname

                    # loop over global attribute filters:
                    for afilter in gattrs_filters:
                        # loop over original attributes:
                        for aname in sfile.attrs.keys():
                            # selected?
                            if fnmatch.fnmatch(aname, afilter):
                                # copy:
                                attrs[gattrs_prefix + aname] = sfile.attrs[aname]
                            # endif
                        # endfor # attributes
                    # endfor # gattr filters

                    # write:
                    csf.Write(filename=output_filename, attrs=attrs, history=history)

                # endif  # any selected

            else:
                # info ..
                logging.info(indent + "    keep   %s ..." % output_filename)
            # endif

            ## testing ...
            # break

        # endfor  # input files

        # info ...
        logging.info(indent + "")
        logging.info(indent + "** end convert")
        logging.info(indent + "")

    # enddef __init__


# endclass CSO_S5p_SuperObs_Convert


########################################################################
###
### create listing file
###
########################################################################


class CSO_S5p_SuperObs_Listing(utopya.UtopyaRc):

    """
    Create *listing* file for converted orbit files.
    
    A *listing* file contains the names of the converted orbit files,
    and the time range of pixels in the file::

        filename                     ;start_time                   ;end_time                     ;orbit
        2018/06/S5p_RPRO_NO2_03272.nc;2018-06-01T01:32:46.673000000;2018-06-01T01:36:12.948000000;03272
        2018/06/S5p_RPRO_NO2_03273.nc;2018-06-01T03:12:53.649000000;2018-06-01T03:17:43.082000000;03273
        2018/06/S5p_RPRO_NO2_03274.nc;2018-06-01T04:52:43.586000000;2018-06-01T04:59:12.377000000;03274
        :

    This file will be used by the observation operator to selects orbits with pixels valid for 
    a desired time range.
    
    In the settings, define the name of the file to be created::

        ! csv file that will hold records per file with:
        ! - timerange of pixels in file
        ! - orbit number
        <rcbase>.file        :  /Scratch/CSO/S5p/RPRO/NO2/Europe/listing.csv
        
    An existing listing file is not replaced,
    unless the following flag is set::

        ! renew table?
        <rcbase>.renew           :  True
        
    Orbit files are searched within a timerange::
    
        <rcbase>.timerange.start        :  2018-06-01 00:00
        <rcbase>.timerange.end          :  2018-06-03 23:59

    Specify filename filters to search for orbit files;
    the patterns are relative to the basedir of the listing file,
    and might contain templates for the time values.
    Multiple patterns could be defined; if for a certain orbit number more than one
    file is found, the first match is used.
    This could be explored to create a listing that combines reprocessed data
    with near-real-time data::

        <rcbase>.patterns            :  RPRO/NO2/Europe/%Y/%m/S5p_*.nc \
                                        OFFL/NO2/Europe/%Y/%m/S5p_*.nc

    Usually the time range is read from the file, but in case the file does
    not have a time accordinate, then the following flag might be used to
    force that the time that matches with the filename is used::
    
        ! adhoc: use the time for which file is valid as timerange;
        ! this is used for the synthetic S4 data that have no time record ...
        <rcbase>.use_t              :  True
    
    Alternatively, the time range could be read from the global attributes if 
    it is specified there::

        ! adhoc: use attributes to extract timerange from?
        <listing>.use_attrs           :  True
        ! if True, define names:
        <listing>.attr_tr1            :  source__time_coverage_start
        <listing>.attr_tr2            :  source__time_coverage_end

    The ``orbit`` column in the listing is extra, and is read from global attributes;
    the list of extra columns is defined with::
    
        ! extra columns to be added, read from global attributes:
        <rcbase>.xcolumns           :  orbit

    """

    def __init__(self, rcfile, rcbase="", env={}, indent=""):

        """
        Convert data.
        """

        # modules:
        import os
        import datetime
        import glob

        # tools:
        import cso_file

        # info ...
        logging.info(indent + "")
        logging.info(indent + "** create listing file")
        logging.info(indent + "")

        # init base object:
        utopya.UtopyaRc.__init__(self, rcfile=rcfile, rcbase=rcbase, env=env)

        # renew output?
        renew = self.GetSetting("renew", totype="bool")

        # table file to be written:
        lst_file = self.GetSetting("file")
        # create?
        if (not os.path.isfile(lst_file)) or renew:
            # info ..
            logging.info(indent + "create %s ..." % lst_file)

            # time range:
            t1 = self.GetSetting("timerange.start", totype="datetime")
            t2 = self.GetSetting("timerange.end", totype="datetime")
            # info ...
            tfmt = "%Y-%m-%d %H:%M"
            logging.info(indent + "  timerange: [%s,%s]" % (t1.strftime(tfmt), t2.strftime(tfmt)))

            # base directory:
            bdir = os.path.dirname(lst_file)
            # create?
            if len(bdir) > 0:
                if not os.path.isdir(bdir):
                    os.makedirs(bdir)
            # endif
            # current directory?
            if len(bdir) == 0:
                bdir = "."
            # info ...
            logging.info(indent + "  base directory: %s ..." % bdir)

            # initiallize for (re)creation:
            listing = cso_file.CSO_Listing(lst_file, indent=indent + "    ")

            # info ...
            logging.info(indent + "  cleanup records if necessary ...")
            # remove entries that do not exist anymore:
            listing.Cleanup(indent=indent + "    ")

            # filename pattern templates:
            pattern_templates = self.GetSetting("patterns").split()
            # collection of scanned patterns:
            patterns = []

            # adhoc: use time of file as time range?
            use_t = self.GetSetting("use_t", totype="bool")

            # adhoc: use attributes to extract time?
            use_attrs = self.GetSetting("use_attrs", totype="bool")
            # enabled?
            if use_attrs:
                # read names:
                attr_tr1 = self.GetSetting("attr_tr1")
                attr_tr2 = self.GetSetting("attr_tr2")
            # endif

            # extra columns to be added:
            xcolumns = self.GetSetting("xcolumns").split()

            # loop over days:
            t = t1
            while t <= t2:

                # loop over patterns:
                for pattern_template in pattern_templates:
                    # expand time values:
                    pattern = t.strftime(pattern_template)
                    # skip if already scanned ...
                    if pattern in patterns:
                        continue
                    # store:
                    patterns.append(pattern)
                    # info ...
                    logging.info(indent + "scan %s ..." % pattern)

                    # list relative to basedir:
                    cwd = os.getcwd()
                    os.chdir(bdir)
                    fnames = glob.glob(pattern)
                    os.chdir(cwd)
                    # empty ?
                    if len(fnames) == 0:
                        logging.info(indent + "  empty ..")
                        continue
                    # endif
                    # sort in place:
                    fnames.sort()
                    # loop over files:
                    for fname in fnames:
                        # absolute path:
                        filename = os.path.join(bdir, fname)
                        # already in table?
                        if fname in listing:
                            # info ...
                            logging.info(indent + "  keep entry %s ..." % fname)
                        else:
                            # open orbit file:
                            csf = cso_file.CSO_File(filename=filename)
                            # get orbit from attribute:
                            orbit = csf.GetAttr("orbit", quiet=True)
                            # already present?
                            if ("orbit" in listing.df.keys()) and (
                                orbit in listing.df["orbit"].values
                            ):
                                # info ...
                                logging.info(
                                    indent
                                    + '  skip file  %s (existing orbit "%s") ...' % (fname, orbit)
                                )
                            else:
                                # info ...
                                logging.info(indent + "  add entry %s ..." % fname)
                                # timerange argument:
                                if use_t:
                                    # use current time instead of reading from file:
                                    tr = (t, t)
                                    #
                                elif use_attrs:
                                    # check ...
                                    for aname in [attr_tr1, attr_tr2]:
                                        if aname not in csf.ds.attrs.keys():
                                            logging.error(
                                                'attribute "%s" not found in: %s'
                                                % (aname, filename)
                                            )
                                            raise Exception
                                        # endif
                                    # endfor
                                    # read time range from attributes:
                                    tfmt = "%Y-%m-%dT%H:%M:%SZ"
                                    tr = (
                                        datetime.datetime.strptime(csf.ds.attrs[attr_tr1], tfmt),
                                        datetime.datetime.strptime(csf.ds.attrs[attr_tr2], tfmt),
                                    )
                                    #
                                else:
                                    # time range will be extracted from 'time' coordinate ..
                                    tr = None
                                # endif
                                # add record to listing for this file, columns are:
                                #  - start_time, end_time
                                #  - extra columns specified by "xcolumns"
                                listing.Update(
                                    fname, csf, tr=tr, xcolumns=xcolumns, indent=indent + "    "
                                )
                            # endif

                        # endif
                    # endfor # filenames

                # endfor # patterns

                ## testing ...
                # break

                # next
                t = t + datetime.timedelta(0, 3600)
            # endwhile

            # save:
            listing.Close(indent=indent + "  ")

        else:
            # info ..
            logging.info(indent + "keep %s ..." % lst_file)
        # endif

        # info ...
        logging.info(indent + "")
        logging.info(indent + "** end listing")
        logging.info(indent + "")

    # enddef __init__


# endclass CSO_S5p_SuperObs_Listing


########################################################################
###
### end
###
########################################################################
