#
# Changes
#
# 2022-09, Arjo Segers
#    Use NaN as fill value for gridded variables to support packed variables on output.
#
# 2022-09, Arjo Segers
#    Added class to create temporal means of gridded averages.
#
# 2022-10, Arjo Segers
#    Use pixel centers as mapping points when mapping level is zero,
#    this solves for the moment a problem with pixels crossing the date line.
#
# 2022-10, Arjo Segers
#    Support packing of output variables.
#
# 2023-08, Arjo Segers
#    Added option to search for input files based on listing file.
#


########################################################################
###
### help
###
########################################################################

"""
.. _cso-gridded:

**********************
``cso_gridded`` module
**********************

The :py:mod:`cso_gridded` module provides classes to average
pixels over a regular grid.


Class hierchy
=============

The classes and are defined according to the following hierchy:


* :py:class:`.UtopyaRc`

  * :py:class:`.CSO_GriddedAverage`
  * :py:class:`.CSO_GriddedAverageMeans`



Classes
=======


"""


########################################################################
###
### modules
###
########################################################################

# modules:
import logging

# tools:
import utopya


########################################################################
###
### gridded average
###
########################################################################

class CSO_GriddedAverage( utopya.UtopyaRc ) :

    """
    Average pixels over regular grid using rcfile settings.

    The value assigned to a cell is a weighted average of the pixel values,
    where the weight is relative to the area of the overlapping part.
    Since a cell might be only partly covered by pixels, the normalization is done
    using the total overlapping area:
    
    .. math::
        x(i_k,j_k) ~=~ \left(\ \sum\limits_{p\in P_k} y_p\ w_{p,k}\ \\right)\ /\ \sum\limits_{p\in P_k}\ w_{p,k}
        
    where:
    
    * :math:`i_k,j_k` are the indices of grid cell :math:`k`;
    * :math:`P_k` is the set of pixels that overlap with cell :math:`k`;
    * :math:`y_p` is the data value of pixel :math:`p`;
    * :math:`w_p` is the footprint area [m\ :sup:`2`] of pixel :math:`p`
    * :math:`w_{p,k}` is the area [m\ :sup:`2`] of pixel :math:`p` that overlaps with the cell :math:`k`.
    
    The overlapping area is computed using the :py:meth:`LonLatPolygonCentroids <cso_mapping.LonLatPolygonCentroids>` method.
    This fractions a footprint area into a large number of triangles, and returns
    for each triangle the centroid and the area.
    The centroids are collected per grid cell, and sum over the associated triangle area's is
    then used as measure for the overlapping area.
    
    The target grid is regular longitude/latitude and defined using the following settings::
        
        ! domain definition:
        <rcbase>.west              :  40.0
        <rcbase>.east              :  60.0
        <rcbase>.south             :  20.0
        <rcbase>.north             :  35.0
        ! grid resolution:
        <rcbase>.dlon              :  0.1
        <rcbase>.dlat              :  0.1

    The level recursive splitting of footprint into triangles and assignment of centroids
    is defined with::

        ! for 4-corner footprints, number of centroids is:
        !  1 (levels=0), 4 (1), 8 (2), 16 (3), 64 (5), 256 (7)
        <rcbase>.mapping.levels         :  7

    A gridded average is created for each orbit within a time range.
    Specify the time range in the settings using::
    
      <rcbase>.timerange.start        :  2018-06-01 00:00
      <rcbase>.timerange.end          :  2018-06-03 23:59
      
    Also specify a time step freqency that is used within the range;
    for each time step an output file is created, 
    based on orbits files that are collected for that time step::
    
      ! step is one of: hour | day | month
      <rcbase>.timerange.step   :  hour

    Input data could originate from different sources, for example a *data* file with footprints
    and retrieval, and a *state* file with simulations by a model.
    Specify a list with keywords describing the sources; the first source should be a file
    that contains the footprints::
    
        ! keywords for source files:
        <rcbase>.sources                :  data

    For each source, specify the input file(s) that should be used to create an output file.    
    There are two options supported:
    
    * Specify the name of a listing file::
    
        ! listing file with input files:
        <rcbase>.source.data.filenames  :  /scratch/CSO-data/S5p/listing-NO2-CAMS.csv
        
      At each time step, the orbits are selected that have their 'middle' time 
      (halfway the ``start_time`` and ``end_time`` values of a record) within (depending on the time step):
      
      * the past hour (ouptut file will have time step of end of this interval)
      * current day
      * current month
    
    * Specify a filename pattern for each source::

        ! filename pattern for source "data":
        <rcbase>.source.data.filenames  :  /scratch/CSO-data/S5p/RPRO/NO2/CAMS/%Y/%m/S5p_RPRO_NO2_%Y%m%d_%H%M.nc
        
      This is typically used when the files have time stamps in their names.

    The input could be filtered by applying selections, for example on quality value;
    see :py:meth:`CSO_File.SelectPixels` method for supported filter types::

        ! keywords for filters:
        <rcbase>.filters                :  quality

        ! minimum quality value required:
        <rcbase>.filter.quality.var       :  qa_value
        <rcbase>.filter.quality.type      :  min
        <rcbase>.filter.quality.min       :  0.8
        <rcbase>.filter.quality.units     :  1

    Specify the name of the target file; could be useful to include a description of the target grid and the applied filters in the path::

        ! target file, might contain templates:
        !   %Y,%m,etc     : time values
        <rcbase>.output.file            :   /scratch/CSO-data-gridded/S5p/RPRO/NO2/CAMS_r01x01_qa08/%Y/%m/S5p_RPRO_NO2_%Y%m%d_%H%M_gridded.nc

    To reduce file size, by default all variables are written to file as short-integers (2 bytes)
    accompanied by ``add_offset`` and ``scale_factor`` attributes.
    A flag is available to disable packing.
    In addition, ``zlib`` compression is enabled.
    The default compression level is 1 (out of 9); set the following flag to a higher level
    to have stronger compression (on expense of computation time), or set to 0 to disable compression::
    
        ! pack floats as shorts:
        <rcbase>.output.packed              :  True
        ! zlib compression level (default 1, 0 for no compression):
        <rcbase>.output.complevel           :  1

    Existing files are replaced if the following flag is set::
    
        ! renew existing files?
        <rcbase>.renew                  :  True

    Specify a list of the variables to be created, and for each of them the source type and source variable::

        ! data variables to be created:
        <rcbase>.output.vars            :  xvmr

        ! input variables:
        !   data:vcd    : from data file
        !   state:hx    : from state file
        <rcbase>.output.xvmr.source     :  data:xvmr

    """
    
    def __init__( self, rcfile, rcbase='', env={}, indent='' ) :
    
        """
        Create catalogue of CSO figures.
        """
                        
        # modules:
        import os
        import numpy
        import xarray
        import datetime
        import glob
        import pandas
        
        # tools:
        import cso_file
        import cso_mapping

        # init base object:
        utopya.UtopyaRc.__init__( self, rcfile, rcbase=rcbase, env=env )
        
        # info ...
        logging.info( indent+'' )
        logging.info( indent+'** Gridded CSO data' )
        logging.info( indent+'' )

        # time range:
        t1 = self.GetSetting("timerange.start", totype="datetime")
        t2 = self.GetSetting("timerange.end", totype="datetime")
        tstep = self.GetSetting("timerange.step")
        # info ...
        tfmt = "%Y-%m-%d %H:%M"
        logging.info(
            indent + "timerange: [%s,%s] by %s" % (t1.strftime(tfmt), t2.strftime(tfmt), tstep)
        )

        # renew?
        renew = self.GetSetting( 'renew', totype='bool' )
        
        # time units to be used:
        tunits = 'seconds since %s' % t1.strftime('%Y-01-01 00:00:00')
        # variable data type:
        dtype = 'f4'

        # regular grid:
        #~ domain:
        west  = self.GetSetting( 'grid.west' , 'float' )
        east  = self.GetSetting( 'grid.east' , 'float' )
        south = self.GetSetting( 'grid.south', 'float' )
        north = self.GetSetting( 'grid.north', 'float' )
        #~ resolution:
        dlon = self.GetSetting( 'grid.dlon', 'float' )
        dlat = self.GetSetting( 'grid.dlat', 'float' )
        #~ shape:
        nlon = int(numpy.round((east -west )/dlon))
        nlat = int(numpy.round((north-south)/dlat))
        # info ...
        logging.info( indent+'define regular grid ...' )
        logging.info( indent+'  domain     :  [%7.2f,%7.2f] x [%7.2f,%7.2f]' % (west,east,south,north) )
        logging.info( indent+'  resolution :  %5.2f x %5.2f' % (dlon,dlat) )
        logging.info( indent+'  resolution :  %5i x %5i' % (nlon,nlat) )
        
        # centers:
        da_lons = xarray.DataArray( west+(numpy.arange(nlon)+0.5)*dlon, dims=('longitude',),
                                     attrs={ 'units' : 'degrees_east', 'standard_name' : 'longitude' } )
        # centers:
        da_lats = xarray.DataArray( south+(numpy.arange(nlat)+0.5)*dlat, dims=('latitude',),
                                     attrs={ 'units' : 'degrees_north', 'standard_name' : 'latitude' } )
        # store:
        grid_coords = { 'longitude' : da_lons, 'latitude' : da_lats }
        
        # info ...
        logging.info( indent+'compute cell_area ...' )
        # corners of first column:
        #    3 o---o 2
        #      |   |
        #    0 o---o 1
        xxc = numpy.zeros((nlat,4),float)
        xxc[:,0] =  west
        xxc[:,1] =  west + dlon
        xxc[:,2] =  west + dlon
        xxc[:,3] =  west
        yyc = numpy.zeros((nlat,4),float)
        yyc[:,0] = south + numpy.arange(0,nlat  )*dlon
        yyc[:,1] = south + numpy.arange(0,nlat  )*dlon
        yyc[:,2] = south + numpy.arange(1,nlat+1)*dlon
        yyc[:,3] = south + numpy.arange(1,nlat+1)*dlon
        # areas:
        aa = cso_mapping.LonLatRectanglesArea( xxc, yyc )
        # fill 2D area map:
        cell_area = numpy.zeros((nlat,nlon),float)
        for i in range(nlon) :
            cell_area[:,i] = aa
        #enfor
        # define grid:
        da_cell_area = xarray.DataArray( numpy.array(cell_area,dtype=dtype),
                                          dims=('latitude','longitude'),
                                          coords=grid_coords,
                                          attrs={ 'long_name' : 'area of grid cell', 'standard_name' : 'area', 
                                                  'units' : 'm2' } )
        # target variable, also used for attributes:
        varname_cell_area = 'cell_area'
        
        # source types:
        stypes = self.GetSetting("sources").split()
        # first source type, this should have the footprints:
        stype0 = stypes[0]

        # storage per source:
        sources = {}
        # loop:
        for stype in stypes :
            # tempate for input file:
            sources[stype] = {}
            sources[stype]["listing"] = self.GetSetting("source.%s.listing" % stype, default=None)
            sources[stype]["filenames"] = self.GetSetting( "source.%s.filenames" % stype, 
                                                             default=None
            )
            # check ..
            if (sources[stype]["listing"] is None) == (sources[stype]["filenames"] is None):
                logging.error(
                    "exactly one of the following input specifications should be defined:"
                )
                logging.error("  %s.source.%s.listing" % (rcbase, stype))
                logging.error("  %s.source.%s.filenames" % (rcbase, stype))
                raise Exception
            # endif
        # endfor

        # listing file specified?
        if sources[stype0]["listing"] is not None:
            # then only single source type allowed ...
            if len(stypes) > 1:
                logging.error(
                    "if input files are specified in a listing file, only  one data source is allowed yet ..."
                )
                raise Exception
            # endif
        # endif

        # number of regridding points:
        mapping_levels = self.GetSetting( 'mapping.levels', 'int' )

        # target file:
        outfile_template = self.GetSetting( 'output.file' )
        # pack floats as shorts?
        packed = self.GetSetting( 'output.packed', 'bool' )
        # compression level:
        complevel = self.GetSetting( 'output.complevel', 'int' )

        # variables:
        vkeys = self.GetSetting( 'output.vars' ).split()
        # storage for input description:
        varsources = {}
        for vkey in vkeys :
            # source description:
            varsources[vkey] = self.GetSetting( 'output.%s.source' % (vkey) )
        #endfor
        
        # collection of processed input file:
        processed_files = []
        # latest listing files:
        listing_latest = None
        # latest input filter:
        infile_latest = None
        # no time step yet:
        dt = None
        
        # info ..
        logging.info( indent+'time loop ...' )
        # time loop:
        t = t1
        while t < t2:
            # info ..
            logging.info(indent + "  %s ..." % t.strftime("%Y-%m-%d %H:%M"))

            # not first?
            if dt is not None:
                # next time step:
                t = t + dt
            #endif

            # timestep:
            if tstep in ["hour", "hourly"]:
                # start of current hour:
                t0 = datetime.datetime(t.year, t.month, t.day, t.hour, 0)
                # if same, gather for previous hour:
                if t == t0:
                    # step:
                    dt = datetime.timedelta(0, 3600)
                    # new start:
                    t0 = t0 - dt
                    # timerange to search orbits:
                    tr = (t0, t0 + dt)
                else:
                    # end time:
                    tend = min(t0 + datetime.timedelta(0, 3600), t2)
                    # time step to end time:
                    dt = tend - t
                    # timerange to search orbits:
                    tr = (t0, tend)
                # endif
                # annotation:
                long_timestamp = t.strftime("%Y-%m-%d %H:%M")
            #
            elif tstep in ["day", "daily"]:
                # start of day:
                t0 = datetime.datetime(t.year, t.month, t.day, 0, 0)
                # end time:
                tend = min(t0 + datetime.timedelta(1), t2)
                # timerange to search orbits:
                tr = (t0, tend)
                # time step to end time:
                dt = tend - t
                # annotation:
                long_timestamp = t.strftime("%Y-%m-%d")
            #
            elif tstep in ["month", "monthly"]:
                # start of month:
                t0 = datetime.datetime(t.year, t.month, 1, 0, 0)
                # number of days in month:
                weekday, nday = calendar.monthrange(t.year, t.month)
                # end time:
                tend = min(t0 + datetime.timedelta(nday), t2)
                # timerange to search orbits:
                tr = (t0, tend)
                # time step to end time:
                dt = tend - t
                # annotation:
                long_timestamp = t0.strftime("%Y-%m")
            #
            else:
                logging.error('unsupported time step "%s"' % tstep)
                raise Exception
            # endif

            # target file:
            outfile = t.strftime(outfile_template)
            # renew?
            if (not os.path.isfile(outfile)) or renew:
                # info ...
                logging.info(indent + "    create %s ..." % outfile)

                # init storage for output arrays:
                das_out = {}
                # init history:
                history = []

                # listing file specified?
                if sources[stype0]["listing"] is not None:

                    # current listing file, path might have time templates:
                    listing_curr = t.strftime(sources[stype0]["listing"])
                    # new file?
                    if listing_curr != listing_latest:
                        # read listing:
                        listing = cso_file.CSO_Listing(listing_curr, indent=indent + "    ")
                        # store name:
                        listing_latest = listing_curr
                    # endif

                    # obtain orbits with middle time within (t,t+dt]:
                    xdf = listing.Select(tr=tr, method="middle")
                    # empty? then leave:
                    if len(xdf) == 0:
                        # info ..
                        logging.info(indent + "    no file(s) found for this time ...")
                        # next time step:
                        continue
                    # endif

                    # extract:
                    datafiles = []
                    for fname in xdf.index.values:
                        datafiles.append(os.path.join(listing.dirname, fname))
                    # endfor
                    # info ..
                    logging.info(indent + "    found %i file(s) in listing .." % (len(datafiles)))

                else:  # filename pattern specified

                    # filename pattern for current time:
                    infile_curr = t.strftime(sources[stype0]["filenames"])
                    # same as before?
                    if infile_curr == infile_latest:
                        # next:
                        continue
                    else:
                        # store:
                        infile_latest = infile_curr
                    # endif

                    # data files including footprints:
                    datafiles = glob.glob(infile_curr)
                    # sorted:
                    datafiles.sort()

                    # info ..
                    logging.info(
                        indent
                        + "    found %i file(s) matching: %s" % (len(datafiles), infile_curr)
                    )

                # endif # listing or filenames

                # loop over data files:
                for datafile in datafiles:

                    # info ...
                    logging.info(indent + "    read input file: %s" % datafile)

                    # check ..
                    if datafile in processed_files:
                        logging.error("input file already processed: %s" % datafile)
                        logging.error("something wrong with input file selection for a timestep?")
                        raise Exception
                    # endif
                    # store filename:
                    processed_files.append(datafile)

                    # loop:
                    for stype in stypes:
                        # first?
                        if stype == stype0:
                            infile = datafile
                        else:
                            # this case only occures when a "filenames" pattern is specified ..
                            infile = t.strftime(sources[stype]["filenames"])
                        # endif
                        # read:
                        sources[stype]["file"] = cso_file.CSO_File(filename=infile)
                        # update history:
                        history.insert(0, "pixel data from %s" % os.path.basename(infile))
                    # endfor

                    # filter:
                    selected, history = sources[stype0]["file"].SelectPixels(
                        self.rcf, rcbase, indent=indent + "    "
                    )
                    # info ..
                    logging.info(
                        indent + "    selected %i / %i pixels .." % (selected.sum(), selected.size)
                    )
                    # selected indices:
                    (iipix,) = numpy.where(selected)

                    # info ...
                    logging.info(indent + "    distribute points over footprints ...")
                    # center points or footprings?
                    if mapping_levels == 0:
                        # centers, just copy:
                        xxp = sources[stype0]["file"].ds["longitude"].values[iipix]
                        yyp = sources[stype0]["file"].ds["latitude"].values[iipix]
                        # unit weights:
                        wwp = numpy.ones(len(xxp))
                    else:
                        # distribute points within footprint by deviding it recursively in triangles;
                        # get points and weights:
                        xxp, yyp, wwp = cso_mapping.LonLatPolygonCentroids(
                            sources[stype0]["file"].ds["longitude_bounds"].values[iipix, :],
                            sources[stype0]["file"].ds["latitude_bounds"].values[iipix, :],
                            maxlevel=mapping_levels,
                        )
                    # endif

                    # convert coordinates to cell indices:
                    ii = numpy.array(numpy.floor((xxp - west) / dlon), int)
                    jj = numpy.array(numpy.floor((yyp - south) / dlat), int)

                    # store in dataframe:
                    df = pandas.DataFrame()
                    df["ii"] = ii.flatten()
                    df["jj"] = jj.flatten()
                    df["ww"] = wwp.flatten()

                    # storage for input arrays
                    das_in = {}
                    # loop over variables to be averaged:
                    for vkey in vkeys:

                        # source format:  state:y
                        source = varsources[vkey]
                        stype, sname = source.split(":")
                        # check ..
                        if stype not in stypes:
                            logging.error('source type "%s" not defined' % stype)
                            raise Exception
                        # endif
                        # check ...
                        if sname not in sources[stype]["file"].ds.keys():
                            logging.error(
                                'data variable "%s" not found in: %s' % (sname, datafile)
                            )
                            raise Exception
                        # endif
                        # link input data array:
                        das_in[vkey] = sources[stype]["file"].ds[sname]

                        # extract dims:
                        vdims = das_in[vkey].dims
                        vshp = das_in[vkey].shape
                        # check ..
                        if vdims[0] != "pixel":
                            logging.error(
                                'expected first dim "pixel", found dims "%s" for variable "%s" in : %s'
                                % (str(vdims), sname, statefile)
                            )
                            raise Exception
                        # endif

                        ## testing ..
                        # print( 'var ', vkey, ';', das_in[vkey].values[ipix0,:] )

                        # number of selected pixels, and number of data values per pixels:
                        npix = len(iipix)
                        nv = int(numpy.product(vshp[1:]))
                        # selection, and reshape to have row of values per pixel;
                        # some arrays (pixel area from "data" file) do not have a "profile" dimension ..
                        if len(vshp) == 1:
                            vshp = (vshp[0], 1)
                            values = das_in[vkey].values[iipix].reshape((npix, nv))
                        else:
                            values = das_in[vkey].values[iipix, :].reshape((npix, nv))
                        # endif
                        # loop over data values:
                        for iv in range(nv):
                            # column name:
                            ckey = "%s_%i" % (vkey, iv)
                            # store weights*values in data frame for summation;
                            # note that '*' gives an element-wise product of wwp (np,npix) and the values (npix):
                            df[ckey] = (wwp * values[:, iv]).flatten()
                        # endfor

                    # endfor # output variables

                    # info ...
                    logging.info(indent + "    average %i pixels over grid cells ..." % len(iipix))

                    # group per target grid cell, sum over weights:
                    df2 = df.groupby(["jj", "ii"]).sum()
                    # extract target indices within grid:
                    ii2 = df2.index.get_level_values("ii").values
                    jj2 = df2.index.get_level_values("jj").values

                    # target array for total pixel weight per grid cell:
                    pat_ww = numpy.zeros((nlat, nlon), float)
                    # within grid:
                    (kk,) = numpy.where(
                        (ii2 >= 0) & (ii2 <= nlon - 1) & (jj2 >= 0) & (jj2 <= nlat - 1)
                    )
                    # info ...
                    logging.info(
                        indent
                        + "      assigned to %i target cells (%i within domain)"
                        % (len(ii2), len(kk))
                    )
                    # any?
                    if len(kk) > 0:
                        # fill:
                        pat_ww[jj2[kk], ii2[kk]] = df2["ww"].values[kk]
                    # endif
                    # non-zero cells:
                    jj0, ii0 = numpy.where(pat_ww > 0.0)
                    # define data array for output:
                    da_ww = xarray.DataArray(
                        pat_ww,
                        dims=("latitude", "longitude"),
                        coords=grid_coords,
                        attrs={"long_name": "area covered by pixels", "units": "m2"},
                    )

                    # loop over output variables:
                    for vkey in vkeys:

                        # extract shape:
                        vdims = das_in[vkey].dims
                        vshp = das_in[vkey].shape
                        # number of data values per pixels:
                        nv = int(numpy.product(vshp[1:]))

                        # storage for gridded field:
                        field = numpy.full((nlat, nlon, nv), fill_value=numpy.nan)
                        # any cells?
                        if len(kk) > 0:
                            # loop over data values:
                            for iv in range(nv):
                                # column name:
                                ckey = "%s_%i" % (vkey, iv)
                                # copy sum per grid cells from dataframe into field:
                                field[jj2[kk], ii2[kk], iv] = df2[ckey].values[kk]
                                # normalize with total weight (non-zero weights only):
                                field[jj0, ii0, iv] /= pat_ww[jj0, ii0]
                            # endfor
                        # endif

                        # new dims:
                        dims = ("time",) + da_cell_area.dims + vdims[1:]
                        # coordinates:
                        coords = da_cell_area.coords
                        # set attributes:
                        attrs = da_cell_area.attrs.copy()
                        attrs.update(das_in[vkey].attrs)
                        attrs["long_name"] = (
                            "grid cell averaged %s" % das_in[vkey].attrs["long_name"]
                        )
                        attrs["units"] = das_in[vkey].attrs["units"]
                        attrs["cell_measures"] = "area: %s" % varname_cell_area
                        # init target array?
                        if vkey not in das_out.keys():
                            # new:
                            das_out[vkey] = xarray.DataArray(
                                field.reshape( (1,nlat,nlon)+vshp[1:] ),
                                dims=dims,
                                coords=coords,
                                attrs=attrs,
                            )
                        else:
                            # combine contributions;
                            # use nansum to interpreted nan as zero,
                            # this requires that the arrays have a new dimension (of length 2):
                            das_out[vkey].values = numpy.nansum(
                                numpy.stack( (das_out[vkey].values, 
                                              field.reshape( (1,nlat,nlon)+vshp[1:] ))),
                                axis=0 )
                        # endif # output array not defined yet

                    # endfor # vkeys

                    # update history list:
                    history.append("averaged %i pixels over grid cells" % len(iipix))

                    ## testing ..
                    # if nfile > 0 :
                    #    logging.warning( 'break after first file' )
                    #    break
                    ##endif

                # endfor # data files

                # init output file:
                outf = cso_file.CSO_File()

                ## testing ...
                # outf.ds['xxp'] = xarray.DataArray( xxp, dims=('pixel','np') )
                # outf.ds['yyp'] = xarray.DataArray( yyp, dims=('pixel','np') )
                # outf.ds['wwp'] = xarray.DataArray( wwp, dims=('pixel','np') )

                # store grid info:
                outf.ds[varname_cell_area] = da_cell_area
                # total weight:
                outf.ds["pixel_area"] = da_ww

                # add time coordinate:
                outf.ds.coords["time"] = xarray.DataArray(
                    [t], dims=("time"), attrs={"standard_name": "time"}
                )
                outf.ds.coords["time"].encoding["units"] = tunits

                # store gridded variables:
                for var in das_out.keys():
                    outf.ds[var] = das_out[var]
                # endfor

                # write:
                outf.Write(
                    outfile,
                    attrs=sources[stype0]["file"].ds.attrs,
                    history=history,
                    unlimited_dims=["time"],
                    packed=packed,
                    complevel=complevel,
                )

            else:
                # info ..
                logging.info("    keep  %s ..." % outfile)
            # endif # renew

            ## testing ..
            # if nfile > 0 :
            #    logging.warning( 'break after first day' )
            #    break
            ##endif

        # endfor # time loop

        # info ...
        logging.info(indent + "")
        logging.info(indent + "processed %i files" % len(processed_files))
        logging.info(indent + "")
        logging.info(indent + "** end")
        logging.info(indent + "")

    # enddef __init__
    
# endclass CSO_GriddedAverage





########################################################################
###
### temporal means over gridded averages
###
########################################################################

class CSO_GriddedAverageMeans( utopya.UtopyaRc ) :

    """
    Create files with temporal means over gridded averages.
    
    The gridded averages are (probably) created using :py:class:`CSO_GriddedAverage` class,
    and stored in files::
    
         2020/06/CSO_gridded_20200601_1100_S5p-no2.nc
                 CSO_gridded_20200601_1300_S5p-no2.nc
                 :

    Temporal means are computed per grid cell over time series,
    weighted using the ``pixel_area`` value that hold the area of a grid cell covered by
    the original pixel footprints.
    
    In the settings, first specify a time range over which temporal means should be computed::
    
      <rcbase>.timerange.start        :  2020-01-01 00:00
      <rcbase>.timerange.end          :  2020-12-31 23:59
      
    Temporal means can be computed over different resolutions.
    Specify a list of resolutions to be used::
    
        ! time resolutions: daily, monthly, yearly
        <rcbase>.resolutions                     :  daily monthly yearly
        
    For each of these resolutions, specify a filename pattern for the input files,
    and a template for the output file.
    Both should have templates to evaluate the target time a temporal mean,
    for example ``%%Y%%m%%d`` for a specific day.
    The input pattern should use ``*`` or ``?`` patterns to denote the higher time resolutions.
    For example for the ``daily`` resolution, specify an input pattern for all hours in a day,
    and an output pattern for a daily file::

        ! daily means from hourly files:
        <rcbase>.resolution.daily.input.file     :  /work/gridded/europe/%Y/%m/CSO_gridded_%Y%m%d_????_S5p-glyox.nc
        <rcbase>.resolution.daily.output.file    :  /work/gridded/europe/daily/CSO_gridded_%Y%m%d_S5p-glyox.nc

    Similar specification should be provided for ``monthly`` and ``yearly`` resolutions.
    To speedup calculations, the ``monthly`` means could use the ``daily`` means as input::
    
        ! monthly means from daily files:
        <rcbase>.resolution.monthly.input.file   :  /work/gridded/europe/daily/CSO_gridded_%Y%m??_S5p-glyox.nc
        <rcbase>.resolution.monthly.output.file  :  /work/gridded/europe/monthly/CSO_gridded_%Y%m_S5p-glyox.nc

     and similar the ``yearly`` means could use the ``monthly`` means as input::

        ! yearly means from monthly files
        <rcbase>.resolution.yearly.input.file    :  /work/gridded/europe/monthly/CSO_gridded_%Y??_S5p-glyox.nc
        <rcbase>.resolution.yearly.output.file   :  /work/gridded/europe/yearly/CSO_gridded_%Y_S5p-glyox.nc

    To reduce file size, by default all variables are written to file as short-integers (2 bytes) 
    accompanied by ``add_offset`` and ``scale_factor`` attributes. 
    A flag is available to disable packing.
    In addition, ``zlib`` compression is enabled.
    The default compression level is 1 (out of 9); set the following flag to a higher level
    to have stronger compression (on expense of computation time), or set to 0 to disable compression::
    
        ! pack floats as shorts:
        <rcbase>.output.packed              :  True
        ! zlib compression level (default 1, 0 for no compression):
        <rcbase>.output.complevel           :  1

    Existing files are replaced if the following flag is set::
    
        ! renew existing files?
        <rcbase>.renew                  :  True

    """
    
    def __init__( self, rcfile, rcbase='', env={}, indent='' ) :
    
        """
        Create temporal means.
        """
                        
        # modules:
        import os
        import numpy
        import xarray
        import datetime
        import glob
        import pandas
        
        # tools:
        import cso_file

        # init base object:
        utopya.UtopyaRc.__init__( self, rcfile, rcbase=rcbase, env=env )
        
        # info ...
        logging.info( indent+'' )
        logging.info( indent+'** Temperoal means over gridded CSO data' )
        logging.info( indent+'' )

        # time range:
        t1 = self.GetSetting( 'timerange.start', totype='datetime' )
        t2 = self.GetSetting( 'timerange.end'  , totype='datetime' )
        # info ...
        tfmt = '%Y-%m-%d %H:%M'
        logging.info( indent+'timerange: [%s,%s]' % (t1.strftime(tfmt),t2.strftime(tfmt)) )
        
        # time units to be used:
        tunits = 'seconds since %s' % t1.strftime('%Y-01-01 00:00:00')

        # pack floats as shorts?
        packed = self.GetSetting( 'output.packed', 'bool' )
        # compression level:
        complevel = self.GetSetting( 'output.complevel', 'int' )

        # renew?
        renew = self.GetSetting( 'renew', totype='bool' )
        
        # resolutions to be created:
        resolutions = self.GetSetting( 'resolutions' ).split()
        # loop:
        for resolution in resolutions :
        
            # info ...
            logging.info( indent+'time resoloution "%s" ...' % resolution )
            
            # time values:
            if resolution == 'daily' :
                tt = pandas.date_range( t1, t2, freq='D' )
                tfmt = '%Y-%m-%d'
            elif resolution == 'monthly' :
                tt = pandas.date_range( t1, t2, freq='M' )
                tfmt = '%Y-%m'
            elif resolution == 'yearly' :
                tt = pandas.date_range( t1, t2, freq='Y' )
                tfmt = '%Y'
            else :
                logging.error( 'unsupported time resolution "%s"' % resolution )
                raise Exception
            #endif
            
            # pattern for input files:
            input_files = self.GetSetting( 'resolution.%s.input.file' % resolution )
            # output file:
            output_file = self.GetSetting( 'resolution.%s.output.file' % resolution )

            # info ..
            logging.info( indent+'time loop ...' )
            # loop:
            for t in tt :
                # info ..
                logging.info( indent+'  %s ..' % t.strftime(tfmt) )

                # target file:
                outfile = t.strftime( output_file )
                # renew?
                if (not os.path.isfile(outfile)) or renew :
                
                    # info ...
                    logging.info( indent+'    create %s ...' % outfile )
                
                    # search pattern for input files:
                    fpattern = t.strftime(input_files)
                    # list:
                    fnames = glob.glob( fpattern )
                    # check ...
                    if len(fnames) == 0 :
                        logging.warning( indent+'no files found matching input pattern:' )
                        logging.warning( indent+'  '+fpattern )
                        continue
                    #endif
                    # open input:
                    ds = xarray.open_mfdataset( fnames )
                    # info ...
                    logging.info( indent+'      read %i records with time range [%s,%s]' % \
                                     (len(ds['time']),ds['time'].values.min(),ds['time'].values.max()) )
                    
                    # shape:
                    nlon = ds.dims['longitude']
                    nlat = ds.dims['latitude']
                
                    # update history list:
                    history = []
                    history.append( 'mean over %i files matching %s' % (len(fnames),os.path.basename(fpattern)) )

                    # init output file:
                    outf = cso_file.CSO_File()
                    
                    # copy grid coordinates:
                    coords = {}
                    for cname in ['longitude','latitude'] :
                        coords[cname] = ds.coords[cname]
                    #endif
                    # new time coordinate:
                    coords['time'] = xarray.DataArray( [t], dims=('time'), attrs={ 'standard_name' : 'time' } ) 
                    coords['time'].encoding['units'] = tunits

                    # cell area, same for all input records, on output only single record needed:
                    vname = 'cell_area'
                    # copy:
                    if len(ds[vname].shape) == 2 :
                        values = ds[vname].values.reshape((1,nlat,nlon))
                    else :
                        values = ds[vname].values[0:1,:,:]
                    #endif
                    # create output variable:
                    outf.ds[vname] = xarray.DataArray( values, dims=('time','latitude','longitude'),
                                                         coords=coords, attrs=ds[vname].attrs )

                    # sum of 'pixel_area' weights
                    vname = 'pixel_area'
                    # copy and update attributes:
                    attrs = ds[vname].attrs
                    attrs['long_name'] = 'sum of areas covered by pixels'
                    # extract, used for weighted averages below, and sum over time:
                    if len(ds[vname].shape) == 2 :
                        pixel_area     = ds[vname].values.reshape((1,nlat,nlon))
                        pixel_area_sum = ds[vname].values.reshape((1,nlat,nlon))
                    else :
                        pixel_area     = ds[vname].values
                        pixel_area_sum = ds[vname].values.sum(axis=0).reshape((1,nlat,nlon))
                    #endif
                    # create target variable:
                    outf.ds[vname] = xarray.DataArray( pixel_area_sum, dims=('time','latitude','longitude'), 
                                                           coords=coords, attrs=attrs )
                    
                    # loop over variables:
                    for vname in ds.keys() :
                        # skip areas:
                        if vname in ['cell_area','pixel_area'] : continue
                        # info ...
                        logging.info( indent+'      variable "%s" ..' % vname )
                        
                        # convert to masked array to avoid warnings about all-nan series:
                        values_in = numpy.ma.array( data=ds[vname].values, mask=numpy.isnan(ds[vname].values) )
                        # input shape: (time,lat,lon,..)
                        shp = values_in.shape
                        # storage for output:
                        values = numpy.full( (1,)+shp[1:], fill_value=numpy.nan )
                        # switch on shape:
                        if len(shp) == 3 :
                            # weighted average, convert output to array with nans:
                            values[0,:,:] = numpy.ma.average( values_in, axis=0, weights=pixel_area ).filled(fill_value=numpy.nan)
                        #
                        elif len(shp) == 4 :
                            # storage for output:
                            values = numpy.full( (1,)+shp[1:], fill_value=numpy.nan )
                            # loop over extra dim:
                            for iv in range(shp[3]) :
                                # weighted average, convert output to array with nans:
                                values[0,:,:,iv] = numpy.ma.average( values_in[:,:,:,iv], axis=0, weights=pixel_area ).filled(fill_value=numpy.nan)
                            #endif
                        #
                        else :
                            logging.error( 'unsupported number of input dimensions %i in input variable "%s"' % (len(shp),vname) )
                            raise Exception
                        #endif
                        
                        # store:
                        outf.ds[vname] = xarray.DataArray( values, dims=ds[vname].dims, attrs=ds[vname].attrs )
                        
                    #endfor # variables

                    # write: copy input attributes, extend history, 
                    # make 'time' unlimitted, and pack:
                    outf.Write( outfile, attrs=ds.attrs, 
                                   history=history, unlimited_dims=['time'],
                                   packed=packed, complevel=complevel )
                    
                    ## testing ..
                    #logging.warning( 'break after first created file' )
                    #break

                #endif # create
                    
            #endfor # time
            
            ## testing ..
            #logging.warning( 'break after first resolution' )
            #break

        #endfor # resolution
            

        # info ...
        logging.info( indent+'' )
        logging.info( indent+'** end' )
        logging.info( indent+'' )
    
    #enddef __init__
    
#endclass CSO_GriddedAverageMeans


########################################################################
###
### end
###
########################################################################

