From 8b1c876cd13bb9559a313002e398a69aa3c6c667 Mon Sep 17 00:00:00 2001 From: Arjo Segers Date: Wed, 28 May 2025 13:40:43 +0200 Subject: [PATCH 1/7] Updated plotting routines. --- doc/source/tutorial.rst | 2 +- src/cso/cso_catalogue.py | 15 +- src/cso/cso_plot.py | 556 +++++++++++++++++++++++++++++---------- src/cso/cso_regions.py | 4 +- 4 files changed, 422 insertions(+), 155 deletions(-) diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst index 823c5a6..3b43105 100644 --- a/doc/source/tutorial.rst +++ b/doc/source/tutorial.rst @@ -333,7 +333,7 @@ The following is a demo Python code that creates an S5p/NO2 map:: title = os.path.basename(filename) label = '%s [%s]' % (vname,attrs['units']) # create map figure (single layer): - fig = cso.QuickMap( values[:,:,0], xx=xx, yy=yy, vmin=0.0, vmax=vmax, + fig = cso.QuickPat( values[:,:,0], x=xx, y=yy, vmin=0.0, vmax=vmax, bmp=dict(resolution='l',countries=True,domain=domain,title=title), cbar=dict(label=label), figsize=(8,6) ) # save: diff --git a/src/cso/cso_catalogue.py b/src/cso/cso_catalogue.py index 1243ed9..42ab0e7 100644 --- a/src/cso/cso_catalogue.py +++ b/src/cso/cso_catalogue.py @@ -21,6 +21,9 @@ # 2024-01, Arjo Segers # Use new plotting routines based on cartopy. # +# 2025-02, Arjo Segers +# Update of plotting routines. +# # 2025-04, Arjo Segers # Updated for latest changes in CSO_Listing class. # @@ -503,8 +506,8 @@ class CSO_Catalogue(CSO_CatalogueBase): # create map figure: fig = cso_plot.QuickPat( values, - xx=xx, - yy=yy, + x=xx, + y=yy, vmin=vmin, vmax=vmax, cmap=dict(colors=colors), # color_bad=color_nan), @@ -858,8 +861,8 @@ class CSO_SimCatalogue(CSO_CatalogueBase): # create map figure: fig = cso_plot.QuickPat( values, - xx=xx, - yy=yy, + x=xx, + y=yy, vmin=vmin, vmax=vmax, cmap=dict(colors=colors, color_bad=color_nan), @@ -1186,8 +1189,8 @@ class CSO_GriddedCatalogue(CSO_CatalogueBase): # create map figure: fig = cso_plot.QuickPat( values, - xm=xm, - ym=ym, + x=xm, + y=ym, vmin=vmin, vmax=vmax, cmap=dict(colors=colors), diff --git a/src/cso/cso_plot.py b/src/cso/cso_plot.py index 6c65574..069c0a2 100644 --- a/src/cso/cso_plot.py +++ b/src/cso/cso_plot.py @@ -13,6 +13,9 @@ # 2024-01, Arjo Segers # Use cartopy for map plots. # +# 2024-07, Arjo Segers +# Fixed automatic value range QuickPat in case the field is a DataArray. +# # 2024-08, Arjo Segers # By default plot country borders. # Made facecolor of lakes transparant. @@ -20,6 +23,24 @@ # 2024-09, Arjo Segers # Support colormaps from `cmcrameri` module. # +# 2024-11, Arjo Segers +# Fixed limitation of map using domain definition. +# Support definition of map raster by list of values rather than step size only. +# Added 'MapFigure' method. +# Support 'dmax' argument for value range in colorbar. +# +# 2025-02, Arjo Segers +# Optionally add multiple patterns with QuickPat. +# Added option to QuickPat to draw border around field. +# Add (gray?) mask over PColor pattern for nan values. +# + +# +# NOTES +# +# * Cartopy downloads maps to: +# ~/.local/share/cartopy/shapefiles/ +# ######################################################################## ### @@ -40,9 +61,16 @@ Plotting tools. Class hierchy ============= -* Figure +* :py:class:`Figure` + + * :py:class:`ColorbarFigure` + - * ColorbarFigure +Methods +======= + +* :py:meth:`QuickPat` +* :py:meth:`MapFigure` Classes and methods @@ -200,17 +228,37 @@ class Figure(object): if "raster" in bkwargs.keys(): # current domain: west, east, south, north = domain - # extract step size: - dmeri, dpara = bkwargs["raster"] - # offset: - meri0 = int(west / dmeri) * dmeri - para0 = int(south / dpara) * dpara - # values: - xlocs = meri0 + (numpy.arange((east - meri0) / dmeri + 1)) * dmeri - ylocs = para0 + (numpy.arange((north - para0) / dpara + 1)) * dpara - # add: - self.ax.gridlines(xlocs=xlocs, ylocs=ylocs) - # endif + # extract values or step size: + meris,paras = bkwargs["raster"] + # draw meridians? + if meris is not None: + # step size instead of values? + if not hasattr(meris,"__len__"): + # value is step size: + dmeri = meris + # offset: + meri0 = int(west / dmeri) * dmeri + # values: + meris = meri0 + (numpy.arange((east - meri0) / dmeri + 1)) * dmeri + # endif + # add lines: + self.ax.gridlines(xlocs=meris) + # endif # meridians defined + # draw parallels? + if paras is not None: + # step size instead of values? + if not hasattr(paras,"__len__"): + # extract step size: + dpara = paras + # offset: + para0 = int(south / dpara) * dpara + # values: + paras = para0 + (numpy.arange((north - para0) / dpara + 1)) * dpara + # endif + # add lines: + self.ax.gridlines(ylocs=paras) + # endif # parallels defined + # endif # "raster" defined # else: # no projection: @@ -221,8 +269,14 @@ class Figure(object): # domain defined? if domain is not None: - # set axes extent: - self.ax.set_extent(domain) + # map projection? + #~ the extent does not always work correct ..? + #if hasattr(self.ax,"set_extent") : + # self.ax.set_extent(domain) + ##endif + #~ just use ax limits ... + self.ax.set_xlim(domain[0:2]) + self.ax.set_ylim(domain[2:4]) # endif # enddef AddAxes @@ -475,6 +529,9 @@ class ColorbarFigure(Figure): self.cmap__color_bad = self.cmap.get_bad() # reset to transparant self.cmap.set_bad(alpha=0) + else : + # not defined: + self.cmap__color_bad = None # endif # no color normalization defined yet, @@ -518,6 +575,9 @@ class ColorbarFigure(Figure): * 'brb' (blue-red-brown) * 'bwr' (blue-white-red) * 'pwb' (purple-white-brown) + * 'kwm' (black-white-magenta) + * 'bgwyr' (blue-green-white-yellow-red) + * 'wjet' ('jet', but start from white) * colormaps from :py:mod:`cmcrameri` module:: @@ -552,6 +612,8 @@ class ColorbarFigure(Figure): # name or values? if type(colors) == str: # our favorite colors: + # + # white-blue-red-brown if colors == "wbrb": colors = [ "white", @@ -564,6 +626,8 @@ class ColorbarFigure(Figure): "magenta", "brown", ] + # + # brown-red-blue-white elif colors == "brbw": colors = [ "brown", @@ -576,6 +640,8 @@ class ColorbarFigure(Figure): "cyan", "white", ] + # + # blue-red-brown elif colors == "brb": colors = [ "cyan", @@ -587,10 +653,26 @@ class ColorbarFigure(Figure): "magenta", "brown", ] + # + # blue-white-red elif colors == "bwr": colors = ["blue", "cyan", "white", "yellow", "red"] + # + # purple-white-brown elif colors == "pwb": colors = ["purple", "blue", "cyan", "white", "yellow", "red", "brown"] + # + # black-white-magenta: + elif colors == "kwm" : + colors = ["black","blue","cyan","lightgreen","white","yellow","orange","red","magenta"] + # + # blue-green-white-yellow-red + elif colors == "bgwyr" : + colors = ["blue","cyan","lightgreen","white","yellow","orange","red"] + # + # white "jet" + elif colors == "wjet" : + colors = ["white","cyan","lightgreen","yellow","orange","red","brown"] # endif # endif @@ -673,20 +755,19 @@ class ColorbarFigure(Figure): # get predefined colormap: if colors.startswith("cmcrameri.cm"): import cmcrameri - - self.cmap = getattr(cmcrameri.cm, colors.lstrip("cmcrameri.cm.")) - else: + self.cmap = getattr(cmcrameri.cm,colors.lstrip("cmcrameri.cm.")) + else : self.cmap = matplotlib.pyplot.get_cmap(colors) # endif # endif # boundaries: - if color_under is not None: + if color_under is not None : self.cmap.set_under(color_under) - if color_over is not None: + if color_over is not None : self.cmap.set_over(color_over) - if color_bad is not None: + if color_bad is not None : self.cmap.set_bad(color_bad) # enddef # Set_ColorMap @@ -721,6 +802,7 @@ class ColorbarFigure(Figure): # external: import matplotlib + import numpy # store value range: self.vmin = vmin @@ -730,9 +812,15 @@ class ColorbarFigure(Figure): if vbounds is None: # set value range if not explicitly done: if self.vmin is None: - self.vmin = values.min() + if hasattr(values,'values') : + self.vmin = numpy.nanmin(values.values) + else : + self.vmin = numpy.nanmin(values) if self.vmax is None: - self.vmax = values.max() + if hasattr(values,'values') : + self.vmax = numpy.nanmax(values.values) + else : + self.vmax = numpy.nanmax(values) # trap constant value ... if self.vmin == self.vmax: @@ -940,6 +1028,7 @@ class ColorbarFigure(Figure): # modules: import numpy + import matplotlib # set mapping from data to color if not done yet: if self.cnorm is None: @@ -958,7 +1047,18 @@ class ColorbarFigure(Figure): p = self.ax.pcolormesh(xx, yy, cc, **pcolor_kwargs) # adhoc: add layer with no-data colors - # .. + if self.cmap__color_bad is not None: + # any nan values? + jj,ii = numpy.where( numpy.isnan(cc) ) + if len(jj) > 0 : + # init as full nan field: + cc2 = numpy.full( cc.shape, numpy.nan ) + # reset: + cc2[jj,ii] = 1.0 + # add mask: + self.ax.pcolormesh(xx, yy, cc2, cmap=matplotlib.colors.ListedColormap([self.cmap__color_bad]) ) + #endif + #endif # add colorbar if not done yet: self.Add_ColorBar(p) @@ -1097,7 +1197,7 @@ class ColorbarFigure(Figure): vmin=vmin, vmax=vmax, vbounds=vbounds, - marker=".", + marker=".", markersize=0, linewidths=0, ) @@ -1106,7 +1206,7 @@ class ColorbarFigure(Figure): # loop: for ip in range(len(values)): # add patch: - self.ax0.fill(xx[ip, :], yy[ip, :], color=colors[ip]) + self.ax.fill(xx[ip, :], yy[ip, :], color=colors[ip]) # endfor # enddef Polygons @@ -1403,7 +1503,7 @@ def mid2bounds(x): # edge: xb[0] = x[0] - 0.5 * (x[1] - x[0]) # mid: - xb[1:nx] = 0.5 * (x[0 : nx - 1] + x[1:nx]) + xb[1:nx] = 0.5 * ( numpy.array(x[:nx-1]) + numpy.array(x[1:nx]) ) # end: xb[nx] = x[nx - 1] + 0.5 * (x[nx - 1] - x[nx - 2]) @@ -1416,9 +1516,10 @@ def mid2bounds(x): # *** -def mid2corners(xx): +def mid2corners(xx, dim=None): """ Return 2D fields with corner values. + Eventually in single dim "x" or "y" only. """ # modules: @@ -1427,19 +1528,31 @@ def mid2corners(xx): # size: ny, nx = xx.shape - # intermediate: - bxx = numpy.zeros((ny, nx + 1), float) - # fill: - for iy in range(ny): - bxx[iy, :] = mid2bounds(xx[iy, :]) - # endfor - - # target: - cxx = numpy.zeros((ny + 1, nx + 1), float) - # fill: - for ix in range(nx + 1): - cxx[:, ix] = mid2bounds(bxx[:, ix]) - # endfor + # extend in x-direction? + if (dim is None) or (dim == "x") : + # intermediate: + bxx = numpy.zeros((ny, nx + 1), float) + # fill: + for iy in range(ny): + bxx[iy, :] = mid2bounds(xx[iy, :]) + # endfor + else : + # create copy: + bxx = xx * 1.0 + #endif + + # extend in x-direction? + if (dim is None) or (dim == "y") : + # target: + cxx = numpy.zeros((ny + 1, nx + 1), float) + # fill: + for ix in range(nx + 1): + cxx[:, ix] = mid2bounds(bxx[:, ix]) + # endfor + else : + # create copy: + cxx = bxx * 1.0 + #endif # ok return cxx @@ -1450,18 +1563,7 @@ def mid2corners(xx): # * -def GetGrid( - shp, - xx=None, - yy=None, - x=None, - y=None, - xm=None, - ym=None, - xxm=None, - yym=None, - domain=None, -): +def GetGrid( shp, x=None, y=None, domain=None ): """ Return 2D grid arrays with corner points. @@ -1471,14 +1573,20 @@ def GetGrid( Optional grid definitions: - * ``xx``, ``yy`` : 2D corner fields, shapes ``(ny+1,nx+1)`` ; - if defined, these are the results - * ``xxm``, ``yym`` : 2D mid of cell fields, shapes ``(nx,ny)`` ; - * ``x``, ``y`` : 1D corner points, shapes ``(nx+1)`` and ``(ny+1)``; used to define 2D corner fields - * ``xm``, ``ym`` : 1D mid of cell, shapes ``(nx)`` and ``(ny)``; used to define 2D corners - - If none of these pairs is defined, a regular grid within the domain is assumed; - if the domain is not specified, the default in ``(0,nx,0,ny)``. + * ``x`` and ``y`` are either 1D or 2D and define the coordinates; + supported shapes: + + * 2D corner field with shape ``(ny+1,nx+1)`` ; the result will have this shape too; + * 2D mid of edges with shape ``(ny,nx+1)`` or ``(ny+1,nx)`` ; + * 2D mid of cell with shape ``(ny,nx)`` ; + * 1D corner points with shape ``(nx+1)`` or ``(ny+1)``; + * 1D mid of cell with shape ``(nx)`` and ``(ny)``. + + * ``domain=(left,right,bottom,top)`` defines the boundaries of the domain; + default = ``(0,nx,0,ny)``. + + If one or both of ``x`` and ``y`` are not defined, the ``domain`` or its default + is used to define a regular grid in one or both of the directions. """ # modules: @@ -1487,79 +1595,154 @@ def GetGrid( # extract: ny, nx = shp - # grid not defined ? - if (xx is None) or (yy is None): - # check: - if (xx is not None) or (yy is not None): - print("ERROR - specify either both or none of xx,yy") - raise Exception - # endif - # 1D axes provided ? - if (x is not None) or (y is not None): - # check ... - if (x is None) or (y is None): - print("ERROR - specify both x and y") - raise Exception - # endif - # 2D corner fields: - xx, yy = numpy.meshgrid(x, y) - elif (xm is not None) or (ym is not None): - # check ... - if (xm is None) or (ym is None): - print("ERROR - specify both xm and ym") - raise Exception - # endif - # create corner axes: - x = mid2bounds(xm) - y = mid2bounds(ym) - # 2D corner fields: - xx, yy = numpy.meshgrid(x, y) - elif (xxm is not None) or (yym is not None): - # interpolate/extrapolate to corners: - xx = mid2corners(xxm) - yy = mid2corners(yym) - else: - # domain definition: - if domain is None: - left, right, bottom, top = 0, nx, 0, ny - else: - left, right, bottom, top = domain - # endif - # regular axes: - x = left + numpy.arange(nx + 1) * (right - left) / float(nx) - y = bottom + numpy.arange(ny + 1) * (top - bottom) / float(ny) - # 2D corner fields: - xx, yy = numpy.meshgrid(x, y) - # endif + # domain definition: + if domain is None: + left, right, bottom, top = 0, nx, 0, ny + else: + left, right, bottom, top = domain # endif - + + # form xx: + if x is None : + # regular boundaries: + xb = left + numpy.arange(nx + 1) * (right - left) / float(nx) + # extend to 2D: + xx = numpy.zeros((ny+1,nx+1)) + for j in range(ny+1) : + xx[j,:] = xb + #endfor + # + elif x.shape == (ny+1,nx+1) : + # copy: + xx = x + # + elif x.shape == (ny,nx) : + # extrapolate both directions: + xx = mid2corners(x) + # + elif x.shape == (ny,nx+1) : + # only in one direction: + xx = mid2corners(x,dim="y") + # + elif x.shape == (ny+1,nx) : + # only in one direction: + xx = mid2corners(x,dim="x") + # + elif x.shape == (nx+1,): + # extend to 2D: + xx = numpy.zeros((ny+1,nx+1)) + for j in range(ny+1) : + xx[j,:] = x + #endfor + # + elif x.shape == (nx,): + # form bounds: + xb = mid2bounds(x) + # extend to 2D: + xx = numpy.zeros((ny+1,nx+1)) + for j in range(ny+1) : + xx[j,:] = xb + #endfor + # + else : + print( f"ERROR - unsupported x shape {x.shape} for field shape {shp}" ) + raise Exception + #endif + + # form yy: + if y is None : + # regular boundaries: + yb = bottom + numpy.arange(ny + 1) * (top - bottom) / float(ny) + # extend to 2D: + yy = numpy.zeros((ny+1,nx+1)) + for i in range(nx+1) : + yy[:,i] = yb + #endfor + # + elif y.shape == (ny+1,nx+1) : + # copy: + yy = y + # + elif y.shape == (ny,nx) : + # extrapolate both directions: + yy = mid2corners(y) + # + elif y.shape == (ny,nx+1) : + # only in one direction: + yy = mid2corners(y,dim="y") + # + elif y.shape == (ny+1,nx) : + # only in one direction: + yy = mid2corners(y,dim="x") + # + elif y.shape == (ny+1,): + # extend to 2D: + yy = numpy.zeros((ny+1,nx+1)) + for i in range(nx+1) : + yy[:,i] = y + #endfor + # + elif y.shape == (ny,): + # form bounds: + yb = mid2bounds(y) + # extend to 2D: + yy = numpy.zeros((ny+1,nx+1)) + for i in range(nx+1) : + yy[:,i] = yb + #endfor + # + else : + print( f"ERROR - unsupported y shape {y.shape} for field shape {shp}" ) + raise Eyception + #endif + # ok return xx, yy - # enddef GetGrid +# * + +def MapFigure( domain=None ): + + """ + Draw map. + + Optional arguments: + + * ``domain=[west,east,south,north]``, default ``[-180,180,-90,90]`` + + Return value: + + * ``fig`` : :py:class:`Figure` object + """ + + # new: + fig = Figure() + # add ax: + fig.AddAxes( domain=domain, bmp=dict(countries=True) ) + +# enddef MapFigure + + # * def QuickPat( cc, - xx=None, - yy=None, x=None, y=None, - xm=None, - ym=None, - xxm=None, - yym=None, domain=None, title=None, vmin=None, vmax=None, + dmax=None, vbounds=None, - bmp=dict(), + bmp=None, cbar=dict(), + border=None, + fig=None, **kwargs, ): """ @@ -1568,10 +1751,9 @@ def QuickPat( Arguments passed to :py:meth:`GetGrid` : - * ``xm``, ``ym`` : 1D mid of cell, shapes (nx) and (ny); used to define 2D corners - * ``x``, ``y`` : 1D corner points, shapes (nx+1) and (ny+1); used to define 2D corner fields - * ``xxm``, ``yym`` : 2D mid of cell, shapes (ny,nx) - * ``xx``, ``yy`` : 2D corner fields, shapes (ny+1,nx+1) + * ``x``, ``y`` : 1D or 2D coordinates; used to define 2D corner fields; + * ``domain=(left,right,bottom,top)`` : coordinate bounds; if ``domain`` is ``None`` or + some elements are ``None``, values are based on the ``x`` or ``y`` coordinates if defined. Arguments passed to :py:meth:`ColorbarFigure.PColor` method: @@ -1579,6 +1761,9 @@ def QuickPat( * ``vmin``,``vmax`` : data range for coloring * ``vbounds`` : value bounds for colorbar (defines interval with same color) + If ``border`` is defined, a border around the 2D field; + the argument might contain a dictionary with line properties. + Other arguments are passed to 'ColorbarFigure' : * ``figsize = (width,height)`` @@ -1594,31 +1779,97 @@ def QuickPat( Return value: * ``fig`` : Figure instance + + If an already existing Figure instance is passed as the ``fig`` argument, + the pattern will be added to the plot. """ - - # domain defined as part of bmp ? - if (domain is None) and ("domain" in bmp.keys()): - domain = bmp["domain"] - + + # modules: + import numpy + + # extract known coordinates: + if hasattr(cc,'coords') : + # longitudes: + if x is None : + for key in ["lon","longitude"] : + if key in cc.coords : + x = cc.coords[key].values + # endif + # endfor + # endif + # latitudes: + if y is None : + for key in ["lat","latitude"] : + if key in cc.coords : + y = cc.coords[key].values + # endif + # endfor + # endif + # endif # cc.coords + + # domain not defined yet? + if domain is None : + # map defined? + if type(bmp) == dict : + # domain defined? + if "domain" in bmp.keys(): + domain = bmp["domain"] + # endif + # endif + # endif + # corner points: - xx, yy = GetGrid( - cc.shape, xx=xx, yy=yy, x=x, y=y, xm=xm, ym=ym, xxm=xxm, yym=yym, domain=domain - ) + xx, yy = GetGrid( cc.shape, x=x, y=y, domain=domain ) # set domain if not present yet: if domain is None: domain = [xx.min(), xx.max(), yy.min(), yy.max()] - - ## replace or add domain in bmp argument: - # bmp["domain"] = domain + else : + if domain[0] is None : + domain[0] = xx.min() + if domain[1] is None : + domain[1] = xx.max() + if domain[2] is None : + domain[2] = yy.min() + if domain[3] is None : + domain[3] = yy.max() + #endif + + # replace or add domain in bmp argument: + if type(bmp) == dict : + bmp["domain"] = domain + + # symetric range? + if dmax is not None : + # range: + vmin = -dmax + vmax = dmax + # colors: + if 'cmap' in kwargs.keys() : + if 'colors' not in kwargs['cmap'].keys() : + kwargs['cmap']['colors'] = 'pwb' + #endif + else : + kwargs['cmap'] = { 'colors' : 'pwb' } + #endif + #endif + + # value range: + if hasattr(cc,'values') : + cmin = numpy.nanmin(cc.values) + cmax = numpy.nanmax(cc.values) + else : + cmin = numpy.nanmin(cc) + cmax = numpy.nanmax(cc) + # endif # auto extend? if "extend" not in cbar.keys(): # extend below minimum? - extend_min = (vmin is not None) and (cc.min() < vmin) + extend_min = (vmin is not None) and (cmin < vmin) # extend above maximum? - extend_max = (vmax is not None) and (cc.max() > vmax) + extend_max = (vmax is not None) and (cmax > vmax) # set keyword: if extend_min and (not extend_max): cbar["extend"] = "min" @@ -1632,21 +1883,34 @@ def QuickPat( # endif # setup figure: - fig = ColorbarFigure(cbar=cbar, bmp=bmp, domain=domain, title=title, **kwargs) - - # add image; no map plot? - if bmp is None: - # use a "pcolormesh" to add the data: - fig.PColor(xx, yy, cc, vmin=vmin, vmax=vmax, vbounds=vbounds) - else: - # a "pcolormesh" on geogrhapical projection does not work very well - # for patches over the date line and/or "bad" values, - # therefore plot as series of patches: - # fig.PolygonsMesh(xx, yy, cc, vmin=vmin, vmax=vmax, vbounds=vbounds) - # TOO SLOW! - fig.PColor(xx, yy, cc, vmin=vmin, vmax=vmax, vbounds=vbounds) - # endif # standard axes or map? - + if fig is None : + fig = ColorbarFigure(cbar=cbar, bmp=bmp, domain=domain, title=title, **kwargs) + #endif + + # TOO SLOW! + # a "pcolormesh" on geogrhapical projection does not work very well + # for patches over the date line and/or "bad" values, + # therefore plot as series of patches: + # fig.PolygonsMesh(xx, yy, cc, vmin=vmin, vmax=vmax, vbounds=vbounds) + + # add "pcolormesh": + fig.PColor(xx, yy, cc, vmin=vmin, vmax=vmax, vbounds=vbounds) + + # add border around pattern? + if border is not None : + # colect points into line: + x = numpy.hstack((xx[0,:],xx[:,-1],numpy.flipud(xx[-1,:]),numpy.flipud(xx[:,0]))) + y = numpy.hstack((yy[0,:],yy[:,-1],numpy.flipud(yy[-1,:]),numpy.flipud(yy[:,0]))) + # plot: + if type(border) == dict: + pkwargs = border + else: + pkwargs = dict(color="black",linestyle="-") + #endif + # add: + fig.ax.plot( x, y, **pkwargs ) + # endif + # ok return fig diff --git a/src/cso/cso_regions.py b/src/cso/cso_regions.py index 3d143bd..ded1a81 100644 --- a/src/cso/cso_regions.py +++ b/src/cso/cso_regions.py @@ -1980,8 +1980,8 @@ class CSO_Catalogue_RegionsMaps(cso_catalogue.CSO_CatalogueBase): # create map figure: fig = cso_plot.QuickPat( values, - xm=xm, - ym=ym, + x=xm, + y=ym, vmin=vmin, vmax=vmax, cmap=dict(colors=colors), -- GitLab From 07d4495212cf3840ba5345a20630f81a5300a18b Mon Sep 17 00:00:00 2001 From: Arjo Segers Date: Wed, 28 May 2025 15:45:39 +0200 Subject: [PATCH 2/7] Updated change logs. --- src/cso/cso_catalogue.py | 3 +++ src/cso/cso_colhub.py | 3 +++ src/cso/cso_colocate.py | 3 +++ src/cso/cso_dataspace.py | 3 +++ src/cso/cso_gridded.py | 3 +++ src/cso/cso_inquire.py | 4 ++++ src/cso/cso_regions.py | 6 ++++++ src/cso/cso_s5p.py | 3 +++ src/cso/cso_superobs.py | 3 +++ 9 files changed, 31 insertions(+) diff --git a/src/cso/cso_catalogue.py b/src/cso/cso_catalogue.py index 42ab0e7..b610981 100644 --- a/src/cso/cso_catalogue.py +++ b/src/cso/cso_catalogue.py @@ -27,6 +27,9 @@ # 2025-04, Arjo Segers # Updated for latest changes in CSO_Listing class. # +# 2025-04, Arjo Segers +# Changed imports for python packaging. +# ######################################################################## diff --git a/src/cso/cso_colhub.py b/src/cso/cso_colhub.py index 1dc8989..88056f9 100644 --- a/src/cso/cso_colhub.py +++ b/src/cso/cso_colhub.py @@ -21,6 +21,9 @@ # Fixed type conversion of "dmode" setting. # Optionally search in multiple mirror archives for missing files. # +# 2025-04, Arjo Segers +# Changed imports for python packaging. +# ######################################################################## ### diff --git a/src/cso/cso_colocate.py b/src/cso/cso_colocate.py index d76b687..3a84e8b 100644 --- a/src/cso/cso_colocate.py +++ b/src/cso/cso_colocate.py @@ -10,6 +10,9 @@ # 2025-04, Arjo Segers # Added option to drop some columns of the location csv after reading, # this prevents these columns from being added to the co-location output. +# +# 2025-04, Arjo Segers +# Changed imports for python packaging. # diff --git a/src/cso/cso_dataspace.py b/src/cso/cso_dataspace.py index 8250df3..7f43c94 100644 --- a/src/cso/cso_dataspace.py +++ b/src/cso/cso_dataspace.py @@ -40,6 +40,9 @@ # 2025-02, Arjo Segers # Use 'shutil.move' instead of 'os.rename' for move over filesystem. # +# 2025-04, Arjo Segers +# Changed imports for python packaging. +# ######################################################################## diff --git a/src/cso/cso_gridded.py b/src/cso/cso_gridded.py index a08ff4b..c0c792d 100644 --- a/src/cso/cso_gridded.py +++ b/src/cso/cso_gridded.py @@ -34,6 +34,9 @@ # Support output frequencies defined in settings. # Support creation of multiple output files. # +# 2025-04, Arjo Segers +# Changed imports for python packaging. +# ######################################################################## diff --git a/src/cso/cso_inquire.py b/src/cso/cso_inquire.py index 155cede..70c9792 100644 --- a/src/cso/cso_inquire.py +++ b/src/cso/cso_inquire.py @@ -1,3 +1,4 @@ +# # CSO data archive inquiry tools. # # CHANGES @@ -27,6 +28,9 @@ # 2025-04, Arjo Segers # Convert datetime time values in listing files after reading to keep default str values in table. # +# 2025-04, Arjo Segers +# Changed imports for python packaging. +# ######################################################################## ### diff --git a/src/cso/cso_regions.py b/src/cso/cso_regions.py index ded1a81..89492e8 100644 --- a/src/cso/cso_regions.py +++ b/src/cso/cso_regions.py @@ -20,6 +20,12 @@ # Store mapping masks in caches for re-use. # Support box-plot timeseries to show variation over region. # +# 2025-02, Arjo Segers +# Updated arguments for plotting routines. +# +# 2025-04, Arjo Segers +# Changed imports for python packaging. +# ######################################################################## ### diff --git a/src/cso/cso_s5p.py b/src/cso/cso_s5p.py index db73c1a..01513f2 100644 --- a/src/cso/cso_s5p.py +++ b/src/cso/cso_s5p.py @@ -67,6 +67,9 @@ # to ensure correct parsing by ncdump and other tools. # Sort input listing by orbit numbers when converting. # +# 2025-04, Arjo Segers +# Changed imports for python packaging. +# ######################################################################## diff --git a/src/cso/cso_superobs.py b/src/cso/cso_superobs.py index 9f7f001..4e18e4c 100644 --- a/src/cso/cso_superobs.py +++ b/src/cso/cso_superobs.py @@ -21,6 +21,9 @@ # Changed size calculation after deprication warning. # Changed insert of new columns to dataframe after efficiency warning. # +# 2025-04, Arjo Segers +# Changed imports for python packaging. +# ######################################################################## -- GitLab From 357b802d7f375f9e4685e12dec89b8148fc94b94 Mon Sep 17 00:00:00 2001 From: Arjo Segers Date: Wed, 28 May 2025 15:47:32 +0200 Subject: [PATCH 3/7] Moved corner interpolation methods into `cso_tools`. --- src/cso/cso_plot.py | 224 +-------------------------------- src/cso/cso_tools.py | 290 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 296 insertions(+), 218 deletions(-) diff --git a/src/cso/cso_plot.py b/src/cso/cso_plot.py index 069c0a2..c105abd 100644 --- a/src/cso/cso_plot.py +++ b/src/cso/cso_plot.py @@ -1484,226 +1484,10 @@ def GetColorMap(colors=None, color_under=None, color_over=None, color_bad=None, # enddef # GetColorMap -# *** - - -def mid2bounds(x): - """ - Return 1D arrays with boundary values. - """ - - # modules: - import numpy - - # size: - nx = len(x) - - # result: - xb = numpy.zeros((nx + 1), float) - # edge: - xb[0] = x[0] - 0.5 * (x[1] - x[0]) - # mid: - xb[1:nx] = 0.5 * ( numpy.array(x[:nx-1]) + numpy.array(x[1:nx]) ) - # end: - xb[nx] = x[nx - 1] + 0.5 * (x[nx - 1] - x[nx - 2]) - - # ok - return xb - - -# enddef mid2bounds # *** -def mid2corners(xx, dim=None): - """ - Return 2D fields with corner values. - Eventually in single dim "x" or "y" only. - """ - - # modules: - import numpy - - # size: - ny, nx = xx.shape - - # extend in x-direction? - if (dim is None) or (dim == "x") : - # intermediate: - bxx = numpy.zeros((ny, nx + 1), float) - # fill: - for iy in range(ny): - bxx[iy, :] = mid2bounds(xx[iy, :]) - # endfor - else : - # create copy: - bxx = xx * 1.0 - #endif - - # extend in x-direction? - if (dim is None) or (dim == "y") : - # target: - cxx = numpy.zeros((ny + 1, nx + 1), float) - # fill: - for ix in range(nx + 1): - cxx[:, ix] = mid2bounds(bxx[:, ix]) - # endfor - else : - # create copy: - cxx = bxx * 1.0 - #endif - - # ok - return cxx - - -# enddef mid2corners - -# * - - -def GetGrid( shp, x=None, y=None, domain=None ): - """ - Return 2D grid arrays with corner points. - - Arguments: - - * ``shp`` : shape (ny,nx) of values - - Optional grid definitions: - - * ``x`` and ``y`` are either 1D or 2D and define the coordinates; - supported shapes: - - * 2D corner field with shape ``(ny+1,nx+1)`` ; the result will have this shape too; - * 2D mid of edges with shape ``(ny,nx+1)`` or ``(ny+1,nx)`` ; - * 2D mid of cell with shape ``(ny,nx)`` ; - * 1D corner points with shape ``(nx+1)`` or ``(ny+1)``; - * 1D mid of cell with shape ``(nx)`` and ``(ny)``. - - * ``domain=(left,right,bottom,top)`` defines the boundaries of the domain; - default = ``(0,nx,0,ny)``. - - If one or both of ``x`` and ``y`` are not defined, the ``domain`` or its default - is used to define a regular grid in one or both of the directions. - """ - - # modules: - import numpy - - # extract: - ny, nx = shp - - # domain definition: - if domain is None: - left, right, bottom, top = 0, nx, 0, ny - else: - left, right, bottom, top = domain - # endif - - # form xx: - if x is None : - # regular boundaries: - xb = left + numpy.arange(nx + 1) * (right - left) / float(nx) - # extend to 2D: - xx = numpy.zeros((ny+1,nx+1)) - for j in range(ny+1) : - xx[j,:] = xb - #endfor - # - elif x.shape == (ny+1,nx+1) : - # copy: - xx = x - # - elif x.shape == (ny,nx) : - # extrapolate both directions: - xx = mid2corners(x) - # - elif x.shape == (ny,nx+1) : - # only in one direction: - xx = mid2corners(x,dim="y") - # - elif x.shape == (ny+1,nx) : - # only in one direction: - xx = mid2corners(x,dim="x") - # - elif x.shape == (nx+1,): - # extend to 2D: - xx = numpy.zeros((ny+1,nx+1)) - for j in range(ny+1) : - xx[j,:] = x - #endfor - # - elif x.shape == (nx,): - # form bounds: - xb = mid2bounds(x) - # extend to 2D: - xx = numpy.zeros((ny+1,nx+1)) - for j in range(ny+1) : - xx[j,:] = xb - #endfor - # - else : - print( f"ERROR - unsupported x shape {x.shape} for field shape {shp}" ) - raise Exception - #endif - - # form yy: - if y is None : - # regular boundaries: - yb = bottom + numpy.arange(ny + 1) * (top - bottom) / float(ny) - # extend to 2D: - yy = numpy.zeros((ny+1,nx+1)) - for i in range(nx+1) : - yy[:,i] = yb - #endfor - # - elif y.shape == (ny+1,nx+1) : - # copy: - yy = y - # - elif y.shape == (ny,nx) : - # extrapolate both directions: - yy = mid2corners(y) - # - elif y.shape == (ny,nx+1) : - # only in one direction: - yy = mid2corners(y,dim="y") - # - elif y.shape == (ny+1,nx) : - # only in one direction: - yy = mid2corners(y,dim="x") - # - elif y.shape == (ny+1,): - # extend to 2D: - yy = numpy.zeros((ny+1,nx+1)) - for i in range(nx+1) : - yy[:,i] = y - #endfor - # - elif y.shape == (ny,): - # form bounds: - yb = mid2bounds(y) - # extend to 2D: - yy = numpy.zeros((ny+1,nx+1)) - for i in range(nx+1) : - yy[:,i] = yb - #endfor - # - else : - print( f"ERROR - unsupported y shape {y.shape} for field shape {shp}" ) - raise Eyception - #endif - - # ok - return xx, yy - -# enddef GetGrid - - -# * - def MapFigure( domain=None ): """ @@ -1749,7 +1533,7 @@ def QuickPat( Plot 2D data as colored pattern. - Arguments passed to :py:meth:`GetGrid` : + Arguments passed to :py:meth:`.cso_tools.GetCornerGridX` and :py:meth:`.cso_tools.GetCornerGridY` : * ``x``, ``y`` : 1D or 2D coordinates; used to define 2D corner fields; * ``domain=(left,right,bottom,top)`` : coordinate bounds; if ``domain`` is ``None`` or @@ -1788,6 +1572,9 @@ def QuickPat( # modules: import numpy + # tools: + from . import cso_tools + # extract known coordinates: if hasattr(cc,'coords') : # longitudes: @@ -1820,7 +1607,8 @@ def QuickPat( # endif # corner points: - xx, yy = GetGrid( cc.shape, x=x, y=y, domain=domain ) + xx = cso_tools.GetCornerGridX( cc.shape, x=x, domain=domain ) + yy = cso_tools.GetCornerGridY( cc.shape, y=y, domain=domain ) # set domain if not present yet: if domain is None: diff --git a/src/cso/cso_tools.py b/src/cso/cso_tools.py index 0321538..f43599b 100644 --- a/src/cso/cso_tools.py +++ b/src/cso/cso_tools.py @@ -4,6 +4,9 @@ # 2024-03, Arjo Segers # Tool module, started with copy of "linearize_avg_kernel" method from Enrico Dammers. # +# 2024-03, Arjo Segers +# Added `mid2bounds`, `mid2corners`, `GetCornerGridX`, and `GetCornerGridY` methods. +# ######################################################################## ### @@ -25,6 +28,293 @@ Methods """ +######################################################################## +### +### grid tools +### +######################################################################## + + +def mid2bounds(x): + """ + Given input ``x`` with shape ``(nx,)``, + return array ``xb`` with shape ``(nx+1,)`` with boundary values + between and outside the input locations. + """ + + # modules: + import numpy + + # size: + nx = len(x) + + # result: + xb = numpy.zeros((nx + 1), float) + # edge: + xb[0] = x[0] - 0.5 * (x[1] - x[0]) + # mid: + xb[1:nx] = 0.5 * ( numpy.array(x[:nx-1]) + numpy.array(x[1:nx]) ) + # end: + xb[nx] = x[nx - 1] + 0.5 * (x[nx - 1] - x[nx - 2]) + + # ok + return xb + + +# enddef mid2bounds + +# *** + + +def mid2corners(xx, dim=None): + """ + Return 2D fields with corner values. + Eventually in single dim "x" or "y" only. + """ + + # modules: + import numpy + + # size: + ny, nx = xx.shape + + # extend in x-direction? + if (dim is None) or (dim == "x") : + # intermediate: + bxx = numpy.zeros((ny, nx + 1), float) + # fill: + for iy in range(ny): + bxx[iy, :] = mid2bounds(xx[iy, :]) + # endfor + else : + # create copy: + bxx = xx * 1.0 + #endif + + # extend in x-direction? + if (dim is None) or (dim == "y") : + # target: + cxx = numpy.zeros((ny + 1, nx + 1), float) + # fill: + for ix in range(nx + 1): + cxx[:, ix] = mid2bounds(bxx[:, ix]) + # endfor + else : + # create copy: + cxx = bxx * 1.0 + #endif + + # ok + return cxx + + +# enddef mid2corners + +# * + + +def GetCornerGridX( shp, x=None, domain=None, bounds=False ): + """ + Return grid array of shape ``(ny+1,nx+1)`` with 'x' locations of corner points. + If ``bounds=True``, the result is a ``(ny,nx,4)`` array. + or + + Arguments: + + * ``shp`` : shape ``(ny,nx)`` of values + + Optional grid definitions: + + * ``x`` is either 1D or 2D and define the coordinates; supported shapes: + + * 2D corner field with shape ``(ny+1,nx+1)`` ; the result will have this shape too; + * 2D mid of edges with shape ``(ny,nx+1)`` ; + * 2D mid of cell with shape ``(ny,nx)`` ; + * 1D corner points with shape ``(nx+1)``; + * 1D mid of cell with shape ``(nx)``. + + * ``domain=(left,right,bottom,top)`` defines the boundaries of the domain; + default = ``(0,nx,0,ny)``. + + If ``x`` is not defined, the ``domain`` or its default is used to define a regular grid. + """ + + # modules: + import numpy + + # extract: + ny, nx = shp + + # domain definition: + if domain is None: + left, right, bottom, top = 0, nx, 0, ny + else: + left, right, bottom, top = domain + # endif + + # form xx: + if x is None : + # regular boundaries: + xb = left + numpy.arange(nx + 1) * (right - left) / float(nx) + # extend to 2D: + xx = numpy.zeros((ny+1,nx+1)) + for j in range(ny+1) : + xx[j,:] = xb + #endfor + # + elif x.shape == (ny+1,nx+1) : + # copy: + xx = x + # + elif x.shape == (ny,nx) : + # extrapolate both directions: + xx = mid2corners(x) + # + elif x.shape == (ny,nx+1) : + # only in one direction: + xx = mid2corners(x,dim="y") + # + elif x.shape == (ny+1,nx) : + # only in one direction: + xx = mid2corners(x,dim="x") + # + elif x.shape == (nx+1,): + # extend to 2D: + xx = numpy.zeros((ny+1,nx+1)) + for j in range(ny+1) : + xx[j,:] = x + #endfor + # + elif x.shape == (nx,): + # form bounds: + xb = mid2bounds(x) + # extend to 2D: + xx = numpy.zeros((ny+1,nx+1)) + for j in range(ny+1) : + xx[j,:] = xb + #endfor + # + else : + print( f"ERROR - unsupported x shape {x.shape} for field shape {shp}" ) + raise Exception + #endif + + # ok + if bounds : + xxb = numpy.zeros((ny,nx,4)) + xxb[:,:,0] = xx[0:ny ,0:nx ] + xxb[:,:,1] = xx[0:ny ,1:nx+1] + xxb[:,:,2] = xx[1:ny+1,1:nx+1] + xxb[:,:,3] = xx[1:ny+1,0:nx ] + return xxb + else : + return xx + #endif + +# enddef GetCornerGridX + +# * + + +def GetCornerGridY( shp, y=None, domain=None, bounds=False ): + """ + Return grid array of shape ``(ny+1,nx+1)`` with 'x' locations of corner points. + If ``bounds=True``, the result is a ``(ny,nx,4)`` array. + + Arguments: + + * ``shp`` : shape (ny,nx) of values + + Optional grid definitions: + + * ``y`` is either 1D or 2D and define the coordinates; supported shapes: + + * 2D corner field with shape ``(ny+1,nx+1)`` ; the result will have this shape too; + * 2D mid of edges with shape ``(ny+1,nx)`` ; + * 2D mid of cell with shape ``(ny,nx)`` ; + * 1D corner points with shape ``(ny+1)``; + * 1D mid of cell with shape ``(ny)``. + + * ``domain=(left,right,bottom,top)`` defines the boundaries of the domain; default = ``(0,nx,0,ny)``. + + If ``y`` is not defined, the ``domain`` or its default is used define a regular grid. + """ + + # modules: + import numpy + + # extract: + ny, nx = shp + + # domain definition: + if domain is None: + left, right, bottom, top = 0, nx, 0, ny + else: + left, right, bottom, top = domain + # endif + + # form yy: + if y is None : + # regular boundaries: + yb = bottom + numpy.arange(ny + 1) * (top - bottom) / float(ny) + # extend to 2D: + yy = numpy.zeros((ny+1,nx+1)) + for i in range(nx+1) : + yy[:,i] = yb + #endfor + # + elif y.shape == (ny+1,nx+1) : + # copy: + yy = y + # + elif y.shape == (ny,nx) : + # extrapolate both directions: + yy = mid2corners(y) + # + elif y.shape == (ny,nx+1) : + # only in one direction: + yy = mid2corners(y,dim="y") + # + elif y.shape == (ny+1,nx) : + # only in one direction: + yy = mid2corners(y,dim="x") + # + elif y.shape == (ny+1,): + # extend to 2D: + yy = numpy.zeros((ny+1,nx+1)) + for i in range(nx+1) : + yy[:,i] = y + #endfor + # + elif y.shape == (ny,): + # form bounds: + yb = mid2bounds(y) + # extend to 2D: + yy = numpy.zeros((ny+1,nx+1)) + for i in range(nx+1) : + yy[:,i] = yb + #endfor + # + else : + print( f"ERROR - unsupported y shape {y.shape} for field shape {shp}" ) + raise Eyception + #endif + + # ok + if bounds : + yyb = numpy.zeros((ny,nx,4)) + yyb[:,:,0] = yy[0:ny ,0:nx ] + yyb[:,:,1] = yy[0:ny ,1:nx+1] + yyb[:,:,2] = yy[1:ny+1,1:nx+1] + yyb[:,:,3] = yy[1:ny+1,0:nx ] + return yyb + else : + return yy + #endif + +# enddef GetCornerGridY + + ######################################################################## ### ### averaging kernel tool -- GitLab From e30e1602c136344e0d5e6e9fbeaba85769920636 Mon Sep 17 00:00:00 2001 From: Arjo Segers Date: Wed, 28 May 2025 15:48:01 +0200 Subject: [PATCH 4/7] Updated change logs. --- src/cso/cso_pal.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/cso/cso_pal.py b/src/cso/cso_pal.py index a879baa..e5132a7 100644 --- a/src/cso/cso_pal.py +++ b/src/cso/cso_pal.py @@ -19,6 +19,9 @@ # 2025-02, Arjo Segers # Use 'shutil.move' instead of 'os.rename' for move over filesystem. # +# 2025-04, Arjo Segers +# Changed imports for python packaging. +# ######################################################################## ### -- GitLab From f65776aaf0fd2a9c9ab5d1fc70defbaeb1621605 Mon Sep 17 00:00:00 2001 From: Arjo Segers Date: Wed, 28 May 2025 15:50:33 +0200 Subject: [PATCH 5/7] Extended processing of listing files. --- src/cso/cso_file.py | 118 ++++++++++++++++++++++++++++---------------- 1 file changed, 75 insertions(+), 43 deletions(-) diff --git a/src/cso/cso_file.py b/src/cso/cso_file.py index 6cd93a7..753dee6 100644 --- a/src/cso/cso_file.py +++ b/src/cso/cso_file.py @@ -1,4 +1,5 @@ -# Changes +# +# CHANGES # # 2022-09, Arjo Segers # Write files with zlib-compression, option to disable this. @@ -39,6 +40,14 @@ # 2025-04, Arjo Segers # Enable zlib compression only for numerical data. # Avoid warnings from packing in case of all-nan values. +# Open a file rather than loading it. +# Extended sort options for listing files. +# Support creation of listing file objects without filename. +# Support selection of multiple records from listing file. +# +# 2025-04, Arjo Segers +# Changed imports for python packaging. +# Change expansion of datetime values from csv files for recent pandas version. # ######################################################################## @@ -189,8 +198,8 @@ def Pack_DataArray(da, dtype="i2"): # only floats ... if da.dtype in [numpy.float32, numpy.float64]: - # should have some values .. - if numpy.any( ~ numpy.isnan(da.values) ): + # any values defined? + if numpy.any( ~ numpy.isnan(da.values) ) : # value range, ignore nan's: vmin = numpy.nanmin(da.values) vmax = numpy.nanmax(da.values) @@ -198,7 +207,7 @@ def Pack_DataArray(da, dtype="i2"): # dummy range: vmin = 0.0 vmax = 0.0 - #end if + #endif # target data type could be integer or float: if dtype.startswith("i"): # use absolute minimum to represent nans: @@ -276,12 +285,14 @@ class CSO_File(object): raise Exception # endif - # access dataset: - with xarray.open_dataset(self.filename) as self.ds: - # load entire file: - self.ds.load() - # endwith # xarray - + # open file: + try: + self.ds = xarray.open_dataset(self.filename) + except: + logging.error(f"could not open (corrupted?) file: {self.filename}") + raise + #endtry + else: # dummy: self.filename = "None" @@ -1291,14 +1302,25 @@ class CSO_Listing(object): # head for index column: self.index_label = "filename" - # read? - if filename is not None: + # store filename: + self.filename = filename + + # directory name: + if self.filename is not None: + # check .. if not os.path.isfile(filename): logging.error("listing file not found: %s" % filename) raise Exception # endif + # base directory: + self.dirname = os.path.dirname(self.filename) + # could be empty .. + if len(self.dirname) == 0 : + self.dirname = os.curdir + # endif + # info ... logging.info(f"{indent} read listing {filename} ...") # read: @@ -1314,7 +1336,11 @@ class CSO_Listing(object): self.df["end_time"] = pandas.to_datetime(self.df["end_time"]) else: - # new table: + + # not defined yet, assume current location: + self.dirname = os.curdir + + # new empty table: self.df = pandas.DataFrame(columns=["start_time", "end_time"]) # endif @@ -1343,7 +1369,7 @@ class CSO_Listing(object): # save, also write the index column: self.df.to_csv(filename, sep=self.sep, columns=columns, index_label=self.index_label) - # enddef Close + # enddef Save # * @@ -1465,7 +1491,7 @@ class CSO_Listing(object): # check .. if fname not in self.df.index: - logging.error('file "%s" is not a record in table: %s' % (fname, filename)) + logging.error(f"file '{fname}' is not a record in table: {self.filename}") raise Exception # endif @@ -1532,9 +1558,9 @@ class CSO_Listing(object): # * - def Select(self, tr=None, method="overlap", expr=None, blacklist=[], indent="", **kwargs): + def Select(self, tr=None, method="overlap", expr=None, blacklist=[], verbose=True, indent="", **kwargs): """ - Return :py:class:`CSO_Listing` objects with selection of records. + Return :py:class:`CSO_Listing` object with selection of records. Optional arguments: @@ -1594,7 +1620,7 @@ class CSO_Listing(object): for key, value in kwargs.items(): # check .. if key not in df.keys(): - logging.error(f"key '{key}' not defined in listing") + logging.error(f"key '{key}' not defined in listing: {self.filename}") raise Exception # endif # select: @@ -1606,6 +1632,9 @@ class CSO_Listing(object): # evaluate selection expression? if expr is not None: # replace templates: + # %{orbit} == '12345' + # to: + # xrec['orbit'] == '12345' for key in self.df.keys(): expr = expr.replace("%{" + key + "}", "xrec['" + key + "']") # endfor @@ -1632,30 +1661,29 @@ class CSO_Listing(object): if eval(selection): selected.append(indx) filestatus[indx] = "selected" - rec = xrec # endif # endfor # records - # exactly one? then leave: - if len(selected) == 1: + + # any selected? + if len(selected) > 0: + # leave: break - elif len(selected) > 1: - logging.error(f"found more than one record matching selection: {selection}") - for fname in selected: - logging.error(f" {fname}") - # endfor - raise Exception - # endif # number found + #endif + # endfor # selection criteria - # info ... - logging.info(f"{indent}available records(s):") - # loop: - for fname, row in df.iterrows(): - line = fname - if fname in filestatus.keys(): - line = line + " [" + filestatus[fname] + "]" - logging.info(f"{indent} {line}") - # endfor + # show selection? + if verbose : + # info ... + logging.info(f"{indent}available records(s):") + # loop: + for fname, row in df.iterrows(): + line = fname + if fname in filestatus.keys(): + line = line + " [" + filestatus[fname] + "]" + logging.info(f"{indent} {line}") + # endfor + #endif # verbose # no match? if len(selected) == 0: @@ -1670,8 +1698,8 @@ class CSO_Listing(object): # create empty dataframe as result: df = pandas.DataFrame(columns=df.columns) else: - # extract selected record: - df = df.loc[[selected[0]]] + # extract selected record(s): + df = df.loc[selected] # endif # endif @@ -1721,13 +1749,17 @@ class CSO_Listing(object): # * - def Sort(self, by="filename"): + def Sort(self, by=None): """ - Sort listing table by filename or other key. + Sort listing table by index (default, this is the "filename") or by a named column. """ - # sort inplace: - self.df.sort_values(by, inplace=True) + # sort index or values: + if by is None: + self.df.sort_index(inplace=True) + else : + self.df.sort_values(by, inplace=True) + # endif # endef Sort -- GitLab From f6e7da382b5be3be3b0079bac60a10070eaa8ef7 Mon Sep 17 00:00:00 2001 From: Arjo Segers Date: Wed, 28 May 2025 15:51:49 +0200 Subject: [PATCH 6/7] Added class to create listing file of downloaded VIIRS files. --- src/cso/cso_earthaccess.py | 222 +++++++++++++++++++++++++++++++++++-- 1 file changed, 215 insertions(+), 7 deletions(-) diff --git a/src/cso/cso_earthaccess.py b/src/cso/cso_earthaccess.py index 032406e..bda43d9 100644 --- a/src/cso/cso_earthaccess.py +++ b/src/cso/cso_earthaccess.py @@ -7,6 +7,12 @@ # 2025-02, Arjo Segers # Use 'shutil.move' instead of 'os.rename' for move over filesystem. # +# 2025-02, Arjo Segers +# Added 'CSO_EarthAccess_Download_Listing' class. +# +# 2025-04, Arjo Segers +# Changed imports for python packaging. +# ######################################################################## ### @@ -34,6 +40,7 @@ The classes and are defined according to the following hierchy: * :py:class:`.CSO_EarthAccess_Inquire` * :py:class:`.CSO_EarthAccess_Download` + * :py:class:`.CSO_EarthAccess_Download_Listing` Classes @@ -66,7 +73,7 @@ import utopya class CSO_EarthAccess_Inquire(utopya.UtopyaRc): """ - Create *listing* table (csv file) with on each line the location and information on + Create *listing* table (csv file) with on each line the location of and information on a data file available via the `EarthAccess `_ package. As example, a file with VIIRS AOD could be available as:: @@ -115,9 +122,9 @@ class CSO_EarthAccess_Inquire(utopya.UtopyaRc): `VIIRS Aerosol `_ page:: .dataset : AERDB_L2_VIIRS_SNPP - .dataset : AERDB_L2_VIIRS_NOAA20 - .dataset : AERDT_L2_VIIRS_SNPP - .dataset : AERDT_L2_VIIRS_NOAA20 + !.dataset : AERDB_L2_VIIRS_NOAA20 + !.dataset : AERDT_L2_VIIRS_SNPP + !.dataset : AERDT_L2_VIIRS_NOAA20 Eventually specify a target area, only orbits with some pixels within the defined box will be downloaded:: @@ -125,7 +132,7 @@ class CSO_EarthAccess_Inquire(utopya.UtopyaRc): .area : !.area : -30,30,35,76 - Name of output csv file:: + Name of the output csv file:: ! output table, here including date of today: .output.file : ${my.work}/AERDB_inquiry_%Y-%m-%d.csv @@ -678,13 +685,214 @@ class CSO_EarthAccess_Download(utopya.UtopyaRc): # info ... logging.info(f"{indent}") - logging.info(f"{indent}** end convert") + logging.info(f"{indent}** end download") + logging.info(f"{indent}") + + # enddef __init__ + + +# endclass CSO_EarthAccess_Download + + +######################################################################## +### +### create listing file for downloaded VIIRS files +### +######################################################################## + + +class CSO_EarthAccess_Download_Listing(utopya.UtopyaRc): + + """ + Create *listing* file for files downloaded from VIIRS data portals. + + A *listing* file contains the names of the converted orbit files, + the time range of pixels in the file, and other information extracted from the filenames or file attributes:: + + filename ;start_time ;end_time ;orbit + 2023/268/AERDB_L2_VIIRS_SNPP.A2023268.0112.002.2023268134001.nc;2023-09-25 00:00:00;2023-09-26 00:00:00;61711 + 2023/268/AERDB_L2_VIIRS_SNPP.A2023268.0248.002.2023268152045.nc;2023-09-25 00:00:00;2023-09-26 00:00:00;61712 + 2023/268/AERDB_L2_VIIRS_SNPP.A2023268.0254.002.2023268154044.nc;2023-09-25 00:00:00;2023-09-26 00:00:00;61712 + 2023/268/AERDB_L2_VIIRS_SNPP.A2023268.0430.002.2023268170054.nc;2023-09-25 00:00:00;2023-09-26 00:00:00;61713 + : + + This file could be used to scan for available files. + + In the settings, define the name of the file to be created:: + + ! create listing of downloaded files; + ! eventully include time templates %Y-%m-%d etc: + .file : /Scratch/Copernicus/VIIRS/listing.csv + + Optionally define a creation mode for the (parent) directories:: + + ! directory creation mode: + .dmode : 0o775 + + An existing listing file is not replaced, + unless the following flag is set:: + + ! renew table? + .renew : True + + Specify filename filters to search for data files: + + .pattern : AER*.nc + + """ + + def __init__(self, rcfile, rcbase="", env={}, indent=""): + """ + Convert data. + """ + + # modules: + import os + import datetime + import fnmatch + import collections + + # tools: + import cso_file + + # info ... + logging.info(f"{indent}") + logging.info(f"{indent}** create listing file") + logging.info(f"{indent}") + + # init base object: + utopya.UtopyaRc.__init__(self, rcfile=rcfile, rcbase=rcbase, env=env) + + # directory creation mode: + dmode = self.GetSetting("dmode", totype="int", default=None) + + # renew output? + renew = self.GetSetting("renew", totype="bool") + + # table file to be written: + lst_file = self.GetSetting("file") + # evaluate current time: + lst_file = datetime.datetime.now().strftime(lst_file) + + # create? + if (not os.path.isfile(lst_file)) or renew: + # info .. + logging.info(f"{indent}create %s ..." % lst_file) + + # pattern for data files: + fpattern = self.GetSetting("pattern") + # info .. + logging.info(f"{indent} scan for datafiles: {fpattern}") + + # path to listing files, data files are search relative to this: + bdir = os.path.dirname(lst_file) + # current directory? + if len(bdir) == 0: + bdir = "." + # info ... + logging.info(f"{indent} scan base directory: %s ..." % bdir) + + # create directory if necessary: + cso_file.CheckDir( lst_file, dmode=dmode ) + + # initiallize for (re)creation: + listing = cso_file.CSO_Listing(indent=f"{indent} ") + + # keep scanned roots for progress info: + subdirs = [] + + # recursively search for files: + for root, dirs, files in os.walk(bdir): + # loop over files: + for fname in files: + + # subdir relative to listing file: + subdir = os.path.relpath(root, start=bdir) + # info ... + if subdir not in subdirs : + # info ... + logging.info(f"{indent} {subdir} ...") + # store: + subdirs.append(subdir) + #endif + + ## testing .. + #if subdir != "2022/007": + # #logging.warning(f"{indent} skip ...") + # continue + ##endif + + # data file? + if fnmatch.fnmatch(fname, fpattern): + + # expected filenames: + # AERDB_L2_VIIRS_SNPP.A2022001.0342.002.2023076013614.nc + parts = fname.split(".") + if len(parts) == 6: + + # second is year-julday, strip the "A" of acquisition: + try: + t1 = datetime.datetime.strptime(parts[1][1:],"%Y%j") + except: + logging.error(f"could not extract date from '{parts[1]}'") + raise Exception + #endtry + + # end time: + t2 = t1 + datetime.timedelta(1) + + else : + logging.error(f"unsupported filename: {fname}") + raise Exception + # endif + + # open for extra info: + sfile = cso_file.CSO_File( os.path.join(root,fname) ) + # extract attributes: + orbit = sfile.GetAttr( "OrbitNumber" ) + # done: + sfile.Close() + + # fill data record: + data = collections.OrderedDict() + data["start_time"] = t1 + data["end_time"] = t2 + data["orbit"] = orbit + + # update record: + listing.UpdateRecord( os.path.join(subdir,fname), data, indent=f"{indent} ") + + # endfor # filename match + + # endfor # filenames + + ## testing ... + #if len(listing) > 10 : + # break + + # endfor # walk over subdirs/files + + # adhoc .. + listing.df = listing.df.astype( { "orbit" : int } ) + # sort on filename: + listing.Sort( by="orbit" ) + # save: + listing.Save(lst_file, dmode=dmode, indent=f"{indent} ") + + else: + # info .. + logging.info(f"{indent}keep %s ..." % lst_file) + # endif + + # info ... + logging.info(f"{indent}") + logging.info(f"{indent}** end listing") logging.info(f"{indent}") # enddef __init__ -# endclass CSO_S5p_Download +# endclass CSO_EarthAccess_Download_Listing ######################################################################## -- GitLab From 53511d8b89ca6132cba2a923b22a5dfd97a0ba34 Mon Sep 17 00:00:00 2001 From: Arjo Segers Date: Wed, 4 Jun 2025 16:08:29 +0200 Subject: [PATCH 7/7] Updated description of VIIRS processing. --- config/VIIRS/cso-user-settings.rc | 112 ++ config/VIIRS/cso-viirs.rc | 416 +++++ config/VIIRS/cso.rc | 182 ++ .../figs/VIIRS/viirs1-aod-db_inquire.png | Bin 0 -> 12471 bytes doc/source/gridding.rst | 4 +- doc/source/obsoper.rst | 2 +- doc/source/pymod-cso_viirs.rst | 5 + doc/source/pymods.rst | 1 + doc/source/s5p-chocho.rst | 10 +- doc/source/s5p-co.rst | 6 +- doc/source/s5p-hcho.rst | 8 +- doc/source/s5p-no2.rst | 10 +- doc/source/s5p-o3.rst | 8 +- doc/source/s5p-so2-cobra.rst | 6 +- doc/source/s5p-so2.rst | 8 +- doc/source/tutorial.rst | 93 +- doc/source/viirs-aod.rst | 92 +- pyproject.toml | 1 + src/cso/__init__.py | 11 + src/cso/cso_earthaccess.py | 32 +- src/cso/cso_file.py | 10 +- src/cso/cso_viirs.py | 1487 +++++++++++++++++ 22 files changed, 2443 insertions(+), 61 deletions(-) create mode 100644 config/VIIRS/cso-user-settings.rc create mode 100644 config/VIIRS/cso-viirs.rc create mode 100644 config/VIIRS/cso.rc create mode 100644 doc/source/figs/VIIRS/viirs1-aod-db_inquire.png create mode 100644 doc/source/pymod-cso_viirs.rst create mode 100644 src/cso/cso_viirs.py diff --git a/config/VIIRS/cso-user-settings.rc b/config/VIIRS/cso-user-settings.rc new file mode 100644 index 0000000..9616ec9 --- /dev/null +++ b/config/VIIRS/cso-user-settings.rc @@ -0,0 +1,112 @@ +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!! +!!! CSO common configuration +!!! +!!! Base settings that are used by multiple tasks: +!!! - time range(s) +!!! - target domain(s) +!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + +!----------------------------------------------------------- +! id's +!----------------------------------------------------------- + +! file format: +my.cso.format : 1.0 + +! file format convention: +my.cso.convention : CF-1.7 + + +!----------------------------------------------------------- +! domain +!----------------------------------------------------------- + +! +! Used for: +! - orbit selection durning download +! - pixel selection +! - map plots +! + +! region name: +my.region : CAMS +! CAMS regional ensemble domain: +my.region.west : -30.0 +my.region.east : 45.0 +my.region.south : 30.0 +my.region.north : 76.0 +! size of map figures for this region: +my.region.figsize : (6,6) + +!! region name: +!my.region : globe +!! global domain: +!my.region.west : -180.0 +!my.region.east : 180.0 +!my.region.south : -90.0 +!my.region.north : 90.0 +!! size of map figures for this region: +!my.region.figsize : (8,6) + + +!---------------------------------------------------------- +! timerange +!---------------------------------------------------------- + +! inquire full timerange: +my.inquire.timerange.start : 2011-10-01 00:00 +my.inquire.timerange.end : 2024-12-31 23:59 + +! testing 3 days +my.timerange.start : 2018-06-01 00:00 +my.timerange.end : 2018-06-03 23:59 + + +!---------------------------------------------------------- +! user specific settings: +!---------------------------------------------------------- + +! Attributes written to output files. +my.attr.author : Your Name +my.attr.institution : CSO +my.attr.email : Your.Name@cso.org + +! base location for work directories: +!my.work : /work/${USER}/CSO-Tutorial +my.work : /Scratch/${USER}/CSO-VIIRS + + +!---------------------------------------------------------- +! job step defaults +!---------------------------------------------------------- + +! run jobs in foreground: +*.script.class : utopya.UtopyaJobScriptForeground + +! dummy value, will be defined when running in virtrual environment: +VIRTUAL_ENV : +! running in virutual envionment? +#if "${VIRTUAL_ENV}" > "" +! interpretor from virtual enviornment, the system path is set automatically: +*.shell : ${VIRTUAL_ENV}/bin/python3 +#else +! search path for python modules: +*.pypath : ${CSO_PREFIX}/src:${CSO_PREFIX}/src/utopya:${CSO_PREFIX}/src/cso +#endif + +! work directory for jobs; +*.workdir : ${my.work}/__NAME2PATH__ + +! for new job files, use jobtree settings from this file: +*.rcfile : ${__filename__} + + + +!---------------------------------------------------------- +! end +!---------------------------------------------------------- + + diff --git a/config/VIIRS/cso-viirs.rc b/config/VIIRS/cso-viirs.rc new file mode 100644 index 0000000..52a02fa --- /dev/null +++ b/config/VIIRS/cso-viirs.rc @@ -0,0 +1,416 @@ +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!! +!!! CSO - CAMS Satellite Operator +!!! +!!! Settings for VIIRS processing. +!!! +!!! Environment: +!!! +!!! MY_PRODUCT : defines the VIIRS product, one of: +!!! viirs1-aod-db viirs1-aod-dt +!!! viirs2-aod-db viirs2-aod-dt +!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + +!----------------------------------------------------------------------- +! user specific settings: +!----------------------------------------------------------------------- + +! include user specfic settings: +#include cso-user-settings.rc + + +!----------------------------------------------------------------------- +! multi-product +!----------------------------------------------------------------------- + +! instrument selection: +#if "${MY_PRODUCT}" in ["viirs1-aod-db","viirs1-aod-dt"] +my.instrument : VIIRS-1 +my.platform : SNPP +#elif "${MY_PRODUCT}" in ["viirs2-aod-db","viirs2-aod-dt"] +my.instrument : VIIRS-2 +my.platform : NOAA20 +#else +#error unsupported product '${MY_PRODUCT}' +#endif + +! product selection: +#if "${MY_PRODUCT}" in ["viirs1-aod-db","viirs2-aod-db"] +!~ "Deep Blue" products: +my.retrieval : AERDB +#elif "${MY_PRODUCT}" in ["viirs1-aod-dt","viirs2-aod-dt"] +!~ "Dark Target" products: +my.retrieval : AERDT +#else +#error unsupported product '${MY_PRODUCT}' +#endif + +! version selection: +my.version : v2.0.0 + +! local archive: +my.arch.dir : ${my.work}/EarthData/${my.instrument}/${my.retrieval}_L2 + + +!====================================================================== +!=== +!=== Inquire +!=== +!====================================================================== + + +!----------------------------------------------------------------------- +! inquire EarthData +!----------------------------------------------------------------------- + +! Obtain names of all available VIIRS files. +! Stored as csv with processing type, processor version, filenames, etc. + +! renew table if file already exists? +cso.inquire-table-earthaccess.renew : True + +! full time range: +cso.inquire-table-earthaccess.timerange.start : ${my.inquire.timerange.start} +cso.inquire-table-earthaccess.timerange.end : ${my.inquire.timerange.end} +!! ... TESTING +!cso.inquire-table-earthaccess.timerange.start : 2022-01-01 12:00:00 +!cso.inquire-table-earthaccess.timerange.end : 2022-01-01 15:00:00 + +!~ dataset name: +cso.inquire-table-earthaccess.dataset : ${my.retrieval}_L2_VIIRS_${my.platform} + +!~ domain specified as: west,south,east,north +cso.inquire-table-earthaccess.area : ${my.region.west},${my.region.south},${my.region.east},${my.region.north} + +! csv file that will hold records per file with: +! - timerange of pixels in file +! - orbit number +cso.inquire-table-earthaccess.output.file : ${my.arch.dir}/inquire/inquire.csv + + + +!----------------------------------------------------------- +! overview plot of versions +!----------------------------------------------------------- + +! renew existing plots? +cso.inquire-plot.renew : True + +! listing files: +cso.inquire-plot.file : ${cso.inquire-table-earthaccess.output.file} +!cso.inquire-plot.filedate : 2023-08-07 + +! annote: +cso.inquire-plot.title : VIIRS-AOD + +! output figure, eventually use templates for time values: +cso.inquire-plot.output.file : ${my.arch.dir}/inquire/inquire.png + + +!====================================================================== +!=== +!=== download +!=== +!====================================================================== + + +! renew existing files (True|False) ? +cso.download.renew : False +!! .. TESTING ... +!cso.download.renew : True + +! time range: +cso.download.timerange.start : ${my.timerange.start} +cso.download.timerange.end : ${my.timerange.end} + +! listing of available source files crated by "inquire" job: +cso.download.inquire.file : ${my.arch.dir}/inquire/inquire.csv +!!~ historic inquire ... +!cso.download.inquire.filedate : 2025-02-01 + +! processor version: +#if "${my.version}" == "v2.0.0" +cso.download.processor_version : 020000 +#else +#error unsupported my.version "${my.version}" +#endif + +! target directory, includiong time values: +cso.download.dir : ${my.arch.dir}/${my.version}/%Y/%j + + + +!----------------------------------------------------------- +! listing of downloaded files +!----------------------------------------------------------- + + +! renew existing file (True|False) ? +cso.download-listing.renew : True + +! create listing of downloaded files; +! eventully include time templates %Y-%m-%d etc: +cso.download-listing.file : ${my.arch.dir}/${my.version}/listing.csv + +! filename pattern for files in subdirs: +cso.download-listing.pattern : AER*.nc + + + +!====================================================================== +!=== +!=== convert (and download) +!=== +!====================================================================== + + +! renew existing files (True|False) ? +!cso.convert.renew : True +! ... TESTING ... +cso.convert.renew : False + +! time range: +cso.convert.timerange.start : ${my.timerange.start} +cso.convert.timerange.end : ${my.timerange.end} + + +!~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +! input files +!~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +! listing file with input data: +cso.convert.input.listing : ${cso.download-listing.file} + +! selection names: +cso.convert.filters : lons lats valid quality + +! filter settings: +cso.convert.filter.lons.type : minmax +cso.convert.filter.lons.minmax : ${my.region.west} ${my.region.east} +cso.convert.filter.lons.var : Longitude +cso.convert.filter.lons.units : degrees_east + +! filter settings: +cso.convert.filter.lats.type : minmax +cso.convert.filter.lats.minmax : ${my.region.south} ${my.region.north} +cso.convert.filter.lats.var : Latitude +cso.convert.filter.lats.units : degrees_north + +! skip pixel with "no data", use the combined land/ocean variable to check: +cso.convert.filter.valid.var : Aerosol_Optical_Thickness_550_Land_Ocean_Best_Estimate +cso.convert.filter.valid.type : valid + +! Descriptions in variables: +! Aerosol_Optical_Thickness_QA_Flag_Land :long_name = "Deep Blue quality assurance flag over land. 0=no retrieval, 1=poor, 2=moderate, 3=good" ; +! Aerosol_Optical_Thickness_QA_Flag_Ocean:long_name = "SOAR quality assurance flag over water. 0=no retrieval, 1=poor, 3=good" ; +! compare for both variables, at least one should have a good quality pixel: +cso.convert.filter.quality.type : min2 +cso.convert.filter.quality.min : 3 +cso.convert.filter.quality.var : Aerosol_Optical_Thickness_QA_Flag_Land +cso.convert.filter.quality.var2 : Aerosol_Optical_Thickness_QA_Flag_Ocean +cso.convert.filter.quality.units : 1 + + + +!~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +! output files +!~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +! output directory and filename: +! - time values taken from start time +! - templates: %{orbit} +cso.convert.output.filename : ${my.work}/${MY_PRODUCT}/data/${my.region}/%Y/%m/${MY_PRODUCT}_%{orbit}.nc + +! pack variables on output: +cso.convert.output.packed : True +! zlib compression level, 0 for no compression: +cso.convert.output.complevel : 1 + +! global attributes: +cso.convert.output.attrs : format Conventions \ + author institution email +! +cso.convert.output.attr.format : ${my.cso.format} +cso.convert.output.attr.Conventions : ${my.cso.convention} +cso.convert.output.attr.author : ${my.attr.author} +cso.convert.output.attr.institution : ${my.attr.institution} +cso.convert.output.attr.email : ${my.attr.email} + +!! no need to swap layes: +!cso.convert.swap_layers : False + + +! ~ variables + +! +! Describe per variable: +! * .dims : dimensions list: +! pixel : selected pixels +! corner : number of footprint bounds (probably 4) +! layer : number of layers in atmospheric profile (layers in kernel) +! layeri : number of layer interfaces in atmospheric profile (layer+1) +! retr : number of layers in retrieval product (1 for columns) ; +! for error covariance use (retr,retr0) to avoid repeated dimensions +! track_scan : original scan index in 2D track +! track_pixel : original ground pixel in 2D track +! * .specal : keyword to select special processing +! * None : no special processing (default) +! * track_longitude : longitudes at centers of original 2D track +! * track_latitude : latitudes at centers of original 2D track +! * .units : target units if different from original +! * .oper : special postprocessing, currently supported: +! * square : fill variable with squared valued (used to form variance from standard deviation) +! In case no special processing is needed: +! * .from : original variable (group path and variable name) +! + +! which fields to be put out ? +cso.convert.output.vars : longitude longitude_bounds \ + latitude latitude_bounds \ + track_longitude track_longitude_bounds \ + track_latitude track_latitude_bounds \ + time \ + aot_412nm aot_488nm aot_550nm aot_670nm aot_865nm aot_1240nm aot_1640nm aot_2250nm + +! qa_value \ +! vcd vcd_errvar +! +!! vertical column density: +!cso.convert.output.var.vcd.dims : pixel retr +!cso.convert.output.var.vcd.from : PRODUCT/nitrogendioxide_tropospheric_column +!cso.convert.output.var.vcd.attrs : { 'ancillary_variables' : None, 'multiplication_factor_to_convert_to_molecules_percm2' : None } +! +!! error variance in vertical column density (after application of kernel), +!! fill with single element 'covariance matrix', from square of standard error: +!! use dims with different names to avoid that cf-checker complains: +!cso.convert.output.var.vcd_errvar.dims : pixel retr retr0 +!cso.convert.output.var.vcd_errvar.special : square +!cso.convert.output.var.vcd_errvar.from : PRODUCT/nitrogendioxide_tropospheric_column_precision_kernel +!!~ skip standard name, modifier "standard_error" is not valid anymore: +!cso.convert.output.var.vcd_errvar.attrs : { 'standard_name' : None, 'multiplication_factor_to_convert_to_molecules_percm2' : None } + +! centers of pixels; +! use special processing to ensure correction at swath edges; +! remove range, gives problems when packed ... +!~ longitudes +cso.convert.output.var.longitude.dims : pixel +cso.convert.output.var.longitude.special : longitude +cso.convert.output.var.longitude.from : Longitude +cso.convert.output.var.longitude.units : degrees_east +cso.convert.output.var.longitude.attrs : { 'valid_range' : None } +!~ latitudes +cso.convert.output.var.latitude.dims : pixel +cso.convert.output.var.latitude.special : latitude +cso.convert.output.var.latitude.from : Latitude +cso.convert.output.var.latitude.units : degrees_north +cso.convert.output.var.latitude.attrs : { 'valid_range' : None } + +! corners of pixels, interpolate/extrapolate from centers; +! use special processing to ensure correction at swath edges; +! remove range, gives problems when packed ... +!~ longitudes +cso.convert.output.var.longitude_bounds.dims : pixel corner +cso.convert.output.var.longitude_bounds.special : longitude_bounds +cso.convert.output.var.longitude_bounds.from : Longitude +cso.convert.output.var.longitude_bounds.units : degrees_east +cso.convert.output.var.longitude_bounds.attrs : { 'valid_range' : None } +!~ latitudes +cso.convert.output.var.latitude_bounds.dims : pixel corner +cso.convert.output.var.latitude_bounds.special : latitude_bounds +cso.convert.output.var.latitude_bounds.from : Latitude +cso.convert.output.var.latitude_bounds.units : degrees_north +cso.convert.output.var.latitude_bounds.attrs : { 'valid_range' : None } + +! original track: +! use special processing to ensure correction at swath edges; +! remove range, gives problems when packed ... +!~ longitudes +cso.convert.output.var.track_longitude.dims : track_scan track_pixel +cso.convert.output.var.track_longitude.special : track_longitude +cso.convert.output.var.track_longitude.from : Longitude +cso.convert.output.var.track_longitude.attrs : { 'valid_range' : None } +!~ latitudes +cso.convert.output.var.track_latitude.dims : track_scan track_pixel +cso.convert.output.var.track_latitude.special : track_latitude +cso.convert.output.var.track_latitude.from : Latitude +cso.convert.output.var.track_latitude.attrs : { 'valid_range' : None } +!~ corner lons +cso.convert.output.var.track_longitude_bounds.dims : track_scan track_pixel corner +cso.convert.output.var.track_longitude_bounds.special : track_longitude_bounds +cso.convert.output.var.track_longitude_bounds.from : Longitude +cso.convert.output.var.track_longitude_bounds.units : degrees_east +!~ corner lats +cso.convert.output.var.track_latitude_bounds.dims : track_scan track_pixel corner +cso.convert.output.var.track_latitude_bounds.special : track_latitude_bounds +cso.convert.output.var.track_latitude_bounds.from : Latitude +cso.convert.output.var.track_latitude_bounds.units : degrees_north + +! time based on TAI93 units:! use time-delta array for shape: +cso.convert.output.var.time.dims : pixel +cso.convert.output.var.time.special : time +cso.convert.output.var.time.from : Scan_Start_Time + +! AOT, merged product: +cso.convert.output.var.aot_550nm.dims : pixel +cso.convert.output.var.aot_550nm.from : Aerosol_Optical_Thickness_550_Land_Ocean_Best_Estimate +cso.convert.output.var.aot_550nm.attrs : { 'valid_range' : None } + +! AOT from land band only: +#for WVL in 412 488 670 865 1240 1640 2250 +cso.convert.output.var.aot_WVLnm.dims : pixel +cso.convert.output.var.aot_WVLnm.special : wavelength=WVL +#if WVL in [412] +cso.convert.output.var.aot_WVLnm.from : Spectral_Aerosol_Optical_Thickness_Land +cso.convert.output.var.aot_WVLnm.attrs : { 'valid_range' : None, 'long_name' : 'aerosol optical thickness at WVL nm over land' } +#elif WVL in [488,670] +cso.convert.output.var.aot_WVLnm.from : Spectral_Aerosol_Optical_Thickness_Land \ + Spectral_Aerosol_Optical_Thickness_Ocean +cso.convert.output.var.aot_WVLnm.attrs : { 'valid_range' : None, 'long_name' : 'aerosol optical thickness at WVL nm over land and water' } +#elif WVL in [865,1240,1640,2250] +cso.convert.output.var.aot_WVLnm.from : Spectral_Aerosol_Optical_Thickness_Ocean +cso.convert.output.var.aot_WVLnm.attrs : { 'valid_range' : None, 'long_name' : 'aerosol optical thickness at WVL nm over water' } +#else +#error unsupported wavelength WVL +#endif +#endfor + + + +!!====================================================================== +!!=== +!!=== listing +!!=== +!!====================================================================== +! +!! csv file that will hold records per file with: +!! - timerange of pixels in file +!! - orbit number +!cso.listing.file : ${my.work}/_PRODUCT_/data/${my.region}/${my.selection}__listing.csv +! +!! renew table if file already exists? +!cso.listing.renew : True +! +!! time range: +!cso.listing.timerange.start : ${my.timerange.start} +!cso.listing.timerange.end : ${my.timerange.end} +! +!! filename filters relative to listing file that should be scanned for orbit files; +!! names could include time templates ; +!! if same orbit is found in multiple directories, the first found is used; +!! remove existing table for safety to ensure that this is done correctly ... +!cso.listing.patterns : ${my.selection}/%Y/%m/_PRODUCT__*.nc +! +!! extra columns to be added, read from global attributes: +!cso.listing.xcolumns : orbit +! +! + + +!====================================================================== +!=== +!=== end +!=== +!====================================================================== + diff --git a/config/VIIRS/cso.rc b/config/VIIRS/cso.rc new file mode 100644 index 0000000..a633c0f --- /dev/null +++ b/config/VIIRS/cso.rc @@ -0,0 +1,182 @@ +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!! +!!! CSO - CAMS Satellite Operator +!!! +!!! Settings for project. +!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +! dummy values for variables defined in 'cso' script, +! these will be replaced by the actual values from the environment, +! but needed for first evaluation of this file ... +CSO_PREFIX : /not/defined/yet +CSO_RCFILE : /not/defined/yet +CSO_RCDIR : /not/defined/yet + + +!---------------------------------------------------------- +! user specific settings: +!---------------------------------------------------------- + +! include user specfic settings: +#include cso-user-settings.rc + + + +!---------------------------------------------------------- +! job tree +!---------------------------------------------------------- + +! class to create a job tree: +cso.class : utopya.UtopyaJobTree +! top level job is virtual: +cso.virtual : True +! list of sub-elements: +!cso.elements : copy viirs1-aod-db viirs2-aod-db viirs1-aod-dt viirs2-aod-dt +cso.elements : copy viirs1-aod-db viirs2-aod-db + + +! ====================================================================== +! === +! === copy to workdir +! === +! ====================================================================== + +! no sub list: +cso.copy.class : utopya.UtopyaJobStep + +! default class is defined in machine specific settings; +! copy is always done in foreground to avoid that files are +! changed before the job is started: +cso.copy.script.class : utopya.UtopyaJobScriptForeground + +!! search path to utopya modules: +!cso.copy.pypath : ${CSO_PREFIX}/py + +! configure and build based on settings in this rcfile; +! the name of this file has been stored in the +! environment variable 'CSO_RCFILE' by the 'cso' script: +cso.copy.task.class : utopya.UtopyaCopy +cso.copy.task.args : '${CSO_RCFILE}', rcbase='cso' + + +! +! * config of "UtopyaCopy" +! + + +! no need to remove existing build directory if present ... +cso.copy.new : False + +! prefix for destination of source and script files +! (base path for subdirectories src, py, etc): +cso.copy.prefix : ${my.work} + +! no extension to build directory need (no objects created) +cso.copy.prefix.extensions : + +! copy will consist of following subdirs: +cso.copy.subdirs : rc + +! directories to be inlcuded in copy, +! otherwise only files are copied: +cso.copy.incdirs : + +! skip files matching these filename patterns +! (tempoary editor files, compiled modules, etc) +cso.copy.skip : .#* *~ *.pyc .DS* + +! list of source directories to be copied; +! here empty, only settings files will be copied: +cso.copy.dirs : +! copy settings to "rc": +cso.copy.rc.dirs : ${CSO_RCDIR} + +! write evaluated rcfile: +cso.copy.rcwrite : ${my.work}/cso.rc + + + + + +! ====================================================================== +! === +! === viirs products +! === +! ====================================================================== + +#for _PRODUCT_ in viirs1-aod-db viirs2-aod-db viirs1-aod-dt viirs2-aod-dt + + +! class to create a job tree: +cso._PRODUCT_.class : utopya.UtopyaJobTree + +!~ sub steps: +!cso._PRODUCT_.elements : inquire download download-listing convert +!~ one by one ... +!cso._PRODUCT_.elements : inquire +!cso._PRODUCT_.elements : download +!cso._PRODUCT_.elements : inquire download +cso._PRODUCT_.elements : download-listing +!cso._PRODUCT_.elements : convert + +! inquire tasks: +!cso._PRODUCT_.inquire.tasks : table-earthaccess plot +cso._PRODUCT_.inquire.tasks : table-earthaccess +!cso._PRODUCT_.inquire.tasks : plot + + + +! single step: +cso._PRODUCT_.inquire.class : utopya.UtopyaJobStep +!~ inquire files downloaded from EarthAccess: +cso._PRODUCT_.inquire.table-earthaccess.class : cso.CSO_EarthAccess_Inquire +cso._PRODUCT_.inquire.table-earthaccess.args : '${my.work}/rc/cso-viirs.rc', \ + rcbase='cso.inquire-table-earthaccess', \ + env={ 'MY_PRODUCT' : '_PRODUCT_' } +!~ create plot of available versions: +cso._PRODUCT_.inquire.plot.class : cso.CSO_Inquire_Plot +cso._PRODUCT_.inquire.plot.args : '${my.work}/rc/cso-viirs.rc', \ + rcbase='cso.inquire-plot', \ + env={ 'MY_PRODUCT' : '_PRODUCT_' } + +!~ download data: +! single step: +cso._PRODUCT_.download.class : utopya.UtopyaJobStep +! download task: +cso._PRODUCT_.download.task.class : cso.CSO_EarthAccess_Download +cso._PRODUCT_.download.task.args : '${my.work}/rc/cso-viirs.rc', \ + rcbase='cso.download', \ + env={ 'MY_PRODUCT' : '_PRODUCT_' } +! single step: +cso._PRODUCT_.download-listing.class : utopya.UtopyaJobStep +! download task: +cso._PRODUCT_.download-listing.task.class : cso.CSO_EarthAccess_Download_Listing +cso._PRODUCT_.download-listing.task.args : '${my.work}/rc/cso-viirs.rc', \ + rcbase='cso.download-listing', \ + env={ 'MY_PRODUCT' : '_PRODUCT_' } + +! * + +! single step: +cso._PRODUCT_.convert.class : utopya.UtopyaJobStep +! convert task: +cso._PRODUCT_.convert.task.class : cso.CSO_VIIRS_Convert +cso._PRODUCT_.convert.task.args : '${my.work}/rc/cso-viirs.rc', \ + rcbase='cso.convert', \ + env={ 'MY_PRODUCT' : '_PRODUCT_' } +!! single step: +!cso._PRODUCT_.listing.class : utopya.UtopyaJobStep +!! listing task: +!cso._PRODUCT_.listing.task.class : cso.CSO_S5p_Listing +!cso._PRODUCT_.listing.task.args : '${my.work}/rc/cso-s5p-ALL.rc', \ +! rcbase='cso._PRODUCT_.listing' +! +#endfor + + +! ====================================================================== +! === +! === end +! === +! ====================================================================== diff --git a/doc/source/figs/VIIRS/viirs1-aod-db_inquire.png b/doc/source/figs/VIIRS/viirs1-aod-db_inquire.png new file mode 100644 index 0000000000000000000000000000000000000000..6f0b8b815f4bc5d5c944eda37815f53b0eec6341 GIT binary patch literal 12471 zcmeAS@N?(olHy`uVBq!ia0y~yVA;UHz&L?}je&t--`1IN3=9mGC9V-A!TD(=<%vb9 z48Dma1v&X8IhjccWvNBQnfZCfdKP*HItm#jB?ZM+`ue$W1$y~K>H4M{x2|SjVBjq9 zh%9Dc;7~!S|0G78i>MOG&4cVbh5x2W}WnJlKEZhJ%5!qO+^e zB(GOB_Lo+wr3p-m-d+|uFMaK9k8?M_-`RWLYU-0ccdAz{T(wGzfq{V`V1?aXTSgG; zh6Jk^h;|K50PzD%DCAyH-t+fuzJGCXv6ybumV<`sa|)bh&6;IY{_c*6iOG{Km;GMf z`F=#$f62myisEqrAQU7#8 zxqr_JOD2XXPK~S#UpR!-d`=Xfx1BEk=K=fcyZb9XKKk=S-TvgmcKLaMi{19FNUs0b z9e>95`<=-k$N&E*|9|4{_xr5bWeObr{oen-dTn9)+|sZ<>vuce?D>3d=bulfKc9~O zXB7YAkhoFSl@)*9q}x{quCF{Q8m{6grV}w?{=YBFU(0^}egD7R?zh{b>%MKC|1??t zcSPR4pU+}SuZG^NdcF4ZyzhI|kIRF@h`dw1E= z5H;UfQ?BoMX1eo16L;Jd8%74N3)^?AGcbH`+VfO5|KyF0$;D5nhCg$b|8>FY_xj!M zcIiZK(-BtlsknH&?&sU({xW? z-}i0noRUkPKhKr#+kEY_zva^@U&TD7jndD}@tJ31$u3(G@NVDlcQJ)WMJ>PGNWLz$ zIpg9Ync6QGGcz+!)_%XcJxufaw(Wa$L7uex`_e!C``z;M%kBTZ+&XXf+ikbY)~S2) zS-;sZr|#EF%U>@R7eAYs{_N`dy3%`}rfuKz;3pG9iTV2jObk^m$BrGFQ}gNMOrumU zxvCcnOTYQgu`osMUt9%-D`~R2a z_D}!)et#Yme#ed-J6kvRxO}~h=clglJ&THOpO~n;`Ej51v)T9mr0MPZ@yO=ai^Z?+ zG{0)y_34zh_33Tj_kG{{^2Li2q2aMxug!?g-)q|2+Y8DQ*Q$Hk+S+6)pG=J0kl?t? zXXdA-xbinQHr`C`w=Md9w>*w?lr}GK+d-8>urAhZ2SK|&(BT0?Z_@WMYimQqOiJO%xekbv@<8dqI0KC-}6LO z+U6Rc^_vTp-&$|w?S8wd=I1A!hz$=|?b#U??AF}C&>)w3sD*Q;eSIA$wW>t!EJ_8r zaL+^D{1d0+|9$ecdbaVn-0Wq`mhs3~82o;_J^t3MTc5h)|0p@P^TlT0c^&tCm(8~u z$vb~On_c|vW_tSk+HWUwx8I#M+0RPTU9K`EopEc{)m3X^c5eFo@z3w%wL2?+&njB8 zaG~PzIYnOcYCfI3Q+7L7KkHt0-1>zJAG(6vr5>2T$Y6WQ`u@*zTefUTd32=HbJEGl z>gn(9?3`Wl`J3s!PgB=FX=Il>5nFzDYW@H3_2=*GEH1th=$`uX)6>^Ixmj7K6x(Gc zeLin*zx_^;_q?iCD`hGkH12%4Z1yKm#`pZR%zUq=udnaie$&*elT^JwEtdbQvA+KA z>&;igqEjy}a*f=Q5qR(4x9!UpE?l_zj#t?Ij=wLL&p)^Myxs1DmG&Qx2zyQ{e0*&0 zgROQypGV7@Jh<_v`E1c%`?U+gSH$^8A#Chgv1g zaw7QbemwZ|Dt!N`^Y#Be@A-ZAeO!7@Xy{Cn%%C=Y`FXbAZaijPTm0v!#LBiR-`e!rK(~irQpP9b@ z&(m{CtM=}-J#X{*Ox5xK`~QFcKWWWA&#Fmpwp{kRS$y8sTK;za{@O`b<>h5;Dhl#; zzuk7H@VIQYt=^6YO*--WY_7*u=T3TZCD`9LJ3ITF)$289W}D~Fy$*8rEO+@@lk|C& zX|ra{`uA|wG~MWF!Tz?Upi*LH`n;cuj2RhT`2D*s&2ZuIo1M?+-OSy7_tVz(wYs69 zxz~=fUthj*rDpt}N8+E?mG9M#|NrZH@j1)qDIXsly;=ABZSfhy<4?}!*A*Y@m7c!; z@7w&HpU+u;{<^;2*82UP;y$}yFL>oXKWvvj_xJn#`TTVsnk`=}XufuH|NXk(PZxFT zoe~X?nFy-SKm~HI`Mnv3`R#40-)wZZ`?}Kqspa^e{w5({aH}P@vnUq z{8}daSdZk+@As-NU%WW+Xt#L!t1ByaembT7`N?GeXCPyLe0-ceOPs+W{4<{p!vgnN z28m9qR-Mw@{bthMZ?{%IJ8u7MM)IcXan(;Bb?cvdbhO(z<-~-VZ=2_z1tok?uKV-j zW94z{dlP5p?b4M0|Ks@Q-S_|IEq>1wU0lEV`~LrbU*Eld<;s*`KTFli%Y0AE|NGFc zvf6tsyL^p7e)a9!pFjcd_;~;MPW5>+md~rwx*k*PyKeWpU0+^qerNw&Z||2&Z#JK| zyIKGLZ}A~c^(V)q^Ur_^BKvJ0VW0p zh6Nd1Iv`tH&)23=9qvtIqh|xqjz!)`J6$cWox$OZA=iP7~CGpR($> zNJRFJ8wt55g&7zaG-CU;H-JLp#wHCgt#L~%IN|J!iGLpQOV^#5aj|s0ZT`JId;b4@ ze}3ETyxIT${!ah$;-ck~3C>05ZNIB{Zo6Oi8`OloSN%TrbtNdx7e8oZFZ=gz&*yXA z&d$zKQc{<0+?Y}L_}J4c!Tx72``h2OOgp$# zlTu!<-G0v9{@2BtXU6kSZc06U>CT-wDw0+u8E(B&Qx%=tp4`6wuWa4^f4_WYnRqs` zGB7ZF@rXX-&awOS0>@^fj0+3?d|7S}YO*BTGz^kO%-`~!LhR03?wemnkCdlub&)Zq^ z+x>W8m3-dv`5c>nKOXP=b}Kvn!bbVpFBfZGN8dm7I{yDJ9yyzexqnYj*FP`5??daB zty`b^=2u-#TMu&N0&9&8uO7(0uW$#I1)h`Do=$M)J2|)f-b_%7c;~BCtFK9IzZw?( zbn^T^DR!S{o=-X2CHmU-nlr!c6vt+^r8{?8?tZh$`&h5^cE5+Ehd9+!X6J4@xxlfx z=;KlG=ZCoU&qUw*G;Lj!94Ki_4@$UtvVoDg=)*zw=f?AY`hc3$bBa!BT7JJ%d|k5q z*URPUH#epJd}h9X=e@_E>Tjl5Zq&ao%k5KdZc5$x_uFky`*NKrxSY?g`#k&197|(v zF`W~V>2ofsW`hdI73{2G<~LKP$9{UV`TV-gwHFpR{_Ow%qkm5Qzn_`e+0p6s`|JL0 zdfaE7)-GG-@&Di7>(y^AE_QzoYDGjvM`zpRgAz_va6;+E^m&zPpmsl~4QKP?L38Dw zkH<->n|Nuk5bI%%*bQ zcKbs^X0|nQ;n~e>yr<%--%j<`+o@9j^K|^Bd-vvna#Zd6>iBbZzu$bd1J$vc|`9Bq3>EHEK=cdk)wl~Mh_w&OiAJF8;Wetm!6KK|FG>7PKZZcd(8@u;(O zn{4@=Ld$nM9$&jvcCYgJ(>pG<0fTU*FfdKLVi2L;aQ#*P4`4%d4WSq9D^_ zPuoJui&;$EAa5_ygm9z!wKuF<_3EAG7p+e$3=9lj*OR%{EL*lr<>aNM-n%Q+_MV<) zbNvZOOTa9-&%qqh($dByFD{&!ZLaTWRPf+{iHS+c=QXFUT)a3@R6Fd+&gb*I=i9x0 zuYo8Ud1S3jdV70I=czIN^_U!+OC(TPoZ2qSI|9wB*fl(I$E zyNgC-o0yqt#hESIxX}>Q=4oJLzH=KiEb`(bsB8t9dcz3nX|ymHJes`%R0?{oxp?i`v}x0( zotbZMuk6-yi`=S}nG^H(e#+ndar7wZNB`m#@6Bb-LZQy?bqopPe~eeSItV z_`lk|rRTR_S)c9hUM;28n?tA&(y$~Ni+uWS;IO($Qx3Vp_#qa;76Y(LW>|SNe zw%hA8mEF~x+tqfzn-x>@@wjo>n;UyR_U50tu+Z7K@X?W&iifQ`-|zeV$^ZWs|H%2F ztHV-3LtmgFlWx6T8s}|3`}o=Yd;%Vi`f|zp`tLa^KOeB$&j3|)pyq{9(vglig~ueJ zBR`Xl9XrNrmmj?5WxAMt+#E&cwkuz5%wFX#7Ga#GV*Pf@t`(qK{)N9p7Os>)AyeNjc$UvV#&w*d?$g1 zT`iwZ2{y{Vx96*vTerCWIgn#P13{pXy~!$@)6SmSQTW)X^3xNUq7#a<_g{Igx$(fO ziSmkb?Bl^zOUc&cNoV|vj(^^!>r>=yW#xC`&(G}scQ0O?0Clixeth7S`#7)q-O1JQ ze_x&Zq~*zH`D6m9qaM7>C-QyqGM|}~zJ2@FBX7U2`1XyB$)8_^@3$(yU;Di~K8*j0 zGB>D$*Ramq=Hx=px(|DD?SI{Wn;(__#Z^3ZO8Wd-v-efkcbiolu`GU;a%DwecCNHp zPQDH}X7nRLSOxA2(yK2?fy!$hJEB*YJg1ekI7AQ|DQTgc_ zcAxw8cF%j8uXRZ0?N9`bBlFk)IK1cQ+5CGiF1q#0?LD`9nV4S8jBnq*mEN2A_0RJ) z5gV1b#r0ycfA6XMZ1!W$JvVT(c!T3v&%Sf7%Hv|EKN0o(_Topi`#%NW?v~&8jSk~q3~}@d;Z46}Bf_8ke7|qk&u6cTE_tehdkOcd-_HdF-MKlI zk$Wl%pU;^8=ZSjdQ|-fnxfcLN`zc#%HdT>HEs_)tg?g+x_owdCgZbaL=@% z*Ggk`{vA8plON|s=kNKrX-&*dBhZ)uXyo?X@#A*iHl8yCrNJ*RFI(5&zj0#*XuL(z zAi?3=?fm_FUdO&a73eNIm0N$$hGL)U>gr|w^XDz~o?e!>`^hBlqAP*!Ph;O#g@e?B z#uPy#n3r$t__lXP)#|oYt5&_6cSI^THy1S6`RC_nOChf@DNfv?E*6+2QX;bM1>K2-qmVVEvT6fj* z-*g%C$>8A*<_iep-UAX8?S8dUnvv!u((x9rmXV&li ze)8l0kkF~B-qTXv+}QZK)~{S+)qz)+)4%HOjEai7RvlzwX0~s|+r72-kIkE`;@rlw zGBR%Inw{F;t{NZro$Ky@L;d)zH`=PleeZ^!viWy$-;<;1n#aFRe*Emc_mhb3TF;x7 zWk+FFg4_}A)i-2bZTtSMR=WD!u`9o}R{h#n z^=!w=&o4NCKZ%%KyY$J9mA}1a9=E@>-B5MS+9hkA-1z$2Y+1|JsZ&Dz&u!HHc5}oScV_fu6vn5xs>7GJk&uZ zSp9N!%Yk2)rtg~qieO2bih`)}SuOfo2{npGsw({obxM?7lYlp2_@w8}H z=-y2}la%{yjBe-c&Sm3~m|&QE?8y;f|1)bMH}5K{EDb&{b3I~TO=a2r+V3VNCMmbJ zWX5h-^=ox{A87tfN=oWlRn4!<^WU9%dewLD`QxC5+Rg0sdqE??6P4Ycy;$7;Zq?GO zp?iz=d_HG=-BSFzjaUt&54f zd&T&;{hx>YmOmaeg9g%iBn*}Q|62cl7HDQ6cITd5D_#3Q!QwHSJ(365n=OR->dwC&dZ#)|NrOB?)UpZn%eke zqxwOvFM9L#y2)1?`PlPU)Q+20J^tzwbYG|Xv#D?Ss?5gH>#^le56k~6c(?cay*pK} z*MerUK~p~I^J}*ip9$ap>uOBR$D==wneUs(E?1$j*uDSUL3a5$s@~H|`XYCgWb)a5 zxiF{p+s)UtbKT`@O=M+dK~t`vr6^A(dF!3rzW?uA-oE#pucoF>0(T`r9SNWLcDA57 zu+P)?|51Hk^W1uON$uC5^D@^hKA$lLbvpGT{mWOD_s#!(=ebe!w>RgGTRxw2IW>Fr z@~US)9?So?P@h+k^zUi>Kh^p_kLAG&PL9jh*GylxKJL}j$NPT2+kLIlqWasLIaRM# zf@Uv4iz&)}TJ4UqJ%4=rzOP_E=I#Ic&1a^O>hAaZs`>2yd^j`Hc)9oW2>!393a_n+ zto%It{v6QKgYS3C<3a0JY(AY(2K6L&RdtpIcFUxnoi#ObbK28G-1=ug!{e*h?b0&8 zUsJ5+Gh@TGY}xuhA7`57Mu9Zmsrh^s#P+lKcx2bQ?W?aC9|w(-&PeV%$!`DS;I4HW zO>J zOW9VHe0{24zq9yE@p;?uty{K)-2eA&`}N9yvgLOIRac7X#r)upf4w^XZ`RhseAZWT z-Ys33KF$93&GW0;+S;;?c8QwKuGidUb^PRH_3-sIPqkOO%T+G%3i6$6wN-oC#kaeR zj=$Y@d)=-lle~9jy>flIO6Khy0Y%?>-h4sNxK5WpH|#VogTV&?b@rWLRVXD z2ifHn^!I!H|MKfP%cOJj`m+CgXt!VU>+9?At=qSUhlYmM|GGSXmCDt<-|yYFd^WH8 z-O5QJ{dT`H#MdtST@u=N-n{V9kyT5VE`3(4?`c~Uv)DZT@)cwE>&CO^J)Tx%8yp%t z`{}e2-(dgPD*iR5`HQ9PYAPl@`FLFZ{2a?-qq;vo&Kdsw{rx;6yUYYo3fLT4jU0e7*ISUVo&5e!Q`e^) z=@7hG`~7b5zn{;)&)R@lxb-W$(SN zd;itUG06;ic7Fc-?0X+Secz>ZTu|99<@wz5bD(uIHoxC&24zFN-ETJaNSSJZCdzZg z0~^+zD*XBB>C5Zu<9~hmxxapwskynjXWjSR_pc_sa$OnP2kKf*P;|buJ^#MSN!#yt zCQq9-?akKfadVTu-K&1TWy_YFZ~I-nuY_9WWxkr3oWF0`%J9B*>1MXabzfGgz5G>N z&l9=za$RE|NAsOrt0O= zp9k3OHoO$pi?H4FXM6qc?R%c6>YwZoR5nUJ)&p7}cKc7K_%iL{HLJ8h%U0|@H1dDx z>KDuP37_9D|NF-AqQ76SKku`CcOrB7+-d7;-)8^3&~6tp`=t01?czW4|Noq?a5|vvrym`(^+wm>r}w`{n_hwJn90C+(??|-C5?pBOrEWva_>u z-rld*eCAr2Hna1e1IjQ~|94sr>)@{{5Y~=jK>u=H|wx=VWJRum9ok`g3V&U-YD^XE$cA z&wMp;@#~pqPZxUcn&-Le-<&%^>oQ-hJ^j!86v#*H@-+o}|A%i_xG-_;{qWGO=eCJw zU0V}r`Q?H$ctORLVE?;IlBcYbi>!MUJpWV+r|>7x%7)GJ|K|8vyt+lIrBs~AcpRakl-%d+B-1cJfw^Wgxqq;cArwDbE#VrM=Ejl0Fve!U7> z!%%!JGJWaBjfV1nU$`4*U0GpbW|sHuj(1QuXeq#_Yuopkg60g**?zy%E0>&n{N>4A zR>yT+_d+VA6BoapdG>UNYV3KH*m}P_&Fdy#uc^L2@au%T-6YWbBq(W^-OjnZv-Zo2 zi=LbOsx9;H?J12qC!Pc9z5nxExmo_D)>Yxtl->KzXvcl*lBxM{aOb~Y zuZ=4|J-Ji;e(!AapG_sDecSKXRf8(JFYoW$gXUQKtl!O00WG^J{&Z6PdID?eit9gr z)KtIUTR!Q@#^Z9g*F2uGWOZ>&Fvpusr}gIE2DOgB6WNBz$DaA@)%a>7ukJTzhM=uYoO{XClw3eE8N+R|LHz=o zXBwwJo0&cjoXVHaud}j#v*GYF{RIMBPZXY}o^Y?BozTG9NU35LRJoSF<_u9bqrp3?BfZJYg zv*TxX3ahUx_3ya1U|!_9_gg1l{eJt!%U4q$g9?Uc{q=v6m-)_~mU($u>Zd0sJtu*N z3ZKufugev?+`H>@%^a)JtVv(G$}_KM9|w)%{r&wt{`GOD?c29M?^K_6qU?6=_MhA@ z)4tltuU>WPNT+b|B~SIQtGIVmJ>60A@{&sA^Y<_9YJZjNi{7CAy3bzTy;o|gV>8>+ zHtD<*pe+62Abb8bDX)OlpmYW*(7(LCu0L7L_f!KT^EE5bBa)N5x@bfa@2GG3zb?I?6eS7~_slB{cyvaQFa>?qbdxHO(<^LpHy>uz) zZsqg2SKr*+Y%T9SUGJ^TXHc!1m6^E`v;^YS!gje;pvAFSXJ#0NMn^|KpK)_@dU)`1 zzqR{*-+f>9@5!Q-!OO!ygW{o~p`c*RIxbtjhR^E7f>)q5b5ZNBMqklB{rlbS^}D`Y z@?H&63~JPt{(ieX{QsZx|6}&PN!4FHuln82UF&yWdo}g({pvp-k5^yyf4d^Qul#;( zdFjWa;$Zv!J+J?_eAmM^=~Ww(kB5O~HSS)&Bl@b+uI^2;{OWD@s$RdE7M-_JuJ+5t ztV1oFq1oBlTN4g8?OL}}`^xX%zpn59S2}gcugmlQg={yy=DfG4=GV*Rs|A(aLfqwR zOH6ZbZOQue<>gg%`=2LQtz5ZsmRWAp-RrU0Uv1>|_kOvQb!v*{)h{nEugSOx`ue@=Ucc4adSc_+wQFx>uiv{Ww*2nasy7>tU$Z)QYO1z%fANcj?P0s$Zi_BG zZ~Hyu`<-I{t*_VZ4r`aMs{lKF=Hto!c9Eb~!ml^!_R+jjCM#xHm1dcizPgfia*}Fk zeEHg|+Q;wJ`3A3-i9LVi-S1yNt8Rbw3B9U+`d*#y<+)i=396CrlOG-FoSSTAWwomK zyzO>SyXorw`u|z~|9)Q&T5r3xulU{0=XqZ&gw_4-Jjnhk_Vv`KgY5Ea?)^As9(K5$ zfBmoT@8fg#|NV9qw15K?Y1y*jw@X1W@jAEu!FBE9n!(FLs^4x62c;EW34;Z_3iH~K=?`%By7_>NK?&Eot&t|@A5%ycb#xEDcD`&IA z>S2rUs#b3CYxx^vSIn~nEh2p_TYOx${7U}8*d=k6-@awfs(SjOYFXL+>*ufEUYEXV zRb3=V@ZFVw(Eq>xd4am^8UgQrPA~!CQnyCmvv4F FO#r4ecHIB~ literal 0 HcmV?d00001 diff --git a/doc/source/gridding.rst b/doc/source/gridding.rst index e8c4464..61f5483 100644 --- a/doc/source/gridding.rst +++ b/doc/source/gridding.rst @@ -25,7 +25,7 @@ where: * :math:`w_p` is the footprint area [m\ :sup:`2`] of pixel :math:`p` * :math:`w_{p,k}` is the area [m\ :sup:`2`] of pixel :math:`p` that overlaps with the cell :math:`k`. -The overlapping area is computed using the :py:meth:`LonLatPolygonCentroids ` method. +The overlapping area is computed using the :py:meth:`LonLatPolygonCentroids <.LonLatPolygonCentroids>` method. This fractions a footprint area into a large number of triangles, and returns for each triangle the centroid and the area. The centroids are collected per grid cell, and sum over the associated triangle area's is @@ -80,7 +80,7 @@ that is configured as:: rcbase='cso.tutorial.gridded-catalogue-index' The actual work is done by the :py:class:`.CSO_GriddedCatalogue` -and :py:class:`utopya.Indexer ` classes, +and :py:class:`utopya.Indexer <.Indexer>` classes, see their description for details on the configuration. The result could be viewed in a browser: diff --git a/doc/source/obsoper.rst b/doc/source/obsoper.rst index 5ada849..785b45f 100644 --- a/doc/source/obsoper.rst +++ b/doc/source/obsoper.rst @@ -1565,7 +1565,7 @@ Figures are saved to files with the basename of the original orbit file and the :alt: Simulated S5p NO2 columns To search for interesting features in the data, -the :py:class:`Indexer ` class could be used to create index pages. +the :py:class:`Indexer <.Indexer>` class could be used to create index pages. Configuration could look like:: ! index creation task: diff --git a/doc/source/pymod-cso_viirs.rst b/doc/source/pymod-cso_viirs.rst new file mode 100644 index 0000000..6afc936 --- /dev/null +++ b/doc/source/pymod-cso_viirs.rst @@ -0,0 +1,5 @@ +.. Documentation for module. + +.. Import documentation from ".py" file: +.. automodule:: cso.cso_viirs + diff --git a/doc/source/pymods.rst b/doc/source/pymods.rst index d738385..ce62f4a 100644 --- a/doc/source/pymods.rst +++ b/doc/source/pymods.rst @@ -33,6 +33,7 @@ Overview of the Python module(s) and classes. pymod-cso_colhub pymod-cso_earthaccess pymod-cso_s5p + pymod-cso_viirs pymod-cso_file pymod-cso_gridded pymod-cso_superobs diff --git a/doc/source/s5p-chocho.rst b/doc/source/s5p-chocho.rst index c30269d..e520e3a 100644 --- a/doc/source/s5p-chocho.rst +++ b/doc/source/s5p-chocho.rst @@ -153,7 +153,7 @@ The portals provide data files created with the same retrieval algorithm, but mo It is therefore necessary to first inquire both archives to see which data is available where, and what the version numbers are. -The :py:class:`CSO_DataSpace_Inquire ` class is available to inquire the +The :py:class:`CSO_DataSpace_Inquire <.CSO_DataSpace_Inquire>` class is available to inquire the *Copernicus DataSpace*. The settings used by this class allow selection on for example time range and intersection area. The result is a csv file which with columns for keywords such as orbit number and processor version, @@ -167,12 +167,12 @@ as well as the filename of the data and the url that should be used to actually See the section on *File name convention* in the *Product User Manual* for the meaning of all parts of the filename. -A similar class :py:class:`CSO_S5p_Download_Listing ` +A similar class :py:class:`CSO_S5p_Download_Listing <.CSO_S5p_Download_Listing>` class is available to list the content of the downloaded GlyRetro files. Also this will produce a table file. To visualize what is available from the portal, the -:py:class:`CSO_Inquire_Plot ` could be used to create an overview figure: +:py:class:`CSO_Inquire_Plot <.CSO_Inquire_Plot>` could be used to create an overview figure: .. figure:: figs/CHOCHO/Copernicus_S5p_CHOCHO.png :scale: 50 % @@ -514,7 +514,7 @@ The *listing* is a csv file that looks something like:: :alt: S5p glyox columns To search for interesting features in the data, - the :py:class:`Indexer ` class could be used to create index pages. + the :py:class:`Indexer <.Indexer>` class could be used to create index pages. Configuration could look like:: ! index creation task: @@ -707,7 +707,7 @@ The *listing* is a csv file that looks something like:: To search for interesting features in the data, - the :py:class:`Indexer ` class could be used to create index pages. + the :py:class:`Indexer <.Indexer>` class could be used to create index pages. Configuration could look like:: ! index creation task: diff --git a/doc/source/s5p-co.rst b/doc/source/s5p-co.rst index 214cb8a..be5b5d0 100644 --- a/doc/source/s5p-co.rst +++ b/doc/source/s5p-co.rst @@ -112,7 +112,7 @@ but with different processor versions. It is therefore necessary to first inquire both archives to see which data is available where, and what the version numbers are. -The :py:class:`CSO_DataSpace_Inquire ` class is available to inquire the +The :py:class:`CSO_DataSpace_Inquire <.CSO_DataSpace_Inquire>` class is available to inquire the *Copernicus DataSpace*. The settings used by this class allow selection on for example time range and intersection area. The result is a csv file which with columns for keywords such orbit number and processor version, @@ -470,7 +470,7 @@ The jobtree configuration to inquire the portals and create the overview figure :alt: S5p co columns To search for interesting features in the data, - the :py:class:`Indexer ` class could be used to create index pages. + the :py:class:`Indexer <.Indexer>` class could be used to create index pages. Configuration could look like:: ! index creation task: @@ -663,7 +663,7 @@ The jobtree configuration to inquire the portals and create the overview figure To search for interesting features in the data, - the :py:class:`Indexer ` class could be used to create index pages. + the :py:class:`Indexer <.Indexer>` class could be used to create index pages. Configuration could look like:: ! index creation task: diff --git a/doc/source/s5p-hcho.rst b/doc/source/s5p-hcho.rst index a538e04..df4fad1 100644 --- a/doc/source/s5p-hcho.rst +++ b/doc/source/s5p-hcho.rst @@ -117,7 +117,7 @@ The portal provides data files created with different processor versions. It is therefore necessary to first inquire both archives to see which data is available where, and what the version numbers are. -The :py:class:`CSO_DataSpace_Inquire ` class is available to inquire the +The :py:class:`CSO_DataSpace_Inquire <.CSO_DataSpace_Inquire>` class is available to inquire the *Copernicus DataSpace*. The settings used by this class allow selection on for example time range and intersection area. The result is a csv file which with columns for keywords such as orbit number and processor version, @@ -132,7 +132,7 @@ See the section on *File name convention* in the *Product User Manual* for the m parts of the filename. To visualize what is available from the various portals, the -:py:class:`CSO_Inquire_Plot ` could be used to create an overview figure: +:py:class:`CSO_Inquire_Plot <.CSO_Inquire_Plot>` could be used to create an overview figure: .. figure:: figs/HCHO/Copernicus_S5p_HCHO.png :scale: 50 % @@ -533,7 +533,7 @@ Figures are saved to files with the basename of the original orbit file and the :alt: S5p hcho columns To search for interesting features in the data, -the :py:class:`Indexer ` class could be used to create index pages. +the :py:class:`Indexer <.Indexer>` class could be used to create index pages. Configuration could look like:: ! index creation task: @@ -720,7 +720,7 @@ Figures are saved to files with the basename of the original orbit file and the To search for interesting features in the data, -the :py:class:`Indexer ` class could be used to create index pages. +the :py:class:`Indexer <.Indexer>` class could be used to create index pages. Configuration could look like:: ! index creation task: diff --git a/doc/source/s5p-no2.rst b/doc/source/s5p-no2.rst index e8da3dc..b876d7e 100644 --- a/doc/source/s5p-no2.rst +++ b/doc/source/s5p-no2.rst @@ -236,7 +236,7 @@ The portals provide data files created with the same retrieval algorithm, but mo It is therefore necessary to first inquire both archives to see which data is available where, and what the version numbers are. -The :py:class:`CSO_DataSpace_Inquire ` class is available to inquire the +The :py:class:`CSO_DataSpace_Inquire <.CSO_DataSpace_Inquire>` class is available to inquire the *Copernicus DataSpace*. The settings used by this class allow selection on for example time range and intersection area. The result is a csv file which with columns for keywords such as orbit number and processor version, @@ -250,11 +250,11 @@ as well as the filename of the data and the url that should be used to actually See the section on *File name convention* in the *Product User Manual* for the meaning of all parts of the filename. -A similar class :py:class:`CSO_PAL_Inquire ` class is available to list the content +A similar class :py:class:`CSO_PAL_Inquire <.CSO_PAL_Inquire>` class is available to list the content of the *Product Algorithm Laboratory* portal. Also this will produce a table file. To visualize what is available from the various portals, the -:py:class:`CSO_Inquire_Plot ` could be used to create an overview figure: +:py:class:`CSO_Inquire_Plot <.CSO_Inquire_Plot>` could be used to create an overview figure: .. figure:: figs/NO2/Copernicus_S5p_NO2.png :scale: 50 % @@ -651,7 +651,7 @@ Figures are saved to files with the basename of the original orbit file and the :alt: S5p NO\ :sub:`2` columns To search for interesting features in the data, -the :py:class:`Indexer ` class could be used to create index pages. +the :py:class:`Indexer <.Indexer>` class could be used to create index pages. Configuration could look like:: ! index creation task: @@ -908,7 +908,7 @@ Figures are saved to files with the basename of the original orbit file and the :alt: S5p NO2 columns To search for interesting features in the data, -the :py:class:`Indexer ` class could be used to create index pages. +the :py:class:`Indexer <.Indexer>` class could be used to create index pages. Configuration could look like:: ! index creation task: diff --git a/doc/source/s5p-o3.rst b/doc/source/s5p-o3.rst index c65d307..fc0a20d 100644 --- a/doc/source/s5p-o3.rst +++ b/doc/source/s5p-o3.rst @@ -162,7 +162,7 @@ Data is available for different processing streams, each identified by a 4-chara * ``OFFL`` : `Offline`, available within weeks after observations; * ``RPRO`` : re-processing of all previously made observations. -The :py:class:`CSO_DataSpace_Inquire ` class is available to inquire the +The :py:class:`CSO_DataSpace_Inquire <.CSO_DataSpace_Inquire>` class is available to inquire the *Copernicus DataSpace*. The settings used by this class allow selection on for example time range and intersection area. The result is a csv file which with columns for keywords such as orbit number and processor version, @@ -177,7 +177,7 @@ See the section on *File name convention* in the *Product User Manual* for the m parts of the filename. To visualize the available data, the -:py:class:`CSO_Inquire_Plot ` could be used to create an overview figure: +:py:class:`CSO_Inquire_Plot <.CSO_Inquire_Plot>` could be used to create an overview figure: .. figure:: figs/O3-PR/Copernicus_S5p_O3-PR.png :scale: 50 % @@ -578,7 +578,7 @@ The example is based on the S5p O\ :sub:`3`-profile file from which the header i :alt: S5p O\ :sub:`3` columns To search for interesting features in the data, - the :py:class:`Indexer ` class could be used to create index pages. + the :py:class:`Indexer <.Indexer>` class could be used to create index pages. Configuration could look like:: ! index creation task: @@ -835,7 +835,7 @@ The example is based on the S5p O\ :sub:`3`-profile file from which the header i :alt: S5p O3 columns To search for interesting features in the data, - the :py:class:`Indexer ` class could be used to create index pages. + the :py:class:`Indexer <.Indexer>` class could be used to create index pages. Configuration could look like:: ! index creation task: diff --git a/doc/source/s5p-so2-cobra.rst b/doc/source/s5p-so2-cobra.rst index 4a4c570..ec32e78 100644 --- a/doc/source/s5p-so2-cobra.rst +++ b/doc/source/s5p-so2-cobra.rst @@ -127,7 +127,7 @@ There might be data available from more than one processor version. It is therefore necessary to inquire the archive first to see which data is available, and what the version numbers are. -The :py:class:`CSO_PAL_Inquire ` class is available to inquire the remote archive. +The :py:class:`CSO_PAL_Inquire <.CSO_PAL_Inquire>` class is available to inquire the remote archive. The settings used by this class allow selection on for example time range and intersection area. The result is a csv file which with columns for keywords such as orbit number and processor version, as well as the filename of the data and the url that should be used to actually download the data:: @@ -141,7 +141,7 @@ See the section on *File name convention* in the *Product User Manual* for the m parts of the filename. To visualize what is available from the various portals, the -:py:class:`CSO_Inquire_Plot ` could be used to create an overview figure: +:py:class:`CSO_Inquire_Plot <.CSO_Inquire_Plot>` could be used to create an overview figure: .. figure:: figs/SO2-COBRA/Copernicus_S5p_SO2-COBRA.png :scale: 50 % @@ -485,7 +485,7 @@ Figures are saved to files with the basename of the original orbit file and the :alt: S5p SO\ :sub:`2` columns To search for interesting features in the data, -the :py:class:`Indexer ` class could be used to create index pages. +the :py:class:`Indexer <.Indexer>` class could be used to create index pages. Configuration could look like:: ! index creation task: diff --git a/doc/source/s5p-so2.rst b/doc/source/s5p-so2.rst index 8571833..52b2aa3 100644 --- a/doc/source/s5p-so2.rst +++ b/doc/source/s5p-so2.rst @@ -128,7 +128,7 @@ The portal provides data files created with different processor versions. It is therefore necessary to first inquire both archives to see which data is available where, and what the version numbers are. -The :py:class:`CSO_DataSpace_Inquire ` class is available to inquire the +The :py:class:`CSO_DataSpace_Inquire <.CSO_DataSpace_Inquire>` class is available to inquire the *Copernicus DataSpace*. The settings used by this class allow selection on for example time range and intersection area. The result is a csv file which with columns for keywords such as orbit number and processor version, @@ -143,7 +143,7 @@ See the section on *File name convention* in the *Product User Manual* for the m parts of the filename. To visualize what is available from the various portals, the -:py:class:`CSO_Inquire_Plot ` could be used to create an overview figure: +:py:class:`CSO_Inquire_Plot <.CSO_Inquire_Plot>` could be used to create an overview figure: .. figure:: figs/SO2/Copernicus_S5p_SO2.png :scale: 50 % @@ -558,7 +558,7 @@ Figures are saved to files with the basename of the original orbit file and the :alt: S5p SO\ :sub:`2` columns To search for interesting features in the data, -the :py:class:`Indexer ` class could be used to create index pages. +the :py:class:`Indexer <.Indexer>` class could be used to create index pages. Configuration could look like:: ! index creation task: @@ -759,7 +759,7 @@ Figures are saved to files with the basename of the original orbit file and the :alt: S5p SO2 columns To search for interesting features in the data, -the :py:class:`Indexer ` class could be used to create index pages. +the :py:class:`Indexer <.Indexer>` class could be used to create index pages. Configuration could look like:: ! index creation task: diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst index 3b43105..453629a 100644 --- a/doc/source/tutorial.rst +++ b/doc/source/tutorial.rst @@ -9,6 +9,85 @@ Tutorial This chapter describes step by step how to run the CSO pre-processor and observation operator. +Setup Python environment +======================== + +There are two ways to setup the correct Python environment for CSO. + + +Running CSO within your own Python environment +---------------------------------------------- + +The CSO tools could be started within your own Python environment. +You should then ensure that the required (versions of) packages have been installed correctly. + +A probablly incomplete list of packages is: + +* ``numpy`` +* ``numba`` +* ``netcdf4`` +* ``xarray`` + +If you want CSO to create plots, you probably also need: + +* ``matplotlib`` +* ``cartopy`` + +To download satellite data, you might need: + +* ``reqests`` : to access Sentinel data; +* ``earthaccess`` : to access VIIRS data. + +To generate a local copy of the User Guide, install: + +* ``sphinx`` +* ``toml`` + + + +Running CSO within a virtual environment +---------------------------------------- + +A *virtual environment* could be created that has all required (versions) of Python packages installed. +For this, the ``pyproject.toml`` file is included which contains the dependencies needed by CSO. + +To create a new virtual enviroment run:: + + python3 -m venv --prompt cso .venv + +This will create a new Python evironment folder called ``.venv``. +To activate it run:: + + source .venv/bin/activate + +The terminal prompt will then be preceded by ``(cso)`` which holds the name specified above +with the ``--prompt`` argument. + +You may need to upgrade ``pip`` with the command:: + + pip install --upgrade pip + +We can then install CSO and its dependencies into the virtual environment using:: + + pip install --editable . + +or, if also the dependencies that are required to generate a local copy of the User Guide:: + + pip install --editable .[docs] + +The ``--editable`` flag installs CSO in editable mode, meaning that the user can change the source code +in the ``src/`` directory and run CSO anywhere using these changes. + +As described below, a typical way to run the CSO tools is then:: + + ./bin/cso config/tutorial/tutorial.rc + +When finished, leave from the virtual environment using:: + + deactivate + + + Run script ========== @@ -56,7 +135,7 @@ This list is actually defined as tree, using lists in which the elements could b For each element in the tree, the configuration file should specify the name of a python class that takes care of the job creation. -If the element is a *tree*, use the :py:class:`utopya.UtopyaJobTree ` class, +If the element is a *tree*, use the :py:class:`utopya.UtopyaJobTree <.UtopyaJobTree>` class, and add a line to specify the names of the sub-elements. For the main ``cso`` job, this looks like:: @@ -79,7 +158,7 @@ in this example again a tree is defined:: cso.tutorial.elements : inquire convert listing catalogue A job that is no container for sub-jobs but should actually do some work should -be defined with the :py:class:`utopya.UtopyaJobStep ` class. +be defined with the :py:class:`utopya.UtopyaJobStep <.UtopyaJobStep>` class. This is for example the case for the ``cso.tutorial.inquire`` job:: ! single step: @@ -169,7 +248,7 @@ The first setting defines that this is single job step that should do some work. The ``tasks`` list defines keywords for the two tasks to be performed. -* For the ``table-dataspace`` task, define that the :py:class:`CSO_DataSpace_Inquire ` class +* For the ``table-dataspace`` task, define that the :py:class:`CSO_DataSpace_Inquire <.CSO_DataSpace_Inquire>` class should be used to do the work; the class is accessible from the :py:mod:`cso` module (implemented in``py/cso.py``). The first arguments that initialize the class specifies the name of an rcfile with settings; @@ -179,7 +258,7 @@ The ``tasks`` list defines keywords for the two tasks to be performed. ``'cso.tutorial.inquire-table-dataspace'``. * Similar for the ``plot`` task, the settings define that the - :py:class:`CSO_Inquire_Plot ` class should be used to do the work. + :py:class:`CSO_Inquire_Plot <.CSO_Inquire_Plot>` class should be used to do the work. The tutorial settings will inquire the time range 2018-2023. @@ -281,7 +360,7 @@ The conversion job is configured with:: cso.tutorial.convert.task.args : '${__filename__}', \ rcbase='cso.tutorial.convert' -The conversion is thus done using the :py:class:`CSO_S5p_Convert ` class +The conversion is thus done using the :py:class:`CSO_S5p_Convert <.CSO_S5p_Convert>` class that can be accessed from the :py:mod:`cso` module. The arguments that initialize the class specify the name of a rcfile with settings (in this case the ``tutorial.rc`` that holds the job-tree definition) @@ -432,7 +511,7 @@ and the variable that is plotted:: Index pages are created to facilitate browsing through the figures. The index is created with the ``index`` task of the job. -As shown above, the work is done by the :py:class:`Indexer ` class +As shown above, the work is done by the :py:class:`Indexer <.Indexer>` class that can be accessed from the :py:mod:`utopya` module. The arguments that initialize the class specify the name of an rcfile with settings (``tutorial.rc``) and that the settings start with keywords ``'cso.tutorial.catalogue-index'``. @@ -664,7 +743,7 @@ and the variable that is plotted:: Index pages are created to facilitate browsing through the figures. The index is created with the ``index`` task of the job. -As shown above, the work is done by the :py:class:`Indexer ` class +As shown above, the work is done by the :py:class:`Indexer <.Indexer>` class that can be accessed from the :py:mod:`utopya` module. The arguments that initialize the class specify the name of an rcfile with settings (``tutorial.rc``) and that the settings start with keywords ``'cso.tutorial.sim-catalogue-index'``. diff --git a/doc/source/viirs-aod.rst b/doc/source/viirs-aod.rst index 73d042b..c0e1342 100644 --- a/doc/source/viirs-aod.rst +++ b/doc/source/viirs-aod.rst @@ -141,11 +141,14 @@ CSO processing An example configuration of the CSO processing of the VIIRS/AOD data is available via the following settings: -* `config/VIIRS/viirs-aod.rc <../../../config/VIIRS/viirs-aod.rc>`_ +* `config/VIIRS/cso.rc <../../../config/VIIRS/cso.rc>`_ defines a job-tree of sub-steps to perform; +* `config/VIIRS/cso-user-settings.rc <../../../config/VIIRS/cso-user-settings.rc>`_ + defines user-settings such as domain, time range, and paths; +* `config/VIIRS/cso-viirs.rc <../../../config/VIIRS/cso-viirs.rc>`_ configures the operations on VIIRS/AOD data. Start the job-tree using:: - ./bin/cso /config/VIIRS/viirs-aod.rc + ./bin/cso /config/VIIRS/cso.rc Selected sub-steps in the processing are described below. @@ -157,6 +160,47 @@ Selected sub-steps in the processing are described below. Inquire VIIRS archives ====================== +The :py:class:`.CSO_EarthAccess_Inquire` class is available to inquire the +*EarthData* portal. The settings used by this class allow selection on the data product, time range, and intersection area. +The result is a csv file which with records for available files and properties of their conentent, +as well as the url that should be used to actually download the data:: + + filename ;start_time ;end_time ;product;platform;processor_version;href + AERDB_L2_VIIRS_SNPP.A2012064.0354.002.2023077205454.nc;20120304T035400;20120304T040000;AERDB ;SNPP ;020000 ;https://data.laadsdaac.earthdatacloud.nasa.gov/prod-lads/AERDB_L2_VIIRS_SNPP/AERDB_L2_VIIRS_SNPP.A2012064.0354.002.2023077205454.nc + AERDB_L2_VIIRS_SNPP.A2012064.0536.002.2023077205330.nc;20120304T053600;20120304T054200;AERDB ;SNPP ;020000 ;https://data.laadsdaac.earthdatacloud.nasa.gov/prod-lads/AERDB_L2_VIIRS_SNPP/AERDB_L2_VIIRS_SNPP.A2012064.0536.002.2023077205330.nc + : + +To visualize what is available from the portal, the +:py:class:`.CSO_Inquire_Plot` could be used to create an overview figure. +The following example shows that for the VIIRS-1 "Deep Blue" AOD product only data from processor version ``v2.0.0`` is available: + +.. figure:: figs/VIIRS/viirs1-aod-db_inquire.png + :scale: 50 % + :align: center + :alt: Example of available VIIRS-1 "Deep Blue" AOD settings. + +The jobtree configuration to inquire the portals and create the overview figure looks like:: + + ! single step: + cso.viirs1-aod-db.inquire.class : utopya.UtopyaJobStep + + ! inquire tasks: + cso.viirs1-aod-db.inquire.tasks : table-earthaccess plot + + + !~ inquire files downloaded from EarthAccess: + cso.viirs1-aod-db.inquire.table-earthaccess.class : cso.CSO_EarthAccess_Inquire + cso.viirs1-aod-db.inquire.table-earthaccess.args : '${my.work}/rc/cso-viirs.rc', \ + rcbase='cso.inquire-table-earthaccess', \ + env={ 'MY_PRODUCT' : 'viirs1-aod-db' } + + !~ create plot of available versions: + cso.viirs1-aod-db.inquire.plot.class : cso.CSO_Inquire_Plot + cso.viirs1-aod-db.inquire.plot.args : '${my.work}/rc/cso-viirs.rc', \ + rcbase='cso.inquire-plot', \ + env={ 'MY_PRODUCT' : 'viirs1-aod-db' } + + .. Label between '.. _' and ':' ; use :ref:`text