diff --git a/doc/source/figs/NO2/Copernicus_S5p_NO2_inquire-avail.png b/doc/source/figs/NO2/Copernicus_S5p_NO2_inquire-avail.png new file mode 100644 index 0000000000000000000000000000000000000000..171bcc0f30fbf52a0e5be27cd54e37daafbe1f00 Binary files /dev/null and b/doc/source/figs/NO2/Copernicus_S5p_NO2_inquire-avail.png differ diff --git a/doc/source/figs/NO2/Copernicus_S5p_NO2.png b/doc/source/figs/NO2/Copernicus_S5p_NO2_inquire-versions.png similarity index 100% rename from doc/source/figs/NO2/Copernicus_S5p_NO2.png rename to doc/source/figs/NO2/Copernicus_S5p_NO2_inquire-versions.png diff --git a/doc/source/s5p-no2.rst b/doc/source/s5p-no2.rst index 55b24fc6edb6dac2f6223ff71839e04ff37ec34f..c465e1611ee9be9fc02b298e9ef3b491a30f2fe2 100644 --- a/doc/source/s5p-no2.rst +++ b/doc/source/s5p-no2.rst @@ -260,7 +260,7 @@ of the *Product Algorithm Laboratory* portal. Also this will produce a table fil To visualize what is available from the various portals, the :py:class:`CSO_Inquire_Plot <.CSO_Inquire_Plot>` could be used to create an overview figure: -.. figure:: figs/NO2/Copernicus_S5p_NO2.png +.. figure:: figs/NO2/Copernicus_S5p_NO2_inquire-versions.png :scale: 50 % :align: center :alt: Overview of available NO\ :sub:`2` processings. diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst index b389fa93b7a139efdd45a5647f83b21bb35dbe61..665ee8168da7442782e5ef904431cf7f0ae37404 100644 --- a/doc/source/tutorial.rst +++ b/doc/source/tutorial.rst @@ -306,7 +306,7 @@ should have created an overview figure next to the table file:: The figure should look like: -.. figure:: figs/NO2/Copernicus_S5p_NO2.png +.. figure:: figs/NO2/Copernicus_S5p_NO2_inquire-versions.png :scale: 50 % :align: center :alt: Overview of available NO2 processings on DataSpace. diff --git a/src/cso/cso_colhub.py b/src/cso/cso_colhub.py index 35c49843e20d1718d111c8f35e2ddf7542359f25..89decabbc074b6d341be95ae541db3e0ca9eb4ef 100644 --- a/src/cso/cso_colhub.py +++ b/src/cso/cso_colhub.py @@ -32,6 +32,9 @@ # Added production time of data files to listings. # Select file with latest production time if multiple are found. # +# 2026-04, Arjo Segers +# Skip files that cannot be accessed, for example a broken link. +# ######################################################################## ### @@ -209,13 +212,22 @@ class CSO_ColHubMirror_Inquire(utopya.UtopyaRc): for fname in files: # data file? if fnmatch.fnmatch(fname, fpattern): - # already in table? - if fname in listing: + # full path: + href = os.path.join(root, fname) + #~ sometimes links to non-existing targets ... + if not os.path.isfile(href): + # info ... + logging.info(f"{indent} WARNING - invalid file (broken link?): {href}") + # + #~ already in table? + elif fname in listing: # info ... - logging.info(f"{indent} keep entry %s ..." % fname) + logging.info(f"{indent} keep entry {fname} ...") + # + #~ new entry: else: # info ... - logging.info(f"{indent} add entry %s ..." % fname) + logging.info(f"{indent} add entry {fname} ...") # Example filename: # S5P_RPRO_L2__CH4____20180430T001851_20180430T020219_02818_01_010301_20190513T141133.nc # @@ -270,7 +282,7 @@ class CSO_ColHubMirror_Inquire(utopya.UtopyaRc): data["collection"] = collection data["processor_version"] = processor_version data["production_time"] = production_time - data["href"] = os.path.join(root, fname) + data["href"] = href # update record: listing.UpdateRecord(fname, data, indent=f"{indent} ") # endif # new record? @@ -290,7 +302,7 @@ class CSO_ColHubMirror_Inquire(utopya.UtopyaRc): # endfor # archive_dir # info ... - logging.warning(f"{indent}save ...") + logging.info(f"{indent}save ...") # sort: listing.Sort(by="orbit") # save: diff --git a/src/cso/cso_dataspace.py b/src/cso/cso_dataspace.py index c793802f12fc940e416c5e09954498d06056024e..aff1bd4acd03f2cd4a7b36ea292ce741a1636653 100644 --- a/src/cso/cso_dataspace.py +++ b/src/cso/cso_dataspace.py @@ -58,6 +58,9 @@ # Trap files that could not be downloaded. # Added production time of data files to listings. # +# 2026-07, Arjo Segers +# Changed counting of files in S3 bucket following deprecation error. +# ######################################################################## @@ -782,7 +785,9 @@ class CSO_DataSpace_Downloader(object): pfiles = bucket.objects.filter( Prefix=product ) # count: - npfile = len(list(pfiles)) + npfile = 0 + for pfile in pfiles: + npfile += 1 # check ... if npfile != 1: print(f"ERROR - found {npfile} files in S3 bucket for product: {product}") diff --git a/src/cso/cso_file.py b/src/cso/cso_file.py index 0ad1114374db6fdacfdc921f65080cffb0d7ea0d..1cb4e51a7b98f2ba1b845448e3969403cbad31d5 100644 --- a/src/cso/cso_file.py +++ b/src/cso/cso_file.py @@ -1688,8 +1688,8 @@ class CSO_Listing(object): for key in self.df.keys(): selection = selection.replace(f"%{{{key}}}", f"df['{key}']").strip() # endfor - # testing ... - logging.info(f"{indent}selection `{selection}` ...") + ## testing ... + #logging.info(f"{indent}selection `{selection}` ...") # adhoc: replace some operators that are not allowded anymore ... selection = selection.replace( " and ", " & " ) # evaluate: diff --git a/src/cso/cso_inquire.py b/src/cso/cso_inquire.py index 0944609ae60d64dddc2168da95f128efa4d94714..772271eb087fbce0f8a5f1c6430cfce8b8e40f8a 100644 --- a/src/cso/cso_inquire.py +++ b/src/cso/cso_inquire.py @@ -34,6 +34,9 @@ # 2026-03, Arjo Segers # Fixed sorting of collections and processors for latests versions of pandas. # +# 2026-03, Arjo Segers +# Added `CSO_Inquire_Plot_Avail` class. +# ######################################################################## ### @@ -59,7 +62,8 @@ The classes and are defined according to the following hierchy: * :py:class:`.UtopyaRc` - * :py:class:`.CSO_Inquire_Plot` + * :py:class:`.CSO_Inquire_Plot_Versions` + * :py:class:`.CSO_Inquire_Plot_Avail` @@ -85,12 +89,12 @@ import utopya ######################################################################## ### -### modules +### plot versions ### ######################################################################## -class CSO_Inquire_Plot(utopya.UtopyaRc): +class CSO_Inquire_Plot_Versions(utopya.UtopyaRc): """ Create plot of data version versus time to indicate the available orbits. @@ -120,10 +124,10 @@ class CSO_Inquire_Plot(utopya.UtopyaRc): The created plot shows a time line and on the vertical ax the processor versions; a bar indicates when a certain version was used to process orbits: - .. figure:: figs/NO2/Copernicus_S5p_NO2.png + .. figure:: figs/NO2/Copernicus_S5p_NO2_inquire-versions.png :scale: 50 % :align: center - :alt: Overview of available NO2 processings. + :alt: Overview of available NO2 processing versions. Specify the name of the target plot:: @@ -164,7 +168,7 @@ class CSO_Inquire_Plot(utopya.UtopyaRc): # info ... logging.info(indent + "") - logging.info(indent + "** create inquire plot") + logging.info(indent + "** create inquire plot of versions") logging.info(indent + "") # init base object: @@ -484,13 +488,260 @@ class CSO_Inquire_Plot(utopya.UtopyaRc): # info ... logging.info(indent + "") - logging.info(indent + "** end inquire plot") + logging.info(indent + "** end inquire plot versions") + logging.info(indent + "") + + # enddef __init__ + + +# endclass CSO_Inquire_Plot_Versions + + +######################################################################## +### +### plot availability +### +######################################################################## + + +class CSO_Inquire_Plot_Avail(utopya.UtopyaRc): + + """ + Create plot with bargraph of number of orbits versus time to check availability. + + The information on orbits is taken from a csv table created by for example + the :py:class:`CSO_DataSpace_Inquire` class. + Specifify the name of the table file in the settings:: + + ! listing file: + .file : ${my.work}/Copernicus/Copernicus_S5p_NO2_%Y-%m-%d.csv + + The date templates are by default filled for the current day. + Alternatively, specify an explicit date:: + + !~ specify dates ("yyyy-mm-dd") to use historic table: + .filedate : 2022-01-28 + + The plot could also be created by combining multiple tables; + use a semi-colon to seperate the file names (and eventually the dates):: + + ! listing files: + .file : ${my.work}/Copernicus/Copernicus_S5p_NO2_%Y-%m-%d.csv ; \\ + ${my.work}/Copernicus/Copernicus_S5p_NO2_pal_%Y-%m-%d.csv + !~ specify dates ("yyyy-mm-dd") to use historic tables: + !.filedate : 2022-01-28 ; 2022-01-28 + + The created plot shows a bar graph with one or more time series with the number of orbits per month: + + .. figure:: figs/NO2/Copernicus_S5p_NO2_inquire-avail.png + :scale: 50 % + :align: center + :alt: Overview of number of available NO2 oribits. + + The time series are defined by keywords:: + + .series : RPRO OFFL + + For each keyword, define the label (used in the legend), a selection filter, and the bar style:: + + .serie.RPRO.label : RPRO + .serie.RPRO.selection : (%{collection} == '03') & (%{processing} == 'RPRO') + .serie.RPRO.style : color="green" + + Specify the name of the target plot:: + + ! output figure, date of today: + .output.file : ${my.work}/Copernicus_S5P_NO2_%Y-%m-%d.png + + Optionally define a creation mode for the (parent) directories:: + + ! directory creation mode: + .dmode : 0o775 + + The following flag is used to ensure that the plot is renewed:: + + ! renew existing plots? + .renew : True + + """ + + def __init__(self, rcfile, rcbase="", env={}, indent=""): + """ + Convert data. + """ + + # modules: + import os + import numpy + import datetime + import pandas + import matplotlib + import matplotlib.pyplot as plt + import matplotlib.dates as mdates + + # tools: + from . import cso_file + + # setup graphics back-end: + matplotlib.use("Agg") + + # info ... + logging.info(indent + "") + logging.info(indent + "** create plot with data availablity") + logging.info(indent + "") + + # init base object: + utopya.UtopyaRc.__init__(self, rcfile=rcfile, rcbase=rcbase, env=env) + + # directory creation mode: + dmode = self.GetSetting("dmode", totype="int", default=None) + + # renew output? + renew = self.GetSetting("renew", totype="bool") + + # table file(s) to be used: + filename_templates = self.GetSetting("file").split(";") + # count: + nfile = len(filename_templates) + # time stamp in file? + filedates = self.GetSetting("filedate", default=(";" * (nfile - 1))).split(";") + + # list with time expanded: + filenames = [] + for ifile in range(len(filename_templates)): + # current: + filename_template = filename_templates[ifile] + # time? + if len(filedates[ifile]) > 0: + t0 = datetime.datetime.strptime(filedates[ifile].strip(), "%Y-%m-%d") + else: + t0 = datetime.datetime.now() + # endif + # expand time templates: + filenames.append(t0.strftime(filename_template.strip())) + # endfor + + # current time: + t = datetime.datetime.now() + + # annotation: + title = t.strftime(self.GetSetting("title")) + + # target file template: + figfile_template = self.GetSetting("output.file") + # target file: + fig_file = t.strftime(figfile_template) + + # create? + if (not os.path.isfile(fig_file)) or renew: + # info .. + logging.info(f"{indent}create {fig_file} ...") + + # not yet for multiple .. + if len(filenames) > 1: + logging.error(f"only single inquiry table file supported yet ..") + raise Exception + #endif + # read: + lst = cso_file.CSO_Listing( filenames[0] ) + + # time range: + t1 = lst.df["start_time"].min() + t2 = lst.df["start_time"].max() + # reset to start/end of year: + t1 = pandas.Timestamp(f"{t1.year}-01-01 00:00") + if not t2.is_year_end: + t2 = pandas.Timestamp(f"{t2.year+1}-01-01 00:00") + #endif + + # collection frequency: + tfreq = "MS" # start of month + tt = pandas.date_range(f"{t1.year}-01",f"{t2.year}-12",freq=tfreq) + # storage at full freqency: + ndf = pandas.DataFrame( index=tt ) + + # keywords for plots: + series = self.GetSetting( "series" ).split() + # loop: + for serie in series: + # info .. + logging.info(f"collect serie `{serie}` ...") + + # settings: + selection = self.GetSetting( f"serie.{serie}.selection" ) + # info .. + logging.info(f" selection: {selection} ...") + + # select records: + xlst = lst.Select( expr=selection, verbose=False ) + # check ... + if len(xlst) == 0: + logger.error(f"no records found matching selection `{selection}`") + raise Exception + #endif + + # group on start time, per start of month: + grp = xlst.df.set_index("start_time").groupby( pandas.Grouper(freq="1MS") ) + # series with months as index and number of grouped records as values: + nn = grp.size() + + # store at full frequency: + ndf.at[ nn.index, serie ] = nn.values + #endfor + + # new: + fig = plt.figure(figsize=(12, 4)) + ax = fig.add_axes([0.05, 0.07, 0.94, 0.85]) + # no bottom yet: + bottom = None + # loop: + for serie in series: + # current values: + nn = ndf[serie].values + + # plot features: + label = self.GetSetting( f"serie.{serie}.label" ) + style = self.GetSetting( f"serie.{serie}.style", totype='dict' ) + # add bar plot, left side of bar at time value (start of month) + ax.bar( ndf.index, nn, width=25, bottom=bottom, align="edge", label=label, **style ) + + # update bottom: + if bottom is None: + bottom = nn + else: + bottom = bottom + nn + #endif + #endfor + # annote: + ax.legend() + ax.set_ylabel(f"number of orbits per month") + # time axis: + ax.set_xlim((t1, t2)) + ax.xaxis.set_minor_locator(mdates.MonthLocator()) + ax.xaxis.set_major_locator(mdates.YearLocator()) + ax.xaxis.set_major_formatter(mdates.DateFormatter("%Y")) + ax.grid(axis="x") + + # directory: + cso_file.CheckDir(fig_file, dmode=dmode) + # save: + fig.savefig(fig_file) + + else: + # info .. + logging.info(indent + "keep %s ..." % fig_file) + + # endif # renew + + # info ... + logging.info(indent + "") + logging.info(indent + "** end inquire plot avail") logging.info(indent + "") # enddef __init__ -# endclass CSO_Inquire_Plot +# endclass CSO_Inquire_Plot_Avail ########################################################################