From 3769a0990e01648549fbcb47b146fc6f23cdd6f5 Mon Sep 17 00:00:00 2001 From: Arjo Segers Date: Fri, 19 Sep 2025 13:39:32 +0200 Subject: [PATCH 1/4] Improved speed of selecting listing records. --- src/cso/cso_file.py | 52 +++++++++++++++++++++++++------------------- src/cso/cso_viirs.py | 4 ++-- 2 files changed, 32 insertions(+), 24 deletions(-) diff --git a/src/cso/cso_file.py b/src/cso/cso_file.py index 1eb4a7f..55b1764 100644 --- a/src/cso/cso_file.py +++ b/src/cso/cso_file.py @@ -52,6 +52,9 @@ # 2025-06, Arjo Segers # Set flag to avoid warnings when decoding time arrays from VIIRS files. # +# 2025-09, Arjo Segers +# Improved speed of CSO_Listing.Selection method. +# ######################################################################## ### @@ -1639,35 +1642,40 @@ class CSO_Listing(object): # evaluate selection expression? if expr is not None: - # replace templates: - # %{orbit} == '12345' - # to: - # xrec['orbit'] == '12345' - for key in self.df.keys(): - expr = expr.replace("%{" + key + "}", "xrec['" + key + "']") - # endfor - # split: + # split into sequnece of critera; first expression with macthing record will be used: selections = expr.split(";") + # initialize empty list of selected record indices (filenames): + selected = [] # storage for status label: "selected", "blacklisted", ... filestatus = {} - # no match yet .. - seleted = [] # loop over selection criteria, # this should give either none or a single file: for selection in selections: - # make empty again: - selected = [] - # loop over records: - for indx, xrec in df.iterrows(): - # skip? - if os.path.basename(indx) in blacklist: - filestatus[indx] = "blacklisted" - continue - # endif - # evaluate expression including 'xrec[key]' values: - if eval(selection): + # replace templates: + # %{orbit} == '12345' + # to: + # df['orbit'] == '12345' + for key in self.df.keys(): + selection = selection.replace(f"%{{{key}}}", f"df['{key}']") + # endfor + # testing ... + logging.info(f"{indent}selection `{selection}` ...") + # evaluate: + xdf = df[ eval(selection) ] + # any? + if len(xdf) > 0: + # selected indices (filenames): + selected = [] + for indx, xrec in xdf.iterrows(): + # skip? + if os.path.basename(indx) in blacklist: + filestatus[indx] = "blacklisted" + continue + # endif + # store: selected.append(indx) + # set status: filestatus[indx] = "selected" # endif # endfor # records @@ -1688,7 +1696,7 @@ class CSO_Listing(object): for fname, row in df.iterrows(): line = fname if fname in filestatus.keys(): - line = line + " [" + filestatus[fname] + "]" + line = line + f" [{filestatus[fname]}]" logging.info(f"{indent} {line}") # endfor # endif # verbose diff --git a/src/cso/cso_viirs.py b/src/cso/cso_viirs.py index 07678e0..69e0a0f 100644 --- a/src/cso/cso_viirs.py +++ b/src/cso/cso_viirs.py @@ -1289,7 +1289,7 @@ class CSO_VIIRS_Convert(utopya.UtopyaRc): # read: listing = cso_file.CSO_Listing(listing_file) # info ... - logging.info(f"{indent}number of files : %i" % len(listing)) + logging.info(f"{indent}number of files : {len(listing)}") # path: listing_dir = os.path.dirname(listing_file) @@ -1325,7 +1325,7 @@ class CSO_VIIRS_Convert(utopya.UtopyaRc): logging.info(f"{indent} orbit '{orbit}' ...") # select orbits: - olst = xlst.Select(expr=f"%{{orbit}} == '{orbit}'", verbose=False) + olst = xlst.Select(expr=f"%{{orbit}} == '{orbit}'", verbose=False, indent=indent+" ") # info .. logging.info(f"{indent} process {len(olst)} files ...") # next orbit if none: -- GitLab From 3fc723bcc4e465e8f814349f90393e0f80095537 Mon Sep 17 00:00:00 2001 From: Arjo Segers Date: Fri, 3 Oct 2025 08:40:38 +0200 Subject: [PATCH 2/4] Expand templates in header and title of index pages. --- src/utopya/utopya_index.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/utopya/utopya_index.py b/src/utopya/utopya_index.py index 66d92e8..9759db2 100644 --- a/src/utopya/utopya_index.py +++ b/src/utopya/utopya_index.py @@ -21,7 +21,9 @@ # Extened option to select range of characters of index value. # Option to provide description text under header. # - +# 2025-09, Arjo Segers +# Expand templates in header and title. +# # ------------------------------------------------- # help @@ -744,6 +746,10 @@ class IndexPart(utopya_rc.UtopyaRc): continue if level_value == "": continue + # replace template? + if "value" in self.current[level].keys(): + level_value = level_value.replace(f"%{{{level}}}",self.current[level]["value"]) + #endif # add link? if alinks and ("alink" in self.current[level]) and (level != self.name): level_value = '%s' % (self.current[level]["alink"], level_value) @@ -844,8 +850,10 @@ class IndexPart(utopya_rc.UtopyaRc): if self.filename is not None: # show info on creation? with_info = self.GetSetting(self.keyname + ".info", totype="bool", default=False) + # expand title: + title = self.ReplaceCurrents( self.header ) # write: - self.html.Write(self.filename, title=self.header, info=with_info) + self.html.Write(self.filename, title=title, info=with_info) # endif # enddef WriteFile -- GitLab From 0026a38fc0281360df7d549ce3b5e688c1a0a092 Mon Sep 17 00:00:00 2001 From: Arjo Segers Date: Fri, 3 Oct 2025 08:48:15 +0200 Subject: [PATCH 3/4] Added templates for debugging. --- src/cso/cso_s5p.py | 6 ++++++ src/cso/cso_viirs.py | 8 +++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/cso/cso_s5p.py b/src/cso/cso_s5p.py index d9b4c04..1acfefb 100644 --- a/src/cso/cso_s5p.py +++ b/src/cso/cso_s5p.py @@ -2860,6 +2860,12 @@ class CSO_S5p_Convert(utopya.UtopyaRc): # info ... logging.info(indent + ' orbit "%s" ...' % orbit) + # TESTING .. + if orbit != "21861": + logging.warning(f"{indent} skip!") + continue + #endif + # select single orbit matching expression: odf = xlst.Select( orbit=orbit, expr=selection_expr, blacklist=blacklist, indent=f" " diff --git a/src/cso/cso_viirs.py b/src/cso/cso_viirs.py index 69e0a0f..f2e7bfe 100644 --- a/src/cso/cso_viirs.py +++ b/src/cso/cso_viirs.py @@ -1058,7 +1058,7 @@ class CSO_VIIRS_File(cso_file.CSO_File): import scipy # loop over pixels: for i in range(nx): - # pixels without data: + # scan lines without data: jj, = numpy.where( values[:,i] < -180.0 ) # any? if len(jj) > 0: @@ -1324,6 +1324,12 @@ class CSO_VIIRS_Convert(utopya.UtopyaRc): # info ... logging.info(f"{indent} orbit '{orbit}' ...") + ## TESTING .. + #if orbit != "60846": + # logging.warning(f"{indent} skip!") + # continue + ##endif + # select orbits: olst = xlst.Select(expr=f"%{{orbit}} == '{orbit}'", verbose=False, indent=indent+" ") # info .. -- GitLab From 94c2c1c0dd0ad4f367c8a6bc06232842ea33eec5 Mon Sep 17 00:00:00 2001 From: Arjo Segers Date: Fri, 3 Oct 2025 08:48:40 +0200 Subject: [PATCH 4/4] Fixed map plots. Fixed error messages. --- src/cso/cso_catalogue.py | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/src/cso/cso_catalogue.py b/src/cso/cso_catalogue.py index 9ab0cbf..0643db5 100644 --- a/src/cso/cso_catalogue.py +++ b/src/cso/cso_catalogue.py @@ -1196,20 +1196,15 @@ class CSO_GriddedCatalogue(CSO_CatalogueBase): if len(values.shape) == 3: # check .. if (vsource_layer < 0) or (values.shape[2] < vsource_layer + 1): - logging.error( - "could not extract source layer %i from values with shape (%s)" - % (vsource_layer, str(vvalues.shape)) - ) + logging.error(f"could not extract source layer {vsource_layer}" + + f" from values with shape ({vvalues.shape})") raise Exception # endif # extract: values = values[:, :, vsource_layer] # elif len(values.shape) != 2: - logging.error( - "could not plot map of array with shape (%s)" - % (str(values.shape)) - ) + logging.error(f"could not plot map of array with shape ({values.shape})") raise Exception # endif @@ -1234,8 +1229,8 @@ class CSO_GriddedCatalogue(CSO_CatalogueBase): # create map figure: fig = cso_plot.QuickPat( values, - xm=xm, - ym=ym, + x=xm, + y=ym, vmin=vmin, vmax=vmax, cmap=cmap, @@ -1255,14 +1250,14 @@ class CSO_GriddedCatalogue(CSO_CatalogueBase): raise Exception # endif # ptype - # endif # renew + else: + logging.info(f"{indent} no input file: {infile1}") + # endif # state file present - ## testing ... - # break + # endif # renew - else: - logging.info(f"{indent} no input file: {infile}") - # endif # state file present + ## testing ... + # break # endfor # variables -- GitLab