diff --git a/src/cso/cso_catalogue.py b/src/cso/cso_catalogue.py index 9ab0cbf5672bab8f37a0128dd2e7e39581a92a3e..0643db50f15d3c23dfb3f646d744c5eb7f91df53 100644 --- a/src/cso/cso_catalogue.py +++ b/src/cso/cso_catalogue.py @@ -1196,20 +1196,15 @@ class CSO_GriddedCatalogue(CSO_CatalogueBase): if len(values.shape) == 3: # check .. if (vsource_layer < 0) or (values.shape[2] < vsource_layer + 1): - logging.error( - "could not extract source layer %i from values with shape (%s)" - % (vsource_layer, str(vvalues.shape)) - ) + logging.error(f"could not extract source layer {vsource_layer}" + + f" from values with shape ({vvalues.shape})") raise Exception # endif # extract: values = values[:, :, vsource_layer] # elif len(values.shape) != 2: - logging.error( - "could not plot map of array with shape (%s)" - % (str(values.shape)) - ) + logging.error(f"could not plot map of array with shape ({values.shape})") raise Exception # endif @@ -1234,8 +1229,8 @@ class CSO_GriddedCatalogue(CSO_CatalogueBase): # create map figure: fig = cso_plot.QuickPat( values, - xm=xm, - ym=ym, + x=xm, + y=ym, vmin=vmin, vmax=vmax, cmap=cmap, @@ -1255,14 +1250,14 @@ class CSO_GriddedCatalogue(CSO_CatalogueBase): raise Exception # endif # ptype - # endif # renew + else: + logging.info(f"{indent} no input file: {infile1}") + # endif # state file present - ## testing ... - # break + # endif # renew - else: - logging.info(f"{indent} no input file: {infile}") - # endif # state file present + ## testing ... + # break # endfor # variables diff --git a/src/cso/cso_file.py b/src/cso/cso_file.py index 1eb4a7f13d61081888c6c97c00105d7d7eb604b5..55b176416f937c855e348fcabf521a9a270ecc50 100644 --- a/src/cso/cso_file.py +++ b/src/cso/cso_file.py @@ -52,6 +52,9 @@ # 2025-06, Arjo Segers # Set flag to avoid warnings when decoding time arrays from VIIRS files. # +# 2025-09, Arjo Segers +# Improved speed of CSO_Listing.Selection method. +# ######################################################################## ### @@ -1639,35 +1642,40 @@ class CSO_Listing(object): # evaluate selection expression? if expr is not None: - # replace templates: - # %{orbit} == '12345' - # to: - # xrec['orbit'] == '12345' - for key in self.df.keys(): - expr = expr.replace("%{" + key + "}", "xrec['" + key + "']") - # endfor - # split: + # split into sequnece of critera; first expression with macthing record will be used: selections = expr.split(";") + # initialize empty list of selected record indices (filenames): + selected = [] # storage for status label: "selected", "blacklisted", ... filestatus = {} - # no match yet .. - seleted = [] # loop over selection criteria, # this should give either none or a single file: for selection in selections: - # make empty again: - selected = [] - # loop over records: - for indx, xrec in df.iterrows(): - # skip? - if os.path.basename(indx) in blacklist: - filestatus[indx] = "blacklisted" - continue - # endif - # evaluate expression including 'xrec[key]' values: - if eval(selection): + # replace templates: + # %{orbit} == '12345' + # to: + # df['orbit'] == '12345' + for key in self.df.keys(): + selection = selection.replace(f"%{{{key}}}", f"df['{key}']") + # endfor + # testing ... + logging.info(f"{indent}selection `{selection}` ...") + # evaluate: + xdf = df[ eval(selection) ] + # any? + if len(xdf) > 0: + # selected indices (filenames): + selected = [] + for indx, xrec in xdf.iterrows(): + # skip? + if os.path.basename(indx) in blacklist: + filestatus[indx] = "blacklisted" + continue + # endif + # store: selected.append(indx) + # set status: filestatus[indx] = "selected" # endif # endfor # records @@ -1688,7 +1696,7 @@ class CSO_Listing(object): for fname, row in df.iterrows(): line = fname if fname in filestatus.keys(): - line = line + " [" + filestatus[fname] + "]" + line = line + f" [{filestatus[fname]}]" logging.info(f"{indent} {line}") # endfor # endif # verbose diff --git a/src/cso/cso_s5p.py b/src/cso/cso_s5p.py index d9b4c0475eefb1e7025471e0fb58573b44f0b8d7..1acfefb0c0fe38280a897f701d8830ef7b221333 100644 --- a/src/cso/cso_s5p.py +++ b/src/cso/cso_s5p.py @@ -2860,6 +2860,12 @@ class CSO_S5p_Convert(utopya.UtopyaRc): # info ... logging.info(indent + ' orbit "%s" ...' % orbit) + # TESTING .. + if orbit != "21861": + logging.warning(f"{indent} skip!") + continue + #endif + # select single orbit matching expression: odf = xlst.Select( orbit=orbit, expr=selection_expr, blacklist=blacklist, indent=f" " diff --git a/src/cso/cso_viirs.py b/src/cso/cso_viirs.py index 07678e017372c8570e840c3d7cf5ce754c5017be..f2e7bfeb2608b61e76539735d8831ede4a936495 100644 --- a/src/cso/cso_viirs.py +++ b/src/cso/cso_viirs.py @@ -1058,7 +1058,7 @@ class CSO_VIIRS_File(cso_file.CSO_File): import scipy # loop over pixels: for i in range(nx): - # pixels without data: + # scan lines without data: jj, = numpy.where( values[:,i] < -180.0 ) # any? if len(jj) > 0: @@ -1289,7 +1289,7 @@ class CSO_VIIRS_Convert(utopya.UtopyaRc): # read: listing = cso_file.CSO_Listing(listing_file) # info ... - logging.info(f"{indent}number of files : %i" % len(listing)) + logging.info(f"{indent}number of files : {len(listing)}") # path: listing_dir = os.path.dirname(listing_file) @@ -1324,8 +1324,14 @@ class CSO_VIIRS_Convert(utopya.UtopyaRc): # info ... logging.info(f"{indent} orbit '{orbit}' ...") + ## TESTING .. + #if orbit != "60846": + # logging.warning(f"{indent} skip!") + # continue + ##endif + # select orbits: - olst = xlst.Select(expr=f"%{{orbit}} == '{orbit}'", verbose=False) + olst = xlst.Select(expr=f"%{{orbit}} == '{orbit}'", verbose=False, indent=indent+" ") # info .. logging.info(f"{indent} process {len(olst)} files ...") # next orbit if none: diff --git a/src/utopya/utopya_index.py b/src/utopya/utopya_index.py index 66d92e8e955925621981bc64fb35669099697ca4..9759db29b999cbf9e86431b4f252ed3d7b2e438a 100644 --- a/src/utopya/utopya_index.py +++ b/src/utopya/utopya_index.py @@ -21,7 +21,9 @@ # Extened option to select range of characters of index value. # Option to provide description text under header. # - +# 2025-09, Arjo Segers +# Expand templates in header and title. +# # ------------------------------------------------- # help @@ -744,6 +746,10 @@ class IndexPart(utopya_rc.UtopyaRc): continue if level_value == "": continue + # replace template? + if "value" in self.current[level].keys(): + level_value = level_value.replace(f"%{{{level}}}",self.current[level]["value"]) + #endif # add link? if alinks and ("alink" in self.current[level]) and (level != self.name): level_value = '%s' % (self.current[level]["alink"], level_value) @@ -844,8 +850,10 @@ class IndexPart(utopya_rc.UtopyaRc): if self.filename is not None: # show info on creation? with_info = self.GetSetting(self.keyname + ".info", totype="bool", default=False) + # expand title: + title = self.ReplaceCurrents( self.header ) # write: - self.html.Write(self.filename, title=self.header, info=with_info) + self.html.Write(self.filename, title=title, info=with_info) # endif # enddef WriteFile