Merge branch 'sesam' into 'master' (8f9c08fc) · Commits · CAMS / CSO

src/cso/cso_catalogue.py

+11 −16

Original line number	Diff line number	Diff line
		@@ -1196,20 +1196,15 @@ class CSO_GriddedCatalogue(CSO_CatalogueBase):
		if len(values.shape) == 3:
		# check ..
		if (vsource_layer < 0) or (values.shape[2] < vsource_layer + 1):
		logging.error(
		"could not extract source layer %i from values with shape (%s)"
		% (vsource_layer, str(vvalues.shape))
		)
		logging.error(f"could not extract source layer {vsource_layer}"
		+ f" from values with shape ({vvalues.shape})")
		raise Exception
		# endif
		# extract:
		values = values[:, :, vsource_layer]
		#
		elif len(values.shape) != 2:
		logging.error(
		"could not plot map of array with shape (%s)"
		% (str(values.shape))
		)
		logging.error(f"could not plot map of array with shape ({values.shape})")
		raise Exception
		# endif

		@@ -1234,8 +1229,8 @@ class CSO_GriddedCatalogue(CSO_CatalogueBase):
		# create map figure:
		fig = cso_plot.QuickPat(
		values,
		xm=xm,
		ym=ym,
		x=xm,
		y=ym,
		vmin=vmin,
		vmax=vmax,
		cmap=cmap,
		@@ -1255,15 +1250,15 @@ class CSO_GriddedCatalogue(CSO_CatalogueBase):
		raise Exception
		# endif # ptype

		else:
		logging.info(f"{indent} no input file: {infile1}")
		# endif # state file present

		# endif # renew

		## testing ...
		# break

		else:
		logging.info(f"{indent} no input file: {infile}")
		# endif # state file present

		# endfor # variables

		# next:

src/cso/cso_file.py

+30 −22

Original line number	Diff line number	Diff line
		@@ -52,6 +52,9 @@
		# 2025-06, Arjo Segers
		# Set flag to avoid warnings when decoding time arrays from VIIRS files.
		#
		# 2025-09, Arjo Segers
		# Improved speed of CSO_Listing.Selection method.
		#

		########################################################################
		###
		@@ -1639,35 +1642,40 @@ class CSO_Listing(object):

		# evaluate selection expression?
		if expr is not None:
		# replace templates:
		# %{orbit} == '12345'
		# to:
		# xrec['orbit'] == '12345'
		for key in self.df.keys():
		expr = expr.replace("%{" + key + "}", "xrec['" + key + "']")
		# endfor
		# split:
		# split into sequnece of critera; first expression with macthing record will be used:
		selections = expr.split(";")

		# initialize empty list of selected record indices (filenames):
		selected = []
		# storage for status label: "selected", "blacklisted", ...
		filestatus = {}
		# no match yet ..
		seleted = []
		# loop over selection criteria,
		# this should give either none or a single file:
		for selection in selections:
		# make empty again:
		# replace templates:
		# %{orbit} == '12345'
		# to:
		# df['orbit'] == '12345'
		for key in self.df.keys():
		selection = selection.replace(f"%{{{key}}}", f"df['{key}']")
		# endfor
		# testing ...
		logging.info(f"{indent}selection `{selection}` ...")
		# evaluate:
		xdf = df[ eval(selection) ]
		# any?
		if len(xdf) > 0:
		# selected indices (filenames):
		selected = []
		# loop over records:
		for indx, xrec in df.iterrows():
		for indx, xrec in xdf.iterrows():
		# skip?
		if os.path.basename(indx) in blacklist:
		filestatus[indx] = "blacklisted"
		continue
		# endif
		# evaluate expression including 'xrec[key]' values:
		if eval(selection):
		# store:
		selected.append(indx)
		# set status:
		filestatus[indx] = "selected"
		# endif
		# endfor # records
		@@ -1688,7 +1696,7 @@ class CSO_Listing(object):
		for fname, row in df.iterrows():
		line = fname
		if fname in filestatus.keys():
		line = line + " [" + filestatus[fname] + "]"
		line = line + f" [{filestatus[fname]}]"
		logging.info(f"{indent} {line}")
		# endfor
		# endif # verbose

src/cso/cso_s5p.py

+6 −0

Original line number	Diff line number	Diff line
		@@ -2860,6 +2860,12 @@ class CSO_S5p_Convert(utopya.UtopyaRc):
		# info ...
		logging.info(indent + ' orbit "%s" ...' % orbit)

		# TESTING ..
		if orbit != "21861":
		logging.warning(f"{indent} skip!")
		continue
		#endif

		# select single orbit matching expression:
		odf = xlst.Select(
		orbit=orbit, expr=selection_expr, blacklist=blacklist, indent=f" "

src/cso/cso_viirs.py

+9 −3

Original line number	Diff line number	Diff line
		@@ -1058,7 +1058,7 @@ class CSO_VIIRS_File(cso_file.CSO_File):
		import scipy
		# loop over pixels:
		for i in range(nx):
		# pixels without data:
		# scan lines without data:
		jj, = numpy.where( values[:,i] < -180.0 )
		# any?
		if len(jj) > 0:
		@@ -1289,7 +1289,7 @@ class CSO_VIIRS_Convert(utopya.UtopyaRc):
		# read:
		listing = cso_file.CSO_Listing(listing_file)
		# info ...
		logging.info(f"{indent}number of files : %i" % len(listing))
		logging.info(f"{indent}number of files : {len(listing)}")

		# path:
		listing_dir = os.path.dirname(listing_file)
		@@ -1324,8 +1324,14 @@ class CSO_VIIRS_Convert(utopya.UtopyaRc):
		# info ...
		logging.info(f"{indent} orbit '{orbit}' ...")

		## TESTING ..
		#if orbit != "60846":
		# logging.warning(f"{indent} skip!")
		# continue
		##endif

		# select orbits:
		olst = xlst.Select(expr=f"%{{orbit}} == '{orbit}'", verbose=False)
		olst = xlst.Select(expr=f"%{{orbit}} == '{orbit}'", verbose=False, indent=indent+" ")
		# info ..
		logging.info(f"{indent} process {len(olst)} files ...")
		# next orbit if none:

src/utopya/utopya_index.py

+10 −2

Original line number	Diff line number	Diff line
		@@ -21,7 +21,9 @@
		# Extened option to select range of characters of index value.
		# Option to provide description text under header.
		#

		# 2025-09, Arjo Segers
		# Expand templates in header and title.
		#

		# -------------------------------------------------
		# help
		@@ -744,6 +746,10 @@ class IndexPart(utopya_rc.UtopyaRc):
		continue
		if level_value == "":
		continue
		# replace template?
		if "value" in self.current[level].keys():
		level_value = level_value.replace(f"%{{{level}}}",self.current[level]["value"])
		#endif
		# add link?
		if alinks and ("alink" in self.current[level]) and (level != self.name):
		level_value = '<a href="%s">%s</a>' % (self.current[level]["alink"], level_value)
		@@ -844,8 +850,10 @@ class IndexPart(utopya_rc.UtopyaRc):
		if self.filename is not None:
		# show info on creation?
		with_info = self.GetSetting(self.keyname + ".info", totype="bool", default=False)
		# expand title:
		title = self.ReplaceCurrents( self.header )
		# write:
		self.html.Write(self.filename, title=self.header, info=with_info)
		self.html.Write(self.filename, title=title, info=with_info)
		# endif

		# enddef WriteFile