Support user defined directory creation mode. (e41bac96) · Commits · CAMS / CSO

py/cso_colhub.py

+11 −1

Original line number	Diff line number	Diff line
		@@ -117,6 +117,11 @@ class CSO_ColHubMirror_Inquire(utopya.UtopyaRc):
		! time templates are replaced with todays date
		<rcbase>.file : /Scratch/Copernicus/S5p/listing-CH4__%Y-%m-%d.csv

		Optionally define a creation mode for the (parent) directories::

		! directory creation mode:
		<rcbase>.dmode : 0o775

		An existing listing file is not replaced,
		unless the following flag is set::

		@@ -338,6 +343,11 @@ class CSO_ColHubMirror_Missing(utopya.UtopyaRc):
		! time templates are replaced with todays date
		<rcbase>.file : /work/inquire/Copernicus_S5p_NO2_colhub-mirror-missing__%Y-%m-%d.csv

		Optionally define a creation mode for the (parent) directories::

		! directory creation mode:
		<rcbase>.dmode : 0o775

		An existing listing file is not replaced,
		unless the following flag is set::

py/cso_dataspace.py

+10 −7

Original line number	Diff line number	Diff line
		@@ -24,13 +24,16 @@
		# Increase waiting time after failed download.
		# Trap "404 Client Error", display messate that inquire table might need an update.
		# Sort listing inplace.
		#
		# s
		# 2024-11, Arjo Segers
		# Inquire Dataspace per month after change in allowed maximum number of records.
		# Sort listing files by processing and processor version.
		# Increased maximum number of records and introduced sort order to avoid that
		# search request return a different result when repeated.
		#
		# 2025-02, Arjo Segers
		# Support user defined directory creation mode.
		#

		########################################################################
		###
		@@ -150,7 +153,7 @@ class CSO_DataSpace_Inquire(utopya.UtopyaRc):
		for a certain time and overlap with a specified region.
		The result is a list with orbit files and instructions on how to download them.

		In the settings, specify the time range over which files should be downloaded::
		In the settings, specify the time range over which files should be searched::

		<rcbase>.timerange.start : 2018-07-01 00:00
		<rcbase>.timerange.end : 2018-07-01 23:59
		@@ -732,13 +735,13 @@ class CSO_DataSpace_Downloader(object):
		# *

		def DownloadFile(
		self, href, output_file, maxtry=10, nsec_wait=5, nsec_wait_max=600, indent=""
		self, href, output_file, maxtry=10, nsec_wait=5, nsec_wait_max=600, dmode=None, indent=""
		):
		"""
		Download file from DataSpace.

		If a request fails it is tried again up to a maximum of ``maxtry`` times,
		with a delay of ``nsec_wait`` between requsts.
		with a delay of initially ``nsec_wait`` between requsts.

		Arguments:

		@@ -752,8 +755,8 @@ class CSO_DataSpace_Downloader(object):

		* ``maxtry`` : number of times to try again if download fails
		* ``nsec_wait`` : initial delay in seconds between requests;
		with every new attempt, this will be increased with a maximum of:
		* ``nsec_wait_max``
		with every new attempt, this will be increased up to a maximum of ``nsec_wait_max``
		* ``dmode``: directory creation mode, e.g. ``777``

		"""

		@@ -830,7 +833,7 @@ class CSO_DataSpace_Downloader(object):
		# info ..
		logging.info(f"{indent}store ...")
		# create target dir if necessary:
		cso_file.CheckDir(output_file)
		cso_file.CheckDir(output_file, dmode=dmode)
		# move to destination:
		os.rename(member, output_file)
		# remove directory tree:

py/cso_file.py

+35 −5

Original line number	Diff line number	Diff line
		@@ -26,6 +26,9 @@
		# 2025-01, Arjo Segers
		# Extened CSO_Listing class with selection and other methods.
		#
		# 2025-02, Arjo Segers
		# Option to CheckDir to set directory creation mode.
		#

		########################################################################
		###
		@@ -74,10 +77,11 @@ import logging
		########################################################################


		def CheckDir(filename):
		def CheckDir(filename, dmode=None):
		"""
		Check if ``filename`` has a directory path;
		if so, create that directory if it does not exist yet.
		Optional ``dmode`` (``0o777``) is used for creation of directories.
		"""

		# modules:
		@@ -89,8 +93,34 @@ def CheckDir(filename):
		if len(dname) > 0:
		# not present yet?
		if not os.path.isdir(dname):
		# create including subdirs:
		os.makedirs(dname)
		# Do not use "os.makedirs", as this will use default creation mode for parent directories ...
		# Therefor pefrom a loop over parent directories.

		# split into sub-directories:
		subdirs = dname.split(os.sep)
		# insert root as first subdir if necessary:
		if dname.startswith(os.sep):
		subdirs.insert(0, os.sep)

		# init parent path:
		pdir = None
		# loop over subdirs:
		for subdir in subdirs:
		# extend parent path:
		if pdir is None:
		pdir = subdir
		else:
		pdir = os.path.join(pdir, subdir)
		# endif

		# parent path not present yet?
		if not os.path.isdir(pdir):
		# create:
		os.mkdir(pdir, mode=dmode)
		# endif

		# endfor # subdir

		# endif # dname present
		# endif # dname defined

		@@ -1261,7 +1291,7 @@ class CSO_Listing(object):

		# *

		def Save(self, filename, indent=""):
		def Save(self, filename, dmode=None, indent=""):
		"""
		Write table to file.
		"""
		@@ -1270,7 +1300,7 @@ class CSO_Listing(object):
		logging.info(f"{indent}save listing {filename} ...")

		# create directory if needed:
		CheckDir(filename)
		CheckDir(filename, dmode=dmode)

		# write all columns:
		columns = list(self.df.keys())

py/cso_s5p.py

+20 −4

Original line number	Diff line number	Diff line
		@@ -2664,6 +2664,11 @@ class CSO_S5p_Convert(utopya.UtopyaRc):
		! %{orbit}, %{processing}, ...
		<rcbase>.output.filename : /Scratch/CSO/S5p/RPRO/NO2/Europe/%Y/%m/S5p_RPRO_NO2_%{orbit}.nc

		Optionally define a creation mode for the (parent) directories::

		! directory creation mode:
		<rcbase>.dmode : 0o775

		A flag is read to decide if existing output files should be renewed or kept::

		<rcbase>.renew : True
		@@ -2724,6 +2729,9 @@ class CSO_S5p_Convert(utopya.UtopyaRc):
		# init base object:
		utopya.UtopyaRc.__init__(self, rcfile=rcfile, rcbase=rcbase, env=env)

		# directory creation mode:
		dmode = self.GetSetting("dmode", totype=int, default=None)

		# renew output?
		renew = self.GetSetting("renew", totype="bool")

		@@ -3009,7 +3017,7 @@ class CSO_S5p_Convert(utopya.UtopyaRc):
		# endif
		# endif
		# download ...
		downloader.DownloadFile(href, input_file, indent=" ")
		downloader.DownloadFile(href, input_file, dmode=dmode, indent=" ")
		# store name:
		downloads.append(input_file)
		# endif
		@@ -3160,7 +3168,7 @@ class CSO_S5p_Download(utopya.UtopyaRc):

		* ``rcfile``, ``rcbase``, ``env`` : settings file, prefix for keys, and environment dictionairy

		A time range is read to select the files to be converted::
		A time range is read to select the files to be downloaded::

		<rcbase>.timerange.start : 2018-06-01 00:00
		<rcbase>.timerange.end : 2018-06-03 23:59
		@@ -3190,7 +3198,7 @@ class CSO_S5p_Download(utopya.UtopyaRc):
		/data/Copernicus/S5P/OFFL/NO2/2018/07/S5P_OFFL_L2__NO2____20180701T005930_20180701T024100_03698_01_010002_20180707T022838.nc
		start_time end_time orbit

		Provide ';' seperated list of to decide if a particular orbit file should be processed.
		Provide a ';'-seperated list of selection criteria to decide if a particular orbit file should be processed.
		If more than one file is available for a particular orbit (from "OFFL" and "RPRO" processing),
		the file with the first match will be used.
		The expressions should include templates '%{header}' for the column values.
		@@ -3493,6 +3501,11 @@ class CSO_S5p_Listing(utopya.UtopyaRc):
		! - orbit number
		<rcbase>.file : /Scratch/CSO/S5p/RPRO/NO2/Europe/listing.csv

		Optionally define a creation mode for the (parent) directories::

		! directory creation mode:
		<rcbase>.dmode : 0o775

		An existing listing file is not replaced,
		unless the following flag is set::

		@@ -3552,6 +3565,9 @@ class CSO_S5p_Listing(utopya.UtopyaRc):
		# init base object:
		utopya.UtopyaRc.__init__(self, rcfile=rcfile, rcbase=rcbase, env=env)

		# directory creation mode:
		dmode = self.GetSetting("dmode", totype=int, default=None)

		# renew output?
		renew = self.GetSetting("renew", totype="bool")

		@@ -3682,7 +3698,7 @@ class CSO_S5p_Listing(utopya.UtopyaRc):
		# endwhile

		# save:
		listing.Save(lst_file, indent=f"{indent} ")
		listing.Save(lst_file, dmode=dmode, indent=f"{indent} ")

		else:
		# info ..