Start jobtree with copy job to install scripts and settings in work directory. (450dff0b) · Commits · CAMS / CSO

CHANGELOG

+7 −0

Original line number	Diff line number	Diff line
		@@ -401,4 +401,11 @@ Increase waiting time after failed download.
		Show extra info on pixel selection.
		py/cso_s5p.py

		Support colormaps from cmcrameri package.
		py/cso_plot.py

		Start jobtree with 'copy' job to install scripts and settings in work directory.
		bin/cso
		config/Copernicus/cso.rc
		doc/source/tutorial.rst

bin/cso

+7 −2

Original line number	Diff line number	Diff line
		@@ -19,7 +19,7 @@ History
		Reformatted using 'black'.

		2023-08, Arjo Segers
		Define environment variable 'CSO_RCFILE'.
		Define environment variables 'CSO_RCFILE' and 'CSO_RCDIR'.

		"""

		@@ -48,6 +48,9 @@ import utopya

		# set environment:
		os.environ["CSO_PREFIX"] = prefix
		# dummy values to ensure that settings could be read first to setup logging:
		os.environ['CSO_RCFILE'] = 'None'
		os.environ['CSO_RCDIR'] = 'None'

		# init script:
		utos = utopya.UtopyaRunScriptRc()
		@@ -61,6 +64,8 @@ args, xargs = utos.ArgumentsParse()

		# store full path to settings:
		os.environ['CSO_RCFILE'] = os.path.abspath( args.rcfile )
		# location of settings file:
		os.environ['CSO_RCDIR'] = os.path.dirname( os.path.abspath( args.rcfile ) )

		# start, shout info:
		logging.info(f"")
		@@ -71,7 +76,7 @@ logging.info(f"input rcfile : {args.rcfile}")
		logging.info(f"settings base : {args.rcbase}")
		logging.info(f"")
		logging.info(f"environment:")
		for key in ["CSO_PREFIX","CSO_RCFILE"]:
		for key in ["CSO_PREFIX","CSO_RCFILE","CSO_RCDIR"]:
		logging.info(f" {key} = {os.environ[key]}")
		# endfor
		logging.info(f"")

config/Copernicus/cso.rc

+87 −22

Original line number	Diff line number	Diff line
		@@ -6,6 +6,10 @@
		!!!
		!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

		! dummy keys, actual values defined in environment by "cso" script:
		CSO_RCFILE :
		CSO_RCDIR :


		!----------------------------------------------------------
		! user specific settings:
		@@ -33,7 +37,7 @@ my.tracers : no2
		! class to create a job tree:
		cso.class : utopya.UtopyaJobTree
		! list of sub-elements:
		cso.elements : s5p
		cso.elements : copy s5p

		! class to create a job tree:
		cso.s5p.class : utopya.UtopyaJobTree
		@@ -54,8 +58,8 @@ cso.s5p.TRACER.class : utopya.UtopyaJobTree
		! catalogue \
		! gridded gridded-catalogue
		!~ preprocessor steps one by one ...
		!cso.s5p.TRACER.elements : inquire
		cso.s5p.TRACER.elements : convert listing catalogue
		cso.s5p.TRACER.elements : inquire
		!cso.s5p.TRACER.elements : convert listing catalogue
		!cso.s5p.TRACER.elements : convert
		!cso.s5p.TRACER.elements : listing
		!cso.s5p.TRACER.elements : catalogue
		@@ -86,19 +90,19 @@ cso.s5p.TRACER.inquire.tasks : table-glyretro table-pal pl
		#endif
		!~ inquire files available on DataSpace:
		cso.s5p.TRACER.inquire.table-dataspace.class : cso.CSO_DataSpace_Inquire
		cso.s5p.TRACER.inquire.table-dataspace.args : '${PWD}/config/Copernicus/cso-s5p-TRACER.rc', \
		cso.s5p.TRACER.inquire.table-dataspace.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.inquire-table-dataspace'
		!~ inquire files available on PAL:
		cso.s5p.TRACER.inquire.table-pal.class : cso.CSO_PAL_Inquire
		cso.s5p.TRACER.inquire.table-pal.args : '${PWD}/config/Copernicus/cso-s5p-TRACER.rc', \
		cso.s5p.TRACER.inquire.table-pal.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.inquire-table-pal'
		!~ inquire files downloaded from GlyRetro:
		cso.s5p.TRACER.inquire.table-glyretro.class : cso.CSO_S5p_Download_Listing
		cso.s5p.TRACER.inquire.table-glyretro.args : '${PWD}/config/Copernicus/cso-s5p-TRACER.rc', \
		cso.s5p.TRACER.inquire.table-glyretro.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.inquire-table-glyretro'
		!~ create plot of available versions:
		cso.s5p.TRACER.inquire.plot.class : cso.CSO_Inquire_Plot
		cso.s5p.TRACER.inquire.plot.args : '${PWD}/config/Copernicus/cso-s5p-TRACER.rc', \
		cso.s5p.TRACER.inquire.plot.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.inquire-plot'

		! *
		@@ -107,19 +111,19 @@ cso.s5p.TRACER.inquire.plot.args : '${PWD}/config/Copernicus/c
		cso.s5p.TRACER.download.class : utopya.UtopyaJobStep
		! convert task:
		cso.s5p.TRACER.download.task.class : cso.CSO_S5p_Download
		cso.s5p.TRACER.download.task.args : '${PWD}/config/Copernicus/cso-s5p-TRACER.rc', \
		cso.s5p.TRACER.download.task.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.download'
		! single step:
		cso.s5p.TRACER.convert.class : utopya.UtopyaJobStep
		! convert task:
		cso.s5p.TRACER.convert.task.class : cso.CSO_S5p_Convert
		cso.s5p.TRACER.convert.task.args : '${PWD}/config/Copernicus/cso-s5p-TRACER.rc', \
		cso.s5p.TRACER.convert.task.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.convert'
		! single step:
		cso.s5p.TRACER.listing.class : utopya.UtopyaJobStep
		! listing task:
		cso.s5p.TRACER.listing.task.class : cso.CSO_S5p_Listing
		cso.s5p.TRACER.listing.task.args : '${PWD}/config/Copernicus/cso-s5p-TRACER.rc', \
		cso.s5p.TRACER.listing.task.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.listing'

		! single step:
		@@ -129,11 +133,11 @@ cso.s5p.TRACER.catalogue.tasks : figs index

		! catalogue creation task:
		cso.s5p.TRACER.catalogue.figs.class : cso.CSO_Catalogue
		cso.s5p.TRACER.catalogue.figs.args : '${PWD}/config/Copernicus/cso-s5p-TRACER.rc', \
		cso.s5p.TRACER.catalogue.figs.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.catalogue'
		! indexer task:
		cso.s5p.TRACER.catalogue.index.class : utopya.Indexer
		cso.s5p.TRACER.catalogue.index.args : '${PWD}/config/Copernicus/cso-s5p-TRACER.rc', \
		cso.s5p.TRACER.catalogue.index.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.catalogue-index'

		! *
		@@ -142,7 +146,7 @@ cso.s5p.TRACER.catalogue.index.args : '${PWD}/config/Copernicus/cs
		cso.s5p.TRACER.gridded.class : utopya.UtopyaJobStep
		! catalogue creation task:
		cso.s5p.TRACER.gridded.task.class : cso.CSO_GriddedAverage
		cso.s5p.TRACER.gridded.task.args : '${PWD}/config/Copernicus/cso-s5p-TRACER.rc', \
		cso.s5p.TRACER.gridded.task.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.gridded'

		! single step:
		@@ -151,11 +155,11 @@ cso.s5p.TRACER.gridded-catalogue.class : utopya.UtopyaJobStep
		cso.s5p.TRACER.gridded-catalogue.tasks : figs index
		! catalogue creation task:
		cso.s5p.TRACER.gridded-catalogue.figs.class : cso.CSO_GriddedCatalogue
		cso.s5p.TRACER.gridded-catalogue.figs.args : '${PWD}/config/Copernicus/cso-s5p-TRACER.rc', \
		cso.s5p.TRACER.gridded-catalogue.figs.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.gridded-catalogue'
		! indexer task:
		cso.s5p.TRACER.gridded-catalogue.index.class : utopya.Indexer
		cso.s5p.TRACER.gridded-catalogue.index.args : '${PWD}/config/Copernicus/cso-s5p-TRACER.rc', \
		cso.s5p.TRACER.gridded-catalogue.index.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.gridded-catalogue-index'

		! *
		@@ -167,11 +171,11 @@ cso.s5p.TRACER.sim-catalogue.tasks : figs index

		! catalogue creation task:
		cso.s5p.TRACER.sim-catalogue.figs.class : cso.CSO_SimCatalogue
		cso.s5p.TRACER.sim-catalogue.figs.args : '${PWD}/config/Copernicus/cso-s5p-TRACER.rc', \
		cso.s5p.TRACER.sim-catalogue.figs.args : , \
		rcbase='cso.s5p.TRACER.sim-catalogue'
		! indexer task:
		cso.s5p.TRACER.sim-catalogue.index.class : utopya.Indexer
		cso.s5p.TRACER.sim-catalogue.index.args : '${PWD}/config/Copernicus/cso-s5p-TRACER.rc', \
		cso.s5p.TRACER.sim-catalogue.index.args : , \
		rcbase='cso.s5p.TRACER.sim-catalogue-index'

		!
		@@ -182,7 +186,7 @@ cso.s5p.TRACER.sim-catalogue.index.args : '${PWD}/config/Copernicus/cs
		cso.s5p.TRACER.sim-gridded.class : utopya.UtopyaJobStep
		! catalogue creation task:
		cso.s5p.TRACER.sim-gridded.task.class : cso.CSO_GriddedAverage
		cso.s5p.TRACER.sim-gridded.task.args : '${PWD}/config/Copernicus/cso-s5p-TRACER.rc', \
		cso.s5p.TRACER.sim-gridded.task.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.sim-gridded'

		! single step:
		@@ -191,11 +195,11 @@ cso.s5p.TRACER.sim-gridded-catalogue.class : utopya.UtopyaJobStep
		cso.s5p.TRACER.sim-gridded-catalogue.tasks : figs index
		! catalogue creation task:
		cso.s5p.TRACER.sim-gridded-catalogue.figs.class : cso.CSO_GriddedCatalogue
		cso.s5p.TRACER.sim-gridded-catalogue.figs.args : '${PWD}/config/Copernicus/cso-s5p-TRACER.rc', \
		cso.s5p.TRACER.sim-gridded-catalogue.figs.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.sim-gridded-catalogue'
		! indexer task:
		cso.s5p.TRACER.sim-gridded-catalogue.index.class : utopya.Indexer
		cso.s5p.TRACER.sim-gridded-catalogue.index.args : '${PWD}/config/Copernicus/cso-s5p-TRACER.rc', \
		cso.s5p.TRACER.sim-gridded-catalogue.index.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.sim-gridded-catalogue-index'

		#endfor
		@@ -209,19 +213,80 @@ cso.s5p.TRACER.sim-gridded-catalogue.index.args : '${PWD}/config/Copernicu
		*.script.class : utopya.UtopyaJobScriptForeground

		! search path for python modules:
		*.pypath : ${PWD}/py
		*.pypath : ${my.work}/py

		! work directory for jobs;
		! here path including subdirectories for job name elements:
		*.workdir : ${my.work}/__NAME2PATH__

		! jobtree settings from this file:
		*.rcfile : ${__filename__}
		*.rcfile : ${my.work}/rc/cso.rc

		! script interpreter:
		*.shell : /usr/bin/env python3


		!----------------------------------------------------------
		! copy
		! - create run directory
		! - copy scripts and settings for jobtree
		!----------------------------------------------------------

		! no sub list:
		cso.copy.class : utopya.UtopyaJobStep

		! default class is defined in machine specific settings;
		! copy is always done in foreground to avoid that files are
		! changed before the job is started:
		cso.copy.script.class : utopya.UtopyaJobScriptForeground

		! utopya jobstep settings: this file,
		! this task will copy it to the workdir:
		cso.copy.rcfile : ${CSO_RCFILE}

		! search path to utopya modules;
		! the 'build.utopya' job will create a copy on the workdir
		! for which the path is included in the '*.pypath' setting:
		cso.copy.pypath : ${CSO_PREFIX}/py

		! configure and build:
		cso.copy.task.class : utopya.UtopyaCopy
		cso.copy.task.args : '${CSO_RCFILE}', rcbase='main'

		! ~

		! prefix for destination of source and script files
		! (base path for subdirectories src, py, etc) ;
		! this should be an absolute path,
		! use ${PWD} for the present dir if necessary:
		main.copy.prefix : ${my.work}

		! space separated extensions:
		main.copy.prefix.extensions :

		! remove existing build directory if present ?
		main.copy.new : False

		! sub directories to be included ;
		! leave empty if no subdirs are defined:
		main.copy.subdirs : py rc

		! directories to be inlcuded in copy,
		! otherwise only files are copied:
		main.copy.incdirs :

		! list of source directories to be copied;
		! files in latest directory overwrite previously copied versions:
		main.copy.dirs : ${CSO_PREFIX}

		! extra directories to be copied into "rc":
		main.copy.rc.dirs : ${CSO_RCDIR}

		! skip files matching these filename patterns
		! (tempoary editor files, compiled modules, etc)
		main.copy.skip : .#* ~ .DS *.pyc


		!----------------------------------------------------------
		! end
		!----------------------------------------------------------

doc/source/tutorial.rst

+1 −0

Original line number	Diff line number	Diff line
		@@ -31,6 +31,7 @@ The ``cso`` script defines environment variables that could be used in the setti

		* ``CSO_PREFIX`` holds the path the CSO installation from where ``bin/cso`` is used;
		* ``CSO_RCFILE`` holds the name of the settings file that was passed as argument to the script.
		* ``CSO_RCDIR`` holds the directory containing ``CSO_RCFILE``.


		Job tree