Updated tutorial settings. (a75ec4b6) · Commits · CAMS / CSO

config/tutorial/tutorial.rc

+170 −111

Original line number	Diff line number	Diff line
		@@ -20,14 +20,15 @@ cso.elements : tutorial
		cso.tutorial.class : utopya.UtopyaJobTree

		! list of sub-elements:
		!cso.tutorial.elements : download convert listing catalogue \
		!cso.tutorial.elements : inquire-scihub \
		! convert listing catalogue \
		! sim-catalogue \
		! gridded gridded-catalogue
		!~ preprocessor steps one by one ...
		cso.tutorial.elements : download
		!cso.tutorial.elements : inquire-scihub
		!cso.tutorial.elements : convert
		!cso.tutorial.elements : listing
		!cso.tutorial.elements : catalogue
		cso.tutorial.elements : catalogue
		! ... after running the obs.oper:
		!cso.tutorial.elements : sim-catalogue
		!cso.tutorial.elements : gridded
		@@ -36,32 +37,38 @@ cso.tutorial.elements : download
		! *

		! single step:
		cso.tutorial.download.class : utopya.UtopyaJobStep
		! download task:
		cso.tutorial.download.task.class : cso.CSO_SciHub_Download
		cso.tutorial.download.task.args : '${PWD}/config/tutorial/tutorial.rc', \
		rcbase='cso.tutorial.download-s5phub'

		! *

		! single step:
		cso.tutorial.listing.class : utopya.UtopyaJobStep
		! conversion task:
		cso.tutorial.listing.task.class : cso.CSO_S5p_Listing
		cso.tutorial.listing.task.args : '${PWD}/config/tutorial/tutorial.rc', \
		rcbase='cso.tutorial.listing'
		cso.tutorial.inquire-scihub.class : utopya.UtopyaJobStep
		! two tasks:
		cso.tutorial.inquire-scihub.tasks : table plot
		!~ inquire available files:
		cso.tutorial.inquire-scihub.table.class : cso.CSO_SciHub_Inquire
		cso.tutorial.inquire-scihub.table.args : '${PWD}/config/tutorial/tutorial.rc', \
		rcbase='cso.tutorial.inquire-s5phub-table'
		!~ create plot of available versions:
		cso.tutorial.inquire-scihub.plot.class : cso.CSO_Inquire_Plot
		cso.tutorial.inquire-scihub.plot.args : '${PWD}/config/tutorial/tutorial.rc', \
		rcbase='cso.tutorial.inquire-s5phub-plot'

		! *

		! single step:
		cso.tutorial.convert.class : utopya.UtopyaJobStep
		! conversion task:
		! task:
		cso.tutorial.convert.task.class : cso.CSO_S5p_Convert
		cso.tutorial.convert.task.args : '${PWD}/config/tutorial/tutorial.rc', \
		rcbase='cso.tutorial.convert'

		! *

		! single step:
		cso.tutorial.listing.class : utopya.UtopyaJobStep
		! task:
		cso.tutorial.listing.task.class : cso.CSO_S5p_Listing
		cso.tutorial.listing.task.args : '${PWD}/config/tutorial/tutorial.rc', \
		rcbase='cso.tutorial.listing'

		! *

		! single step:
		cso.tutorial.catalogue.class : utopya.UtopyaJobStep
		! two tasks:
		@@ -154,7 +161,7 @@ my.region.figsize : (8,6)
		! timerange
		!----------------------------------------------------------

		! testing S5P data:
		! testing 3 days
		my.timerange.start : 2018-06-01 00:00
		my.timerange.end : 2018-06-03 23:59

		@@ -169,9 +176,8 @@ my.attr.institution : CSO
		my.attr.email : Your.Name@cso.org

		! base location for work directories:
		!my.work : /work/yourname/CSO-Tests
		!~ testing ...
		my.work : ${SCRATCH}/CSO-Tests
		!~ running on ppi:
		my.work : /lustre/storeB/users/${USER}/work/projects/eCSO


		!----------------------------------------------------------
		@@ -185,24 +191,25 @@ my.work : ${SCRATCH}/CSO-Tests
		*.pypath : ${PWD}/py

		! work directory for jobs;
		! here path including subdirectories for job name elements:
		!*.workdir : /work/${USER}/CAMS/__NAME2PATH__
		!~ use the 'my.work' definition included from "tutorial-user-settings.rc":
		*.workdir : ${my.work}/__NAME2PATH__

		! for new job files, use jobtree settings from this file:
		*.rcfile : ${__filename__}



		!======================================================================
		!===
		!=== download - S-5P Hub
		!=== Inquire - S-5P Hub
		!===
		!======================================================================

		! Obtain names of all available S5p files.
		! Stored as csv with processing type, processor version, filenames, etc.

		! time range:
		cso.tutorial.download-s5phub.timerange.start : ${my.timerange.start}
		cso.tutorial.download-s5phub.timerange.end : ${my.timerange.end}
		cso.tutorial.inquire-s5phub-table.timerange.start : ${my.timerange.start}
		cso.tutorial.inquire-s5phub-table.timerange.end : ${my.timerange.end}

		!
		! server url ;
		@@ -210,48 +217,46 @@ cso.tutorial.download-s5phub.timerange.end : ${my.timerange.end}
		!
		! machine s5phub.copernicus.eu login s5pguest password s5pguest
		!
		cso.tutorial.download-s5phub.url : https://s5phub.copernicus.eu/dhus
		cso.tutorial.inquire-s5phub-table.url : https://s5phub.copernicus.eu/dhus

		! target area;
		!~ empty for no limitation:
		!cso.tutorial.download-s5phub.area :
		!!~ empty for no limitation:
		!cso.tutorial.inquire-s5phub-table.area :
		!~ domain specified as: west,south:east,north
		cso.tutorial.download-s5phub.area : ${my.region.west},${my.region.south}:${my.region.east},${my.region.north}
		cso.tutorial.inquire-s5phub-table.area : ${my.region.west},${my.region.south}:${my.region.east},${my.region.north}

		!
		! search query, obtained from interactive download:
		!
		! platformname : Sentinel-5
		! producttype : L2__NO2___ \| L2__CO____ (always 10 characters!)
		! processinglevel : L2
		! processingmode : Reprocessing \| Offline
		!
		cso.tutorial.download-s5phub.query : platformname:Sentinel-5 AND \
		cso.tutorial.inquire-s5phub-table.query : platformname:Sentinel-5 AND \
		producttype:L2__NO2___ AND \
		processinglevel:L2 AND \
		processingmode:Reprocessing
		processinglevel:L2

		! output archive, store in subdirs per month:
		cso.tutorial.download-s5phub.output.dir : ${my.work}/Copernicus/S5P/RPRO/NO2/%Y/%m
		! output table, date of today:
		cso.tutorial.inquire-s5phub-table.output.file : ${my.work}/Copernicus/Copernicus_S5p_NO2_s5phub_%Y-%m-%d.csv

		! renew existing files?
		cso.tutorial.download-s5phub.renew : False

		! * create plot

		!! * extra when using "CSO_SciHub_Download_DHuS" ;
		!! note that "renew" flag is not used then ...
		!
		!! location of script:
		!cso.tutorial.download-s5phub.script : ${PWD}/bin/dhusget.sh
		!
		!! work directory, will contain log files etc:
		!cso.tutorial.download-s5phub.work.dir : ${my.work}/tmp.DHuS
		! renew existing plots?
		cso.tutorial.inquire-s5phub-plot.renew : True

		! listing files:
		cso.tutorial.inquire-s5phub-plot.file : ${cso.tutorial.inquire-s5phub-table.output.file}
		!!~ specify dates ("yyyy-mm-dd") to use historic tables,
		!! default is table of today:
		!cso.tutorial.inquire-s5phub-plot.filedate : 2022-01-28

		! output figure, date of today:
		cso.tutorial.inquire-s5phub-plot.output.file : ${my.work}/Copernicus/Copernicus_S5p_NO2_${my.region}__%Y-%m-%d.png


		!======================================================================
		!===
		!=== convert
		!=== convert (and download)
		!===
		!======================================================================

		@@ -269,14 +274,26 @@ cso.tutorial.convert.timerange.end : ${my.timerange.end}
		! input files
		!~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

		! input directory, same as where 'download' has stored the files:
		cso.tutorial.convert.files.dir : ${cso.tutorial.download-s5phub.output.dir}
		! listing of available source files,
		! created by 'inquire-s5phub' job:
		cso.tutorial.convert.inquire.file : ${my.work}/Copernicus/Copernicus_S5p_NO2_s5phub_%Y-%m-%d.csv
		!!~ historic inquire ...
		!cso.tutorial.convert.inquire.filedate : 2022-02-02

		! filename filters (space seperated list):
		cso.tutorial.convert.files.filters : S5P_*.nc
		! list of processings:
		cso.tutorial.convert.processings : RPRO

		! target component:
		!cso.tutorial.convert.component : ${my.tracer}
		! list of processor versions, empty for all:
		cso.tutorial.convert.processor_versions :

		! input directory;
		! files are searched here or downloaded to if not present yet;
		! supported templates:
		! %{processing}
		cso.tutorial.convert.input.dir : ${my.work}/Copernicus/S5P/%{processing}/NO2/%Y/%m

		! remove downloaded input files after convert?
		cso.tutorial.convert.downloads.cleanup : False

		! selection names:
		cso.tutorial.convert.filters : lons lats valid quality
		@@ -301,11 +318,11 @@ cso.tutorial.convert.filter.valid.type : valid
		! "A continuous quality descriptor,
		! varying between 0 (no data) and 1 (full quality data).
		! Recommend to ignore data with qa_value < 0.5"
		! Tests suggest that better threshold is 0.54,
		! this removes the kernels with very high values.
		! Recomended is qa_value>=0.75 to remove clouded and other problematic pixels;
		! here use 0.5 to include retrievals over clouds and snow/ice.
		cso.tutorial.convert.filter.quality.var : PRODUCT/qa_value
		cso.tutorial.convert.filter.quality.type : min
		cso.tutorial.convert.filter.quality.min : 0.54
		cso.tutorial.convert.filter.quality.min : 0.5
		cso.tutorial.convert.filter.quality.units : 1


		@@ -316,9 +333,14 @@ cso.tutorial.convert.filter.quality.units : 1

		! output directory and filename:
		! - times are taken from mid of selection, rounded to hours
		! - use '%{processing}' for the processing name
		! - use '%{orbit}' for orbit number
		cso.tutorial.convert.output.dir : ${my.work}/CSO-data/S5p/RPRO/NO2/${my.region}/%Y/%m
		cso.tutorial.convert.output.filename : S5p_RPRO_NO2_%{orbit}.nc
		cso.tutorial.convert.output.filename : ${my.work}/CSO-data/S5p/%{processing}/NO2/${my.region}/%Y/%m/S5p_%{processing}_NO2_%{orbit}.nc

		! pack variables on output:
		cso.tutorial.convert.output.packed : True
		! zlib compression level, 0 for no compression:
		cso.tutorial.convert.output.complevel : 1

		! global attributes:
		cso.tutorial.convert.output.attrs : format Conventions \
		@@ -523,17 +545,19 @@ cso.tutorial.listing.xcolumns : orbit
		!===
		!======================================================================

		! target directory for catalogue:
		cso.tutorial.catalogue.output.dir : ${my.work}/CSO-data-catalogue

		! listing file with filenames/timerange.
		cso.tutorial.catalogue.input.listing : ${cso.tutorial.listing.file}


		! time range:
		cso.tutorial.catalogue.timerange.start : ${my.timerange.start}
		cso.tutorial.catalogue.timerange.end : ${my.timerange.end}

		! target filenames; templates:
		! - time values
		! - %{orbit} : from listing
		! - %{varname} for variable
		cso.tutorial.catalogue.output.file : ${my.work}/CSO-data-catalogue/S5p/NO2/${my.region}/%Y/%m/%d/S5p_NO2_%{orbit}__%{varname}.png

		! map domain (west east south north)
		cso.tutorial.catalogue.domain : ${my.region.west} ${my.region.east} ${my.region.south} ${my.region.north}

		@@ -591,7 +615,7 @@ cso.tutorial.catalogue.var.cloud_radiance_fraction.colors : ['blue','cyan',
		!-----------------------------------------------------------

		! target location:
		cso.tutorial.catalogue-index.outdir : ${my.work}/CSO-data-catalogue
		cso.tutorial.catalogue-index.outdir : ${my.work}/CSO-data-catalogue/S5p/NO2/${my.region}

		! title:
		cso.tutorial.catalogue-index.header : CSO catalogue
		@@ -624,8 +648,8 @@ cso.tutorial.catalogue-index.date.orbit.values : ${cso.tutorial.ca
		! content type:
		cso.tutorial.catalogue-index.date.orbit.var.type : img
		! define image:
		cso.tutorial.catalogue-index.date.orbit.var.img : %{date[0:4]}/%{date[4:6]}/%{date[6:8]}/S5p_RPRO_NO2_%{orbit}__%{var}.png
		cso.tutorial.catalogue-index.date.orbit.var.kwargs : height=200
		cso.tutorial.catalogue-index.date.orbit.var.img : %{date[0:4]}/%{date[4:6]}/%{date[6:8]}/S5p_NO2_%{orbit}__%{var}.png
		cso.tutorial.catalogue-index.date.orbit.var.kwargs : height=300



		@@ -643,16 +667,14 @@ cso.tutorial.catalogue-index.date.orbit.var.kwargs : height=200
		! time range:
		cso.tutorial.sim-catalogue.timerange.start : ${my.timerange.start}
		cso.tutorial.sim-catalogue.timerange.end : ${my.timerange.end}
		cso.tutorial.sim-catalogue.timerange.step : hour

		! input files:
		cso.tutorial.sim-catalogue.input.data.file : ${my.work}/CSO-oper/CSO_output_%Y%m%d_%H%M_data.nc
		cso.tutorial.sim-catalogue.input.state.file : ${my.work}/CSO-oper/CSO_output_%Y%m%d_%H%M_state.nc

		! target directory for catalogue:
		cso.tutorial.sim-catalogue.output.dir : ${my.work}/CSO-oper/sim-catalogue

		! target files, time tempates and variable name are replaced:
		cso.tutorial.sim-catalogue.output.file : %Y/%m/%d/S5p_RPRO_NO2_%Y%m%d_%H%M_%{var}.png
		cso.tutorial.sim-catalogue.output.file : ${my.work}/CSO-data-sim-catalogue/S5p/NO2/${my.region}/%Y/%m/%d/S5p_NO2_%Y%m%d_%H%M_%{var}.png

		! map domain used for simulations (west east south north):
		cso.tutorial.sim-catalogue.domain : -10 30 35 65
		@@ -747,7 +769,7 @@ cso.tutorial.sim-catalogue-index.date.time.values : ${cso.tutorial.sim
		! content type:
		cso.tutorial.sim-catalogue-index.date.time.var.type : img
		! define image:
		cso.tutorial.sim-catalogue-index.date.time.var.img : %{date[0:4]}/%{date[4:6]}/%{date[6:8]}/S5p_RPRO_NO2_%{date}_%{time}_%{var}.png
		cso.tutorial.sim-catalogue-index.date.time.var.img : %{date[0:4]}/%{date[4:6]}/%{date[6:8]}/S5p_NO2_%{date}_%{time}_%{var}.png
		cso.tutorial.sim-catalogue-index.date.time.var.kwargs : height=300


		@@ -771,36 +793,72 @@ cso.tutorial.gridded.timerange.end : ${my.timerange.end}
		cso.tutorial.gridded.renew : True
		!cso.tutorial.gridded.renew : False

		! file with regular grid description,
		! which should have at least variable with 2D cell area:
		cso.tutorial.gridded.input.grid.file : ${my.work}/CSO-oper/CSO_grid.nc
		! variable with cell area:
		cso.tutorial.gridded.input.grid.area : cell_area

		! data file:
		cso.tutorial.gridded.input.data : ${my.work}/CSO-oper/CSO_output_%Y%m%d_%H%M_data.nc

		! keywords for output files to be created:
		cso.tutorial.gridded.outputs : all

		! ~ "all"

		! target file:
		cso.tutorial.gridded.output.all.file : ${my.work}/CSO-gridded/CSO_output_%Y%m%d_%H%M_gridded.nc

		! state file used as input:
		cso.tutorial.gridded.output.all.state : ${my.work}/CSO-oper/CSO_output_%Y%m%d_%H%M_state.nc
		! keyword used for subdir for resolution and filters:
		my.gridded.subdir : ${my.region}__r01x01__qa08

		!~

		! testing ..
		my.gridded.region : ${my.region}
		! grid definition:
		!!~ same as pixel selection on conversion:
		!cso.tutorial.gridded.grid.west : ${my.region.west}
		!cso.tutorial.gridded.grid.east : ${my.region.east}
		!cso.tutorial.gridded.grid.south : ${my.region.south}
		!cso.tutorial.gridded.grid.north : ${my.region.north}
		!~ observation operator tutorial:
		cso.tutorial.gridded.grid.west : -10
		cso.tutorial.gridded.grid.east : 30
		cso.tutorial.gridded.grid.south : 35
		cso.tutorial.gridded.grid.north : 65
		! resolution:
		cso.tutorial.gridded.grid.dlon : 0.1
		cso.tutorial.gridded.grid.dlat : 0.1

		! level of recursive splitting of footprint into triangles,
		! and assignment of centroids to grid cells;
		! for 4-corner footprints, number of centroids is:
		! 1 (levels=0), 4 (1), 8 (2), 16 (3), 64 (5), 256 (7)
		cso.tutorial.gridded.mapping.levels : 7

		!~

		! keywords for source files;
		! the first one should have the footprints:
		cso.tutorial.gridded.sources : data state

		! input files for each source type:
		cso.tutorial.gridded.source.data : ${my.work}/CSO-oper/CSO_output_%Y%m%d_%H%M_data.nc
		cso.tutorial.gridded.source.state : ${my.work}/CSO-oper/CSO_output_%Y%m%d_%H%M_state.nc

		!~

		! keywords for filters:
		cso.tutorial.gridded.filters : quality

		! minimum quality value required:
		cso.tutorial.gridded.filter.quality.var : qa_value
		cso.tutorial.gridded.filter.quality.type : min
		cso.tutorial.gridded.filter.quality.min : 0.8
		cso.tutorial.gridded.filter.quality.units : 1

		!~

		! target file, might contain templates:
		! %Y,%m,etc : time values
		! %{basename} : basename (without extension) of first source file
		cso.tutorial.gridded.output.file : ${my.work}/CSO-gridded/${my.gridded.subdir}/%Y/%m/CSO_output_%Y%m%d_%H%M_gridded.nc

		! data variables to be created:
		cso.tutorial.gridded.output.all.vars : yr ys yr_m ys_m
		cso.tutorial.gridded.output.vars : yr ys yr_m ys_m

		! input variables:
		! data:yr : from data file
		! state:ys : from state file
		cso.tutorial.gridded.output.all.yr.source : data:yr
		cso.tutorial.gridded.output.all.ys.source : state:ys
		cso.tutorial.gridded.output.all.yr_m.source : state:yr_m
		cso.tutorial.gridded.output.all.ys_m.source : state:ys_m
		cso.tutorial.gridded.output.yr.source : data:yr
		cso.tutorial.gridded.output.ys.source : state:ys
		cso.tutorial.gridded.output.yr_m.source : state:yr_m
		cso.tutorial.gridded.output.ys_m.source : state:ys_m


		!-----------------------------------------------------------
		@@ -811,13 +869,17 @@ cso.tutorial.gridded.output.all.ys_m.source : state:ys_m
		cso.tutorial.gridded-catalogue.timerange.start : ${my.timerange.start}
		cso.tutorial.gridded-catalogue.timerange.end : ${my.timerange.end}

		! renew existing files?
		cso.tutorial.gridded-catalogue.renew : True
		!cso.tutorial.gridded-catalogue.renew : False

		! target directory for catalogue:
		cso.tutorial.gridded-catalogue.output.dir : ${my.work}/CSO-gridded/catalogue
		cso.tutorial.gridded-catalogue.output.dir : ${my.work}/CSO-gridded/${my.gridded.subdir}/catalogue

		! input files per "output" key of the gridding operator:
		cso.tutorial.gridded-catalogue.input.all.file : ../CSO_output_%Y%m%d_%H%M_gridded.nc
		! input files:
		cso.tutorial.gridded-catalogue.input.file : ../%Y/%m/CSO_output_%Y%m%d_%H%M_gridded.nc
		!!~ idem for daily average:
		!cso.tutorial.gridded-catalogue.input.all.file : ../CSO_output_%Y%m%d_aver_gridded.nc
		!cso.tutorial.gridded-catalogue.input.file : ../CSO_output_%Y%m%d_aver_gridded.nc

		! target files, time tempates and variable name are replaced:
		cso.tutorial.gridded-catalogue.output.file : %Y/%m/%d/S5p_RPRO_NO2_%Y%m%d_%H%M_gridded_%{var}.png
		@@ -827,15 +889,11 @@ cso.tutorial.gridded-catalogue.output.file : %Y/%m/%d/S5p_RPRO_NO2_%
		! figure size (inches), default is (8,6):
		cso.tutorial.gridded-catalogue.figsize : (6,6)

		! renew existing files?
		cso.tutorial.gridded-catalogue.renew : True
		!cso.tutorial.gridded-catalogue.renew : False

		! variables to be plotted:
		cso.tutorial.gridded-catalogue.vars : yr ys yr_m ys_m

		! variable:
		cso.tutorial.gridded-catalogue.var.yr.source : all:yr
		cso.tutorial.gridded-catalogue.var.yr.source : yr
		! convert units:
		cso.tutorial.gridded-catalogue.var.yr.units : 1e15 mlc/cm2
		! style:
		@@ -843,7 +901,7 @@ cso.tutorial.gridded-catalogue.var.yr.vmin : 0.0
		cso.tutorial.gridded-catalogue.var.yr.vmax : 10.0

		! variable:
		cso.tutorial.gridded-catalogue.var.ys.source : all:ys
		cso.tutorial.gridded-catalogue.var.ys.source : ys
		! convert units:
		cso.tutorial.gridded-catalogue.var.ys.units : 1e15 mlc/cm2
		! style:
		@@ -851,7 +909,7 @@ cso.tutorial.gridded-catalogue.var.ys.vmin : 0.0
		cso.tutorial.gridded-catalogue.var.ys.vmax : 10.0

		! variable:
		cso.tutorial.gridded-catalogue.var.yr_m.source : all:yr_m
		cso.tutorial.gridded-catalogue.var.yr_m.source : yr_m
		! convert units:
		cso.tutorial.gridded-catalogue.var.yr_m.units : 1e15 mlc/cm2
		! style:
		@@ -859,7 +917,7 @@ cso.tutorial.gridded-catalogue.var.yr_m.vmin : 0.0
		cso.tutorial.gridded-catalogue.var.yr_m.vmax : 10.0

		! variable:
		cso.tutorial.gridded-catalogue.var.ys_m.source : all:ys_m
		cso.tutorial.gridded-catalogue.var.ys_m.source : ys_m
		! convert units:
		cso.tutorial.gridded-catalogue.var.ys_m.units : 1e15 mlc/cm2
		! style:
		@@ -873,7 +931,7 @@ cso.tutorial.gridded-catalogue.var.ys_m.vmax : 10.0
		!-----------------------------------------------------------

		! target location:
		cso.tutorial.gridded-catalogue-index.outdir : ${my.work}/CSO-gridded/catalogue
		cso.tutorial.gridded-catalogue-index.outdir : ${my.work}/CSO-gridded/${my.gridded.subdir}/catalogue

		! title:
		cso.tutorial.gridded-catalogue-index.header : CSO catalogue
		@@ -916,3 +974,4 @@ cso.tutorial.gridded-catalogue-index.date.time.var.kwargs : height=300
		!===
		!======================================================================