Updated template settings. (288ddd12) · Commits · CAMS / CSO

config/CrIS/cso-cris-nh3.rc

+64 −34

Original line number	Diff line number	Diff line
		@@ -137,22 +137,19 @@ cso.convert.downloads.dir : ${my.arch.dir}/download

		! remove files extracted from collection files after convert?
		cso.convert.downloads.cleanup.extracts : True
		!!~ TESTING ...
		!cso.convert.downloads.cleanup.extracts : False

		! only extract datafiles that overlap with target region (w/e/s/n):
		cso.convert.downloads.region : ${my.region.west} ${my.region.east} ${my.region.south} ${my.region.north}

		!! input directory, same as where 'download' has stored the files, including unpacked structure:
		!!cso.convert.files.dir : ${cso.cris2.nh3.download-CrIS2.output.dir}/${cso.cris2.nh3.processing.inputdomain}/%Y/%m/%d
		!cso.convert.files.dir : /tsn.tno.nl/RA-Data/Express/ra_express_modasuscratch_unix/observations/cris2/${cso.cris2.nh3.processing.version}/${cso.cris2.nh3.processing.inputdomain}/%Y/%m/%d/

		!
		!! processing mode /database
		!cso.convert.processing : ${cso.cris2.nh3.processing}
		! * pixel selection
		!
		!! filename filters (space seperated list):
		!cso.convert.files.filters : Combined_*.nc

		! selection names:
		cso.convert.filters : lons lats valid quality cloud_flag AM_or_PM
		cso.convert.filters : lons lats daytime landfraction pressure valid cloud_flag quality

		! filter settings:
		cso.convert.filter.lons.var : Longitude
		@@ -166,6 +163,25 @@ cso.convert.filter.lats.type : minmax
		cso.convert.filter.lats.minmax : ${my.region.south} ${my.region.north}
		cso.convert.filter.lats.units : degrees_north

		! 1 gives daytime obs, 0 nighttime
		cso.convert.filter.daytime.var : Daylight_Flag
		cso.convert.filter.daytime.type : equal
		cso.convert.filter.daytime.value : 1
		cso.convert.filter.daytime.units : 1

		! exclude water pixels, at least 80% land
		cso.convert.filter.landfraction.var : LandFraction
		cso.convert.filter.landfraction.type : min
		cso.convert.filter.landfraction.min : 0.5
		cso.convert.filter.landfraction.units : 1

		! exclude pixels over mountains;
		! "surface" pressure in profile should be at least 900 mbar (~1000m)
		cso.convert.filter.pressure.var : pressure
		cso.convert.filter.pressure.type : min-of-max-in-profile
		cso.convert.filter.pressure.min : 900.0
		cso.convert.filter.pressure.units : hPa

		! some pixels have incorrect meteo values ("tm", "zm", and "h2o" variables)
		! with values of about 9.969e36 ; remove these pixels based on the "zm" profiles:
		cso.convert.filter.valid.var : zm
		@@ -197,7 +213,7 @@ cso.convert.filter.valid.units : km
		! More conservative retrieval quality flag with information content
		! Quality flag 4 & also filtered to have a minimum degree-of-freedom for signal of 0.1
		!
		! Following recomendations, use pixels with quality flag >= 4
		! Following recomendations, use pixels with quality flag >= 5
		cso.convert.filter.quality.var : Quality_Flag
		cso.convert.filter.quality.type : min
		cso.convert.filter.quality.min : 5
		@@ -217,18 +233,12 @@ cso.convert.filter.cloud_flag.var : Cloud_Flag
		cso.convert.filter.cloud_flag.type : in
		cso.convert.filter.cloud_flag.value : 0 3

		! 1 gives daytime obs, 0 nighttime
		cso.convert.filter.AM_or_PM.var : Daylight_Flag
		cso.convert.filter.AM_or_PM.type : equal
		cso.convert.filter.AM_or_PM.value : 1
		cso.convert.filter.AM_or_PM.units : 1


		!~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
		! output files
		!~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

		! output directory and filename:
		! output filename:
		!cso.convert.output.filename : ${my.arch.dir}/${my.version}/%{stream}/%Y/%m/${my.instrument}_NH3_%Y%m%d.nc
		!~ no need to distinguish between "OFFL" and "NRT" ...
		cso.convert.output.filename : ${my.arch.dir}/${my.version}/${my.region}/%Y/%m/${my.instrument}_NH3_%Y%m%d.nc
		@@ -273,10 +283,10 @@ cso.convert.output.attr.email : ${my.attr.email}
		! satellite_range day_night_flag

		cso.convert.output.vars : longitude latitude pixel_bounds time \
		pressure hpressure altitude isfc \
		pressure hpressure altitude isfc pressure_sfc landfraction \
		tot_col tot_col_meas_errvar tot_col_total_errvar xr yr \
		xa ya \
		lin_kernel A \
		log_kernel lin_kernel A \
		quality_flag cloud_flag

		! Extra specialized variables:
		@@ -315,11 +325,13 @@ cso.convert.output.vars : longitude latitude pixel_bounds time \
		cso.convert.output.var.longitude.dims : pixel
		cso.convert.output.var.longitude.from : Longitude
		cso.convert.output.var.longitude.units : degrees_east
		cso.convert.output.var.longitude.attrs : { 'standard_name' : 'longitude' }

		! description:
		cso.convert.output.var.latitude.dims : pixel
		cso.convert.output.var.latitude.from : Latitude
		cso.convert.output.var.latitude.units : degrees_north
		cso.convert.output.var.latitude.attrs : { 'standard_name' : 'latitude' }


		! corner longitudes and latitudes:
		@@ -349,6 +361,13 @@ cso.convert.output.var.pressure.dims : pixel layer
		cso.convert.output.var.pressure.from : pressure
		cso.convert.output.var.pressure.units : Pa

		! retrieved surface conc;
		! this is the lowest value that is not nan:
		cso.convert.output.var.pressure_sfc.dims : pixel
		cso.convert.output.var.pressure_sfc.special : surface_non_fill
		cso.convert.output.var.pressure_sfc.from : pressure
		cso.convert.output.var.pressure_sfc.units : Pa

		! half level pressures:
		! - lowest pressure
		! - mid of pressure levels
		@@ -358,16 +377,21 @@ cso.convert.output.var.hpressure.special : pressure_to_hpressure
		cso.convert.output.var.hpressure.from : pressure
		cso.convert.output.var.hpressure.units : Pa

		! altitude
		cso.convert.output.var.altitude.dims : pixel layer
		cso.convert.output.var.altitude.from : zm
		cso.convert.output.var.altitude.units : m

		! land fraction
		cso.convert.output.var.landfraction.dims : pixel
		cso.convert.output.var.landfraction.from : LandFraction
		cso.convert.output.var.landfraction.units : 1

		! 0-based index of surface layer:
		cso.convert.output.var.isfc.dims : pixel
		cso.convert.output.var.isfc.special : sfc_layer_index
		cso.convert.output.var.isfc.from : pressure

		! altitude
		cso.convert.output.var.altitude.from : zm
		cso.convert.output.var.altitude.units : m
		cso.convert.output.var.altitude.dims : pixel layer

		! ~ retrieved

		! description:
		@@ -443,11 +467,11 @@ cso.convert.output.var.ya.attrs : { 'long_name' : 'Aprio

		! ~ kernels

		!! kernel:
		!! THIS HAS ORIGINAL DIMS, IS THAT ACTUALLY "(pixel,layer,retr_profile)" ?
		!cso.convert.output.var.kernel.dims : pixel layer retr_profile
		!cso.convert.output.var.kernel.from : avg_kernel
		!cso.convert.output.var.kernel.units : ln(ppmv)/ln(ppmv)
		! kernel:
		cso.convert.output.var.log_kernel.dims : pixel layer retr_profile
		cso.convert.output.var.log_kernel.from : avg_kernel
		cso.convert.output.var.log_kernel.units : 1
		cso.convert.output.var.log_kernel.attrs : { 'long_units' : 'ln(ppmv)/ln(ppmv)' }

		! linearization from units "ln(ppmv)/ln(ppmv)" to "ppmv/ppmv":
		! kernel := averaging_kernel * amf/amft
		@@ -615,9 +639,9 @@ cso.catalogue.timerange.end : ${my.timerange.end}
		!cso.catalogue.timerange.end : 2024-02-29 23:59

		! map domain (west east south north)
		cso.catalogue.domain : ${my.region.west} ${my.region.east} ${my.region.south} ${my.region.north}
		!! ~ zooom:
		!cso.catalogue.domain : 3.0 8.0 50.0 54.0
		!cso.catalogue.domain : ${my.region.west} ${my.region.east} ${my.region.south} ${my.region.north}
		! ~ zooom:
		cso.catalogue.domain : 7 12 54 60
		!!~ TESTING TJK domain:
		!cso.catalogue.domain : 67 75.5 36.5 41.5

		@@ -625,12 +649,12 @@ cso.catalogue.domain : ${my.region.west} ${my.region.east}
		cso.catalogue.figsize : ${my.region.figsize}

		! renew existing files?
		!cso.catalogue.renew : True
		cso.catalogue.renew : False
		cso.catalogue.renew : True
		!cso.catalogue.renew : False

		! variables to be plotted:
		!cso.catalogue.vars : tot_col yr ya quality_flag
		cso.catalogue.vars : yr ya quality_flag cloud_flag
		cso.catalogue.vars : yr ya quality_flag cloud_flag landfraction

		! convert units:
		cso.catalogue.var.tot_col.units : umol/m2
		@@ -677,6 +701,12 @@ cso.catalogue.var.cloud_flag.vmax : 3.5
		cso.catalogue.var.cloud_flag.colors : ['gray','green','orange','red','magenta']
		cso.catalogue.var.cloud_flag.ncolor : 5

		! style:
		!cso.catalogue.var.landfraction.vmin : 0.0
		!cso.catalogue.var.landfraction.vmax : 1.0
		cso.catalogue.var.landfraction.colors : ["blue","cyan","yellow","green"]
		cso.catalogue.var.landfraction.ncolor : 4


		!-----------------------------------------------------------
		! catalogue index

config/CrIS/cso-user-settings.rc

+2 −35

Original line number	Diff line number	Diff line
		@@ -17,7 +17,7 @@
		my.cso.format : 1.0

		! file format convention:
		my.cso.convention : CF-1.7
		my.cso.convention : CF-1.8


		!-----------------------------------------------------------
		@@ -71,18 +71,10 @@ my.region.figsize : (8,6)
		my.inquire.timerange.start : 2011-10-01 00:00
		my.inquire.timerange.end : 2026-12-31 23:59

		!! testing 3 days
		!my.timerange.start : 2024-03-01 00:00
		!my.timerange.end : 2024-03-03 23:59

		! Reporting2026
		! single year:
		my.timerange.start : 2024-01-01 00:00
		my.timerange.end : 2024-12-31 23:59

		! SESAM archive
		!!~ completion of CrIS2 archive:
		!my.timerange.start : 2019-03-01 00:00
		!my.timerange.end : 2023-12-31 23:59


		!----------------------------------------------------------
		@@ -101,31 +93,6 @@ my.attr.email : Your.Name@cso.org
		! base location for work directories:
		my.work : /Scratch/${USER}/CSO-Test-CrIS

		!..............................
		#elif "${USER}" == "arjos"
		!..............................

		! Attributes written to output files.
		my.attr.author : Arjo Segers
		my.attr.institution : MET Norway
		my.attr.email : arjos@met.no

		! base location for work directories:
		!my.work : /work/${USER}/CSO-Test-CrIS
		my.work : /lustre/storeB/project/fou/kl/sesam/work/users/${USER}/CSO-Test-CrIS

		!..............................
		#elif "${USER}" == "segersaj"
		!..............................

		! Attributes written to output files.
		my.attr.author : Arjo Segers
		my.attr.institution : TNO
		my.attr.email : Arjo.Segers@tno.nl

		! base location for work directories:
		my.work : ${SCRATCH}/CSO-Test-CrIS

		!..............................
		#else
		#error unsupported USER "${USER}"

config/CrIS/cso.rc

+3 −167

Original line number	Diff line number	Diff line
		@@ -111,19 +111,16 @@ cso.copy.rcwrite : ${my.work}/cso.rc
		cso._PRODUCT_.class : utopya.UtopyaJobTree

		!~ sub steps:
		!cso._PRODUCT_.elements : inquire download download-listing convert listing
		!cso._PRODUCT_.elements : inquire convert listing
		!~ one by one ...
		!cso._PRODUCT_.elements : inquire
		cso._PRODUCT_.elements : convert listing
		!cso._PRODUCT_.elements : listing
		cso._PRODUCT_.elements : inquire
		!cso._PRODUCT_.elements : convert listing
		!cso._PRODUCT_.elements : catalogue

		! *

		! inquire tasks:
		cso._PRODUCT_.inquire.tasks : table-cris plot
		!cso._PRODUCT_.inquire.tasks : table-cris
		!cso._PRODUCT_.inquire.tasks : plot

		! single step:
		cso._PRODUCT_.inquire.class : utopya.UtopyaJobStep
		@@ -184,164 +181,3 @@ cso._PRODUCT_.catalogue.index.args : '${my.work}/rc/cso-cris-nh3.
		! ===
		! ======================================================================


		!! class to create a job tree:
		!cso.cris2.class : utopya.UtopyaJobTree
		!! list of sub-elements:
		!cso.cris2.elements : ${my.tracers}
		!
		!
		!! *
		!
		!#for TRACER in ${my.tracers}
		!
		!! class to create a job tree:
		!cso.cris2.TRACER.class : utopya.UtopyaJobTree
		!! list of sub-elements:
		!!cso.cris2.TRACER.elements : download convert catalogue catalogue-index \
		!! sim-catalogue sim-catalogue-index
		!!~ preprocessor steps one by one ...
		!!cso.cris2.TRACER.elements : download convert catalogue catalogue-index
		!!cso.cris2.TRACER.elements : convert catalogue catalogue-index
		!!cso.cris2.TRACER.elements : catalogue catalogue-index
		!cso.cris2.TRACER.elements : listing
		!!cso.cris2.TRACER.elements : download_CrIS
		!!cso.cris2.TRACER.elements : convert_cris
		!!cso.cris2.TRACER.elements : convert_cris gridded gridded-catalogue
		!!cso.cris2.TRACER.elements : gridded-catalogue
		!!cso.cris2.TRACER.elements : catalogue catalogue-index
		!
		!!
		!!!~ process observation simulator output:
		!!cso.cris2.TRACER.elements : sim-catalogue sim-catalogue-index
		!
		!! *
		!
		!! single step:
		!cso.cris2.TRACER.download.class : utopya.UtopyaJobStep
		!! download task:
		!cso.cris2.TRACER.download.task.class : cso.CSO_SciHub_Download
		!cso.cris2.TRACER.download.task.args : '${PWD}/config/CrIS2/cso-cris-TRACER.rc', \
		! rcbase='cso.cris2.TRACER.download-s5phub'
		!
		!! single step:
		!cso.cris2.TRACER.download_PAL.class : utopya.UtopyaJobStep
		!! download task:
		!cso.cris2.TRACER.download_PAL.task.class : cso.CSO_PAL_Download
		!cso.cris2.TRACER.download_PAL.task.args : '${PWD}/config/CrIS2/cso-cris-TRACER.rc', \
		! rcbase='cso.cris2.TRACER.download-PAL'
		!! single step:
		!cso.cris2.TRACER.download_CrIS.class : utopya.UtopyaJobStep
		!! download task:
		!cso.cris2.TRACER.download_CrIS.task.class : cso.CSO_CrIS_Download
		!cso.cris2.TRACER.download_CrIS.task.args : '${PWD}/config/CrIS2/cso-cris-TRACER.rc', \
		! rcbase='cso.cris2.TRACER.download-cris'
		!
		!! single step:
		!cso.cris2.TRACER.convert.class : utopya.UtopyaJobStep
		!! dummy task:
		!cso.cris2.TRACER.convert.task.class : cso.CSO_S5p_Convert
		!cso.cris2.TRACER.convert.task.args : '${PWD}/config/CrIS2/cso-cris-TRACER.rc', \
		! rcbase='cso.cris2.TRACER.convert'
		!! single step:
		!cso.cris2.TRACER.convert_cris.class : utopya.UtopyaJobStep
		!! dummy task:
		!cso.cris2.TRACER.convert_cris.task.class : cso.CSO_CrIS_Convert
		!cso.cris2.TRACER.convert_cris.task.args : '${PWD}/config/CrIS2/cso-cris-TRACER.rc', \
		! rcbase='cso.cris2.TRACER.convert'
		!! single step:
		!cso.cris2.TRACER.catalogue.class : utopya.UtopyaJobStep
		!! dummy task:
		!cso.cris2.TRACER.catalogue.task.class : cso.CSO_Catalogue
		!cso.cris2.TRACER.catalogue.task.args : '${PWD}/config/CrIS2/cso-cris-TRACER.rc', \
		! rcbase='cso.cris2.TRACER.catalogue'
		!
		!! single step:
		!cso.cris2.TRACER.catalogue-index.class : utopya.UtopyaJobStep
		!! dummy task:
		!cso.cris2.TRACER.catalogue-index.task.class : utopya.Indexer
		!cso.cris2.TRACER.catalogue-index.task.args : '${PWD}/config/CrIS2/cso-cris-TRACER.rc', \
		! rcbase='cso.cris2.TRACER.catalogue-index'
		!
		!! single step:
		!cso.cris2.TRACER.sim-catalogue.class : utopya.UtopyaJobStep
		!! dummy task:
		!cso.cris2.TRACER.sim-catalogue.task.class : cso.CSO_SimCatalogue
		!cso.cris2.TRACER.sim-catalogue.task.args : '${PWD}/config/CrIS2/cso-cris-TRACER.rc', \
		! rcbase='cso.cris2.TRACER.sim-catalogue'
		!
		!! single step:
		!cso.cris2.TRACER.sim-catalogue-index.class : utopya.UtopyaJobStep
		!! dummy task:
		!cso.cris2.TRACER.sim-catalogue-index.task.class : utopya.Indexer
		!cso.cris2.TRACER.sim-catalogue-index.task.args : '${PWD}/config/CrIS2/cso-cris-TRACER.rc', \
		! rcbase='cso.cris2.TRACER.sim-catalogue-index'
		!
		!! single step:
		!cso.cris2.TRACER.listing.class : utopya.UtopyaJobStep
		!! conversion task:
		!cso.cris2.TRACER.listing.task.class : cso.CSO_S5p_Listing
		!cso.cris2.TRACER.listing.task.args : '${PWD}/config/CrIS2/cso-cris-TRACER.rc', \
		! rcbase='cso.cris2.TRACER.listing'
		!! single step:
		!cso.cris2.TRACER.gridded-aver.class : utopya.UtopyaJobStep
		!! gridded average creation task:
		!cso.cris2.TRACER.gridded-aver.task.class : cso.CSO_GriddedAverageOverFiles
		!cso.cris2.TRACER.gridded-aver.task.args : '${PWD}/CrIS/cso-cris-TRACER.rc', \
		! rcbase='cso.cris2.TRACER.gridded-aver'
		!! single step:
		!cso.cris2.TRACER.gridded-compare.class : utopya.UtopyaJobStep
		!! gridded average creation task:
		!cso.cris2.TRACER.gridded-compare.task.class : cso.CSO_GriddedCompare
		!cso.cris2.TRACER.gridded-compare.task.args : '${PWD}/CrIS/cso-cris-TRACER.rc', \
		! rcbase='cso.cris2.TRACER.gridded-compare'
		!!
		!! * gridded fields
		!!
		!
		!! single step:
		!cso.cris2.TRACER.gridded.class : utopya.UtopyaJobStep
		!! catalogue creation task:
		!cso.cris2.TRACER.gridded.task.class : cso.CSO_GriddedAverage
		!cso.cris2.TRACER.gridded.task.args : '${PWD}/config/CrIS2/cso-cris-TRACER.rc', \
		! rcbase='cso.cris2.TRACER.gridded'
		!
		!! *
		!
		!! single step:
		!cso.cris2.TRACER.gridded-catalogue.class : utopya.UtopyaJobStep
		!! two tasks:
		!cso.cris2.TRACER.gridded-catalogue.tasks : figs index
		!! catalogue creation task:
		!cso.cris2.TRACER.gridded-catalogue.figs.class : cso.CSO_GriddedCatalogue
		!cso.cris2.TRACER.gridded-catalogue.figs.args : '${PWD}/config/CrIS2/cso-cris-TRACER.rc', \
		! rcbase='cso.cris2.TRACER.gridded-catalogue'
		!! indexer task:
		!cso.cris2.TRACER.gridded-catalogue.index.class : utopya.Indexer
		!cso.cris2.TRACER.gridded-catalogue.index.args : '${PWD}/config/CrIS2/cso-cris-TRACER.rc', \
		! rcbase='cso.cris2.TRACER.gridded-catalogue-index'
		!
		!#endfor


		!----------------------------------------------------------
		! job step defaults
		!----------------------------------------------------------

		!! run jobs in foreground:
		!*.script.class : utopya.UtopyaJobScriptForeground

		!! search path for python modules:
		!*.pypath : ${PWD}/py

		!! work directory for jobs;
		!! here path including subdirectories for job name elements:
		!*.workdir : ${my.work}/__NAME2PATH__
		!
		!! jobtree settings from this file:
		!*.rcfile : ${__filename__}


		!----------------------------------------------------------
		! end
		!----------------------------------------------------------