Support ColHub mirror. (dd495cfa) · Commits · CAMS / CSO

CHANGELOG

+8 −0

Original line number	Diff line number	Diff line
		@@ -466,3 +466,11 @@ Increased maximum number of records and introduced sort order to avoid that
		search request return a different result when repeated.
		py/cso_dataspace.py

		colhub branch
		~~~~~~~~~~~~~

		Support inquiry and use of ColHub mirror archive.
		py/cso_colhub.py
		py/cso_file.py
		py/cso_s5p.py
		py/cso.py

config/_test-colhub/cso-s5p-no2.rc

0 → 100644

+1232 −0

File added.

Preview size limit exceeded, changes collapsed.

config/_test-colhub/cso-user-settings.rc

0 → 100644

+169 −0

Original line number	Diff line number	Diff line
		!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
		!!!
		!!! CSO common configuration
		!!!
		!!! Base settings that are used by multiple tasks:
		!!! - time range(s)
		!!! - target domain(s)
		!!!
		!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!


		!-----------------------------------------------------------
		! id's
		!-----------------------------------------------------------

		! file format:
		my.cso.format : 1.0

		! file format convention:
		my.cso.convention : CF-1.7


		!-----------------------------------------------------------
		! domain
		!-----------------------------------------------------------

		!
		! Used for:
		! - orbit selection durning download
		! - pixel selection
		! - map plots
		!

		! region name:
		!my.region : globe
		!my.region : CAMS
		my.region : xEMEP

		! switch:
		!....................................
		#if "${my.region}" == "globe"
		!....................................

		! global domain:
		my.region.west : -180.0
		my.region.east : 180.0
		my.region.south : -90.0
		my.region.north : 90.0

		! size of map figures for this region, default size is (8,6)
		my.region.figsize : (8,6)

		!....................................
		#elif "${my.region}" == "CAMS"
		!....................................

		! CAMS regional ensemble domain:
		my.region.west : -30.0
		my.region.east : 45.0
		my.region.south : 30.0
		my.region.north : 76.0

		! size of map figures for this region, default size is (8,6)
		my.region.figsize : (8,6)

		!....................................
		#elif "${my.region}" == "xEMEP"
		!....................................

		! outer bound of all known domains:
		my.region.west : -50.0
		my.region.east : 90.0
		my.region.south : 25.0
		my.region.north : 85.0

		! size of map figures for this region, default size is (8,6)
		my.region.figsize : (8,5)

		!....................................
		#else
		#error unsupported my.region "${my.region}"
		#endif
		!....................................




		!----------------------------------------------------------
		! timerange
		!----------------------------------------------------------

		!! full range, used for inquiry jobs:
		!my.full-timerange.start : 2018-01-01 00:00
		!my.full-timerange.end : 2024-12-31 23:59

		! TESTING: 1 month ...
		my.full-timerange.start : 2023-01-01 00:00
		my.full-timerange.end : 2023-01-31 23:59

		! processing for selected period:
		my.timerange.start : 2018-06-01 00:00
		my.timerange.end : 2018-06-01 23:59

		!! SO2-COBRA
		!my.timerange.start : 2023-06-01 00:00
		!my.timerange.end : 2023-06-01 23:59


		!----------------------------------------------------------
		! user specific settings:
		!----------------------------------------------------------

		!..............................
		#if "${USER}" == "you"
		!..............................

		! Attributes written to output files.
		my.attr.author : Your Name
		my.attr.institution : CSO
		my.attr.email : Your.Name@cso.org

		! base location for work directories:
		my.work : /Scratch/${USER}/CSO-Copernicus

		! storage for downloaded observations:
		my.observations : /Scratch/${USER}/observations

		!..............................
		#elif "${USER}" == "arjos"
		!..............................

		! Attributes written to output files.
		my.attr.author : Arjo Segers
		my.attr.institution : MET Norway
		my.attr.email : arjos@met.no

		! base location for work directories:
		my.work : ${WORK}/projects/SESAM/test-colhub

		! storage for downloaded observations:
		my.observations : ${WORK}/../observations

		!..............................
		#elif "${USER}" == "segersaj"
		!..............................

		! Attributes written to output files.
		my.attr.author : Arjo Segers
		my.attr.institution : TNO
		my.attr.email : Arjo.Segers@tno.nl

		! base location for work directories:
		my.work : ${SCRATCH}/CSO-Copernicus

		! storage for downloaded observations:
		my.observations : ${SCRATCH}/observations

		!..............................
		#else
		#error unsupported USER "${USER}"
		#endif
		!..............................


		!----------------------------------------------------------
		! end
		!----------------------------------------------------------

config/_test-colhub/cso.rc

0 → 100644

+311 −0

Original line number	Diff line number	Diff line
		!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
		!!!
		!!! CSO - CAMS Satellite Operator
		!!!
		!!! Settings for project.
		!!!
		!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

		! dummy keys, actual values defined in environment by "cso" script:
		CSO_RCFILE :
		CSO_RCDIR :


		!----------------------------------------------------------
		! user specific settings:
		!----------------------------------------------------------

		! include user specfic settings:
		#include cso-user-settings.rc

		! selected tracers:
		!my.tracers : no2 so2 so2-cobra hcho co chocho
		!~ one by one ..
		my.tracers : no2
		!my.tracers : so2
		!my.tracers : so2-cobra
		!my.tracers : hcho
		!my.tracers : co
		!my.tracers : o3-col
		!my.tracers : o3-pr
		!my.tracers : chocho


		!----------------------------------------------------------
		! job tree
		!----------------------------------------------------------

		! class to create a job tree:
		cso.class : utopya.UtopyaJobTree
		! list of sub-elements:
		cso.elements : copy s5p

		! class to create a job tree:
		cso.s5p.class : utopya.UtopyaJobTree
		! list of sub-elements:
		cso.s5p.elements : ${my.tracers}


		! *

		#for TRACER in ${my.tracers}

		! class to create a job tree:
		cso.s5p.TRACER.class : utopya.UtopyaJobTree

		! list of sub-elements:
		!cso.s5p.TRACER.elements : inquire \
		! convert listing \
		! catalogue \
		! gridded gridded-catalogue
		!~ preprocessor steps one by one ...
		cso.s5p.TRACER.elements : inquire
		!cso.s5p.TRACER.elements : convert listing catalogue
		!cso.s5p.TRACER.elements : convert
		!cso.s5p.TRACER.elements : listing
		!cso.s5p.TRACER.elements : catalogue
		!cso.s5p.TRACER.elements : gridded
		!cso.s5p.TRACER.elements : gridded-catalogue

		!!~ downloading only, no convert:
		!cso.s5p.TRACER.elements : download
		!
		!~ process simulator output:
		!cso.s5p.TRACER.elements : sim-catalogue
		!cso.s5p.TRACER.elements : sim-gridded
		!cso.s5p.TRACER.elements : sim-gridded-catalogue

		! *

		! single step:
		cso.s5p.TRACER.inquire.class : utopya.UtopyaJobStep
		! two or more tasks:
		#if "TRACER" in ["no2","so2","hcho","co","o3-pr","o3-col"]
		!cso.s5p.TRACER.inquire.tasks : table-dataspace plot
		cso.s5p.TRACER.inquire.tasks : table-dataspace
		!cso.s5p.TRACER.inquire.tasks : table-mirror
		!cso.s5p.TRACER.inquire.tasks : table-missing
		!cso.s5p.TRACER.inquire.tasks : plot
		#elif "TRACER" in ["so2-cobra"]
		cso.s5p.TRACER.inquire.tasks : table-pal plot
		#elif "TRACER" in ["chocho"]
		cso.s5p.TRACER.inquire.tasks : table-glyretro table-pal plot
		#else
		#error unsupported tracer "TRACER"
		#endif
		!~ inquire files available on DataSpace:
		cso.s5p.TRACER.inquire.table-dataspace.class : cso.CSO_DataSpace_Inquire
		cso.s5p.TRACER.inquire.table-dataspace.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.inquire-table-dataspace'
		!~ inquire files available on ColHub:
		cso.s5p.TRACER.inquire.table-colhub.class : cso.CSO_ColHub_Inquire
		cso.s5p.TRACER.inquire.table-colhub.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.inquire-table-colhub'
		!~ inquire file archive:
		cso.s5p.TRACER.inquire.table-mirror.class : cso.CSO_ColHubMirror_Inquire
		cso.s5p.TRACER.inquire.table-mirror.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.inquire-table-mirror'
		!~ create table with files that are missing:
		cso.s5p.TRACER.inquire.table-missing.class : cso.CSO_ColHubMirror_Missing
		cso.s5p.TRACER.inquire.table-missing.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.inquire-table-missing'
		!~ inquire files available on PAL:
		cso.s5p.TRACER.inquire.table-pal.class : cso.CSO_PAL_Inquire
		cso.s5p.TRACER.inquire.table-pal.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.inquire-table-pal'
		!~ inquire files downloaded from GlyRetro:
		cso.s5p.TRACER.inquire.table-glyretro.class : cso.CSO_S5p_Download_Listing
		cso.s5p.TRACER.inquire.table-glyretro.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.inquire-table-glyretro'
		!~ create plot of available versions:
		cso.s5p.TRACER.inquire.plot.class : cso.CSO_Inquire_Plot
		cso.s5p.TRACER.inquire.plot.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.inquire-plot'

		! *

		! single step:
		cso.s5p.TRACER.download.class : utopya.UtopyaJobStep
		! convert task:
		cso.s5p.TRACER.download.task.class : cso.CSO_S5p_Download
		cso.s5p.TRACER.download.task.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.download'
		! single step:
		cso.s5p.TRACER.convert.class : utopya.UtopyaJobStep
		! convert task:
		cso.s5p.TRACER.convert.task.class : cso.CSO_S5p_Convert
		cso.s5p.TRACER.convert.task.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.convert'
		! single step:
		cso.s5p.TRACER.listing.class : utopya.UtopyaJobStep
		! listing task:
		cso.s5p.TRACER.listing.task.class : cso.CSO_S5p_Listing
		cso.s5p.TRACER.listing.task.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.listing'

		! single step:
		cso.s5p.TRACER.catalogue.class : utopya.UtopyaJobStep
		! two tasks:
		cso.s5p.TRACER.catalogue.tasks : figs index

		! catalogue creation task:
		cso.s5p.TRACER.catalogue.figs.class : cso.CSO_Catalogue
		cso.s5p.TRACER.catalogue.figs.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.catalogue'
		! indexer task:
		cso.s5p.TRACER.catalogue.index.class : utopya.Indexer
		cso.s5p.TRACER.catalogue.index.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.catalogue-index'

		! *

		! single step:
		cso.s5p.TRACER.gridded.class : utopya.UtopyaJobStep
		! catalogue creation task:
		cso.s5p.TRACER.gridded.task.class : cso.CSO_GriddedAverage
		cso.s5p.TRACER.gridded.task.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.gridded'

		! single step:
		cso.s5p.TRACER.gridded-catalogue.class : utopya.UtopyaJobStep
		! two tasks:
		cso.s5p.TRACER.gridded-catalogue.tasks : figs index
		! catalogue creation task:
		cso.s5p.TRACER.gridded-catalogue.figs.class : cso.CSO_GriddedCatalogue
		cso.s5p.TRACER.gridded-catalogue.figs.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.gridded-catalogue'
		! indexer task:
		cso.s5p.TRACER.gridded-catalogue.index.class : utopya.Indexer
		cso.s5p.TRACER.gridded-catalogue.index.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.gridded-catalogue-index'

		! *

		! single step:
		cso.s5p.TRACER.sim-catalogue.class : utopya.UtopyaJobStep
		! two tasks:
		cso.s5p.TRACER.sim-catalogue.tasks : figs index

		! catalogue creation task:
		cso.s5p.TRACER.sim-catalogue.figs.class : cso.CSO_SimCatalogue
		cso.s5p.TRACER.sim-catalogue.figs.args : , \
		rcbase='cso.s5p.TRACER.sim-catalogue'
		! indexer task:
		cso.s5p.TRACER.sim-catalogue.index.class : utopya.Indexer
		cso.s5p.TRACER.sim-catalogue.index.args : , \
		rcbase='cso.s5p.TRACER.sim-catalogue-index'

		!
		! *
		!

		! single step:
		cso.s5p.TRACER.sim-gridded.class : utopya.UtopyaJobStep
		! catalogue creation task:
		cso.s5p.TRACER.sim-gridded.task.class : cso.CSO_GriddedAverage
		cso.s5p.TRACER.sim-gridded.task.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.sim-gridded'

		! single step:
		cso.s5p.TRACER.sim-gridded-catalogue.class : utopya.UtopyaJobStep
		! two tasks:
		cso.s5p.TRACER.sim-gridded-catalogue.tasks : figs index
		! catalogue creation task:
		cso.s5p.TRACER.sim-gridded-catalogue.figs.class : cso.CSO_GriddedCatalogue
		cso.s5p.TRACER.sim-gridded-catalogue.figs.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.sim-gridded-catalogue'
		! indexer task:
		cso.s5p.TRACER.sim-gridded-catalogue.index.class : utopya.Indexer
		cso.s5p.TRACER.sim-gridded-catalogue.index.args : '${my.work}/rc/cso-s5p-TRACER.rc', \
		rcbase='cso.s5p.TRACER.sim-gridded-catalogue-index'

		#endfor


		!----------------------------------------------------------
		! job step defaults
		!----------------------------------------------------------

		! run jobs in foreground:
		*.script.class : utopya.UtopyaJobScriptForeground

		! search path for python modules:
		*.pypath : ${my.work}/py

		! work directory for jobs;
		! here path including subdirectories for job name elements:
		*.workdir : ${my.work}/__NAME2PATH__
		!! TESTING ...
		!*.workdir : ${my.work}/test-difflist-__NAME2PATH__

		! jobtree settings from this file:
		*.rcfile : ${my.work}/rc/cso.rc

		! script interpreter:
		*.shell : /usr/bin/env python3


		!----------------------------------------------------------
		! copy
		! - create run directory
		! - copy scripts and settings for jobtree
		!----------------------------------------------------------

		! no sub list:
		cso.copy.class : utopya.UtopyaJobStep

		! default class is defined in machine specific settings;
		! copy is always done in foreground to avoid that files are
		! changed before the job is started:
		cso.copy.script.class : utopya.UtopyaJobScriptForeground

		! utopya jobstep settings: this file,
		! this task will copy it to the workdir:
		cso.copy.rcfile : ${CSO_RCFILE}

		! search path to utopya modules;
		! the 'build.utopya' job will create a copy on the workdir
		! for which the path is included in the '*.pypath' setting:
		cso.copy.pypath : ${CSO_PREFIX}/py

		! configure and build:
		cso.copy.task.class : utopya.UtopyaCopy
		cso.copy.task.args : '${CSO_RCFILE}', rcbase='main'

		! ~

		! prefix for destination of source and script files
		! (base path for subdirectories src, py, etc) ;
		! this should be an absolute path,
		! use ${PWD} for the present dir if necessary:
		main.copy.prefix : ${my.work}

		! space separated extensions:
		main.copy.prefix.extensions :

		! remove existing build directory if present ?
		main.copy.new : False

		! sub directories to be included ;
		! leave empty if no subdirs are defined:
		main.copy.subdirs : py rc

		! directories to be inlcuded in copy,
		! otherwise only files are copied:
		main.copy.incdirs :

		! list of source directories to be copied;
		! files in latest directory overwrite previously copied versions:
		main.copy.dirs : ${CSO_PREFIX}

		! extra directories to be copied into "rc":
		main.copy.rc.dirs : ${CSO_RCDIR}

		! skip files matching these filename patterns
		! (tempoary editor files, compiled modules, etc)
		main.copy.skip : .#* ~ .DS *.pyc


		!----------------------------------------------------------
		! end
		!----------------------------------------------------------

doc/source/history.rst

+3 −0

Original line number	Diff line number	Diff line
		@@ -110,6 +110,9 @@ A summary of the versions and changes.
		* \| v2.9
		\| Initial support for S5P O3-profile product.

		* \| colhub branch
		\| Support inquiry of ColHub mirror archive.


		To be included
		==============