.gitignore

+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Fortran objects
+*.mod
+*.o
+*.x

CHANGELOG

@@ -244,9 +244,13 @@ Use pandas concat to combine dataframes.
   py/cso_pal.py
 
 Replaced `where` concstructs by loops after memory errors in output filling.
-  oper/src/cso_ncfile
+  oper/src/cso_ncfile.F90
   
 Check on `pix_all` value to know if pixels are assigned to any of the sub-domains.
   src/tutorial_oper_S5p.F90
 
-
+Migration from Copernicus SciHub to DataSpace.
+  bin/dhusget.sh
+  doc/source/*
+  py/*
+  config/*

bin/cso

@@ -15,12 +15,15 @@ History
 2023-08, Arjo Segers
   Define 'CSO_PREFIX' environment variable for use in rcfile settings.
 
+2023-11, Arjo Segers
+  Reformatted using 'black'.
+
 """
 
 
-#-------------------------------------------------
+# -------------------------------------------------
 # modules
-#-------------------------------------------------
+# -------------------------------------------------
 
 # modules:
 import sys
@@ -28,56 +31,56 @@ import os
 import logging
 
 # prefix of CSO installation:
-prefix = os.path.abspath( os.path.join( os.path.dirname(sys.argv[0]), os.pardir ) )
+prefix = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), os.pardir))
 # extend path:
-sys.path.insert( 0, os.path.join(prefix,'py') )
+sys.path.insert(0, os.path.join(prefix, "py"))
 
 # tools:
 import utopya
 
 
-#-------------------------------------------------
+# -------------------------------------------------
 # begin
-#-------------------------------------------------
+# -------------------------------------------------
 
 # set environment:
-os.environ['CSO_PREFIX'] = prefix
+os.environ["CSO_PREFIX"] = prefix
 
 # init script:
 utos = utopya.UtopyaRunScriptRc()
 
 # setup standard command line arguments, and also enable rcfile arguments:
-utos.ArgumentsSetup( description='CAMS Satellite Operator', rcbase='cso' )
+utos.ArgumentsSetup(description="CAMS Satellite Operator", rcbase="cso")
 
 # evaluate known arguments, store the other ;
 # might show help text and exit:
-args,xargs = utos.ArgumentsParse()
+args, xargs = utos.ArgumentsParse()
 
 # start, shout info:
-logging.info( '' )
-logging.info( '** CSO - CAMS Satellite Operator **' )
-logging.info( '' )
-logging.info( 'installation prefix  : %s' % prefix )
-logging.info( 'input rcfile         : %s' % args.rcfile )
-logging.info( 'settings base        : %s' % args.rcbase )
-logging.info( '' )
+logging.info(f"")
+logging.info(f"** CSO - CAMS Satellite Operator **")
+logging.info(f"")
+logging.info(f"installation prefix  : {prefix}")
+logging.info(f"input rcfile         : {args.rcfile}")
+logging.info(f"settings base        : {args.rcbase}")
+logging.info(f"")
 
 # info ...
-logging.info( 'start job tree ...' )
+logging.info(f"start job tree ...")
 # import class as defined in settings by 'cso.class' key;
 # should be (derived from) 'utopya.JobStep' :
-jbs_cls = utos.ImportClass( 'class' )
+jbs_cls = utos.ImportClass("class")
 # init job step object, use the 'rcbase' as name:
-jbs = jbs_cls( args.rcbase, args.rcfile )
+jbs = jbs_cls(args.rcbase, args.rcfile)
 # start first:
-jbs.Start( single=args.single )
+jbs.Start(single=args.single)
 
 # info:
-logging.info( '' )
-logging.info( '** end **' )
-logging.info( '' )
+logging.info(f"")
+logging.info(f"** end **")
+logging.info(f"")
 
 
-#-------------------------------------------------
+# -------------------------------------------------
 # end
-#-------------------------------------------------
+# -------------------------------------------------

config/Copernicus/cso-s5p-chocho.rc

@@ -2,7 +2,7 @@
 !!!
 !!! CSO - CAMS Satellite Operator
 !!!
-!!! Settings for S5p/Glyoxal processing.
+!!! Settings for S5p/CHOCHO (glyoxal) processing.
 !!!
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
@@ -38,7 +38,7 @@ cso.s5p.chocho.inquire-table-glyretro.timerange.end          :  ${my.full-timera
 
 ! output table, time templates are filled with date of today;
 ! base path is used for searching input files:
-cso.s5p.chocho.inquire-table-glyretro.file                   :  ${my.observations}/GlyRetro/Copernicus_S5p_GLYOX_2022-09-22.csv
+cso.s5p.chocho.inquire-table-glyretro.file                   :  ${my.work}/Copernicus-inquire/Copernicus_S5p_CHOCHO_glyretro__2022-09-22.csv
 
 ! filename filters relative to listing file that should be scanned for orbit files;
 ! names could include time templates ;
@@ -72,7 +72,7 @@ cso.s5p.chocho.inquire-table-pal.producttype            :  L2__CHOCHO
 cso.s5p.chocho.inquire-table-pal.area                   :  ${my.region.west},${my.region.south}:${my.region.east},${my.region.north}
 
 ! output table, date of today:
-cso.s5p.chocho.inquire-table-pal.output.file            :  ${my.work}/Copernicus-inquire/Copernicus_S5p_CHOCHO_pal_%Y-%m-%d.csv
+cso.s5p.chocho.inquire-table-pal.output.file            :  ${my.work}/Copernicus-inquire/Copernicus_S5p_CHOCHO_pal__%Y-%m-%d.csv
 
 
 
@@ -86,12 +86,12 @@ cso.s5p.chocho.inquire-plot.renew                    :  True
 ! listing files, here from two inquires:
 cso.s5p.chocho.inquire-plot.file                     :  ${cso.s5p.chocho.inquire-table-glyretro.file} ; \
                                                         ${cso.s5p.chocho.inquire-table-pal.output.file}
-!!~ specify dates ("yyyy-mm-dd") to use historic tables,
-!!  default is table of today:
-!cso.s5p.chocho.inquire-plot.filedate                 :  2022-01-28 ; 2023-08-08
+!~ specify dates ("yyyy-mm-dd") to use historic tables,
+!  default is table of today:
+cso.s5p.chocho.inquire-plot.filedate                 :  2022-09-22 ;
 
 ! annote:
-cso.s5p.chocho.inquire-plot.title                    :  S5p/chochoal %Y-%m-%d
+cso.s5p.chocho.inquire-plot.title                    :  S5p/CHOCHO %Y-%m-%d
 
 ! output figure, date of today:
 cso.s5p.chocho.inquire-plot.output.file              :  ${my.work}/Copernicus-inquire/Copernicus_S5p_CHOCHO__%Y-%m-%d.png
@@ -118,9 +118,9 @@ cso.s5p.chocho.convert.timerange.end          :  ${my.timerange.end}
 !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 ! listing file or ';'-seperated list of listing files:
-cso.s5p.chocho.convert.inquire.file                   :   ${cso.s5p.chocho.inquire-plot.file}
-!!~ historic inquire ...
-!cso.s5p.chocho.convert.inquire.filedate               : ${cso.s5p.chocho.inquire-plot.filedate}
+cso.s5p.chocho.convert.inquire.file                   :  ${cso.s5p.chocho.inquire-plot.file}
+!~ historic inquire ...
+cso.s5p.chocho.convert.inquire.filedate               :  ${cso.s5p.chocho.inquire-plot.filedate}
 
 !! testing: produced only selected files ...
 !!~ these files have "no-data" in "PRODUCT/delta_time", while qa_value is 1.0 ...
@@ -149,8 +149,8 @@ my.s5p.chocho.selection                                :  Cxx
 ! strange archive, need to combine collections;
 ! prefere PAL_ for later years, otherwise RPRO or OFFL:
 cso.s5p.chocho.convert.selection                      :  (%{processing} == 'PAL_') ; \
-                                                        (%{processing} == 'RPRO') ; \
-                                                        (%{processing} == 'OFFL')
+                                                         (%{processing} == 'RPRO') ; \
+                                                         (%{processing} == 'OFFL')
 #else
 #error unsupported my.s5p.chocho.selection "${my.s5p.chocho.selection}"
 #endif
@@ -646,9 +646,9 @@ cso.s5p.chocho.gridded-catalogue.input.file             :  ${my.gridded.dir}/%Y/
 !cso.s5p.chocho.gridded-catalogue.input.file             :  ${my.gridded.dir}/S5p_CHOCHO_%Y%m%d_aver_gridded.nc
 
 ! target files, time tempates and variable name are replaced:
-cso.s5p.chocho.gridded-catalogue.output.file            :  ${my.gridded-catalogue.output.dir}/%Y/%m/%d/S5p_CHOCHO_%Y%m%d_%H%M_gridded_%{var}.png
+cso.s5p.chocho.gridded-catalogue.output.file            :  ${my.chocho.gridded-catalogue.output.dir}/%Y/%m/%d/S5p_CHOCHO_%Y%m%d_%H%M_gridded_%{var}.png
 !!~ idem for daily average:
-!cso.s5p.chocho.gridded-catalogue.output.file            :  ${my.gridded-catalogue.output.dir}/%Y/%m/%d/S5p_CHOCHO_%Y%m%d_aver_gridded_%{var}.png
+!cso.s5p.chocho.gridded-catalogue.output.file            :  ${my.chocho.gridded-catalogue.output.dir}/%Y/%m/%d/S5p_CHOCHO_%Y%m%d_aver_gridded_%{var}.png
 
 ! figure size (inches), default is (8,6):
 cso.s5p.chocho.gridded-catalogue.figsize                :  (6,6)
@@ -673,7 +673,7 @@ cso.s5p.chocho.gridded-catalogue.var.yr.vmax            : None
 !-----------------------------------------------------------
 
 ! target location:
-cso.s5p.chocho.gridded-catalogue-index.outdir                    :  ${my.gridded-catalogue.output.dir}
+cso.s5p.chocho.gridded-catalogue-index.outdir                    :  ${my.chocho.gridded-catalogue.output.dir}
 
 ! title:
 cso.s5p.chocho.gridded-catalogue-index.header                    :  CSO catalogue

config/Copernicus/cso-s5p-co.rc

@@ -23,42 +23,39 @@
 
 
 !-----------------------------------------------------------------------
-! inquire SciHub portal
+! inquire DataSpace
 !-----------------------------------------------------------------------
 
 ! Obtain names of all available S5p files.
 ! Stored as csv with processing type, processor version, filenames, etc.
 
-! inquire full time range:
-cso.s5p.co.inquire-table-scihub.timerange.start        :  ${my.full-timerange.start}
-cso.s5p.co.inquire-table-scihub.timerange.end          :  ${my.full-timerange.end}
+! full time range:
+cso.s5p.co.inquire-table-dataspace.timerange.start        :  ${my.full-timerange.start}
+cso.s5p.co.inquire-table-dataspace.timerange.end          :  ${my.full-timerange.end}
 
-!
-! server url ; 
-! provide login/password in ~/.netrc:
-!
-!   machine s5phub.copernicus.eu      login s5pguest        password s5pguest
-!
-cso.s5p.co.inquire-table-scihub.url                    :  https://s5phub.copernicus.eu/dhus
+! API url:
+cso.s5p.co.inquire-table-dataspace.url                    :  https://finder.creodias.eu/resto/api
+
+! collection name:
+cso.s5p.co.inquire-table-dataspace.collection             :  Sentinel5P
+
+! product type, always 10 characters!
+!   L2__CO___
+!   L2__CO____
+!   ...
+cso.s5p.co.inquire-table-dataspace.producttype            :  L2__CO____
 
 ! target area;
 !!~ empty for no limitation:
-!cso.s5p.co.inquire-table-scihub.area                   :  
-!~ domain specified as:  west,south:east,north
-cso.s5p.co.inquire-table-scihub.area                   :  ${my.region.west},${my.region.south}:${my.region.east},${my.region.north}
+!cso.s5p.co.inquire-table-dataspace.area                   :  
+!~ domain specified as:  west,south,east,north
+cso.s5p.co.inquire-table-dataspace.area                   :  ${my.region.west},${my.region.south},${my.region.east},${my.region.north}
 
-! search query, obtained from interactive download:
-!
-!   platformname     : Sentinel-5
-!   producttype      : L2__CO___ | L2__CO____    (always 10 characters!)
-!   processinglevel  : L2
-!
-cso.s5p.co.inquire-table-scihub.query                  :      platformname:Sentinel-5     AND \
-                                                               producttype:L2__CO____    AND \
-                                                           processinglevel:L2
+! template for download url given "{product_id}":
+cso.s5p.co.inquire-table-dataspace.download_url           :  https://zipper.dataspace.copernicus.eu/odata/v1/Products({product_id})/$value
 
 ! output table, date of today:
-cso.s5p.co.inquire-table-scihub.output.file            :  ${my.work}/Copernicus-inquire/Copernicus_S5p_CO_scihub__%Y-%m-%d.csv
+cso.s5p.co.inquire-table-dataspace.output.file            :  ${my.work}/Copernicus-inquire/Copernicus_S5p_CO_dataspace__%Y-%m-%d.csv
 
 
 !-----------------------------------------------------------
@@ -69,7 +66,7 @@ cso.s5p.co.inquire-table-scihub.output.file            :  ${my.work}/Copernicus-
 cso.s5p.co.inquire-plot.renew                    :  True
 
 ! listing files:
-cso.s5p.co.inquire-plot.file                     :  ${cso.s5p.co.inquire-table-scihub.output.file}
+cso.s5p.co.inquire-plot.file                     :  ${cso.s5p.co.inquire-table-dataspace.output.file}
 !!~ specify dates ("yyyy-mm-dd") to use historic tables,
 !!  default is table of today:
 !cso.s5p.co.inquire-plot.filedate                 :  2023-08-07
@@ -78,12 +75,12 @@ cso.s5p.co.inquire-plot.file                     :  ${cso.s5p.co.inquire-table-s
 cso.s5p.co.inquire-plot.title                    :  S5p/CO %Y-%m-%d
 
 ! output figure, date of today:
-cso.s5p.co.inquire-plot.output.file              :  ${my.work}/Copernicus-inquire/Copernicus_S5p_CO_scihub__%Y-%m-%d.png
+cso.s5p.co.inquire-plot.output.file              :  ${my.work}/Copernicus-inquire/Copernicus_S5p_CO_dataspace__%Y-%m-%d.png
 
 
 !======================================================================
 !===
-!=== convert
+!=== convert (and download)
 !===
 !======================================================================
 
@@ -102,8 +99,8 @@ cso.s5p.co.convert.timerange.end          :  ${my.timerange.end}
 !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 ! listing of available source files,
-! created by 'inquire-scihub' job:
-cso.s5p.co.convert.inquire.file                   :  ${my.work}/Copernicus-inquire/Copernicus_S5p_CO_scihub__%Y-%m-%d.csv
+! created by 'inquire-dataspace' job:
+cso.s5p.co.convert.inquire.file                   :  ${my.work}/Copernicus-inquire/Copernicus_S5p_CO_dataspace__%Y-%m-%d.csv
 !!~ historic inquire ...
 !cso.s5p.co.convert.inquire.filedate               :  ${cso.s5p.co.inquire-plot.filedate}
 
@@ -119,7 +116,7 @@ my.s5p.co.selection                                :  C03
 !   (%{collection} == '03') and (%{processing} == 'OFFL')
 !
 #if "${my.s5p.co.selection}" == "C03"
-cso.s5p.co.convert.selection                      :  (%{collection} == '03') and (%{processing} == 'RPRO') ; \
+cso.s5p.co.convert.selection                     :  (%{collection} == '03') and (%{processing} == 'RPRO') ; \
                                                      (%{collection} == '03') and (%{processing} == 'OFFL')
 #else
 #error unsupported my.s5p.co.selection "${my.s5p.co.selection}"

config/Copernicus/cso-s5p-hcho.rc

@@ -23,42 +23,39 @@
 
 
 !-----------------------------------------------------------------------
-! inquire SciHub portal
+! inquire DataSpace
 !-----------------------------------------------------------------------
 
 ! Obtain names of all available S5p files.
 ! Stored as csv with processing type, processor version, filenames, etc.
 
-! inquire full time range:
-cso.s5p.hcho.inquire-table-scihub.timerange.start        :  ${my.full-timerange.start}
-cso.s5p.hcho.inquire-table-scihub.timerange.end          :  ${my.full-timerange.end}
+! full time range:
+cso.s5p.hcho.inquire-table-dataspace.timerange.start        :  ${my.full-timerange.start}
+cso.s5p.hcho.inquire-table-dataspace.timerange.end          :  ${my.full-timerange.end}
 
-!
-! server url ; 
-! provide login/password in ~/.netrc:
-!
-!   machine s5phub.copernicus.eu      login s5pguest        password s5pguest
-!
-cso.s5p.hcho.inquire-table-scihub.url                    :  https://s5phub.copernicus.eu/dhus
+! API url:
+cso.s5p.hcho.inquire-table-dataspace.url                    :  https://finder.creodias.eu/resto/api
+
+! collection name:
+cso.s5p.hcho.inquire-table-dataspace.collection             :  Sentinel5P
+
+! product type, always 10 characters!
+!   L2__HCHO___
+!   L2__CO____
+!   ...
+cso.s5p.hcho.inquire-table-dataspace.producttype            :  L2__HCHO__
 
 ! target area;
 !!~ empty for no limitation:
-!cso.s5p.hcho.inquire-table-scihub.area                   :  
-!~ domain specified as:  west,south:east,north
-cso.s5p.hcho.inquire-table-scihub.area                   :  ${my.region.west},${my.region.south}:${my.region.east},${my.region.north}
+!cso.s5p.hcho.inquire-table-dataspace.area                   :  
+!~ domain specified as:  west,south,east,north
+cso.s5p.hcho.inquire-table-dataspace.area                   :  ${my.region.west},${my.region.south},${my.region.east},${my.region.north}
 
-! search query, obtained from interactive download:
-!
-!   platformname     : Sentinel-5
-!   producttype      : L2__HCHO___ | L2__CO____    (always 10 characters!)
-!   processinglevel  : L2
-!
-cso.s5p.hcho.inquire-table-scihub.query                  :     platformname:Sentinel-5    AND \
-                                                                producttype:L2__HCHO__    AND \
-                                                            processinglevel:L2
+! template for download url given "{product_id}":
+cso.s5p.hcho.inquire-table-dataspace.download_url           :  https://zipper.dataspace.copernicus.eu/odata/v1/Products({product_id})/$value
 
 ! output table, date of today:
-cso.s5p.hcho.inquire-table-scihub.output.file            :  ${my.work}/Copernicus-inquire/Copernicus_S5p_HCHO_scihub__%Y-%m-%d.csv
+cso.s5p.hcho.inquire-table-dataspace.output.file            :  ${my.work}/Copernicus-inquire/Copernicus_S5p_HCHO_dataspace__%Y-%m-%d.csv
 
 
 !-----------------------------------------------------------
@@ -66,24 +63,24 @@ cso.s5p.hcho.inquire-table-scihub.output.file            :  ${my.work}/Copernicu
 !-----------------------------------------------------------
 
 ! renew existing plots?
-cso.s5p.hcho.inquire-scihub-plot.renew                    :  True
+cso.s5p.hcho.inquire-plot.renew                    :  True
 
 ! listing files:
-cso.s5p.hcho.inquire-scihub-plot.file                     :  ${cso.s5p.hcho.inquire-table-scihub.output.file}
+cso.s5p.hcho.inquire-plot.file                     :  ${cso.s5p.hcho.inquire-table-dataspace.output.file}
 !!~ specify dates ("yyyy-mm-dd") to use historic tables,
 !!  default is table of today:
-!cso.s5p.hcho.inquire-scihub-plot.filedate                 :  2023-08-07
+!cso.s5p.hcho.inquire-plot.filedate                 :  2023-08-07
 
 ! annote:
-cso.s5p.hcho.inquire-plot.title                           :  S5p/HCHO %Y-%m-%d
+cso.s5p.hcho.inquire-plot.title                    :  S5p/HCHO %Y-%m-%d
 
 ! output figure, date of today:
-cso.s5p.hcho.inquire-scihub-plot.output.file              :  ${my.work}/Copernicus-inquire/Copernicus_S5p_HCHO_scihub__%Y-%m-%d.png
+cso.s5p.hcho.inquire-plot.output.file              :  ${my.work}/Copernicus-inquire/Copernicus_S5p_HCHO_dataspace__%Y-%m-%d.png
 
 
 !======================================================================
 !===
-!=== convert
+!=== convert (and download)
 !===
 !======================================================================
 
@@ -102,10 +99,10 @@ cso.s5p.hcho.convert.timerange.end          :  ${my.timerange.end}
 !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 ! listing of available source files,
-! created by 'inquire-scihub' job:
-cso.s5p.hcho.convert.inquire.file                   :  ${my.work}/Copernicus-inquire/Copernicus_S5p_HCHO_scihub__%Y-%m-%d.csv
+! created by 'inquire-dataspace' job:
+cso.s5p.hcho.convert.inquire.file                   :  ${my.work}/Copernicus-inquire/Copernicus_S5p_HCHO_dataspace__%Y-%m-%d.csv
 !!~ historic inquire ...
-!cso.s5p.hcho.convert.inquire.filedate               :  ${cso.s5p.hcho.inquire-scihub-plot.filedate}
+!cso.s5p.hcho.convert.inquire.filedate               :  ${cso.s5p.hcho.inquire-plot.filedate}
 
 ! selection keyword:
 my.s5p.hcho.selection                                :  C03
@@ -473,8 +470,8 @@ cso.s5p.hcho.catalogue.domain                      :  ${my.region.west} ${my.reg
 cso.s5p.hcho.catalogue.figsize                     :  ${my.region.figsize}
 
 ! renew existing files?
-cso.s5p.hcho.catalogue.renew                       :  False
-!cso.s5p.hcho.catalogue.renew                       :  True
+!cso.s5p.hcho.catalogue.renew                       :  False
+cso.s5p.hcho.catalogue.renew                       :  True
 
 ! variables to be plotted:
 cso.s5p.hcho.catalogue.vars                        :  vcd vcd_errvar qa_value cloud_fraction
@@ -623,16 +620,17 @@ cso.s5p.hcho.gridded.filter.quality.var       :  qa_value
 cso.s5p.hcho.gridded.filter.quality.type      :  min
 cso.s5p.hcho.gridded.filter.quality.min       :  0.8
 cso.s5p.hcho.gridded.filter.quality.units     :  1
-       
+
 #else
 #error unsupported  my.s5p.hcho.gridded-selection "${my.s5p.hcho.gridded-selection}"
 #endif
- 
+
+        
 !~
 
 ! target file, might contain templates:
 !   %Y,%m,etc     : time values
-cso.s5p.hcho.gridded.output.file               :  ${my.gridded.dir}/%Y/%m/S5p_HCHO_%Y%m%d_%H%M_gridded.nc
+cso.s5p.hcho.gridded.output.file              :  ${my.gridded.dir}/%Y/%m/S5p_HCHO_%Y%m%d_%H%M_gridded.nc
 
 ! pack floats as short values?
 cso.s5p.hcho.gridded.output.packed            :  True
@@ -658,11 +656,11 @@ cso.s5p.hcho.gridded-catalogue.timerange.end          :  ${my.timerange.end}
 cso.s5p.hcho.gridded-catalogue.timerange.step         :  hour
 
 ! renew existing files?
-!cso.s5p.hcho.gridded-catalogue.renew                  :  True
-cso.s5p.hcho.gridded-catalogue.renew                  :  False
+cso.s5p.hcho.gridded-catalogue.renew                  :  True
+!cso.s5p.hcho.gridded-catalogue.renew                  :  False
 
 ! target directory for catalogue:
-my.gridded-catalogue.output.dir                       :  ${my.gridded.dir}/catalogue
+my.hcho.gridded-catalogue.output.dir                  :  ${my.gridded.dir}/catalogue
 
 ! input files:
 cso.s5p.hcho.gridded-catalogue.input.file             :  ${my.gridded.dir}/%Y/%m/S5p_HCHO_%Y%m%d_%H%M_gridded.nc
@@ -670,9 +668,9 @@ cso.s5p.hcho.gridded-catalogue.input.file             :  ${my.gridded.dir}/%Y/%m
 !cso.s5p.hcho.gridded-catalogue.input.file             :  ${my.gridded.dir}/S5p_HCHO_%Y%m%d_aver_gridded.nc
 
 ! target files, time tempates and variable name are replaced:
-cso.s5p.hcho.gridded-catalogue.output.file            :  ${my.gridded-catalogue.output.dir}/%Y/%m/%d/S5p_HCHO_%Y%m%d_%H%M_gridded_%{var}.png
+cso.s5p.hcho.gridded-catalogue.output.file            :  ${my.hcho.gridded-catalogue.output.dir}/%Y/%m/%d/S5p_HCHO_%Y%m%d_%H%M_gridded_%{var}.png
 !!~ idem for daily average:
-!cso.s5p.hcho.gridded-catalogue.output.file            :  ${my.gridded-catalogue.output.dir}/%Y/%m/%d/S5p_HCHO_%Y%m%d_aver_gridded_%{var}.png
+!cso.s5p.hcho.gridded-catalogue.output.file            :  ${my.hcho.gridded-catalogue.output.dir}/%Y/%m/%d/S5p_HCHO_%Y%m%d_aver_gridded_%{var}.png
 
 ! figure size (inches), default is (8,6):
 cso.s5p.hcho.gridded-catalogue.figsize                :  (6,6)
@@ -686,7 +684,6 @@ cso.s5p.hcho.gridded-catalogue.var.yr.source          :  yr
 cso.s5p.hcho.gridded-catalogue.var.yr.units           :  umol/m2
 ! style:
 cso.s5p.hcho.gridded-catalogue.var.yr.vmin            :   0.0
-!gcso.s5p.hcho.gridded-catalogue.var.yr.vmax            : 100.0
 cso.s5p.hcho.gridded-catalogue.var.yr.vmax            : None
 
 
@@ -697,7 +694,7 @@ cso.s5p.hcho.gridded-catalogue.var.yr.vmax            : None
 !-----------------------------------------------------------
 
 ! target location:
-cso.s5p.hcho.gridded-catalogue-index.outdir                    :  ${my.gridded-catalogue.output.dir}
+cso.s5p.hcho.gridded-catalogue-index.outdir                    :  ${my.hcho.gridded-catalogue.output.dir}
 
 ! title:
 cso.s5p.hcho.gridded-catalogue-index.header                    :  CSO catalogue

config/Copernicus/cso-s5p-no2.rc

@@ -23,42 +23,39 @@
 
 
 !-----------------------------------------------------------------------
-! inquire SciHub portal
+! inquire DataSpace
 !-----------------------------------------------------------------------
 
 ! Obtain names of all available S5p files.
 ! Stored as csv with processing type, processor version, filenames, etc.
 
-! inquire full time range:
-cso.s5p.no2.inquire-table-scihub.timerange.start        :  ${my.full-timerange.start}
-cso.s5p.no2.inquire-table-scihub.timerange.end          :  ${my.full-timerange.end}
+! full time range:
+cso.s5p.no2.inquire-table-dataspace.timerange.start        :  ${my.full-timerange.start}
+cso.s5p.no2.inquire-table-dataspace.timerange.end          :  ${my.full-timerange.end}
 
-!
-! server url ; 
-! provide login/password in ~/.netrc:
-!
-!   machine s5phub.copernicus.eu      login s5pguest        password s5pguest
-!
-cso.s5p.no2.inquire-table-scihub.url                    :  https://s5phub.copernicus.eu/dhus
+! API url:
+cso.s5p.no2.inquire-table-dataspace.url                    :  https://finder.creodias.eu/resto/api
+
+! collection name:
+cso.s5p.no2.inquire-table-dataspace.collection             :  Sentinel5P
+
+! product type, always 10 characters!
+!   L2__NO2___
+!   L2__CO____
+!   ...
+cso.s5p.no2.inquire-table-dataspace.producttype            :  L2__NO2___
 
 ! target area;
 !!~ empty for no limitation:
-!cso.s5p.no2.inquire-table-scihub.area                   :  
-!~ domain specified as:  west,south:east,north
-cso.s5p.no2.inquire-table-scihub.area                   :  ${my.region.west},${my.region.south}:${my.region.east},${my.region.north}
+!cso.s5p.no2.inquire-table-dataspace.area                   :  
+!~ domain specified as:  west,south,east,north
+cso.s5p.no2.inquire-table-dataspace.area                   :  ${my.region.west},${my.region.south},${my.region.east},${my.region.north}
 
-! search query, obtained from interactive download:
-!
-!   platformname     : Sentinel-5
-!   producttype      : L2__NO2___ | L2__CO____    (always 10 characters!)
-!   processinglevel  : L2
-!
-cso.s5p.no2.inquire-table-scihub.query                  :     platformname:Sentinel-5    AND \
-                                                                producttype:L2__NO2___    AND \
-                                                            processinglevel:L2
+! template for download url given "{product_id}":
+cso.s5p.no2.inquire-table-dataspace.download_url           :  https://zipper.dataspace.copernicus.eu/odata/v1/Products({product_id})/$value
 
 ! output table, date of today:
-cso.s5p.no2.inquire-table-scihub.output.file            :  ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_scihub__%Y-%m-%d.csv
+cso.s5p.no2.inquire-table-dataspace.output.file            :  ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_dataspace__%Y-%m-%d.csv
 
 
 !!-----------------------------------------------------------------------
@@ -96,7 +93,7 @@ cso.s5p.no2.inquire-table-scihub.output.file            :  ${my.work}/Copernicus
 cso.s5p.no2.inquire-plot.renew                    :  True
 
 ! listing files:
-cso.s5p.no2.inquire-plot.file                     :  ${cso.s5p.no2.inquire-table-scihub.output.file}
+cso.s5p.no2.inquire-plot.file                     :  ${cso.s5p.no2.inquire-table-dataspace.output.file}
 !!~ specify dates ("yyyy-mm-dd") to use historic tables,
 !!  default is table of today:
 !cso.s5p.no2.inquire-plot.filedate                 :  2023-08-07
@@ -105,7 +102,7 @@ cso.s5p.no2.inquire-plot.file                     :  ${cso.s5p.no2.inquire-table
 cso.s5p.no2.inquire-plot.title                    :  S5p/NO2 %Y-%m-%d
 
 ! output figure, date of today:
-cso.s5p.no2.inquire-plot.output.file              :  ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_scihub__%Y-%m-%d.png
+cso.s5p.no2.inquire-plot.output.file              :  ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_dataspace__%Y-%m-%d.png
 
 
 !======================================================================
@@ -129,8 +126,8 @@ cso.s5p.no2.convert.timerange.end          :  ${my.timerange.end}
 !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 ! listing of available source files,
-! created by 'inquire-scihub' job:
-cso.s5p.no2.convert.inquire.file                   :  ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_scihub__%Y-%m-%d.csv
+! created by 'inquire-dataspace' job:
+cso.s5p.no2.convert.inquire.file                   :  ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_dataspace__%Y-%m-%d.csv
 !!~ historic inquire ...
 !cso.s5p.no2.convert.inquire.filedate               :  ${cso.s5p.no2.inquire-plot.filedate}
 
@@ -717,9 +714,9 @@ cso.s5p.no2.gridded-catalogue.input.file             :  ${my.gridded.dir}/%Y/%m/
 !cso.s5p.no2.gridded-catalogue.input.file             :  ${my.gridded.dir}/S5p_NO2_%Y%m%d_aver_gridded.nc
 
 ! target files, time tempates and variable name are replaced:
-cso.s5p.no2.gridded-catalogue.output.file            :  ${my.gridded-catalogue.output.dir}/%Y/%m/%d/S5p_NO2_%Y%m%d_%H%M_gridded_%{var}.png
+cso.s5p.no2.gridded-catalogue.output.file            :  ${my.no2.gridded-catalogue.output.dir}/%Y/%m/%d/S5p_NO2_%Y%m%d_%H%M_gridded_%{var}.png
 !!~ idem for daily average:
-!cso.s5p.no2.gridded-catalogue.output.file            :  ${my.gridded-catalogue.output.dir}/%Y/%m/%d/S5p_NO2_%Y%m%d_aver_gridded_%{var}.png
+!cso.s5p.no2.gridded-catalogue.output.file            :  ${my.no2.gridded-catalogue.output.dir}/%Y/%m/%d/S5p_NO2_%Y%m%d_aver_gridded_%{var}.png
 
 ! figure size (inches), default is (8,6):
 cso.s5p.no2.gridded-catalogue.figsize                :  (6,6)
@@ -743,7 +740,7 @@ cso.s5p.no2.gridded-catalogue.var.yr.vmax            : 100.0
 !-----------------------------------------------------------
 
 ! target location:
-cso.s5p.no2.gridded-catalogue-index.outdir                    :  ${my.gridded-catalogue.output.dir}
+cso.s5p.no2.gridded-catalogue-index.outdir                    :  ${my.no2.gridded-catalogue.output.dir}
 
 ! title:
 cso.s5p.no2.gridded-catalogue-index.header                    :  CSO catalogue

config/Copernicus/cso-s5p-so2.rc

@@ -23,42 +23,39 @@
 
 
 !-----------------------------------------------------------------------
-! inquire SciHub portal
+! inquire DataSpace
 !-----------------------------------------------------------------------
 
 ! Obtain names of all available S5p files.
 ! Stored as csv with processing type, processor version, filenames, etc.
 
-! inquire full time range:
-cso.s5p.so2.inquire-table-scihub.timerange.start        :  ${my.full-timerange.start}
-cso.s5p.so2.inquire-table-scihub.timerange.end          :  ${my.full-timerange.end}
+! full time range:
+cso.s5p.so2.inquire-table-dataspace.timerange.start        :  ${my.full-timerange.start}
+cso.s5p.so2.inquire-table-dataspace.timerange.end          :  ${my.full-timerange.end}
 
-!
-! server url ; 
-! provide login/password in ~/.netrc:
-!
-!   machine s5phub.copernicus.eu      login s5pguest        password s5pguest
-!
-cso.s5p.so2.inquire-table-scihub.url                    :  https://s5phub.copernicus.eu/dhus
+! API url:
+cso.s5p.so2.inquire-table-dataspace.url                    :  https://finder.creodias.eu/resto/api
+
+! collection name:
+cso.s5p.so2.inquire-table-dataspace.collection             :  Sentinel5P
+
+! product type, always 10 characters!
+!   L2__SO2___
+!   L2__CO____
+!   ...
+cso.s5p.so2.inquire-table-dataspace.producttype            :  L2__SO2___
 
 ! target area;
 !!~ empty for no limitation:
-!cso.s5p.so2.inquire-table-scihub.area                   :  
-!~ domain specified as:  west,south:east,north
-cso.s5p.so2.inquire-table-scihub.area                   :  ${my.region.west},${my.region.south}:${my.region.east},${my.region.north}
+!cso.s5p.so2.inquire-table-dataspace.area                   :  
+!~ domain specified as:  west,south,east,north
+cso.s5p.so2.inquire-table-dataspace.area                   :  ${my.region.west},${my.region.south},${my.region.east},${my.region.north}
 
-! search query, obtained from interactive download:
-!
-!   platformname     : Sentinel-5
-!   producttype      : L2__SO2___ | L2__CO____    (always 10 characters!)
-!   processinglevel  : L2
-!
-cso.s5p.so2.inquire-table-scihub.query                  :     platformname:Sentinel-5    AND \
-                                                                producttype:L2__SO2___    AND \
-                                                            processinglevel:L2
+! template for download url given "{product_id}":
+cso.s5p.so2.inquire-table-dataspace.download_url           :  https://zipper.dataspace.copernicus.eu/odata/v1/Products({product_id})/$value
 
 ! output table, date of today:
-cso.s5p.so2.inquire-table-scihub.output.file            :  ${my.work}/Copernicus-inquire/Copernicus_S5p_SO2_scihub__%Y-%m-%d.csv
+cso.s5p.so2.inquire-table-dataspace.output.file            :  ${my.work}/Copernicus-inquire/Copernicus_S5p_SO2_dataspace__%Y-%m-%d.csv
 
 
 !-----------------------------------------------------------
@@ -69,7 +66,7 @@ cso.s5p.so2.inquire-table-scihub.output.file            :  ${my.work}/Copernicus
 cso.s5p.so2.inquire-plot.renew                    :  True
 
 ! listing files:
-cso.s5p.so2.inquire-plot.file                     :  ${cso.s5p.so2.inquire-table-scihub.output.file}
+cso.s5p.so2.inquire-plot.file                     :  ${cso.s5p.so2.inquire-table-dataspace.output.file}
 !!~ specify dates ("yyyy-mm-dd") to use historic tables,
 !!  default is table of today:
 !cso.s5p.so2.inquire-plot.filedate                 :  2023-08-07
@@ -78,7 +75,7 @@ cso.s5p.so2.inquire-plot.file                     :  ${cso.s5p.so2.inquire-table
 cso.s5p.so2.inquire-plot.title                    :  S5p/SO2 %Y-%m-%d
 
 ! output figure, date of today:
-cso.s5p.so2.inquire-plot.output.file              :  ${my.work}/Copernicus-inquire/Copernicus_S5p_SO2_scihub__%Y-%m-%d.png
+cso.s5p.so2.inquire-plot.output.file              :  ${my.work}/Copernicus-inquire/Copernicus_S5p_SO2_dataspace__%Y-%m-%d.png
 
 
 !======================================================================
@@ -102,8 +99,8 @@ cso.s5p.so2.convert.timerange.end          :  ${my.timerange.end}
 !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 ! listing of available source files,
-! created by 'inquire-scihub' job:
-cso.s5p.so2.convert.inquire.file                   :  ${my.work}/Copernicus-inquire/Copernicus_S5p_SO2_scihub__%Y-%m-%d.csv
+! created by 'inquire-dataspace' job:
+cso.s5p.so2.convert.inquire.file                   :  ${my.work}/Copernicus-inquire/Copernicus_S5p_SO2_dataspace__%Y-%m-%d.csv
 !!~ historic inquire ...
 !cso.s5p.so2.convert.inquire.filedate               :  ${cso.s5p.so2.inquire-plot.filedate}
 
@@ -503,6 +500,7 @@ cso.s5p.so2.catalogue-index.date.orbit.var.img             :  %{date[0:4]}/%{dat
 cso.s5p.so2.catalogue-index.date.orbit.var.kwargs          :  height=300
 
 
+
 !======================================================================
 !===
 !=== gridded orbits
@@ -575,11 +573,12 @@ cso.s5p.so2.gridded.filter.quality.var       :  qa_value
 cso.s5p.so2.gridded.filter.quality.type      :  min
 cso.s5p.so2.gridded.filter.quality.min       :  0.8
 cso.s5p.so2.gridded.filter.quality.units     :  1
-  
+
 #else
 #error unsupported  my.s5p.so2.gridded-selection "${my.s5p.so2.gridded-selection}"
 #endif
-      
+
+        
 !~
 
 ! target file, might contain templates:
@@ -610,8 +609,8 @@ cso.s5p.so2.gridded-catalogue.timerange.end          :  ${my.timerange.end}
 cso.s5p.so2.gridded-catalogue.timerange.step         :  hour
 
 ! renew existing files?
-!cso.s5p.so2.gridded-catalogue.renew                  :  True
-cso.s5p.so2.gridded-catalogue.renew                  :  False
+cso.s5p.so2.gridded-catalogue.renew                  :  True
+!cso.s5p.so2.gridded-catalogue.renew                  :  False
 
 ! target directory for catalogue:
 my.so2.gridded-catalogue.output.dir                  :  ${my.gridded.dir}/catalogue
@@ -622,9 +621,9 @@ cso.s5p.so2.gridded-catalogue.input.file             :  ${my.gridded.dir}/%Y/%m/
 !cso.s5p.so2.gridded-catalogue.input.file             :  ${my.gridded.dir}/S5p_SO2_%Y%m%d_aver_gridded.nc
 
 ! target files, time tempates and variable name are replaced:
-cso.s5p.so2.gridded-catalogue.output.file            :  ${my.gridded-catalogue.output.dir}/%Y/%m/%d/S5p_SO2_%Y%m%d_%H%M_gridded_%{var}.png
+cso.s5p.so2.gridded-catalogue.output.file            :  ${my.so2.gridded-catalogue.output.dir}/%Y/%m/%d/S5p_SO2_%Y%m%d_%H%M_gridded_%{var}.png
 !!~ idem for daily average:
-!cso.s5p.so2.gridded-catalogue.output.file            :  ${my.gridded-catalogue.output.dir}/%Y/%m/%d/S5p_SO2_%Y%m%d_aver_gridded_%{var}.png
+!cso.s5p.so2.gridded-catalogue.output.file            :  ${my.so2.gridded-catalogue.output.dir}/%Y/%m/%d/S5p_SO2_%Y%m%d_aver_gridded_%{var}.png
 
 ! figure size (inches), default is (8,6):
 cso.s5p.so2.gridded-catalogue.figsize                :  (6,6)
@@ -648,7 +647,7 @@ cso.s5p.so2.gridded-catalogue.var.yr.vmax            : 100.0
 !-----------------------------------------------------------
 
 ! target location:
-cso.s5p.so2.gridded-catalogue-index.outdir                    :  ${my.gridded-catalogue.output.dir}
+cso.s5p.so2.gridded-catalogue-index.outdir                    :  ${my.so2.gridded-catalogue.output.dir}
 
 ! title:
 cso.s5p.so2.gridded-catalogue-index.header                    :  CSO catalogue

config/Copernicus/cso-user-settings.rc

@@ -98,9 +98,9 @@ my.attr.institution             :  CSO
 my.attr.email                   :  Your.Name@cso.org
 
 ! base location for work directories:
-my.work                         :  /Scratch/${USER}/CSO-work
+my.work                         :  /Scratch/${USER}/CSO-Copernicus
 
-! storage for downloaded observations, used for glyoxal:
+! storage for downloaded observations:
 my.observations                 :  /Scratch/${USER}/observations
 
 !..............................
@@ -113,11 +113,26 @@ my.attr.institution             :  MET Norway
 my.attr.email                   :  arjos@met.no
 
 ! base location for work directories:
-my.work                         :  ${WORK}/projects/CSO
+my.work                         :  ${WORK}/projects/CSO-Copernicus
 
-! storage for downloaded observations, used for glyoxal:
+! storage for downloaded observations:
 my.observations                 :  ${WORK}/../observations
 
+!..............................
+#elif "${USER}" == "segersaj"
+!..............................
+
+! Attributes written to output files.
+my.attr.author                  :  Arjo Segers
+my.attr.institution             :  TNO
+my.attr.email                   :  Arjo.Segers@tno.nl
+
+! base location for work directories:
+my.work                         :  ${SCRATCH}/CSO-Copernicus
+
+! storage for downloaded observations:
+my.observations                 :  ${SCRATCH}/observations
+
 !..............................
 #else
 #error unsupported USER "${USER}"

config/Copernicus/cso.rc

@@ -54,7 +54,6 @@ cso.s5p.TRACER.elements                          :  inquire \
                                                     convert listing \
                                                     catalogue \
                                                     gridded gridded-catalogue
-!
 !~ preprocessor steps one by one ...
 !cso.s5p.TRACER.elements                          :  inquire
 !cso.s5p.TRACER.elements                          :  convert
@@ -74,7 +73,7 @@ cso.s5p.TRACER.elements                          :  inquire \
 cso.s5p.TRACER.inquire.class                      :  utopya.UtopyaJobStep
 ! two or more tasks:
 #if "TRACER" in ["no2","so2","hcho","co"]
-cso.s5p.TRACER.inquire.tasks                      :  table-scihub plot
+cso.s5p.TRACER.inquire.tasks                      :  table-dataspace plot
 #elif "TRACER" in ["so2-cobra"]
 cso.s5p.TRACER.inquire.tasks                      :  table-pal plot
 #elif "TRACER" in ["chocho"]
@@ -82,17 +81,17 @@ cso.s5p.TRACER.inquire.tasks                      :  table-glyretro table-pal pl
 #else
 #error unsupported tracer "TRACER"
 #endif
-!~ inquire files available on SciHub:
-cso.s5p.TRACER.inquire.table-scihub.class         :  cso.CSO_SciHub_Inquire
-cso.s5p.TRACER.inquire.table-scihub.args          :  '${PWD}/config/Copernicus/cso-s5p-TRACER.rc', \
-                                                     rcbase='cso.s5p.TRACER.inquire-table-scihub'
+!~ inquire files available on DataSpace:
+cso.s5p.TRACER.inquire.table-dataspace.class      :  cso.CSO_DataSpace_Inquire
+cso.s5p.TRACER.inquire.table-dataspace.args       :  '${PWD}/config/Copernicus/cso-s5p-TRACER.rc', \
+                                                     rcbase='cso.s5p.TRACER.inquire-table-dataspace'
 !~ inquire files available on PAL:
-cso.s5p.TRACER.inquire.table-pal.class           :  cso.CSO_PAL_Inquire
-cso.s5p.TRACER.inquire.table-pal.args            :  '${PWD}/config/Copernicus/cso-s5p-TRACER.rc', \
+cso.s5p.TRACER.inquire.table-pal.class            :  cso.CSO_PAL_Inquire
+cso.s5p.TRACER.inquire.table-pal.args             :  '${PWD}/config/Copernicus/cso-s5p-TRACER.rc', \
                                                       rcbase='cso.s5p.TRACER.inquire-table-pal'
 !~ inquire files downloaded from GlyRetro:
-cso.s5p.TRACER.inquire.table-glyretro.class      :  cso.CSO_SciHub_Listing
-cso.s5p.TRACER.inquire.table-glyretro.args       :  '${PWD}/config/Copernicus/cso-s5p-TRACER.rc', \
+cso.s5p.TRACER.inquire.table-glyretro.class       :  cso.CSO_S5p_Download_Listing
+cso.s5p.TRACER.inquire.table-glyretro.args        :  '${PWD}/config/Copernicus/cso-s5p-TRACER.rc', \
                                                       rcbase='cso.s5p.TRACER.inquire-table-glyretro'
 !~ create plot of available versions:
 cso.s5p.TRACER.inquire.plot.class                 :  cso.CSO_Inquire_Plot

config/tutorial/tutorial.rc

@@ -21,11 +21,11 @@ cso.tutorial.class                             :  utopya.UtopyaJobTree
 
 ! list of sub-elements:
 cso.tutorial.elements                          :  inquire \
-                                                  convert listing 
+                                                  convert listing \
                                                   catalogue \
-                                                  gridded gridded-catalogue \
-                                                  sim-catalogue \
-                                                  sim-gridded sim-gridded-catalogue
+                                                  gridded gridded-catalogue
+!                                                  sim-catalogue \
+!                                                  sim-gridded sim-gridded-catalogue
 
 ! ... preprocessor steps one by one:
 !cso.tutorial.elements                          :  inquire
@@ -47,12 +47,12 @@ cso.tutorial.elements                          :  inquire \
 ! single step:
 cso.tutorial.inquire.class                     :  utopya.UtopyaJobStep
 ! two tasks:
-cso.tutorial.inquire.tasks                     :  table-scihub plot
-!~ inquire available files:
-cso.tutorial.inquire.table-scihub.class        :  cso.CSO_SciHub_Inquire
-cso.tutorial.inquire.table-scihub.args         :  '${__filename__}', \
-                                                      rcbase='cso.tutorial.inquire-table-scihub'
-!~ create plot of available versions:
+cso.tutorial.inquire.tasks                     :  table-dataspace plot
+!~ task: inquire available files:
+cso.tutorial.inquire.table-dataspace.class     :  cso.CSO_DataSpace_Inquire
+cso.tutorial.inquire.table-dataspace.args      :  '${__filename__}', \
+                                                      rcbase='cso.tutorial.inquire-table-dataspace'
+!~ task: create plot of available versions:
 cso.tutorial.inquire.plot.class                :  cso.CSO_Inquire_Plot
 cso.tutorial.inquire.plot.args                 :  '${__filename__}', \
                                                       rcbase='cso.tutorial.inquire-plot'
@@ -222,8 +222,8 @@ my.attr.institution             :  CSO
 my.attr.email                   :  Your.Name@cso.org
 
 ! base location for work directories:
-!my.work                         :  /Scratch/${USER}/CSO-tutorial
-my.work                         :  ${WORK}/projects/CSO-tutorial
+!my.work                         :  /work/${USER}/CSO-Tutorial
+my.work                         :  /Scratch/${USER}/CSO-Tutorial
 
 
 !----------------------------------------------------------
@@ -254,35 +254,32 @@ my.work                         :  ${WORK}/projects/CSO-tutorial
 ! Stored as csv with processing type, processor version, filenames, etc.
 
 ! full time range:
-cso.tutorial.inquire-table-scihub.timerange.start        :  2018-01-01 00:00:00
-cso.tutorial.inquire-table-scihub.timerange.end          :  2024-01-01 00:00:00
+cso.tutorial.inquire-table-dataspace.timerange.start        :  2018-01-01 00:00:00
+cso.tutorial.inquire-table-dataspace.timerange.end          :  2024-01-01 00:00:00
 
-!
-! server url ; 
-! provide login/password in ~/.netrc:
-!
-!   machine s5phub.copernicus.eu      login s5pguest        password s5pguest
-!
-cso.tutorial.inquire-table-scihub.url                    :  https://s5phub.copernicus.eu/dhus
+! API url:
+cso.tutorial.inquire-table-dataspace.url                    :  https://finder.creodias.eu/resto/api
+
+! collection name:
+cso.tutorial.inquire-table-dataspace.collection             :  Sentinel5P
+
+! product type, always 10 characters!
+!   L2__NO2___
+!   L2__CO____
+!   ...
+cso.tutorial.inquire-table-dataspace.producttype            :  L2__NO2___
 
 ! target area;
 !!~ empty for no limitation:
-!cso.tutorial.inquire-table-scihub.area                   :  
-!~ domain specified as:  west,south:east,north
-cso.tutorial.inquire-table-scihub.area                   :  ${my.region.west},${my.region.south}:${my.region.east},${my.region.north}
+!cso.tutorial.inquire-table-dataspace.area                   :  
+!~ domain specified as:  west,south,east,north
+cso.tutorial.inquire-table-dataspace.area                   :  ${my.region.west},${my.region.south},${my.region.east},${my.region.north}
 
-! search query, obtained from interactive download:
-!
-!   platformname     : Sentinel-5
-!   producttype      : L2__NO2___ | L2__CO____    (always 10 characters!)
-!   processinglevel  : L2
-!
-cso.tutorial.inquire-table-scihub.query                  :     platformname:Sentinel-5    AND \
-                                                                producttype:L2__NO2___    AND \
-                                                            processinglevel:L2
+! template for download url given "{product_id}":
+cso.tutorial.inquire-table-dataspace.download_url           :  https://zipper.dataspace.copernicus.eu/odata/v1/Products({product_id})/$value
 
 ! output table, date of today:
-cso.tutorial.inquire-table-scihub.output.file            :  ${my.work}/Copernicus/Copernicus_S5p_NO2_scihub_%Y-%m-%d.csv
+cso.tutorial.inquire-table-dataspace.output.file            :  ${my.work}/Copernicus/Copernicus_S5p_NO2_dataspace__%Y-%m-%d.csv
 
 
 ! * create plot
@@ -291,7 +288,7 @@ cso.tutorial.inquire-table-scihub.output.file            :  ${my.work}/Copernicu
 cso.tutorial.inquire-plot.renew                    :  True
 
 ! listing files:
-cso.tutorial.inquire-plot.file                     :  ${cso.tutorial.inquire-table-scihub.output.file}
+cso.tutorial.inquire-plot.file                     :  ${cso.tutorial.inquire-table-dataspace.output.file}
 !!~ specify dates ("yyyy-mm-dd") to use historic tables,
 !!  default is table of today:
 !cso.tutorial.inquire-plot.filedate                 :  2022-01-28
@@ -300,7 +297,7 @@ cso.tutorial.inquire-plot.file                     :  ${cso.tutorial.inquire-tab
 cso.tutorial.inquire-plot.title                    :  S5p/NO2 %Y-%m-%d
 
 ! output figure, date of today:
-cso.tutorial.inquire-plot.output.file              :  ${my.work}/Copernicus/Copernicus_S5p_NO2_${my.region}__%Y-%m-%d.png
+cso.tutorial.inquire-plot.output.file              :  ${my.work}/Copernicus/Copernicus_S5p_NO2_dataspace__%Y-%m-%d.png
 
 
 !======================================================================
@@ -324,12 +321,9 @@ cso.tutorial.convert.timerange.end          :  ${my.timerange.end}
 !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 ! listing of available source files,  created by 'inquire' job:
-cso.tutorial.convert.inquire.file                   :  ${my.work}/Copernicus/Copernicus_S5p_NO2_scihub_%Y-%m-%d.csv
-!!~ historic inquire ...
-!cso.tutorial.convert.inquire.filedate               :  2022-02-02
-
-! selection keyword:
-my.s5p.no2.selection                                :  C03
+cso.tutorial.convert.inquire.file                   :  ${my.work}/Copernicus/Copernicus_S5p_NO2_dataspace__%Y-%m-%d.csv
+!~ historic inquire ...
+cso.tutorial.convert.inquire.filedate               :  2023-11-06
 
 ! Provide ';' seperated list of to decide if a particular orbit file should be processed.
 ! If more than one file is available for a particular orbit (from "OFFL" and "RPRO" processing),
@@ -339,12 +333,8 @@ my.s5p.no2.selection                                :  C03
 !   (%{collection} == '03') and (%{processing} == 'RPRO') ; \
 !   (%{collection} == '03') and (%{processing} == 'OFFL')
 !
-#if "${my.s5p.no2.selection}" == "C03"
 cso.tutorial.convert.selection                     :  (%{collection} == '03') and (%{processing} == 'RPRO') ; \
                                                       (%{collection} == '03') and (%{processing} == 'OFFL')
-#else
-#error unsupported my.s5p.no2.selection "${my.s5p.no2.selection}"
-#endif
 
 ! input directory;
 ! files are searched here or downloaded to if not present yet;

doc/source/conf.py

@@ -22,7 +22,7 @@ copyright = '2020-2023, Arjo Segers'
 author = 'Arjo Segers'
 
 # The full version, including alpha/beta/rc tags
-release = 'v2.4'
+release = 'v2.5'
 
 
 # -- General configuration ---------------------------------------------------

doc/source/documentation.rst

@@ -53,7 +53,7 @@ The source tree of the documentation files is:
   * `pymod-cso_colocate.rst <../../source/pymod-cso_colocate.rst>`_
   * `pymod-cso_plot.rst <../../source/pymod-cso_plot.rst>`_
   * `pymod-cso_s5p.rst <../../source/pymod-cso_s5p.rst>`_
-  * `pymod-cso_scihub.rst <../../source/pymod-cso_scihub.rst>`_
+  * `pymod-cso_dataspace.rst <../../source/pymod-cso_dataspace.rst>`_
   * `pymods.rst <../../source/pymods.rst>`_
   * `documentation.rst <../../source/documentation.rst>`_
 
@@ -173,7 +173,7 @@ Example taken from introduction of the 'cso_s5p' module.
     .. toctree::
        :maxdepth: 1
 
-       pymod-cso_scihub
+       pymod-cso_dataspace
        pymod-cso_s5p              <---
        :