TNO Intern

Updates for Sentinel-5P processing.

Arjo Segersrequested to merge
sesam into master

Following processsing of all available TROPOMI data, the following updates were made:

  • Removed depricated SO2-COBRA processing.
  • Added delays to limit the number of errors from STAC inquiry.
  • Updated python dependencies.

Merge request reports

config/Copernicus/cso-s5p-chocho.rc

+4 −34
Changes for config/Copernicus/cso-s5p-chocho.rc: 4 added lines, 34 removed lines.
Original line number Diff line number Diff line
@@ -22,32 +22,6 @@ 
!======================================================================





!-----------------------------------------------------------

! inquire downloaded GlyRetro data

!-----------------------------------------------------------



! Obtain names of all available S5p files.

! Stored as csv with processing type, processor version, filenames, etc.



! renew table if file already exists?

cso.s5p.chocho.inquire-table-glyretro.renew                  :  False



! time range:

cso.s5p.chocho.inquire-table-glyretro.timerange.start        :  ${my.full-timerange.start}

cso.s5p.chocho.inquire-table-glyretro.timerange.end          :  ${my.full-timerange.end}



! output table, time templates are filled with date of today;

! base path is used for searching input files:

cso.s5p.chocho.inquire-table-glyretro.file                   :  ${my.work}/Copernicus-inquire/Copernicus_S5p_CHOCHO_glyretro__2022-09-22.csv



! filename filters relative to listing file that should be scanned for orbit files;

! names could include time templates ;

! if same orbit is found in multiple directories, the first found is used;

! remove existing table for safety to ensure that this is done correctly ...

cso.s5p.chocho.inquire-table-glyretro.patterns               :   %Y/%m/%d/S5P_RPRO_L2__CHOCHO_*.nc \

                                                                 %Y/%m/%d/S5P_OFFL_L2__CHOCHO_*.nc





!-----------------------------------------------------------------------

! inquire PAL portal

!-----------------------------------------------------------------------
@@ -84,8 +58,7 @@ cso.s5p.chocho.inquire-table-pal.output.file : ${my.work}/Copernicus 
cso.s5p.chocho.inquire-plot.renew                    :  True



! listing files, here from two inquires:

cso.s5p.chocho.inquire-plot.file                     :  ${cso.s5p.chocho.inquire-table-glyretro.file} ; \

                                                        ${cso.s5p.chocho.inquire-table-pal.output.file}

cso.s5p.chocho.inquire-plot.file                     :  ${cso.s5p.chocho.inquire-table-pal.output.file}

!~ specify dates ("yyyy-mm-dd") to use historic tables,

!  default is table of today:

cso.s5p.chocho.inquire-plot.filedate                 :  2022-09-22 ;
@@ -146,18 +119,15 @@ my.s5p.chocho.selection : Cxx 
!   (%{collection} == '03') and (%{processing} == 'OFFL')

!

#if "${my.s5p.chocho.selection}" == "Cxx"

! strange archive, need to combine collections;

! prefere PAL_ for later years, otherwise RPRO or OFFL:

cso.s5p.chocho.convert.selection                      :  (%{processing} == 'PAL_') ; \

                                                         (%{processing} == 'RPRO') ; \

                                                         (%{processing} == 'OFFL')

! only PAL archive:

cso.s5p.chocho.convert.selection                      :  (%{processing} == 'PAL_')

#else

#error unsupported my.s5p.chocho.selection "${my.s5p.chocho.selection}"

#endif



! target dir for downloads;

! here the local archive:

cso.s5p.chocho.convert.input.dir                      :  ${my.observations}/GlyRetro

cso.s5p.chocho.convert.input.dir                      :  ${my.work}/Copernicus/S5P/%{processing}/CHOCHO/%Y/%m



! remove downloaded files?

! here all data comes from local archive, set False anyway ...

config/Copernicus/cso-s5p-so2-cobra.rc

deleted100644 → 0
+0 −1021

File deleted.

Preview size limit exceeded, changes collapsed.

config/Copernicus/cso.rc

+2 −9
Changes for config/Copernicus/cso.rc: 2 added lines, 9 removed lines.
Original line number Diff line number Diff line
@@ -19,11 +19,10 @@ CSO_RCDIR : 
#include cso-user-settings.rc



! selected tracers:

!my.tracers                :  no2 so2 so2-cobra hcho co chocho

!my.tracers                :  no2 so2 hcho co chocho

!~ one by one ..

my.tracers                :  no2

!my.tracers                :  so2

!my.tracers                :  so2-cobra

!my.tracers                :  hcho

!my.tracers                :  co

!my.tracers                :  o3-col
@@ -83,10 +82,8 @@ cso.s5p.TRACER.inquire.class : utopya.UtopyaJobStep 
#if "TRACER" in ["no2","so2","hcho","co","o3-pr","o3-col"]

cso.s5p.TRACER.inquire.tasks                      :  table-dataspace plot

!cso.s5p.TRACER.inquire.tasks                      :  table-colhub-mirror table-colhub-missing

#elif "TRACER" in ["so2-cobra"]

cso.s5p.TRACER.inquire.tasks                      :  table-pal plot

#elif "TRACER" in ["chocho"]

cso.s5p.TRACER.inquire.tasks                      :  table-glyretro table-pal plot

cso.s5p.TRACER.inquire.tasks                      :  table-pal plot

#else

#error unsupported tracer "TRACER"

#endif
@@ -106,10 +103,6 @@ cso.s5p.TRACER.inquire.table-colhub-missing.args : '${my.work}/rc/cso-s5p-TRAC 
cso.s5p.TRACER.inquire.table-pal.class            :  cso.CSO_PAL_Inquire

cso.s5p.TRACER.inquire.table-pal.args             :  '${my.work}/rc/cso-s5p-TRACER.rc', \

                                                      rcbase='cso.s5p.TRACER.inquire-table-pal'

!~ inquire files downloaded from GlyRetro:

cso.s5p.TRACER.inquire.table-glyretro.class       :  cso.CSO_S5p_Download_Listing

cso.s5p.TRACER.inquire.table-glyretro.args        :  '${my.work}/rc/cso-s5p-TRACER.rc', \

                                                      rcbase='cso.s5p.TRACER.inquire-table-glyretro'

!~ create plot of available versions:

cso.s5p.TRACER.inquire.plot.class                 :  cso.CSO_Inquire_Plot

cso.s5p.TRACER.inquire.plot.args                  :  '${my.work}/rc/cso-s5p-TRACER.rc', \

config/tutorial/tutorial.rc

+1 −1
Changes for config/tutorial/tutorial.rc: 1 added line, 1 removed line.
Original line number Diff line number Diff line
@@ -299,7 +299,7 @@ VIRTUAL_ENV : 
! Stored as csv with processing type, processor version, filenames, etc.



! full time range:

cso.tutorial.inquire-table-dataspace.timerange.start        :  2018-01-01 00:00:00

cso.tutorial.inquire-table-dataspace.timerange.start        :  2018-04-30 00:00:00

cso.tutorial.inquire-table-dataspace.timerange.end          :  2025-01-01 00:00:00



! API url:

doc/source/index.rst

+0 −1
Changes for doc/source/index.rst: 0 added lines, 1 removed line.
Original line number Diff line number Diff line
@@ -32,7 +32,6 @@ Contents 
   tutorial

   s5p-no2

   s5p-so2

   s5p-so2-cobra

   s5p-hcho

   s5p-co

   s5p-chocho