TNO Intern

Commit effd7ff7 authored by Arjo Segers's avatar Arjo Segers
Browse files

Added templates for ColHub archive to settings.

parent 581c5146

config/Copernicus/cso-s5p-no2.rc

+41 −0
Original line number Diff line number Diff line
@@ -58,6 +58,47 @@ cso.s5p.no2.inquire-table-dataspace.download_url : https://zipper.dat 
cso.s5p.no2.inquire-table-dataspace.output.file            :  ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_dataspace__%Y-%m-%d.csv





!-----------------------------------------------------------------------

! inquire colhub mirror archive

!-----------------------------------------------------------------------



! csv file that will hold records per file with:

! - timerange of pixels in file

! - orbit number

cso.s5p.no2.inquire-table-colhub-mirror.output.file         :  ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_colhub-listing__%Y-%m-%d.csv



! renew table if file already exists?

cso.s5p.no2.inquire-table-colhub-mirror.renew               :  True



! base path; example file:

!   <dir>/x0/x0/x1/S5P_OFFL_L2__NO2____20230701T002408_20230701T020537_29604_03_020500_20230702T161349.nc

cso.s5p.no2.inquire-table-colhub-mirror.dir                 :  /lustre/storeB/project/ESAcolhub/production-backend-AOI/S5p/all

! filename filter:

cso.s5p.no2.inquire-table-colhub-mirror.pattern             :  S5P_*_L2__NO2_*.nc





! ** create list of missing files



! renew table if file already exists?

cso.s5p.no2.inquire-table-colhub-missing.renew               :  True



! csv file that will hold records per file with:

! - timerange of pixels in file

! - orbit number

cso.s5p.no2.inquire-table-colhub-missing.output.file         :  ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_colhub-missing__%Y-%m-%d.csv



! listing from DataSpace with all potentially available files:

cso.s5p.no2.inquire-table-colhub-missing.all.file            :  ${cso.s5p.no2.inquire-table-dataspace.output.file}

!cso.s5p.no2.inquire-table-colhub-missing.all.filedate        :  2025-01-23



! listing from current archive:

cso.s5p.no2.inquire-table-colhub-missing.curr.file           :  ${cso.s5p.no2.inquire-table-colhub-mirror.output.file}

!cso.s5p.no2.inquire-table-colhub-missing.curr.filedate       :  2025-01-24



! selection of orbits, see "convert" below:

cso.s5p.no2.inquire-table-colhub-missing.selection           :  ${cso.s5p.no2.convert.selection}





!!-----------------------------------------------------------------------

!! inquire PAL portal

!!-----------------------------------------------------------------------

config/Copernicus/cso-user-settings.rc

+15 −1
Original line number Diff line number Diff line
@@ -34,6 +34,7 @@ my.cso.convention : CF-1.7 
! region name:

!my.region                       :  globe

my.region                       :  CAMS

!my.region                       :  xEMEP



! switch:

!....................................
@@ -62,6 +63,19 @@ my.region.north : 76.0 
! size of map figures for this region, default size is (8,6)

my.region.figsize               :   (8,6)



!....................................

#elif "${my.region}" == "xEMEP"

!....................................



! outer bound of all known domains:

my.region.west                  :  -50.0

my.region.east                  :   90.0

my.region.south                 :   25.0

my.region.north                 :   85.0



! size of map figures for this region, default size is (8,6)

my.region.figsize               :   (8,5)



!....................................

#else

#error unsupported my.region "${my.region}"
@@ -77,7 +91,7 @@ my.region.figsize : (8,6) 


! full range, used for inquiry jobs:

my.full-timerange.start         :  2018-01-01 00:00

my.full-timerange.end           :  2023-12-31 23:59

my.full-timerange.end           :  2024-12-31 23:59



! processing for selected period:

my.timerange.start              :  2018-06-01 00:00

config/Copernicus/cso.rc

+9 −0
Original line number Diff line number Diff line
@@ -82,6 +82,7 @@ cso.s5p.TRACER.inquire.class : utopya.UtopyaJobStep 
! two or more tasks:

#if "TRACER" in ["no2","so2","hcho","co","o3-pr","o3-col"]

cso.s5p.TRACER.inquire.tasks                      :  table-dataspace plot

!cso.s5p.TRACER.inquire.tasks                      :  table-colhub-mirror table-colhub-missing

#elif "TRACER" in ["so2-cobra"]

cso.s5p.TRACER.inquire.tasks                      :  table-pal plot

#elif "TRACER" in ["chocho"]
@@ -93,6 +94,14 @@ cso.s5p.TRACER.inquire.tasks : table-glyretro table-pal pl 
cso.s5p.TRACER.inquire.table-dataspace.class      :  cso.CSO_DataSpace_Inquire

cso.s5p.TRACER.inquire.table-dataspace.args       :  '${my.work}/rc/cso-s5p-TRACER.rc', \

                                                     rcbase='cso.s5p.TRACER.inquire-table-dataspace'

!~ inquire file archive:

cso.s5p.TRACER.inquire.table-colhub-mirror.class  :  cso.CSO_ColHubMirror_Inquire

cso.s5p.TRACER.inquire.table-colhub-mirror.args   :  '${my.work}/rc/cso-s5p-TRACER.rc', \

                                                     rcbase='cso.s5p.TRACER.inquire-table-colhub-mirror'

!~ create table with files that are missing:

cso.s5p.TRACER.inquire.table-colhub-missing.class :  cso.CSO_ColHubMirror_Missing

cso.s5p.TRACER.inquire.table-colhub-missing.args  :  '${my.work}/rc/cso-s5p-TRACER.rc', \

                                                     rcbase='cso.s5p.TRACER.inquire-table-colhub-missing'

!~ inquire files available on PAL:

cso.s5p.TRACER.inquire.table-pal.class            :  cso.CSO_PAL_Inquire

cso.s5p.TRACER.inquire.table-pal.args             :  '${my.work}/rc/cso-s5p-TRACER.rc', \

config/_test-colhub/cso-s5p-no2.rc

deleted100644 → 0
+0 −1232

File deleted.

Preview size limit exceeded, changes collapsed.

config/_test-colhub/cso-user-settings.rc

deleted100644 → 0
+0 −169
Original line number Diff line number Diff line 
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

!!!

!!! CSO common configuration

!!!

!!! Base settings that are used by multiple tasks:

!!! - time range(s)

!!! - target domain(s)

!!!

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!





!-----------------------------------------------------------

! id's

!-----------------------------------------------------------



! file format:

my.cso.format         :   1.0



! file format convention:

my.cso.convention     :   CF-1.7





!-----------------------------------------------------------

! domain

!-----------------------------------------------------------



!

! Used for:

! - orbit selection durning download

! - pixel selection

! - map plots

!



! region name:

!my.region                       :  globe

!my.region                       :  CAMS

my.region                       :  xEMEP



! switch:

!....................................

#if "${my.region}" == "globe"

!....................................



! global domain:

my.region.west                  : -180.0

my.region.east                  :  180.0

my.region.south                 :  -90.0

my.region.north                 :   90.0



! size of map figures for this region, default size is (8,6)

my.region.figsize               :   (8,6)



!....................................

#elif "${my.region}" == "CAMS"

!....................................



! CAMS regional ensemble domain:

my.region.west                  :  -30.0

my.region.east                  :   45.0

my.region.south                 :   30.0

my.region.north                 :   76.0



! size of map figures for this region, default size is (8,6)

my.region.figsize               :   (8,6)



!....................................

#elif "${my.region}" == "xEMEP"

!....................................



! outer bound of all known domains:

my.region.west                  :  -50.0

my.region.east                  :   90.0

my.region.south                 :   25.0

my.region.north                 :   85.0



! size of map figures for this region, default size is (8,6)

my.region.figsize               :   (8,5)



!....................................

#else

#error unsupported my.region "${my.region}"

#endif

!....................................









!----------------------------------------------------------

! timerange

!----------------------------------------------------------



!! full range, used for inquiry jobs:

!my.full-timerange.start         :  2018-01-01 00:00

!my.full-timerange.end           :  2024-12-31 23:59



! TESTING: 1 month ...

my.full-timerange.start         :  2023-01-01 00:00

my.full-timerange.end           :  2023-01-31 23:59



! processing for selected period:

my.timerange.start              :  2018-06-01 00:00

my.timerange.end                :  2018-06-01 23:59



!! SO2-COBRA

!my.timerange.start              :  2023-06-01 00:00

!my.timerange.end                :  2023-06-01 23:59





!----------------------------------------------------------

! user specific settings:

!----------------------------------------------------------



!..............................

#if "${USER}" == "you"

!..............................



! Attributes written to output files.

my.attr.author                  :  Your Name

my.attr.institution             :  CSO

my.attr.email                   :  Your.Name@cso.org



! base location for work directories:

my.work                         :  /Scratch/${USER}/CSO-Copernicus



! storage for downloaded observations:

my.observations                 :  /Scratch/${USER}/observations



!..............................

#elif "${USER}" == "arjos"

!..............................



! Attributes written to output files.

my.attr.author                  :  Arjo Segers

my.attr.institution             :  MET Norway

my.attr.email                   :  arjos@met.no



! base location for work directories:

my.work                         :  ${WORK}/projects/SESAM/test-colhub



! storage for downloaded observations:

my.observations                 :  ${WORK}/../observations



!..............................

#elif "${USER}" == "segersaj"

!..............................



! Attributes written to output files.

my.attr.author                  :  Arjo Segers

my.attr.institution             :  TNO

my.attr.email                   :  Arjo.Segers@tno.nl



! base location for work directories:

my.work                         :  ${SCRATCH}/CSO-Copernicus



! storage for downloaded observations:

my.observations                 :  ${SCRATCH}/observations



!..............................

#else

#error unsupported USER "${USER}"

#endif

!..............................





!----------------------------------------------------------

! end

!----------------------------------------------------------