!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!!!
!!! CSO - CAMS Satellite Operator
!!!
!!! Settings for Tutorial.
!!!
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!


!----------------------------------------------------------
! job tree
!----------------------------------------------------------

! class to create a job tree:
cso.class                                     :  utopya.UtopyaJobTree
! list of sub-elements:
cso.elements                                  :  tutorial

! class to create a job tree:
cso.tutorial.class                             :  utopya.UtopyaJobTree

! list of sub-elements:
!cso.tutorial.elements                          :  inquire-scihub \
!                                                  convert listing catalogue \
!                                                  sim-catalogue \
!                                                  gridded gridded-catalogue
!~ preprocessor steps one by one ...
!cso.tutorial.elements                          :  inquire-scihub
!cso.tutorial.elements                          :  convert
!cso.tutorial.elements                          :  listing
!cso.tutorial.elements                          :  catalogue
! ... after running the obs.oper:
!cso.tutorial.elements                          :  sim-catalogue
!cso.tutorial.elements                          :  gridded
cso.tutorial.elements                          :  gridded-catalogue

! *

! single step:
cso.tutorial.inquire-scihub.class              :  utopya.UtopyaJobStep
! two tasks:
cso.tutorial.inquire-scihub.tasks              :  table plot
!~ inquire available files:
cso.tutorial.inquire-scihub.table.class        :  cso.CSO_SciHub_Inquire
cso.tutorial.inquire-scihub.table.args         :  '${__filename__}', \
                                                      rcbase='cso.tutorial.inquire-scihub-table'
!~ create plot of available versions:
cso.tutorial.inquire-scihub.plot.class         :  cso.CSO_Inquire_Plot
cso.tutorial.inquire-scihub.plot.args          :  '${__filename__}', \
                                                      rcbase='cso.tutorial.inquire-scihub-plot'

! *

! single step:
cso.tutorial.convert.class                     :  utopya.UtopyaJobStep
! task:
cso.tutorial.convert.task.class                :  cso.CSO_S5p_Convert
cso.tutorial.convert.task.args                 :  '${__filename__}', \
                                                   rcbase='cso.tutorial.convert'

! *

! single step:
cso.tutorial.listing.class                     :  utopya.UtopyaJobStep
! task:
cso.tutorial.listing.task.class                :  cso.CSO_S5p_Listing
cso.tutorial.listing.task.args                 :  '${__filename__}', \
                                                   rcbase='cso.tutorial.listing'

! *

! single step:
cso.tutorial.catalogue.class                   :  utopya.UtopyaJobStep
! two tasks:
cso.tutorial.catalogue.tasks                   :  figs index
! catalogue creation task:
cso.tutorial.catalogue.figs.class              :  cso.CSO_Catalogue
cso.tutorial.catalogue.figs.args               :  '${__filename__}', \
                                                   rcbase='cso.tutorial.catalogue'
! indexer task:
cso.tutorial.catalogue.index.class             :  utopya.Indexer
cso.tutorial.catalogue.index.args              :  '${__filename__}', \
                                                    rcbase='cso.tutorial.catalogue-index'

! *

! single step:
cso.tutorial.sim-catalogue.class               :  utopya.UtopyaJobStep
! two tasks:
cso.tutorial.sim-catalogue.tasks               :  figs index
! catalogue creation task:
cso.tutorial.sim-catalogue.figs.class          :  cso.CSO_SimCatalogue
cso.tutorial.sim-catalogue.figs.args           :  '${__filename__}', \
                                                   rcbase='cso.tutorial.sim-catalogue'
! indexer task:
cso.tutorial.sim-catalogue.index.class         :  utopya.Indexer
cso.tutorial.sim-catalogue.index.args          :  '${__filename__}', \
                                                   rcbase='cso.tutorial.sim-catalogue-index'

!
! * gridded fields
!

! single step:
cso.tutorial.gridded.class                     :  utopya.UtopyaJobStep
! catalogue creation task:
cso.tutorial.gridded.task.class                :  cso.CSO_GriddedAverage
cso.tutorial.gridded.task.args                 :  '${__filename__}', \
                                                   rcbase='cso.tutorial.gridded'

! *

! single step:
cso.tutorial.gridded-catalogue.class           :  utopya.UtopyaJobStep
! two tasks:
cso.tutorial.gridded-catalogue.tasks           :  figs index
! catalogue creation task:
cso.tutorial.gridded-catalogue.figs.class      :  cso.CSO_GriddedCatalogue
cso.tutorial.gridded-catalogue.figs.args       :  '${__filename__}', \
                                                   rcbase='cso.tutorial.gridded-catalogue'
! indexer task:
cso.tutorial.gridded-catalogue.index.class     :  utopya.Indexer
cso.tutorial.gridded-catalogue.index.args      :  '${__filename__}', \
                                                   rcbase='cso.tutorial.gridded-catalogue-index'



!======================================================================
!===
!=== settings
!===
!======================================================================


!-----------------------------------------------------------
! id's
!-----------------------------------------------------------

! file format:
my.cso.format         :   1.0

! file format convention:
my.cso.convention     :   CF-1.7


!-----------------------------------------------------------
! domain
!-----------------------------------------------------------

!
! Used for:
! - orbit selection durning download
! - pixel selection
! - map plots
!

! region name:
my.region                       :  CAMS

! CAMS regional ensemble domain:
my.region.west                  :  -30.0
my.region.east                  :   45.0
my.region.south                 :   30.0
my.region.north                 :   76.0

! size of map figures for this region,
! default size is (8,6)
my.region.figsize               :   (8,6)


!----------------------------------------------------------
! timerange
!----------------------------------------------------------

! testing 3 days
my.timerange.start              :  2018-06-01 00:00
my.timerange.end                :  2018-06-03 23:59


!----------------------------------------------------------
! user specific settings:
!----------------------------------------------------------

! Attributes written to output files.
my.attr.author                  :  Your Name
my.attr.institution             :  CSO
my.attr.email                   :  Your.Name@cso.org

! base location for work directories:
my.work                         :  /Scratch/${USER}/CSO-tutorial


!----------------------------------------------------------
! job step defaults
!----------------------------------------------------------

! run jobs in foreground:
*.script.class                      :  utopya.UtopyaJobScriptForeground

! search path for python modules:
*.pypath                            :  ${CSO_PREFIX}/py
        
! work directory for jobs; 
*.workdir                           :  ${my.work}/__NAME2PATH__

! for new job files, use jobtree settings from this file:
*.rcfile                            :  ${__filename__}



!======================================================================
!===
!=== Inquire - S-5P Hub
!===
!======================================================================

! Obtain names of all available S5p files.
! Stored as csv with processing type, processor version, filenames, etc.

! time range:
cso.tutorial.inquire-scihub-table.timerange.start        :  ${my.timerange.start}
cso.tutorial.inquire-scihub-table.timerange.end          :  ${my.timerange.end}

!
! server url ; 
! provide login/password in ~/.netrc:
!
!   machine s5phub.copernicus.eu      login s5pguest        password s5pguest
!
cso.tutorial.inquire-scihub-table.url                    :  https://s5phub.copernicus.eu/dhus

! target area;
!!~ empty for no limitation:
!cso.tutorial.inquire-scihub-table.area                   :  
!~ domain specified as:  west,south:east,north
cso.tutorial.inquire-scihub-table.area                   :  ${my.region.west},${my.region.south}:${my.region.east},${my.region.north}

! search query, obtained from interactive download:
!
!   platformname     : Sentinel-5
!   producttype      : L2__NO2___ | L2__CO____    (always 10 characters!)
!   processinglevel  : L2
!
cso.tutorial.inquire-scihub-table.query                  :     platformname:Sentinel-5    AND \
                                                                producttype:L2__NO2___    AND \
                                                            processinglevel:L2

! output table, date of today:
cso.tutorial.inquire-scihub-table.output.file            :  ${my.work}/Copernicus/Copernicus_S5p_NO2_scihub_%Y-%m-%d.csv


! * create plot

! renew existing plots?
cso.tutorial.inquire-scihub-plot.renew                    :  True

! listing files:
cso.tutorial.inquire-scihub-plot.file                     :  ${cso.tutorial.inquire-scihub-table.output.file}
!!~ specify dates ("yyyy-mm-dd") to use historic tables,
!!  default is table of today:
!cso.tutorial.inquire-scihub-plot.filedate                 :  2022-01-28

! output figure, date of today:
cso.tutorial.inquire-scihub-plot.output.file              :  ${my.work}/Copernicus/Copernicus_S5p_NO2_${my.region}__%Y-%m-%d.png


!======================================================================
!===
!=== convert (and download)
!===
!======================================================================


! renew existing files (True|False) ?
cso.tutorial.convert.renew                  :  True
!cso.tutorial.convert.renew                  :  False

! time range:
cso.tutorial.convert.timerange.start        :  ${my.timerange.start}
cso.tutorial.convert.timerange.end          :  ${my.timerange.end}


!~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
! input files
!~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

! listing of available source files,
! created by 'inquire-scihub' job:
cso.tutorial.convert.inquire.file                   :  ${my.work}/Copernicus/Copernicus_S5p_NO2_scihub_%Y-%m-%d.csv
!!~ historic inquire ...
!cso.tutorial.convert.inquire.filedate               :  2022-02-02

! selection keyword:
my.s5p.no2.selection                                :  C03

! Provide ';' seperated list of to decide if a particular orbit file should be processed.
! If more than one file is available for a particular orbit (from "OFFL" and "RPRO" processing),
! the file with the first match will be used.
! The expressions should include templates '%{header}' for the column values.
! Example to select files from collection '03', preferably from processing 'RPRO' but otherwise from 'OFFL':
!   (%{collection} == '03') and (%{processing} == 'RPRO') ; \
!   (%{collection} == '03') and (%{processing} == 'OFFL')
!
#if "${my.s5p.no2.selection}" == "C03"
cso.tutorial.convert.selection                     :  (%{collection} == '03') and (%{processing} == 'RPRO') ; \
                                                      (%{collection} == '03') and (%{processing} == 'OFFL')
#else
#error unsupported my.s5p.no2.selection "${my.s5p.no2.selection}"
#endif

! input directory;
! files are searched here or downloaded to if not present yet;
! supported templates:
!   %{processing}
cso.tutorial.convert.input.dir                      :  ${my.work}/Copernicus/S5P/%{processing}/NO2/%Y/%m

! remove downloaded input files after convert?
cso.tutorial.convert.downloads.cleanup              :  False

! selection names:
cso.tutorial.convert.filters                        :  lons lats valid quality

! filter settings:
cso.tutorial.convert.filter.lons.var                :  PRODUCT/longitude
cso.tutorial.convert.filter.lons.type               :  minmax
cso.tutorial.convert.filter.lons.minmax             :  ${my.region.west} ${my.region.east}
cso.tutorial.convert.filter.lons.units              :  degrees_east

! filter settings:
cso.tutorial.convert.filter.lats.var                :  PRODUCT/latitude
cso.tutorial.convert.filter.lats.type               :  minmax
cso.tutorial.convert.filter.lats.minmax             :  ${my.region.south} ${my.region.north}
cso.tutorial.convert.filter.lats.units              :  degrees_north

! skip pixel with "no data", use the "qa_value" variable to check:
cso.tutorial.convert.filter.valid.var               :  PRODUCT/qa_value
cso.tutorial.convert.filter.valid.type              :  valid

! Comment in "PRODUCT/qa_value" variable:
!   "A continuous quality descriptor, 
!   varying between 0 (no data) and 1 (full quality data). 
!   Recommend to ignore data with qa_value < 0.5" 
! Recomended is qa_value>=0.75 to remove clouded and other problematic pixels;
! here use 0.5 to include retrievals over clouds and snow/ice.
cso.tutorial.convert.filter.quality.var             :  PRODUCT/qa_value
cso.tutorial.convert.filter.quality.type            :  min
cso.tutorial.convert.filter.quality.min             :  0.5
cso.tutorial.convert.filter.quality.units           :  1



!~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
! output files
!~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

! output directory and filename:
! - times are taken from mid of selection, rounded to hours
! - use '%{processing}' for the processing name
! - use '%{orbit}' for orbit number
cso.tutorial.convert.output.filename            :  ${my.work}/CSO-data/S5p/%{processing}/NO2/${my.region}/%Y/%m/S5p_%{processing}_NO2_%{orbit}.nc

! pack variables on output:
cso.tutorial.convert.output.packed              :  True
! zlib compression level, 0 for no compression:
cso.tutorial.convert.output.complevel           :  1

! global attributes:
cso.tutorial.convert.output.attrs               :  format Conventions \
                                                     author institution email
!
cso.tutorial.convert.output.attr.format         :  ${my.cso.format}
cso.tutorial.convert.output.attr.Conventions    :  ${my.cso.convention}
cso.tutorial.convert.output.attr.author         :  ${my.attr.author}
cso.tutorial.convert.output.attr.institution    :  ${my.attr.institution}
cso.tutorial.convert.output.attr.email          :  ${my.attr.email}

! no need to swap layes:
cso.tutorial.convert.swap_layers                :  False


! ~ variables

! which fields to be put out ?
cso.tutorial.convert.output.vars    :  longitude longitude_bounds \
                                       latitude  latitude_bounds \
                                       track_longitude track_longitude_bounds \
                                       track_latitude  track_latitude_bounds \
                                       time \
                                       qa_value \
                                       pressure kernel_trop amf amf_trop nla \
                                       vcd vcd_errvar \
                                       cloud_fraction cloud_radiance_fraction

! 
! Describe per variable:
! * .dims :  dimensions list:
!     pixel       : selected pixels
!     corner      : number of footprint bounds (probably 4)
!     layer       : number of layers in atmospheric profile (layers in kernel)
!     layeri      : number of layer interfaces in atmospheric profile (layer+1)
!     retr        : number of layers in retrieval product (1 for columns) ;
!                   for error covariance use (retr,retr0) to avoid repeated dimensions
!     track_scan  : original scan index   in 2D track
!     track_pixel : original ground pixel in 2D track
! * .specal  : keyword to select special processing
!     * None                     : no special processing (default)
!     * track_longitude          : longiudes at centers of original 2D track
!     * track_latitude           : latiudes  at centers of original 2D track
!     * track_longitude_bounds   : longiude bounds at centers of original 2D track
!     * track_latitude_bounds    : latiude  bounds at centers of original 2D track
! * .units   : target units if different from original
! * .oper    : special postprocessing, currently supported:
!     * square      :  fill variable with squared valued (used to form variance from standard deviation)
! In case no special processing is needed:
! * .from : original variable (group path and variable name)
!

! center longitudes; remove bounds attribute, no coordinate ...
cso.tutorial.convert.output.var.longitude.dims                   :   pixel
cso.tutorial.convert.output.var.longitude.from                   :   PRODUCT/longitude
cso.tutorial.convert.output.var.longitude.attrs                  :   { 'bounds' : None }
! center latitudes; remove bounds attribute, no coordinate ...
cso.tutorial.convert.output.var.latitude.dims                    :   pixel
cso.tutorial.convert.output.var.latitude.from                    :   PRODUCT/latitude
cso.tutorial.convert.output.var.latitude.attrs                   :   { 'bounds' : None }

! corner longitudes; no units in file:
cso.tutorial.convert.output.var.longitude_bounds.dims            :   pixel corner
cso.tutorial.convert.output.var.longitude_bounds.from            :   PRODUCT/SUPPORT_DATA/GEOLOCATIONS/longitude_bounds
cso.tutorial.convert.output.var.longitude_bounds.units           :   degrees_east
! corner latitudes, no units in file:
cso.tutorial.convert.output.var.latitude_bounds.dims             :   pixel corner
cso.tutorial.convert.output.var.latitude_bounds.from             :   PRODUCT/SUPPORT_DATA/GEOLOCATIONS/latitude_bounds
cso.tutorial.convert.output.var.latitude_bounds.units            :   degrees_north

! original track:
!~ center lon; remove bounds attribute, no coordinate ...
cso.tutorial.convert.output.var.track_longitude.dims             :   track_scan track_pixel
cso.tutorial.convert.output.var.track_longitude.special          :   track_longitude
cso.tutorial.convert.output.var.track_longitude.from             :   PRODUCT/longitude
cso.tutorial.convert.output.var.track_longitude.attrs            :   { 'bounds' : None }
!~ center lat; remove bounds attribute, no coordinate ...
cso.tutorial.convert.output.var.track_latitude.dims              :   track_scan track_pixel
cso.tutorial.convert.output.var.track_latitude.special           :   track_latitude
cso.tutorial.convert.output.var.track_latitude.from              :   PRODUCT/latitude
cso.tutorial.convert.output.var.track_latitude.attrs             :   { 'bounds' : None }
!~ corner lons
cso.tutorial.convert.output.var.track_longitude_bounds.dims      :   track_scan track_pixel corner
cso.tutorial.convert.output.var.track_longitude_bounds.special   :   track_longitude_bounds
cso.tutorial.convert.output.var.track_longitude_bounds.from      :   PRODUCT/SUPPORT_DATA/GEOLOCATIONS/longitude_bounds
cso.tutorial.convert.output.var.track_longitude_bounds.units     :   degrees_east
!~ corner lats
cso.tutorial.convert.output.var.track_latitude_bounds.dims       :   track_scan track_pixel corner
cso.tutorial.convert.output.var.track_latitude_bounds.special    :   track_latitude_bounds
cso.tutorial.convert.output.var.track_latitude_bounds.from       :   PRODUCT/SUPPORT_DATA/GEOLOCATIONS/latitude_bounds
cso.tutorial.convert.output.var.track_latitude_bounds.units      :   degrees_north

! time value per scan line
cso.tutorial.convert.output.var.time.dims                        :   pixel
cso.tutorial.convert.output.var.time.special                     :   time-delta
cso.tutorial.convert.output.var.time.tref                        :   PRODUCT/time
cso.tutorial.convert.output.var.time.dt                          :   PRODUCT/delta_time

! vertical column density:
cso.tutorial.convert.output.var.vcd.dims                         :   pixel retr
cso.tutorial.convert.output.var.vcd.from                         :   PRODUCT/nitrogendioxide_tropospheric_column
cso.tutorial.convert.output.var.vcd.attrs                        :   { 'ancillary_variables' : None }

! error variance in vertical column density (after application of kernel),
! fill with single element 'covariance matrix', from square of standard error:
! use dims with different names to avoid that cf-checker complains:
cso.tutorial.convert.output.var.vcd_errvar.dims                  :   pixel retr retr0
cso.tutorial.convert.output.var.vcd_errvar.special               :   square
cso.tutorial.convert.output.var.vcd_errvar.from                  :   PRODUCT/nitrogendioxide_tropospheric_column_precision_kernel
!~ skip standard name, modifier "standard_error" is not valid anymore:
cso.tutorial.convert.output.var.vcd_errvar.attrs                 :   { 'standard_name' : None }

! Convert from hybride coefficient bounds in (2,nlev) aray to 3D half level pressure:
cso.tutorial.convert.output.var.pressure.dims                    :   pixel layeri
cso.tutorial.convert.output.var.pressure.special                 :   hybounds_to_pressure
cso.tutorial.convert.output.var.pressure.sp                      :   PRODUCT/SUPPORT_DATA/INPUT_DATA/surface_pressure
cso.tutorial.convert.output.var.pressure.hyab                    :   PRODUCT/tm5_constant_a
cso.tutorial.convert.output.var.pressure.hybb                    :   PRODUCT/tm5_constant_b
cso.tutorial.convert.output.var.pressure.units                   :   Pa

! (total) airmass factor:
cso.tutorial.convert.output.var.amf.dims                         :   pixel retr
cso.tutorial.convert.output.var.amf.from                         :   PRODUCT/air_mass_factor_total
cso.tutorial.convert.output.var.amf.attrs                        :   { 'coordinates' : None, 'ancillary_variables' : None }

! tropospheric airmass factor:
cso.tutorial.convert.output.var.amf_trop.dims                    :   pixel retr
cso.tutorial.convert.output.var.amf_trop.from                    :   PRODUCT/air_mass_factor_troposphere
cso.tutorial.convert.output.var.amf_trop.attrs                   :   { 'coordinates' : None, 'ancillary_variables' : None }

! number of apriori layers in retrieval layer,
! enforce that it is stored as a short integer:
cso.tutorial.convert.output.var.nla.dims                         :   pixel retr
cso.tutorial.convert.output.var.nla.dtype                        :   i2
cso.tutorial.convert.output.var.nla.from                         :   PRODUCT/tm5_tropopause_layer_index
cso.tutorial.convert.output.var.nla.attrs                        :   { 'coordinates' : None, 'ancillary_variables' : None }

! description:
!   kernel := averaging_kernel * amf/amf_trop
! for layers up to l_trop, zero above
cso.tutorial.convert.output.var.kernel_trop.dims                 :   pixel layer retr
cso.tutorial.convert.output.var.kernel_trop.special              :   kernel_trop
cso.tutorial.convert.output.var.kernel_trop.avk                  :   PRODUCT/averaging_kernel
cso.tutorial.convert.output.var.kernel_trop.amf                  :   PRODUCT/air_mass_factor_total
cso.tutorial.convert.output.var.kernel_trop.amft                 :   PRODUCT/air_mass_factor_troposphere
cso.tutorial.convert.output.var.kernel_trop.troplayer            :   PRODUCT/tm5_tropopause_layer_index
cso.tutorial.convert.output.var.kernel_trop.attrs                :   { 'coordinates' : None, 'ancillary_variables' : None }

! cloud property:
cso.tutorial.convert.output.var.cloud_fraction.dims             :   pixel
cso.tutorial.convert.output.var.cloud_fraction.from             :   PRODUCT/SUPPORT_DATA/INPUT_DATA/cloud_fraction_crb
cso.tutorial.convert.output.var.cloud_fraction.attrs            :   { 'coordinates' : None, 'source' : None }

! cloud property:
cso.tutorial.convert.output.var.cloud_radiance_fraction.dims    :   pixel
cso.tutorial.convert.output.var.cloud_radiance_fraction.from    :   PRODUCT/SUPPORT_DATA/DETAILED_RESULTS/cloud_radiance_fraction_nitrogendioxide_window
cso.tutorial.convert.output.var.cloud_radiance_fraction.attrs   :   { 'coordinates' : None, 'ancillary_variables' : None }

! quality flag:
cso.tutorial.convert.output.var.qa_value.dims                   :   pixel
cso.tutorial.convert.output.var.qa_value.from                   :   PRODUCT/qa_value
!~ skip some attributes, cf-checker complains ...
cso.tutorial.convert.output.var.qa_value.attrs                  :   { 'valid_min' : None, 'valid_max' : None }


!======================================================================
!===
!=== listing
!===
!======================================================================

! csv file that will hold records per file with:
! - timerange of pixels in file
! - orbit number
cso.tutorial.listing.file                :  ${my.work}/CSO-data/S5p/listing-NO2-${my.region}.csv

! renew table if file already exists?
cso.tutorial.listing.renew               :  True

! time range:
cso.tutorial.listing.timerange.start     :  ${my.timerange.start}
cso.tutorial.listing.timerange.end       :  ${my.timerange.end}

! filename filters relative to listing file that should be scanned for orbit files;
! names could include time templates ;
! if same orbit is found in multiple directories, the first found is used;
! remove existing table for safety to ensure that this is done correctly ...
cso.tutorial.listing.patterns            :  RPRO/NO2/${my.region}/%Y/%m/S5p_*.nc \
                                            OFFL/NO2/${my.region}/%Y/%m/S5p_*.nc

! extra columns to be added, read from global attributes:
cso.tutorial.listing.xcolumns            :  orbit


!======================================================================
!===
!=== catalogue
!===
!======================================================================

! listing file with filenames/timerange.
cso.tutorial.catalogue.input.listing               :  ${cso.tutorial.listing.file}

! time range:
cso.tutorial.catalogue.timerange.start             :  ${my.timerange.start}
cso.tutorial.catalogue.timerange.end               :  ${my.timerange.end}

! target filenames; templates:
! - time values
! - %{orbit}   : from listing
! - %{varname} for variable
cso.tutorial.catalogue.output.file                 :  ${my.work}/CSO-data-catalogue/S5p/NO2/${my.region}/%Y/%m/%d/S5p_NO2_%{orbit}__%{varname}.png

! map domain (west east south north)
cso.tutorial.catalogue.domain                      :  ${my.region.west} ${my.region.east} ${my.region.south} ${my.region.north}

! figure size (inches), default is (8,6):
cso.tutorial.catalogue.figsize                     :  ${my.region.figsize}

! renew existing files?
cso.tutorial.catalogue.renew                       :  False
!cso.tutorial.catalogue.renew                       :  True

! variables to be plotted:
cso.tutorial.catalogue.vars                        :  vcd vcd_errvar qa_value \
                                                      amf amf_trop nla \
                                                      cloud_fraction cloud_radiance_fraction

!! color for no-data values in track, default '0.8' (gray):
!cso.tutorial.catalogue.color_nan                   :  white

!! extra keyword arguments for map:
!cso.tutorial.catalogue.map                         :  resolution='h'

! convert units:
cso.tutorial.catalogue.var.vcd.units                         :  umol/m2
! style:
cso.tutorial.catalogue.var.vcd.vmin                          :   0.0
cso.tutorial.catalogue.var.vcd.vmax                          :  100.0

! show error as std.dev, convert to vcd units:
cso.tutorial.catalogue.var.vcd_errvar.units                  :  umol/m2
! style:
cso.tutorial.catalogue.var.vcd_errvar.vmax                   :  100.0

! style:
cso.tutorial.catalogue.var.amf.vmin                          :   0.0
cso.tutorial.catalogue.var.amf.vmax                          :   4.0

! style:
cso.tutorial.catalogue.var.amf_trop.vmin                     :   0.0
cso.tutorial.catalogue.var.amf_trop.vmax                     :   4.0

! style:
cso.tutorial.catalogue.var.nla.vmin                          :   1
cso.tutorial.catalogue.var.nla.vmax                          :   34

! style:
cso.tutorial.catalogue.var.qa_value.vmin                     :   0.5
cso.tutorial.catalogue.var.qa_value.vmax                     :   1.0
cso.tutorial.catalogue.var.qa_value.colors                   :   ['red','yellow','green']

! style:
cso.tutorial.catalogue.var.cloud_fraction.vmax               :   1.0
cso.tutorial.catalogue.var.cloud_fraction.colors             :   ['blue','cyan','white']

! style:
cso.tutorial.catalogue.var.cloud_radiance_fraction.vmax      :   1.0
cso.tutorial.catalogue.var.cloud_radiance_fraction.colors    :   ['blue','cyan','white']


!-----------------------------------------------------------
! catalogue index
!-----------------------------------------------------------

! target location:
cso.tutorial.catalogue-index.outdir                    :  ${my.work}/CSO-data-catalogue/S5p/NO2/${my.region}

! title:
cso.tutorial.catalogue-index.header                    :  CSO catalogue
! show info on created page?
cso.tutorial.catalogue-index.info                      :  True

! create new page for each value?
cso.tutorial.catalogue-index.newpage                   :  True

! content type:
cso.tutorial.catalogue-index.type                      :  list
! define row values:
cso.tutorial.catalogue-index.name                      :  date
cso.tutorial.catalogue-index.values                    :  TimeSeries( ${my.timerange.start}, ${my.timerange.end}, '1 day', '%Y%m%d' )

! create new page for each value:
cso.tutorial.catalogue-index.date.newpage                   :  True
! content type:
cso.tutorial.catalogue-index.date.type                      :  table-row
! define row values:
cso.tutorial.catalogue-index.date.name                      :  orbit
cso.tutorial.catalogue-index.date.values                    :  CsvFile( '%{date[0:4]}/%{date[4:6]}/%{date[6:8]}/orbits.csv' )

! content type:
cso.tutorial.catalogue-index.date.orbit.type                :  table-col
! define column values:
cso.tutorial.catalogue-index.date.orbit.name                :  var
cso.tutorial.catalogue-index.date.orbit.values              :  ${cso.tutorial.catalogue.vars}

! content type:
cso.tutorial.catalogue-index.date.orbit.var.type            :  img
! define image:
cso.tutorial.catalogue-index.date.orbit.var.img             :  %{date[0:4]}/%{date[4:6]}/%{date[6:8]}/S5p_NO2_%{orbit}__%{var}.png
cso.tutorial.catalogue-index.date.orbit.var.kwargs          :  height=300



!======================================================================
!===
!=== simulation catalogue
!===
!======================================================================


!-----------------------------------------------------------
! simulation catalogue
!-----------------------------------------------------------

! time range:
cso.tutorial.sim-catalogue.timerange.start        :  ${my.timerange.start}
cso.tutorial.sim-catalogue.timerange.end          :  ${my.timerange.end}
cso.tutorial.sim-catalogue.timerange.step         :  hour

! input files:
cso.tutorial.sim-catalogue.input.data.file        :  ${my.work}/CSO-oper/CSO_output_%Y%m%d_%H%M_data.nc
cso.tutorial.sim-catalogue.input.state.file       :  ${my.work}/CSO-oper/CSO_output_%Y%m%d_%H%M_state.nc

! target files, time tempates and variable name are replaced:
cso.tutorial.sim-catalogue.output.file            :  ${my.work}/CSO-data-sim-catalogue/S5p/NO2/${my.region}/%Y/%m/%d/S5p_NO2_%Y%m%d_%H%M_%{var}.png

! map domain used for simulations (west east south north):
cso.tutorial.sim-catalogue.domain                 :  -10 30 35 65

! figure size (inches), default is (8,6):
cso.tutorial.sim-catalogue.figsize                :  (6,6)

! renew existing files?
!cso.tutorial.sim-catalogue.renew                  :  False
cso.tutorial.sim-catalogue.renew                  :  True

! variables to be plotted:
cso.tutorial.sim-catalogue.vars                   :  yr ys yr_m ys_m tcc

! variable:
cso.tutorial.sim-catalogue.var.yr.source          :  data:yr
! convert units:
cso.tutorial.sim-catalogue.var.yr.units           :  1e15 mlc/cm2
! style:
cso.tutorial.sim-catalogue.var.yr.vmin            :   0.0
cso.tutorial.sim-catalogue.var.yr.vmax            :  10.0

! variable:
cso.tutorial.sim-catalogue.var.ys.source          :  state:ys
! convert units:
cso.tutorial.sim-catalogue.var.ys.units           :  1e15 mlc/cm2
! style:
cso.tutorial.sim-catalogue.var.ys.vmin            :   0.0
cso.tutorial.sim-catalogue.var.ys.vmax            :  10.0

! variable:
cso.tutorial.sim-catalogue.var.yr_m.source        :  state:yr_m
! convert units:
cso.tutorial.sim-catalogue.var.yr_m.units         :  1e15 mlc/cm2
! style:
cso.tutorial.sim-catalogue.var.yr_m.vmin          :   0.0
cso.tutorial.sim-catalogue.var.yr_m.vmax          :  10.0

! variable:
cso.tutorial.sim-catalogue.var.ys_m.source        :  state:ys_m
! convert units:
cso.tutorial.sim-catalogue.var.ys_m.units         :  1e15 mlc/cm2
! style:
cso.tutorial.sim-catalogue.var.ys_m.vmin          :   0.0
cso.tutorial.sim-catalogue.var.ys_m.vmax          :  10.0

! variable:
cso.tutorial.sim-catalogue.var.tcc.source         :  state:mod_tcc
! convert units:
cso.tutorial.sim-catalogue.var.tcc.units          :  1
! style:
cso.tutorial.sim-catalogue.var.tcc.vmin           :   0.0
cso.tutorial.sim-catalogue.var.tcc.vmax           :   1.0



!-----------------------------------------------------------
! sim catalogue index
!-----------------------------------------------------------

! target location:
cso.tutorial.sim-catalogue-index.outdir                    :  ${my.work}/CSO-data-sim-catalogue/S5p/NO2/CAMS

! title:
cso.tutorial.sim-catalogue-index.header                    :  CSO catalogue
! show info on created page?
cso.tutorial.sim-catalogue-index.info                      :  True

! create new page for each value?
cso.tutorial.sim-catalogue-index.newpage                   :  True

! content type:
cso.tutorial.sim-catalogue-index.type                      :  list
! define row values:
cso.tutorial.sim-catalogue-index.name                      :  date
cso.tutorial.sim-catalogue-index.values                    :  TimeSeries( ${my.timerange.start}, ${my.timerange.end}, '1 day', '%Y%m%d' )

! create new page for each value:
cso.tutorial.sim-catalogue-index.date.newpage              :  True
! content type:
cso.tutorial.sim-catalogue-index.date.type                 :  table-row
! define row values:
cso.tutorial.sim-catalogue-index.date.name                 :  time
cso.tutorial.sim-catalogue-index.date.values               :  Range( 0, 23, 1, '%2.2i00' )

! content type:
cso.tutorial.sim-catalogue-index.date.time.type            :  table-col
! define column values:
cso.tutorial.sim-catalogue-index.date.time.name            :  var
cso.tutorial.sim-catalogue-index.date.time.values          :  ${cso.tutorial.sim-catalogue.vars}

! content type:
cso.tutorial.sim-catalogue-index.date.time.var.type        :  img
! define image:
cso.tutorial.sim-catalogue-index.date.time.var.img         :  %{date[0:4]}/%{date[4:6]}/%{date[6:8]}/S5p_NO2_%{date}_%{time}_%{var}.png
cso.tutorial.sim-catalogue-index.date.time.var.kwargs      :  height=300



!======================================================================
!===
!=== gridded orbits
!===
!======================================================================


!-----------------------------------------------------------
! gridded orbits
!-----------------------------------------------------------

! time range:
cso.tutorial.gridded.timerange.start        :  ${my.timerange.start}
cso.tutorial.gridded.timerange.end          :  ${my.timerange.end}

! renew existing files?
cso.tutorial.gridded.renew                  :  True
!cso.tutorial.gridded.renew                  :  False

! target directory, incl. subdir for resolution and filters:
my.gridded.dir                              :  ${my.work}/CSO-gridded/${my.region}__r01x01__qa08

!~

! testing ..
my.gridded.region                           :  ${my.region}
! grid definition:
!!~ same as pixel selection on conversion:
!cso.tutorial.gridded.grid.west              :  ${my.region.west}
!cso.tutorial.gridded.grid.east              :  ${my.region.east}
!cso.tutorial.gridded.grid.south             :  ${my.region.south}
!cso.tutorial.gridded.grid.north             :  ${my.region.north}
!~ observation operator tutorial:
cso.tutorial.gridded.grid.west              :  -10
cso.tutorial.gridded.grid.east              :   30
cso.tutorial.gridded.grid.south             :   35
cso.tutorial.gridded.grid.north             :   65
! resolution:
cso.tutorial.gridded.grid.dlon              :  0.1
cso.tutorial.gridded.grid.dlat              :  0.1

! level of recursive splitting of footprint into triangles,
! and assignment of centroids to grid cells;
! for 4-corner footprints, number of centroids is:
!  1 (levels=0), 4 (1), 8 (2), 16 (3), 64 (5), 256 (7)
cso.tutorial.gridded.mapping.levels         :  7

!~

! keywords for source files; 
! the first one should have the footprints:
cso.tutorial.gridded.sources                :  data state

! input files for each source type:
cso.tutorial.gridded.source.data            :  ${my.work}/CSO-oper/CSO_output_%Y%m%d_%H%M_data.nc
cso.tutorial.gridded.source.state           :  ${my.work}/CSO-oper/CSO_output_%Y%m%d_%H%M_state.nc

!~

! keywords for filters:
cso.tutorial.gridded.filters                  :  quality

! minimum quality value required:
cso.tutorial.gridded.filter.quality.var       :  qa_value
cso.tutorial.gridded.filter.quality.type      :  min
cso.tutorial.gridded.filter.quality.min       :  0.8
cso.tutorial.gridded.filter.quality.units     :  1
        
!~

! target file, might contain templates:
!   %Y,%m,etc     : time values
!   %{basename}   : basename (without extension) of first source file
cso.tutorial.gridded.output.file              :  ${my.gridded.dir}/%Y/%m/CSO_output_%Y%m%d_%H%M_gridded.nc

! pack variables on output:
cso.tutorial.gridded.output.packed            :  True
! zlib compression level, 0 for no compression:
cso.tutorial.gridded.output.complevel         :  1

! data variables to be created:
cso.tutorial.gridded.output.vars              :  yr ys yr_m ys_m

! input variables:
!   data:yr    : from data file
!   state:ys    : from state file
cso.tutorial.gridded.output.yr.source         :  data:yr
cso.tutorial.gridded.output.ys.source         :  state:ys
cso.tutorial.gridded.output.yr_m.source       :  state:yr_m
cso.tutorial.gridded.output.ys_m.source       :  state:ys_m


!-----------------------------------------------------------
! catalogue of gridded fields
!-----------------------------------------------------------

! time range:
cso.tutorial.gridded-catalogue.timerange.start        :  ${my.timerange.start}
cso.tutorial.gridded-catalogue.timerange.end          :  ${my.timerange.end}
! hourly fields:
cso.tutorial.gridded-catalogue.timerange.step         :  hour

! renew existing files?
cso.tutorial.gridded-catalogue.renew                  :  True
!cso.tutorial.gridded-catalogue.renew                  :  False

! target directory for catalogue:
cso.tutorial.gridded-catalogue.output.dir             :  ${my.gridded.dir}/catalogue

! input files:
cso.tutorial.gridded-catalogue.input.file             :  ${my.gridded.dir}/%Y/%m/CSO_output_%Y%m%d_%H%M_gridded.nc
!!~ idem for daily average:
!cso.tutorial.gridded-catalogue.input.file             :  ${my.gridded.dir}/CSO_output_%Y%m%d_aver_gridded.nc

! target files, time tempates and variable name are replaced:
cso.tutorial.gridded-catalogue.output.file            :  ${my.gridded.dir}/catalogue/%Y/%m/%d/S5p_RPRO_NO2_%Y%m%d_%H%M_gridded_%{var}.png
!!~ idem for daily average:
!cso.tutorial.gridded-catalogue.output.file            :  ${my.gridded.dir}/catalogue/%Y/%m/%d/S5p_RPRO_NO2_%Y%m%d_aver_gridded_%{var}.png

! figure size (inches), default is (8,6):
cso.tutorial.gridded-catalogue.figsize                :  (6,6)

! variables to be plotted:
cso.tutorial.gridded-catalogue.vars                   :  yr ys yr_m ys_m

! variable:
cso.tutorial.gridded-catalogue.var.yr.source          :  yr
! convert units:
cso.tutorial.gridded-catalogue.var.yr.units           :  umol/m2
! style:
cso.tutorial.gridded-catalogue.var.yr.vmin            :   0.0
cso.tutorial.gridded-catalogue.var.yr.vmax            : 100.0

! variable:
cso.tutorial.gridded-catalogue.var.ys.source          :  ys
! convert units:
cso.tutorial.gridded-catalogue.var.ys.units           :  umol/m2
! style:
cso.tutorial.gridded-catalogue.var.ys.vmin            :   0.0
cso.tutorial.gridded-catalogue.var.ys.vmax            : 100.0

! variable:
cso.tutorial.gridded-catalogue.var.yr_m.source        :  yr_m
! convert units:
cso.tutorial.gridded-catalogue.var.yr_m.units         :  umol/m2
! style:
cso.tutorial.gridded-catalogue.var.yr_m.vmin          :   0.0
cso.tutorial.gridded-catalogue.var.yr_m.vmax          : 100.0

! variable:
cso.tutorial.gridded-catalogue.var.ys_m.source        :  ys_m
! convert units:
cso.tutorial.gridded-catalogue.var.ys_m.units         :  umol/m2
! style:
cso.tutorial.gridded-catalogue.var.ys_m.vmin          :   0.0
cso.tutorial.gridded-catalogue.var.ys_m.vmax          : 100.0



!-----------------------------------------------------------
! gridded catalogue index
!-----------------------------------------------------------

! target location:
cso.tutorial.gridded-catalogue-index.outdir                    :  ${my.gridded.dir}/catalogue

! title:
cso.tutorial.gridded-catalogue-index.header                    :  CSO catalogue
! show info on created page?
cso.tutorial.gridded-catalogue-index.info                      :  True

! create new page for each value?
cso.tutorial.gridded-catalogue-index.newpage                   :  True

! content type:
cso.tutorial.gridded-catalogue-index.type                      :  list
! define row values:
cso.tutorial.gridded-catalogue-index.name                      :  date
cso.tutorial.gridded-catalogue-index.values                    :  TimeSeries( ${my.timerange.start}, ${my.timerange.end}, '1 day', '%Y%m%d' )

! create new page for each value:
cso.tutorial.gridded-catalogue-index.date.newpage              :  True
! content type:
cso.tutorial.gridded-catalogue-index.date.type                 :  table-row
! define row values:
cso.tutorial.gridded-catalogue-index.date.name                 :  time
cso.tutorial.gridded-catalogue-index.date.values               :  Range( 0, 23, 1, '%2.2i00' )

! content type:
cso.tutorial.gridded-catalogue-index.date.time.type            :  table-col
! define column values:
cso.tutorial.gridded-catalogue-index.date.time.name            :  var
cso.tutorial.gridded-catalogue-index.date.time.values          :  ${cso.tutorial.gridded-catalogue.vars}

! content type:
cso.tutorial.gridded-catalogue-index.date.time.var.type        :  img
! define image:
cso.tutorial.gridded-catalogue-index.date.time.var.img         :  %{date[0:4]}/%{date[4:6]}/%{date[6:8]}/S5p_RPRO_NO2_%{date}_%{time}_gridded_%{var}.png
cso.tutorial.gridded-catalogue-index.date.time.var.kwargs      :  height=300


!======================================================================
!===
!=== end
!===
!======================================================================


