!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!!!
!!! CSO - CAMS Satellite Operator
!!!
!!! Settings for S5p/HCHO processing.
!!!
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!


!-----------------------------------------------------------------------
! user specific settings:
!-----------------------------------------------------------------------

! include user specfic settings:
#include cso-user-settings.rc


!======================================================================
!===
!=== Inquire
!===
!======================================================================


!-----------------------------------------------------------------------
! inquire SciHub portal
!-----------------------------------------------------------------------

! Obtain names of all available S5p files.
! Stored as csv with processing type, processor version, filenames, etc.

! inquire full time range:
cso.s5p.hcho.inquire-table-scihub.timerange.start        :  ${my.full-timerange.start}
cso.s5p.hcho.inquire-table-scihub.timerange.end          :  ${my.full-timerange.end}

!
! server url ; 
! provide login/password in ~/.netrc:
!
!   machine s5phub.copernicus.eu      login s5pguest        password s5pguest
!
cso.s5p.hcho.inquire-table-scihub.url                    :  https://s5phub.copernicus.eu/dhus

! target area;
!!~ empty for no limitation:
!cso.s5p.hcho.inquire-table-scihub.area                   :  
!~ domain specified as:  west,south:east,north
cso.s5p.hcho.inquire-table-scihub.area                   :  ${my.region.west},${my.region.south}:${my.region.east},${my.region.north}

! search query, obtained from interactive download:
!
!   platformname     : Sentinel-5
!   producttype      : L2__HCHO___ | L2__CO____    (always 10 characters!)
!   processinglevel  : L2
!
cso.s5p.hcho.inquire-table-scihub.query                  :     platformname:Sentinel-5    AND \
                                                                producttype:L2__HCHO__    AND \
                                                            processinglevel:L2

! output table, date of today:
cso.s5p.hcho.inquire-table-scihub.output.file            :  ${my.work}/Copernicus-inquire/Copernicus_S5p_HCHO_scihub__%Y-%m-%d.csv


!-----------------------------------------------------------
! overview plot of versions
!-----------------------------------------------------------

! renew existing plots?
cso.s5p.hcho.inquire-scihub-plot.renew                    :  True

! listing files:
cso.s5p.hcho.inquire-scihub-plot.file                     :  ${cso.s5p.hcho.inquire-table-scihub.output.file}
!!~ specify dates ("yyyy-mm-dd") to use historic tables,
!!  default is table of today:
!cso.s5p.hcho.inquire-scihub-plot.filedate                 :  2023-08-07

! annote:
cso.s5p.hcho.inquire-plot.title                           :  S5p/HCHO %Y-%m-%d

! output figure, date of today:
cso.s5p.hcho.inquire-scihub-plot.output.file              :  ${my.work}/Copernicus-inquire/Copernicus_S5p_HCHO_scihub__%Y-%m-%d.png


!======================================================================
!===
!=== convert
!===
!======================================================================


! renew existing files (True|False) ?
cso.s5p.hcho.convert.renew                  :  True
!cso.s5p.hcho.convert.renew                  :  False

! time range:
cso.s5p.hcho.convert.timerange.start        :  ${my.timerange.start}
cso.s5p.hcho.convert.timerange.end          :  ${my.timerange.end}


!~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
! input files
!~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

! listing of available source files,
! created by 'inquire-scihub' job:
cso.s5p.hcho.convert.inquire.file                   :  ${my.work}/Copernicus-inquire/Copernicus_S5p_HCHO_scihub__%Y-%m-%d.csv
!!~ historic inquire ...
!cso.s5p.hcho.convert.inquire.filedate               :  ${cso.s5p.hcho.inquire-scihub-plot.filedate}

! selection keyword:
my.s5p.hcho.selection                                :  C03

! Provide ';' seperated list of to decide if a particular orbit file should be processed.
! If more than one file is available for a particular orbit (from "OFFL" and "RPRO" processing),
! the file with the first match will be used.
! The expressions should include templates '%{header}' for the column values.
! Example to select files from collection '03', preferably from processing 'RPRO' but otherwise from 'OFFL':
!   (%{collection} == '03') and (%{processing} == 'RPRO') ; \
!   (%{collection} == '03') and (%{processing} == 'OFFL')
!
#if "${my.s5p.hcho.selection}" == "C03"
cso.s5p.hcho.convert.selection                     :  (%{collection} == '03') and (%{processing} == 'RPRO') ; \
                                                     (%{collection} == '03') and (%{processing} == 'OFFL')
#else
#error unsupported my.s5p.hcho.selection "${my.s5p.hcho.selection}"
#endif

! input directory;
! files are searched here or downloaded to if not present yet;
! supported templates:
!   %{processing}
cso.s5p.hcho.convert.input.dir                      :  ${my.work}/Copernicus/S5P/%{processing}/HCHO/%Y/%m

! remove downloaded input files after convert?
cso.s5p.hcho.convert.downloads.cleanup              :  False

! selection names:
cso.s5p.hcho.convert.filters                        :  lons lats valid quality sza error_ratio ground_pixel

! filter settings:
cso.s5p.hcho.convert.filter.lons.var                :  PRODUCT/longitude
cso.s5p.hcho.convert.filter.lons.type               :  minmax
cso.s5p.hcho.convert.filter.lons.minmax             :  ${my.region.west} ${my.region.east}
cso.s5p.hcho.convert.filter.lons.units              :  degrees_east

! filter settings:
cso.s5p.hcho.convert.filter.lats.var                :  PRODUCT/latitude
cso.s5p.hcho.convert.filter.lats.type               :  minmax
cso.s5p.hcho.convert.filter.lats.minmax             :  ${my.region.south} ${my.region.north}
cso.s5p.hcho.convert.filter.lats.units              :  degrees_north

! skip pixel with "no data", use the "qa_value" variable to check:
cso.s5p.hcho.convert.filter.valid.var               :  PRODUCT/qa_value
cso.s5p.hcho.convert.filter.valid.type              :  valid

! Comment in "PRODUCT/qa_value" variable:
!   "A continuous quality descriptor, 
!   varying between 0 (no data) and 1 (full quality data). 
!   Recommend to ignore data with qa_value < 0.5" 
! 0.5 is the recommended use so what we will use here
cso.s5p.hcho.convert.filter.quality.var             :  PRODUCT/qa_value
cso.s5p.hcho.convert.filter.quality.type            :  min
cso.s5p.hcho.convert.filter.quality.min             :  0.5
cso.s5p.hcho.convert.filter.quality.units           :  1

! recommended 0.3 cloud fraction max (https://agupubs.onlinelibrary.wiley.com/doi/full/10.1029/2018GL081095)
cso.s5p.hcho.convert.filter.cloud_fraction.var             :  PRODUCT/SUPPORT_DATA/DETAILED_RESULTS/cloud_fraction_crb_nitrogendioxide_window
cso.s5p.hcho.convert.filter.cloud_fraction.type            :  max
cso.s5p.hcho.convert.filter.cloud_fraction.max             :  0.3
cso.s5p.hcho.convert.filter.cloud_fraction.units           :  1

! solar zenith angle
! this removes the obs at very high SZA.
cso.s5p.hcho.convert.filter.sza.var             :  PRODUCT/SUPPORT_DATA/GEOLOCATIONS/solar_zenith_angle
cso.s5p.hcho.convert.filter.sza.type            :  max
cso.s5p.hcho.convert.filter.sza.max             :  75
cso.s5p.hcho.convert.filter.sza.units           :  degree

! solar zenith angle
! this removes the obs with very high uncertainties (often for the negative and low conc observations)
cso.s5p.hcho.convert.filter.error_ratio.var             :  PRODUCT/formaldehyde_tropospheric_vertical_column
cso.s5p.hcho.convert.filter.error_ratio.type            :  ratio_square
cso.s5p.hcho.convert.filter.error_ratio.var2            :  PRODUCT/SUPPORT_DATA/DETAILED_RESULTS/formaldehyde_tropospheric_vertical_column_kernel_trueness
cso.s5p.hcho.convert.filter.error_ratio.var3            :  PRODUCT/formaldehyde_tropospheric_vertical_column_precision
cso.s5p.hcho.convert.filter.error_ratio.max             :  1.
cso.s5p.hcho.convert.filter.error_ratio.units           :  1

! filter ground pixel sides, more uncertain observations 50-400, here a few more:
!Theys, N., Hedelt, P., De Smedt, I. et al. Global monitoring of volcanic SO2
! degassing with unprecedented resolution from TROPOMI onboard Sentinel-5 Precursor. Sci Rep 9, 2643 (2019).
! https://doi.org/10.1038/s41598-019-39279-y
cso.s5p.hcho.convert.filter.ground_pixel.var        :  PRODUCT/ground_pixel
cso.s5p.hcho.convert.filter.ground_pixel.type       :  minmax
cso.s5p.hcho.convert.filter.ground_pixel.minmax     :  25 425
cso.s5p.hcho.convert.filter.ground_pixel.units      :  1

!~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
! output files
!~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

! output directory and filename:
! - times are taken from mid of selection, rounded to hours
! - use '%{processing}' for the processing name
! - use '%{orbit}' for orbit number
cso.s5p.hcho.convert.output.filename            :  ${my.work}/CSO-data/${my.region}/S5p/HCHO/${my.s5p.hcho.selection}/%Y/%m/S5p_HCHO_%{orbit}.nc

! pack variables on output:
cso.s5p.hcho.convert.output.packed              :  True
! zlib compression level, 0 for no compression:
cso.s5p.hcho.convert.output.complevel           :  1

! global attributes:
cso.s5p.hcho.convert.output.attrs               :  format Conventions \
                                                     author institution email
!
cso.s5p.hcho.convert.output.attr.format         :  ${my.cso.format}
cso.s5p.hcho.convert.output.attr.Conventions    :  ${my.cso.convention}
cso.s5p.hcho.convert.output.attr.author         :  ${my.attr.author}
cso.s5p.hcho.convert.output.attr.institution    :  ${my.attr.institution}
cso.s5p.hcho.convert.output.attr.email          :  ${my.attr.email}

! no need to swap layes:
cso.s5p.hcho.convert.swap_layers                :  False


! ~ variables

! which fields to be put out ?
cso.s5p.hcho.convert.output.vars    :  longitude longitude_bounds \
                                       latitude latitude_bounds \
                                       track_longitude track_longitude_bounds \
                                       track_latitude  track_latitude_bounds \
                                       time \
                                       pressure  qa_value \
                                       vcd vcd_errvar \
                                       cloud_fraction \                           
                                       ground_pixel \
                                       cloud_pressure_crb \
                                       surface_albedo \
                                       viewing_zenith_angle \
                                       viewing_azimuth_angle \
                                       solar_azimuth_angle \
                                       solar_zenith_angle \
                                       amf_troposphere \
                                       quality \
                                       kernel\									   
                                       surface_classification \
                                       snow_ice_flag         


! 
! Describe per variable:
! * .dims :  dimensions list:
!     pixel       : selected pixels
!     corner      : number of footprint bounds (probably 4)
!     layer       : number of layers in atmospheric profile (layers in kernel)
!     layeri      : number of layer interfaces in atmospheric profile (layer+1)
!     retr        : number of layers in retrieval product (1 for columns) ;
!                   for error covariance use (retr,retr0) to avoid repeated dimensions
!     track_scan  : original scan index   in 2D track
!     track_pixel : original ground pixel in 2D track
! * .specal  : keyword to select special processing
!     * None                     : no special processing (default)
!     * track_longitude          : longitudes at centers of original 2D track
!     * track_latitude           : latitudes  at centers of original 2D track
!     * track_longitude_bounds   : longitude bounds at centers of original 2D track
!     * track_latitude_bounds    : latitude  bounds at centers of original 2D track
! * .units   : target units if different from original
! * .oper    : special postprocessing, currently supported:
!     * square      :  fill variable with squared valued (used to form variance from standard deviation)
! In case no special processing is needed:
! * .from : original variable (group path and variable name)
!

! center longitudes; remove bounds attribute, no coordinate ...
cso.s5p.hcho.convert.output.var.longitude.dims                   :   pixel
cso.s5p.hcho.convert.output.var.longitude.from                   :   PRODUCT/longitude
cso.s5p.hcho.convert.output.var.longitude.attrs                  :   { 'bounds' : None }
! center latitudes; remove bounds attribute, no coordinate ...
cso.s5p.hcho.convert.output.var.latitude.dims                    :   pixel
cso.s5p.hcho.convert.output.var.latitude.from                    :   PRODUCT/latitude
cso.s5p.hcho.convert.output.var.latitude.attrs                   :   { 'bounds' : None }

! corner longitudes; no units in file:
cso.s5p.hcho.convert.output.var.longitude_bounds.dims            :   pixel corner
cso.s5p.hcho.convert.output.var.longitude_bounds.from            :   PRODUCT/SUPPORT_DATA/GEOLOCATIONS/longitude_bounds
cso.s5p.hcho.convert.output.var.longitude_bounds.units           :   degrees_east
! corner latitudes, no units in file:
cso.s5p.hcho.convert.output.var.latitude_bounds.dims             :   pixel corner
cso.s5p.hcho.convert.output.var.latitude_bounds.from             :   PRODUCT/SUPPORT_DATA/GEOLOCATIONS/latitude_bounds
cso.s5p.hcho.convert.output.var.latitude_bounds.units            :   degrees_north

! original track:
!~ center lon; remove bounds attribute, no coordinate ...
cso.s5p.hcho.convert.output.var.track_longitude.dims             :   track_scan track_pixel
cso.s5p.hcho.convert.output.var.track_longitude.special          :   track_longitude
cso.s5p.hcho.convert.output.var.track_longitude.from             :   PRODUCT/longitude
cso.s5p.hcho.convert.output.var.track_longitude.attrs            :   { 'bounds' : None }
!~ center lat; remove bounds attribute, no coordinate ...
cso.s5p.hcho.convert.output.var.track_latitude.dims              :   track_scan track_pixel
cso.s5p.hcho.convert.output.var.track_latitude.special           :   track_latitude
cso.s5p.hcho.convert.output.var.track_latitude.from              :   PRODUCT/latitude
cso.s5p.hcho.convert.output.var.track_latitude.attrs             :   { 'bounds' : None }
!~ corner lons
cso.s5p.hcho.convert.output.var.track_longitude_bounds.dims      :   track_scan track_pixel corner
cso.s5p.hcho.convert.output.var.track_longitude_bounds.special   :   track_longitude_bounds
cso.s5p.hcho.convert.output.var.track_longitude_bounds.from      :   PRODUCT/SUPPORT_DATA/GEOLOCATIONS/longitude_bounds
cso.s5p.hcho.convert.output.var.track_longitude_bounds.units     :   degrees_east
!~ corner lats
cso.s5p.hcho.convert.output.var.track_latitude_bounds.dims       :   track_scan track_pixel corner
cso.s5p.hcho.convert.output.var.track_latitude_bounds.special    :   track_latitude_bounds
cso.s5p.hcho.convert.output.var.track_latitude_bounds.from       :   PRODUCT/SUPPORT_DATA/GEOLOCATIONS/latitude_bounds
cso.s5p.hcho.convert.output.var.track_latitude_bounds.units      :   degrees_north

! time value per scan line
cso.s5p.hcho.convert.output.var.time.dims                        :   pixel
cso.s5p.hcho.convert.output.var.time.special                     :   time-delta
cso.s5p.hcho.convert.output.var.time.tref                        :   PRODUCT/time
cso.s5p.hcho.convert.output.var.time.dt                          :   PRODUCT/delta_time

! vertical column density:
cso.s5p.hcho.convert.output.var.vcd.dims                         :   pixel retr
cso.s5p.hcho.convert.output.var.vcd.from                         :   PRODUCT/formaldehyde_tropospheric_vertical_column

! error variance in vertical column density (after application of kernel),
! fill with single element 'covariance matrix', from square of standard error:
! use dims with different names to avoid that cf-checker complains:
cso.s5p.hcho.convert.output.var.vcd_errvar.dims                  :   pixel retr retr0
cso.s5p.hcho.convert.output.var.vcd_errvar.special               :   square_sum
cso.s5p.hcho.convert.output.var.vcd_errvar.from                  :   PRODUCT/formaldehyde_tropospheric_vertical_column_precision
cso.s5p.hcho.convert.output.var.vcd_errvar.from2                 :   PRODUCT/SUPPORT_DATA/DETAILED_RESULTS/formaldehyde_tropospheric_vertical_column_kernel_trueness
!~ skip standard name, modifier "standard_error" is not valid anymore:
cso.s5p.hcho.convert.output.var.vcd_errvar.attrs                 :   { 'standard_name' : None }

! Convert from hybride coefficient for mid of layer to 3D half level pressure:
cso.s5p.hcho.convert.output.var.pressure.dims                    :   pixel layeri
cso.s5p.hcho.convert.output.var.pressure.special                 :   hymid_to_pressure
cso.s5p.hcho.convert.output.var.pressure.sp                      :   PRODUCT/SUPPORT_DATA/INPUT_DATA/surface_pressure
cso.s5p.hcho.convert.output.var.pressure.hyam                    :   PRODUCT/SUPPORT_DATA/INPUT_DATA/tm5_constant_a
cso.s5p.hcho.convert.output.var.pressure.hybm                    :   PRODUCT/SUPPORT_DATA/INPUT_DATA/tm5_constant_b
cso.s5p.hcho.convert.output.var.pressure.units                   :   Pa

cso.s5p.hcho.convert.output.var.cloud_top_pressure.from         :   Data Fields/CloudPressure
cso.s5p.hcho.convert.output.var.cloud_top_pressure.units        :   hPa
cso.s5p.hcho.convert.output.var.cloud_top_pressure.dims         :   pixel

!! description:
cso.s5p.hcho.convert.output.var.cloud_pressure_crb.from         :   PRODUCT/SUPPORT_DATA/INPUT_DATA/cloud_pressure_crb
cso.s5p.hcho.convert.output.var.cloud_pressure_crb.units        :   Pa
cso.s5p.hcho.convert.output.var.cloud_pressure_crb.dims         :   pixel
!! description:
cso.s5p.hcho.convert.output.var.surface_albedo.from         :   PRODUCT/SUPPORT_DATA/INPUT_DATA/surface_albedo
cso.s5p.hcho.convert.output.var.surface_albedo.units        :   1
cso.s5p.hcho.convert.output.var.surface_albedo.dims         :   pixel
!! description:
cso.s5p.hcho.convert.output.var.amf_troposphere.from            :   PRODUCT/SUPPORT_DATA/DETAILED_RESULTS/formaldehyde_tropospheric_air_mass_factor
cso.s5p.hcho.convert.output.var.amf_troposphere.units           :   1
cso.s5p.hcho.convert.output.var.amf_troposphere.dims         :   pixel
!! description:
cso.s5p.hcho.convert.output.var.viewing_zenith_angle.from         :   PRODUCT/SUPPORT_DATA/GEOLOCATIONS/viewing_zenith_angle
cso.s5p.hcho.convert.output.var.viewing_zenith_angle.units        :   degree
cso.s5p.hcho.convert.output.var.viewing_zenith_angle.dims         :   pixel
!! description:
cso.s5p.hcho.convert.output.var.viewing_azimuth_angle.from         :   PRODUCT/SUPPORT_DATA/GEOLOCATIONS/viewing_azimuth_angle
cso.s5p.hcho.convert.output.var.viewing_azimuth_angle.units        :   degree
cso.s5p.hcho.convert.output.var.viewing_azimuth_angle.dims         :   pixel
!! description:
cso.s5p.hcho.convert.output.var.solar_azimuth_angle.from         :   PRODUCT/SUPPORT_DATA/GEOLOCATIONS/solar_azimuth_angle
cso.s5p.hcho.convert.output.var.solar_azimuth_angle.units        :   degree
cso.s5p.hcho.convert.output.var.solar_azimuth_angle.dims         :   pixel
!! description:
cso.s5p.hcho.convert.output.var.solar_zenith_angle.from         :   PRODUCT/SUPPORT_DATA/GEOLOCATIONS/solar_zenith_angle
cso.s5p.hcho.convert.output.var.solar_zenith_angle.units        :   degree
cso.s5p.hcho.convert.output.var.solar_zenith_angle.dims         :   pixel
!
!! description:
cso.s5p.hcho.convert.output.var.cloud_fraction.from             :   PRODUCT/SUPPORT_DATA/INPUT_DATA/cloud_fraction_crb
cso.s5p.hcho.convert.output.var.cloud_fraction.units            :   1
cso.s5p.hcho.convert.output.var.cloud_fraction.dims         :   pixel
!
!! description:
cso.s5p.hcho.convert.output.var.quality.from             :  PRODUCT/qa_value
cso.s5p.hcho.convert.output.var.quality.units           :  1
cso.s5p.hcho.convert.output.var.quality.dims         :   pixel
!! description:
cso.s5p.hcho.convert.output.var.snow_ice_flag.from             :  PRODUCT/SUPPORT_DATA/INPUT_DATA/snow_ice_flag
cso.s5p.hcho.convert.output.var.snow_ice_flag.units           :  None
cso.s5p.hcho.convert.output.var.snow_ice_flag.dims         :   pixel
!! description:
cso.s5p.hcho.convert.output.var.surface_classification.from             :  PRODUCT/SUPPORT_DATA/INPUT_DATA/surface_classification
cso.s5p.hcho.convert.output.var.surface_classification.units           :  None
cso.s5p.hcho.convert.output.var.surface_classification.dims         :   pixel
!
!! for traceback:
!cso.s5p.hcho.convert.output.var.orbit_number.special            :   orbit_number
!cso.s5p.hcho.convert.output.var.orbit_number.units              :   1
!cso.s5p.hcho.convert.output.var.orbit_number.dims         :   pixel
!
!! for traceback:
!cso.s5p.hcho.convert.output.var.image_number.special            :   scan_number
!cso.s5p.hcho.convert.output.var.image_number.units              :   1
!cso.s5p.hcho.convert.output.var.image_number.dims         :   pixel
!
!! for traceback:
! compression index:
cso.s5p.hcho.convert.output.var.ground_pixel.dims                   :   track_pixel
cso.s5p.hcho.convert.output.var.ground_pixel.special   			   :   ground_pixel
cso.s5p.hcho.convert.output.var.ground_pixel.from                   :   PRODUCT/ground_pixel
!! for traceback:

! description:
cso.s5p.hcho.convert.output.var.kernel.dims                      :   pixel layer retr
cso.s5p.hcho.convert.output.var.kernel.from                      :   PRODUCT/SUPPORT_DATA/DETAILED_RESULTS/averaging_kernel

! quality flag:
cso.s5p.hcho.convert.output.var.qa_value.dims                   :   pixel
cso.s5p.hcho.convert.output.var.qa_value.from                   :   PRODUCT/qa_value
!~ skip some attributes, cf-checker complains ...
cso.s5p.hcho.convert.output.var.qa_value.attrs                  :   { 'valid_min' : None, 'valid_max' : None }


!======================================================================
!===
!=== listing
!===
!======================================================================

! csv file that will hold records per file with:
! - timerange of pixels in file
! - orbit number
cso.s5p.hcho.listing.file                :  ${my.work}/CSO-data/${my.region}/S5p/HCHO/${my.s5p.hcho.selection}__listing.csv

! renew table if file already exists?
cso.s5p.hcho.listing.renew               :  True

! time range:
cso.s5p.hcho.listing.timerange.start     :  ${my.timerange.start}
cso.s5p.hcho.listing.timerange.end       :  ${my.timerange.end}

! filename filters relative to listing file that should be scanned for orbit files;
! names could include time templates ;
! if same orbit is found in multiple directories, the first found is used;
! remove existing table for safety to ensure that this is done correctly ...
cso.s5p.hcho.listing.patterns            :  ${my.s5p.hcho.selection}/%Y/%m/S5p_*.nc

! extra columns to be added, read from global attributes:
cso.s5p.hcho.listing.xcolumns            :  orbit


!======================================================================
!===
!=== catalogue
!===
!======================================================================

! listing file with filenames/timerange.
cso.s5p.hcho.catalogue.input.listing               :  ${cso.s5p.hcho.listing.file}

! time range:
cso.s5p.hcho.catalogue.timerange.start             :  ${my.timerange.start}
cso.s5p.hcho.catalogue.timerange.end               :  ${my.timerange.end}

! target filenames; templates:
! - time values
! - %{orbit}   : from listing
! - %{varname} for variable
cso.s5p.hcho.catalogue.output.file                 :  ${my.work}/CSO-data-catalogue/${my.region}/S5p/HCHO/${my.s5p.hcho.selection}/%Y/%m/%d/S5p_HCHO_%{orbit}__%{varname}.png

! map domain (west east south north)
cso.s5p.hcho.catalogue.domain                      :  ${my.region.west} ${my.region.east} ${my.region.south} ${my.region.north}

! figure size (inches), default is (8,6):
cso.s5p.hcho.catalogue.figsize                     :  ${my.region.figsize}

! renew existing files?
cso.s5p.hcho.catalogue.renew                       :  False
!cso.s5p.hcho.catalogue.renew                       :  True

! variables to be plotted:
cso.s5p.hcho.catalogue.vars                        :  vcd vcd_errvar qa_value cloud_fraction

!! color for no-data values in track, default '0.8' (gray):
!cso.s5p.hcho.catalogue.color_nan                   :  white

!! extra keyword arguments for map:
!cso.s5p.hcho.catalogue.map                         :  resolution='h'

! convert units:
cso.s5p.hcho.catalogue.var.vcd.units                         :  umol/m2
! style:
cso.s5p.hcho.catalogue.var.vcd.vmin                          :   0.0
!cso.s5p.hcho.catalogue.var.vcd.vmax                          :  100.0
cso.s5p.hcho.catalogue.var.vcd.vmax                          :  None

! show error as std.dev, convert to vcd units:
cso.s5p.hcho.catalogue.var.vcd_errvar.units                  :  umol/m2
! style:
!cso.s5p.hcho.catalogue.var.vcd_errvar.vmax                   :  100.0
cso.s5p.hcho.catalogue.var.vcd_errvar.vmax                   :  None

! style:
cso.s5p.hcho.catalogue.var.qa_value.vmin                     :   0.5
cso.s5p.hcho.catalogue.var.qa_value.vmax                     :   1.0
cso.s5p.hcho.catalogue.var.qa_value.colors                   :   ['red','yellow','green']

! style:
cso.s5p.hcho.catalogue.var.cloud_fraction.vmax               :   1.0
cso.s5p.hcho.catalogue.var.cloud_fraction.colors             :   ['blue','cyan','white']


!-----------------------------------------------------------
! catalogue index
!-----------------------------------------------------------

! target location:
cso.s5p.hcho.catalogue-index.outdir                    :  ${my.work}/CSO-data-catalogue/${my.region}/S5p/HCHO/${my.s5p.hcho.selection}

! title:
cso.s5p.hcho.catalogue-index.header                    :  CSO catalogue
! show info on created page?
cso.s5p.hcho.catalogue-index.info                      :  True

! create new page for each value?
cso.s5p.hcho.catalogue-index.newpage                   :  True

! content type:
cso.s5p.hcho.catalogue-index.type                      :  list
! define row values:
cso.s5p.hcho.catalogue-index.name                      :  date
cso.s5p.hcho.catalogue-index.values                    :  TimeSeries( ${my.timerange.start}, ${my.timerange.end}, '1 day', '%Y%m%d' )

! create new page for each value:
cso.s5p.hcho.catalogue-index.date.newpage                   :  True
! content type:
cso.s5p.hcho.catalogue-index.date.type                      :  table-row
! define row values:
cso.s5p.hcho.catalogue-index.date.name                      :  orbit
cso.s5p.hcho.catalogue-index.date.values                    :  CsvFile( '%{date[0:4]}/%{date[4:6]}/%{date[6:8]}/orbits.csv' )

! content type:
cso.s5p.hcho.catalogue-index.date.orbit.type                :  table-col
! define column values:
cso.s5p.hcho.catalogue-index.date.orbit.name                :  var
cso.s5p.hcho.catalogue-index.date.orbit.values              :  ${cso.s5p.hcho.catalogue.vars}

! content type:
cso.s5p.hcho.catalogue-index.date.orbit.var.type            :  img
! define image:
cso.s5p.hcho.catalogue-index.date.orbit.var.img             :  %{date[0:4]}/%{date[4:6]}/%{date[6:8]}/S5p_HCHO_%{orbit}__%{var}.png
cso.s5p.hcho.catalogue-index.date.orbit.var.kwargs          :  height=300



!======================================================================
!===
!=== gridded orbits
!===
!======================================================================


!-----------------------------------------------------------
! gridded orbits
!-----------------------------------------------------------

! time range:
cso.s5p.hcho.gridded.timerange.start        :  ${my.timerange.start}
cso.s5p.hcho.gridded.timerange.end          :  ${my.timerange.end}
! create one gridded file per hour:
cso.s5p.hcho.gridded.timerange.step         :  hour

! renew existing files?
cso.s5p.hcho.gridded.renew                  :  True

! target directory, incl. subdir for resolution and filters:
my.gridded.dir                              :  ${my.work}/CSO-gridded/${my.region}__r01x01/S5p/HCHO/${my.s5p.hcho.selection}__${my.s5p.hcho.gridded-selection}

!~

! grid definition:
!~ same as pixel selection on conversion:
cso.s5p.hcho.gridded.grid.west              :  ${my.region.west}
cso.s5p.hcho.gridded.grid.east              :  ${my.region.east}
cso.s5p.hcho.gridded.grid.south             :  ${my.region.south}
cso.s5p.hcho.gridded.grid.north             :  ${my.region.north}
! resolution:
cso.s5p.hcho.gridded.grid.dlon              :  0.1
cso.s5p.hcho.gridded.grid.dlat              :  0.1

! level of recursive splitting of footprint into triangles,
! and assignment of centroids to grid cells;
! for 4-corner footprints, number of centroids is:
!  1 (levels=0), 4 (1), 8 (2), 16 (3), 64 (5), 256 (7)
cso.s5p.hcho.gridded.mapping.levels         :  5

!~

! keywords for source files; 
! the first one should have the footprints;
! here the only source are the converted files:
cso.s5p.hcho.gridded.sources                  :  data

! input files for each source type:
!~ here: specify listing file, 
!  this is only supported for a single source
cso.s5p.hcho.gridded.source.data.listing      :  ${cso.s5p.hcho.listing.file}
!!~ alternative: filename patterns with time templates,
!!  but here the source files do not have time stamps
!cso.s5p.hcho.gridded.source.data.filenames    :  ${my.work}/CSO-data/S5p/RPRO/HCHO/${my.region}/%Y/%m/S5p_*.nc

!~

! filter description:
my.s5p.hcho.gridded-selection                 :  qa08

! switch:
#if "${my.s5p.hcho.gridded-selection}" == "qa08"

! keywords for filters:
cso.s5p.hcho.gridded.filters                  :  quality

! minimum quality value required:
cso.s5p.hcho.gridded.filter.quality.var       :  qa_value
cso.s5p.hcho.gridded.filter.quality.type      :  min
cso.s5p.hcho.gridded.filter.quality.min       :  0.8
cso.s5p.hcho.gridded.filter.quality.units     :  1
       
#else
#error unsupported  my.s5p.hcho.gridded-selection "${my.s5p.hcho.gridded-selection}"
#endif
 
!~

! target file, might contain templates:
!   %Y,%m,etc     : time values
cso.s5p.hcho.gridded.output.file               :  ${my.gridded.dir}/%Y/%m/S5p_HCHO_%Y%m%d_%H%M_gridded.nc

! pack floats as short values?
cso.s5p.hcho.gridded.output.packed            :  True
! zlib compression level (default 1, 0 for no compression):
cso.s5p.hcho.gridded.output.complevel         :  1

! data variables to be created:
cso.s5p.hcho.gridded.output.vars              :  yr

! input variables:
!   data:vcd    : variable "vcd" from source "data"
cso.s5p.hcho.gridded.output.yr.source         :  data:vcd


!-----------------------------------------------------------
! catalogue of gridded fields
!-----------------------------------------------------------

! time range:
cso.s5p.hcho.gridded-catalogue.timerange.start        :  ${my.timerange.start}
cso.s5p.hcho.gridded-catalogue.timerange.end          :  ${my.timerange.end}
! create one gridded file per hour:
cso.s5p.hcho.gridded-catalogue.timerange.step         :  hour

! renew existing files?
!cso.s5p.hcho.gridded-catalogue.renew                  :  True
cso.s5p.hcho.gridded-catalogue.renew                  :  False

! target directory for catalogue:
cso.s5p.hcho.gridded-catalogue.output.dir             :  ${my.gridded.dir}/catalogue

! input files:
cso.s5p.hcho.gridded-catalogue.input.file             :  ${my.gridded.dir}/%Y/%m/S5p_HCHO_%Y%m%d_%H%M_gridded.nc
!!~ idem for daily average:
!cso.s5p.hcho.gridded-catalogue.input.file             :  ${my.gridded.dir}/S5p_HCHO_%Y%m%d_aver_gridded.nc

! target files, time tempates and variable name are replaced:
cso.s5p.hcho.gridded-catalogue.output.file            :  ${my.gridded.dir}/catalogue/%Y/%m/%d/S5p_HCHO_%Y%m%d_%H%M_gridded_%{var}.png
!!~ idem for daily average:
!cso.s5p.hcho.gridded-catalogue.output.file            :  ${my.gridded.dir}/catalogue/%Y/%m/%d/S5p_HCHO_%Y%m%d_aver_gridded_%{var}.png

! figure size (inches), default is (8,6):
cso.s5p.hcho.gridded-catalogue.figsize                :  (6,6)

! variables to be plotted:
cso.s5p.hcho.gridded-catalogue.vars                   :  yr

! variable:
cso.s5p.hcho.gridded-catalogue.var.yr.source          :  yr
! convert units:
cso.s5p.hcho.gridded-catalogue.var.yr.units           :  umol/m2
! style:
cso.s5p.hcho.gridded-catalogue.var.yr.vmin            :   0.0
!gcso.s5p.hcho.gridded-catalogue.var.yr.vmax            : 100.0
cso.s5p.hcho.gridded-catalogue.var.yr.vmax            : None




!-----------------------------------------------------------
! gridded catalogue index
!-----------------------------------------------------------

! target location:
cso.s5p.hcho.gridded-catalogue-index.outdir                    :  ${cso.s5p.hcho.gridded-catalogue.output.dir}

! title:
cso.s5p.hcho.gridded-catalogue-index.header                    :  CSO catalogue
! show info on created page?
cso.s5p.hcho.gridded-catalogue-index.info                      :  True

! create new page for each value?
cso.s5p.hcho.gridded-catalogue-index.newpage                   :  True

! content type:
cso.s5p.hcho.gridded-catalogue-index.type                      :  list
! define row values:
cso.s5p.hcho.gridded-catalogue-index.name                      :  date
cso.s5p.hcho.gridded-catalogue-index.values                    :  TimeSeries( ${my.timerange.start}, ${my.timerange.end}, '1 day', '%Y%m%d' )

! create new page for each value:
cso.s5p.hcho.gridded-catalogue-index.date.newpage              :  True
! content type:
cso.s5p.hcho.gridded-catalogue-index.date.type                 :  table-row
! define row values:
cso.s5p.hcho.gridded-catalogue-index.date.name                 :  time
cso.s5p.hcho.gridded-catalogue-index.date.values               :  Range( 0, 23, 1, '%2.2i00' )

! content type:
cso.s5p.hcho.gridded-catalogue-index.date.time.type            :  table-col
! define column values:
cso.s5p.hcho.gridded-catalogue-index.date.time.name            :  var
cso.s5p.hcho.gridded-catalogue-index.date.time.values          :  ${cso.s5p.hcho.gridded-catalogue.vars}

! content type:
cso.s5p.hcho.gridded-catalogue-index.date.time.var.type        :  img
! define image:
cso.s5p.hcho.gridded-catalogue-index.date.time.var.img         :  %{date[0:4]}/%{date[4:6]}/%{date[6:8]}/S5p_HCHO_%{date}_%{time}_gridded_%{var}.png
cso.s5p.hcho.gridded-catalogue-index.date.time.var.kwargs      :  height=300



!======================================================================
!===
!=== simulation catalogue
!===
!======================================================================


!-----------------------------------------------------------
! simulation catalogue
!-----------------------------------------------------------

! time range:
cso.s5p.hcho.sim-catalogue.timerange.start        :  ${my.timerange.start}
cso.s5p.hcho.sim-catalogue.timerange.end          :  ${my.timerange.end}
cso.s5p.hcho.sim-catalogue.timerange.step         :  hour

! input files:
cso.s5p.hcho.sim-catalogue.input.data.file        :  ${my.work}/CSO-oper/CSO_output_%Y%m%d_%H%M_data.nc
cso.s5p.hcho.sim-catalogue.input.state.file       :  ${my.work}/CSO-oper/CSO_output_%Y%m%d_%H%M_state.nc

! target files, time tempates and variable name are replaced:
cso.s5p.hcho.sim-catalogue.output.file            :  ${my.work}/CSO-data-sim-catalogue/S5p/HCHO/${my.region}/%Y/%m/%d/S5p_HCHO_%Y%m%d_%H%M_%{var}.png

! map domain used for simulations (west east south north):
cso.s5p.hcho.sim-catalogue.domain                 :  ${my.region.west} ${my.region.east} ${my.region.south} ${my.region.north}

! figure size (inches), default is (8,6):
cso.s5p.hcho.sim-catalogue.figsize                :  ${my.region.figsize}

! renew existing files?
cso.s5p.hcho.sim-catalogue.renew                  :  False
!cso.s5p.hcho.sim-catalogue.renew                  :  True

! variables to be plotted:
cso.s5p.hcho.sim-catalogue.vars                   :  yr ys yr_m ys_m

! variable:
cso.s5p.hcho.sim-catalogue.var.yr.source          :  data:yr
! convert units:
cso.s5p.hcho.sim-catalogue.var.yr.units           :  umol/m2
! style:
cso.s5p.hcho.sim-catalogue.var.yr.vmin            :   0.0
!cso.s5p.hcho.sim-catalogue.var.yr.vmax            : 100.0
cso.s5p.hcho.sim-catalogue.var.yr.vmax            : None

! variable:
cso.s5p.hcho.sim-catalogue.var.ys.source          :  state:ys
! convert units:
cso.s5p.hcho.sim-catalogue.var.ys.units           :  umol/m2
! style:
cso.s5p.hcho.sim-catalogue.var.ys.vmin            :   0.0
!cso.s5p.hcho.sim-catalogue.var.ys.vmax            : 100.0
cso.s5p.hcho.sim-catalogue.var.ys.vmax            : None



!-----------------------------------------------------------
! sim catalogue index
!-----------------------------------------------------------

! target location:
cso.s5p.hcho.sim-catalogue-index.outdir                    :  ${my.work}/CSO-sim-catalogue

! title:
cso.s5p.hcho.sim-catalogue-index.header                    :  CSO catalogue
! show info on created page?
cso.s5p.hcho.sim-catalogue-index.info                      :  True

! create new page for each value?
cso.s5p.hcho.sim-catalogue-index.newpage                   :  True

! content type:
cso.s5p.hcho.sim-catalogue-index.type                      :  list
! define row values:
cso.s5p.hcho.sim-catalogue-index.name                      :  date
cso.s5p.hcho.sim-catalogue-index.values                    :  TimeSeries( ${my.timerange.start}, ${my.timerange.end}, '1 day', '%Y%m%d' )

! create new page for each value:
cso.s5p.hcho.sim-catalogue-index.date.newpage              :  True
! content type:
cso.s5p.hcho.sim-catalogue-index.date.type                 :  table-row
! define row values:
cso.s5p.hcho.sim-catalogue-index.date.name                 :  time
cso.s5p.hcho.sim-catalogue-index.date.values               :  Range( 0, 23, 1, '%2.2i00' )

! content type:
cso.s5p.hcho.sim-catalogue-index.date.time.type            :  table-col
! define column values:
cso.s5p.hcho.sim-catalogue-index.date.time.name            :  var
cso.s5p.hcho.sim-catalogue-index.date.time.values          :  ${cso.s5p.hcho.sim-catalogue.vars}

! content type:
cso.s5p.hcho.sim-catalogue-index.date.time.var.type        :  img
! define image:
cso.s5p.hcho.sim-catalogue-index.date.time.var.img         :  %{date[0:4]}/%{date[4:6]}/%{date[6:8]}/S5p_HCHO_%{date}_%{time}_%{var}.png
cso.s5p.hcho.sim-catalogue-index.date.time.var.kwargs      :  height=300



!======================================================================
!===
!=== gridded simulated orbits
!===
!======================================================================


!-----------------------------------------------------------
! gridded orbits
!-----------------------------------------------------------

! time range:
cso.s5p.hcho.sim-gridded.timerange.start        :  ${my.timerange.start}
cso.s5p.hcho.sim-gridded.timerange.end          :  ${my.timerange.end}

! renew existing files?
cso.s5p.hcho.sim-gridded.renew                  :  True

! keyword used for subdir for resolution and filters:
my.gridded.subdir                           :  ${my.region}__r01x01__qa08

!~

! testing ..
my.gridded.region                           :  ${my.region}
! grid definition:
!~ same as pixel selection on conversion:
cso.s5p.hcho.sim-gridded.grid.west              :  ${my.region.west}
cso.s5p.hcho.sim-gridded.grid.east              :  ${my.region.east}
cso.s5p.hcho.sim-gridded.grid.south             :  ${my.region.south}
cso.s5p.hcho.sim-gridded.grid.north             :  ${my.region.north}
! resolution:
cso.s5p.hcho.sim-gridded.grid.dlon              :  0.1
cso.s5p.hcho.sim-gridded.grid.dlat              :  0.1

!~

! keywords for source files; 
! the first one should have the footprints:
cso.s5p.hcho.sim-gridded.sources                :  data state

! input files for each source type:
cso.s5p.hcho.sim-gridded.source.data            :  ${my.work}/CSO-oper/CSO_output_%Y%m%d_%H%M_data.nc
cso.s5p.hcho.sim-gridded.source.state           :  ${my.work}/CSO-oper/CSO_output_%Y%m%d_%H%M_state.nc

!~

! keywords for filters:
cso.s5p.hcho.sim-gridded.filters                  :  quality

! minimum quality value required:
cso.s5p.hcho.sim-gridded.filter.quality.var       :  qa_value
cso.s5p.hcho.sim-gridded.filter.quality.type      :  min
cso.s5p.hcho.sim-gridded.filter.quality.min       :  0.8
cso.s5p.hcho.sim-gridded.filter.quality.units     :  1
        
!~

! target file, might contain templates:
!   %Y,%m,etc     : time values
!   %{basename}   : basename (without extension) of first source file
cso.s5p.hcho.sim-gridded.output.file              :  ${my.work}/CSO-gridded/${my.gridded.subdir}/%Y/%m/CSO_output_%Y%m%d_%H%M_gridded.nc

! data variables to be created:
cso.s5p.hcho.sim-gridded.output.vars              :  yr ys yr_m ys_m

! input variables:
!   data:yr    : from data file
!   state:ys    : from state file
cso.s5p.hcho.sim-gridded.output.yr.source         :  data:yr
cso.s5p.hcho.sim-gridded.output.ys.source         :  state:ys
cso.s5p.hcho.sim-gridded.output.yr_m.source       :  state:yr_m
cso.s5p.hcho.sim-gridded.output.ys_m.source       :  state:ys_m


!-----------------------------------------------------------
! catalogue of gridded fields
!-----------------------------------------------------------

! time range:
cso.s5p.hcho.sim-gridded-catalogue.timerange.start        :  ${my.timerange.start}
cso.s5p.hcho.sim-gridded-catalogue.timerange.end          :  ${my.timerange.end}

! renew existing files?
cso.s5p.hcho.sim-gridded-catalogue.renew                  :  True
!cso.s5p.hcho.sim-gridded-catalogue.renew                  :  False

! target directory for catalogue:
cso.s5p.hcho.sim-gridded-catalogue.output.dir             :  ${my.work}/CSO-gridded/${my.gridded.subdir}/catalogue

! input files:
cso.s5p.hcho.sim-gridded-catalogue.input.file             :  ../%Y/%m/CSO_output_%Y%m%d_%H%M_gridded.nc
!!~ idem for daily average:
!cso.s5p.hcho.sim-gridded-catalogue.input.file             :  ../CSO_output_%Y%m%d_aver_gridded.nc

! target files, time tempates and variable name are replaced:
cso.s5p.hcho.sim-gridded-catalogue.output.file            :  %Y/%m/%d/S5p_RPRO_HCHO_%Y%m%d_%H%M_gridded_%{var}.png
!!~ idem for daily average:
!cso.s5p.hcho.sim-gridded-catalogue.output.file            :  %Y/%m/%d/S5p_RPRO_HCHO_%Y%m%d_aver_gridded_%{var}.png

! figure size (inches), default is (8,6):
cso.s5p.hcho.sim-gridded-catalogue.figsize                :  (6,6)

! variables to be plotted:
cso.s5p.hcho.sim-gridded-catalogue.vars                   :  yr ys yr_m ys_m

! variable:
cso.s5p.hcho.sim-gridded-catalogue.var.yr.source          :  yr
! convert units:
cso.s5p.hcho.sim-gridded-catalogue.var.yr.units           :  umol/m2
! style:
cso.s5p.hcho.sim-gridded-catalogue.var.yr.vmin            :   0.0
!cso.s5p.hcho.sim-gridded-catalogue.var.yr.vmax            : 100.0
cso.s5p.hcho.sim-gridded-catalogue.var.yr.vmax            : None

! variable:
cso.s5p.hcho.sim-gridded-catalogue.var.ys.source          :  ys
! convert units:
cso.s5p.hcho.sim-gridded-catalogue.var.ys.units           :  umol/m2
! style:
cso.s5p.hcho.sim-gridded-catalogue.var.ys.vmin            :   0.0
!cso.s5p.hcho.sim-gridded-catalogue.var.ys.vmax            : 100.0
cso.s5p.hcho.sim-gridded-catalogue.var.ys.vmax            : None



!-----------------------------------------------------------
! gridded catalogue index
!-----------------------------------------------------------

! target location:
cso.s5p.hcho.sim-gridded-catalogue-index.outdir                    :  ${cso.s5p.hcho.sim-gridded-catalogue.output.dir}

! title:
cso.s5p.hcho.sim-gridded-catalogue-index.header                    :  CSO catalogue
! show info on created page?
cso.s5p.hcho.sim-gridded-catalogue-index.info                      :  True

! create new page for each value?
cso.s5p.hcho.sim-gridded-catalogue-index.newpage                   :  True

! content type:
cso.s5p.hcho.sim-gridded-catalogue-index.type                      :  list
! define row values:
cso.s5p.hcho.sim-gridded-catalogue-index.name                      :  date
cso.s5p.hcho.sim-gridded-catalogue-index.values                    :  TimeSeries( ${my.timerange.start}, ${my.timerange.end}, '1 day', '%Y%m%d' )

! create new page for each value:
cso.s5p.hcho.sim-gridded-catalogue-index.date.newpage              :  True
! content type:
cso.s5p.hcho.sim-gridded-catalogue-index.date.type                 :  table-row
! define row values:
cso.s5p.hcho.sim-gridded-catalogue-index.date.name                 :  time
cso.s5p.hcho.sim-gridded-catalogue-index.date.values               :  Range( 0, 23, 1, '%2.2i00' )

! content type:
cso.s5p.hcho.sim-gridded-catalogue-index.date.time.type            :  table-col
! define column values:
cso.s5p.hcho.sim-gridded-catalogue-index.date.time.name            :  var
cso.s5p.hcho.sim-gridded-catalogue-index.date.time.values          :  ${cso.s5p.hcho.sim-gridded-catalogue.vars}

! content type:
cso.s5p.hcho.sim-gridded-catalogue-index.date.time.var.type        :  img
! define image:
cso.s5p.hcho.sim-gridded-catalogue-index.date.time.var.img         :  %{date[0:4]}/%{date[4:6]}/%{date[6:8]}/S5p_RPRO_HCHO_%{date}_%{time}_gridded_%{var}.png
cso.s5p.hcho.sim-gridded-catalogue-index.date.time.var.kwargs      :  height=300


!======================================================================
!===
!=== end
!===
!======================================================================


